From e7f89204631cc09a0de3b9d8895663c5502c006d Mon Sep 17 00:00:00 2001 From: panos-xyz Date: Mon, 13 Jul 2026 21:48:22 +0800 Subject: [PATCH 1/6] refactor: migrate engine validation to reth main hooks --- Cargo.lock | 1624 ++++++------ Cargo.toml | 170 +- crates/chainspec/src/hardfork.rs | 11 +- crates/consensus/src/validation.rs | 5 +- crates/engine-api/src/builder.rs | 3 +- crates/engine-tree-ext/Cargo.toml | 80 - crates/engine-tree-ext/src/gate.rs | 67 - crates/engine-tree-ext/src/lib.rs | 13 - .../engine-tree-ext/src/payload_validator.rs | 2193 ----------------- crates/engine-tree-ext/src/trie_updates.rs | 357 --- crates/engine-tree-ext/tests/jade_boundary.rs | 212 -- crates/evm/Cargo.toml | 1 + crates/evm/src/block/mod.rs | 16 +- crates/evm/src/context.rs | 3 +- crates/evm/src/evm.rs | 5 +- crates/node/Cargo.toml | 6 +- crates/node/build.rs | 15 +- crates/node/src/node.rs | 1 + crates/node/src/test_utils.rs | 4 +- crates/node/src/validator.rs | 502 +++- crates/node/tests/it/engine.rs | 40 +- crates/node/tests/it/helpers.rs | 11 +- crates/payload/builder/src/builder.rs | 33 +- crates/payload/types/src/attributes.rs | 1 + crates/payload/types/src/lib.rs | 9 +- crates/revm/src/evm.rs | 73 +- crates/revm/src/precompiles.rs | 5 +- crates/rpc/src/eth/call.rs | 4 + crates/rpc/src/eth/mod.rs | 25 +- 29 files changed, 1494 insertions(+), 3995 deletions(-) delete mode 100644 crates/engine-tree-ext/Cargo.toml delete mode 100644 crates/engine-tree-ext/src/gate.rs delete mode 100644 crates/engine-tree-ext/src/lib.rs delete mode 100644 crates/engine-tree-ext/src/payload_validator.rs delete mode 100644 crates/engine-tree-ext/src/trie_updates.rs delete mode 100644 crates/engine-tree-ext/tests/jade_boundary.rs diff --git a/Cargo.lock b/Cargo.lock index 5e07217b..52398dd5 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -14,7 +14,7 @@ version = "0.5.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d122413f284cf2d62fb1b7db97e02edb8cda96d769b16e443a4f6195e35662b0" dependencies = [ - "crypto-common", + "crypto-common 0.1.7", "generic-array", ] @@ -101,14 +101,14 @@ dependencies = [ [[package]] name = "alloy-consensus" -version = "2.0.4" +version = "2.1.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e3d64da86c616b5092ea64eea648f311bbd58630a0b384c42d699175d6f9122b" +checksum = "a6ff0c4adba2abdcd9fb5829ae5f4394c06f8585ed283a9ba79aa33763c802e1" dependencies = [ - "alloy-eips 2.0.4", + "alloy-eips", "alloy-primitives", "alloy-rlp", - "alloy-serde 2.0.4", + "alloy-serde", "alloy-trie", "alloy-tx-macros", "arbitrary", @@ -129,24 +129,24 @@ dependencies = [ [[package]] name = "alloy-consensus-any" -version = "2.0.4" +version = "2.1.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8fd98696ca3617d3a9ba1a6f2011880cbfd5618228dab6400c9f8bca457859a8" +checksum = "7cdf48932b1db3216175e19a2b476d89d53076e004850ee7983c6807ba0fde74" dependencies = [ "alloy-consensus", - "alloy-eips 2.0.4", + "alloy-eips", "alloy-primitives", "alloy-rlp", - "alloy-serde 2.0.4", + "alloy-serde", "arbitrary", "serde", ] [[package]] name = "alloy-dyn-abi" -version = "1.5.7" +version = "1.6.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "cc2db5c583aaef0255aa63a4fe827f826090142528bba48d1bf4119b62780cad" +checksum = "a475bb02d9cef2dbb99065c1664ab3fe1f9352e21d6d5ed3f02cdbfc06ed1abc" dependencies = [ "alloy-json-abi", "alloy-primitives", @@ -156,7 +156,7 @@ dependencies = [ "itoa", "serde", "serde_json", - "winnow 0.7.15", + "winnow 1.0.2", ] [[package]] @@ -207,9 +207,9 @@ dependencies = [ [[package]] name = "alloy-eip7928" -version = "0.3.6" +version = "0.4.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "72bc95fd73591644f0022065bef3c027d1a5ed875a53cb19a898d71a00818c1d" +checksum = "b3b12337f74cbfa451cb04dac173974814a6ff463079e1793aa09600ba8813ab" dependencies = [ "alloy-primitives", "alloy-rlp", @@ -222,32 +222,9 @@ dependencies = [ [[package]] name = "alloy-eips" -version = "1.8.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e6ef28c9fdad22d4eec52d894f5f2673a0895f1e5ef196734568e68c0f6caca8" -dependencies = [ - "alloy-eip2124", - "alloy-eip2930", - "alloy-eip7702", - "alloy-eip7928", - "alloy-primitives", - "alloy-rlp", - "alloy-serde 1.8.3", - "auto_impl", - "borsh", - "c-kzg", - "derive_more", - "either", - "serde", - "serde_with", - "sha2", -] - -[[package]] -name = "alloy-eips" -version = "2.0.4" +version = "2.1.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "64c0456f5f7a4497e9342d20f528e30f5288ddfa0d6a012bd5044afee46cd8a0" +checksum = "ea4c0453065b9206acc0f869a258dc8dcbbd595e144b4446f2c493a24a814d1f" dependencies = [ "alloy-eip2124", "alloy-eip2930", @@ -255,7 +232,7 @@ dependencies = [ "alloy-eip7928", "alloy-primitives", "alloy-rlp", - "alloy-serde 2.0.4", + "alloy-serde", "arbitrary", "auto_impl", "borsh", @@ -271,12 +248,12 @@ dependencies = [ [[package]] name = "alloy-evm" -version = "0.34.0" +version = "0.37.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c1ceeea6dcbbcd4e546b27700763a6f6c3b3fee30054209884f521078b6fda4f" +checksum = "acde665074b478c97047dc2a461b4916cac77922d690e5b7415d1941ae7ff7bf" dependencies = [ "alloy-consensus", - "alloy-eips 2.0.4", + "alloy-eips", "alloy-hardforks", "alloy-primitives", "alloy-rpc-types-engine", @@ -291,13 +268,13 @@ dependencies = [ [[package]] name = "alloy-genesis" -version = "2.0.4" +version = "2.1.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a71ff8b55d2b8aa05259f474cae7dea0e4991724dc18936b81cb23ec492a0c2a" +checksum = "027ba57264c5d05e4ff52e6090b3592e7526f5d5f5a844add6bbdd17c071c911" dependencies = [ - "alloy-eips 2.0.4", + "alloy-eips", "alloy-primitives", - "alloy-serde 2.0.4", + "alloy-serde", "alloy-trie", "borsh", "serde", @@ -320,9 +297,9 @@ dependencies = [ [[package]] name = "alloy-json-abi" -version = "1.5.7" +version = "1.6.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e9dbe713da0c737d9e5e387b0ba790eb98b14dd207fe53eef50e19a5a8ec3dac" +checksum = "7c36c9d7f9021601b04bfef14a4b64849f6d73116a4e91e071d7fbfe10247901" dependencies = [ "alloy-primitives", "alloy-sol-type-parser", @@ -332,9 +309,9 @@ dependencies = [ [[package]] name = "alloy-json-rpc" -version = "2.0.4" +version = "2.1.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "19e352478b756bad5d7203148e4b461861282ea2ded3da406ba24868b52cd098" +checksum = "c4691c60de5d628533752cd07e102d17c47874c06c04c91af33960fd94c484f4" dependencies = [ "alloy-primitives", "alloy-sol-types", @@ -347,19 +324,19 @@ dependencies = [ [[package]] name = "alloy-network" -version = "2.0.4" +version = "2.1.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ed08ae169869e08370ed121612e0d3dadac33d1a256e9f2465926b23f0bd7d95" +checksum = "9d912bb639bf4ac31e83095afe9e907c8b9774ce0c405966228368309fcfc45f" dependencies = [ "alloy-consensus", "alloy-consensus-any", - "alloy-eips 2.0.4", + "alloy-eips", "alloy-json-rpc", "alloy-network-primitives", "alloy-primitives", "alloy-rpc-types-any", "alloy-rpc-types-eth", - "alloy-serde 2.0.4", + "alloy-serde", "alloy-signer", "alloy-sol-types", "async-trait", @@ -373,22 +350,22 @@ dependencies = [ [[package]] name = "alloy-network-primitives" -version = "2.0.4" +version = "2.1.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "02e6c7ad28afe348a9a9c5624b67ee5b3607b8de98d5816b3056ecdfa6fa2697" +checksum = "d875a11bd98595f57f73890f2e36f4a1cca0fa670623e59c9fb08b2389979c36" dependencies = [ "alloy-consensus", - "alloy-eips 2.0.4", + "alloy-eips", "alloy-primitives", - "alloy-serde 2.0.4", + "alloy-serde", "serde", ] [[package]] name = "alloy-primitives" -version = "1.5.7" +version = "1.6.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "de3b431b4e72cd8bd0ec7a50b4be18e73dab74de0dba180eef171055e5d5926e" +checksum = "4885c1409b6936c4898e646ef58baf6ec54edaf6d8179f79df805a7b85b7cf3e" dependencies = [ "alloy-rlp", "arbitrary", @@ -398,31 +375,32 @@ dependencies = [ "derive_more", "foldhash 0.2.0", "getrandom 0.4.2", - "hashbrown 0.16.1", + "hashbrown 0.17.0", "indexmap 2.14.0", "itoa", "k256", "keccak-asm", "paste", "proptest", - "proptest-derive", + "proptest-derive 0.8.0", "rand 0.9.4", "rapidhash", "ruint", "rustc-hash", + "secp256k1 0.31.1", "serde", - "sha3", + "sha3 0.11.0", ] [[package]] name = "alloy-provider" -version = "2.0.4" +version = "2.1.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "93a7c17472b55482d4734154c2f5ed13f72e03f6752cebb927f6a2d8b52e646c" +checksum = "0e7d526d184d392a8fbcf4293e457789b307bb70646e4f8b902989a246529450" dependencies = [ "alloy-chains", "alloy-consensus", - "alloy-eips 2.0.4", + "alloy-eips", "alloy-json-rpc", "alloy-network", "alloy-network-primitives", @@ -448,7 +426,7 @@ dependencies = [ "lru", "parking_lot", "pin-project", - "reqwest 0.13.3", + "reqwest", "serde", "serde_json", "thiserror 2.0.18", @@ -460,9 +438,9 @@ dependencies = [ [[package]] name = "alloy-pubsub" -version = "2.0.4" +version = "2.1.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a8d86958b02bca85103d64fa60d7b364a8b017c6e40f2b02c3f50ca22964a738" +checksum = "6fa5976a77e32a63152e937fa90d0dae1f8142b41a9a99a6386d6ea58934cfa6" dependencies = [ "alloy-json-rpc", "alloy-primitives", @@ -482,9 +460,9 @@ dependencies = [ [[package]] name = "alloy-rlp" -version = "0.3.15" +version = "0.3.16" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "dc90b1e703d3c03f4ff7f48e82dd0bc1c8211ab7d079cd836a06fcfeb06651cb" +checksum = "24671b1f62edcf0f9b62994c7bf72cd621a04a4b99f5020ece1a647b40e2f103" dependencies = [ "alloy-rlp-derive", "arrayvec", @@ -493,9 +471,9 @@ dependencies = [ [[package]] name = "alloy-rlp-derive" -version = "0.3.15" +version = "0.3.16" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f36834a5c0a2fa56e171bf256c34d70fca07d0c0031583edea1c4946b7889c9e" +checksum = "9d4311c03125e8a18296504560b9de3d75ecbd0dcda7f71e6cf2a196d57e6fba" dependencies = [ "proc-macro2", "quote", @@ -504,9 +482,9 @@ dependencies = [ [[package]] name = "alloy-rpc-client" -version = "2.0.4" +version = "2.1.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5beb5c2fe6b960c8e8b038e69fd502a90a2e930afa4770efb748b163b0767729" +checksum = "755447dc13a04c6fa6db8dedb5d32ab5edfa4dd7aa34487ff192a3d873e0e8cf" dependencies = [ "alloy-json-rpc", "alloy-primitives", @@ -517,7 +495,7 @@ dependencies = [ "alloy-transport-ws", "futures", "pin-project", - "reqwest 0.13.3", + "reqwest", "serde", "serde_json", "tokio", @@ -530,22 +508,22 @@ dependencies = [ [[package]] name = "alloy-rpc-types" -version = "2.0.4" +version = "2.1.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4ee1257a278f6d293e05c5162c5940a1561b1aa85ded0028b464c81de37ebfa5" +checksum = "96deb317fe224e98a8ae17a7ed7ea63478867385bf50b7abd53642b24cfd811a" dependencies = [ "alloy-primitives", "alloy-rpc-types-engine", "alloy-rpc-types-eth", - "alloy-serde 2.0.4", + "alloy-serde", "serde", ] [[package]] name = "alloy-rpc-types-admin" -version = "2.0.4" +version = "2.1.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e2144d5b2866e651796eac0a997d3b5a056449c12e0d91be3184129e0c722885" +checksum = "3279cb55f7069c2fb1dbcd9ab2c6e79a02d63c92fd0b850addea0a3715d6b05f" dependencies = [ "alloy-genesis", "alloy-primitives", @@ -555,38 +533,38 @@ dependencies = [ [[package]] name = "alloy-rpc-types-anvil" -version = "2.0.4" +version = "2.1.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "df32156f085e74eac942b6103744be49b817c302341aaa8cb0c1c88dc29228d9" +checksum = "9f89d27756bf3effdac25d07692724e840ce90ca1ff956a6b47dd2ae1612f597" dependencies = [ "alloy-primitives", "alloy-rpc-types-eth", - "alloy-serde 2.0.4", + "alloy-serde", "serde", ] [[package]] name = "alloy-rpc-types-any" -version = "2.0.4" +version = "2.1.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6a234bfbdf7a76c3d13808f729af5321852de3dedcaa6fc6d5f54787aaf54c6a" +checksum = "911a513723ef0b90c3b072f65eebf54d6ebc8b651d06734e252d024bd89b88ac" dependencies = [ "alloy-consensus-any", "alloy-network-primitives", "alloy-primitives", "alloy-rpc-types-eth", - "alloy-serde 2.0.4", + "alloy-serde", "serde", "serde_json", ] [[package]] name = "alloy-rpc-types-beacon" -version = "2.0.4" +version = "2.1.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "296450f5e76bece0116c939b9437b0421a5da9c5d40031bf4cf9b38d3d94e475" +checksum = "81133671b8da58169589aac996f3c4da23bbdee85b13ecee7098065f3eae718a" dependencies = [ - "alloy-eips 2.0.4", + "alloy-eips", "alloy-primitives", "alloy-rpc-types-engine", "derive_more", @@ -602,9 +580,9 @@ dependencies = [ [[package]] name = "alloy-rpc-types-debug" -version = "2.0.4" +version = "2.1.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0ab075ac1c25bcf697f133b7cd92e2fb26afe213e872ef79fdf77f0d7bcb3793" +checksum = "b9e9c1f9ac81c56b2d96901201db7eb95c4e9fc3c21237a4690f8c652aa62089" dependencies = [ "alloy-primitives", "alloy-rlp", @@ -615,15 +593,15 @@ dependencies = [ [[package]] name = "alloy-rpc-types-engine" -version = "2.0.4" +version = "2.1.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "73b12366c96f4013e1aeebc96c6b56e5f33f07853c42ea2f485045c0c157a4a1" +checksum = "5b1f682c4881aa7000ac7cc86d7aeeb3b09836a77a90b329820140233651aeea" dependencies = [ "alloy-consensus", - "alloy-eips 2.0.4", + "alloy-eips", "alloy-primitives", "alloy-rlp", - "alloy-serde 2.0.4", + "alloy-serde", "derive_more", "ethereum_ssz", "ethereum_ssz_derive", @@ -635,17 +613,17 @@ dependencies = [ [[package]] name = "alloy-rpc-types-eth" -version = "2.0.4" +version = "2.1.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "56a282daf869eeb7383d3d5c2deb35b0b3fb45ecb329513af4090fc61245ee18" +checksum = "4e1bd19904581ee2075b61faabab1a4cefa8b96d8f2c85343adeae8ecf615e2b" dependencies = [ "alloy-consensus", "alloy-consensus-any", - "alloy-eips 2.0.4", + "alloy-eips", "alloy-network-primitives", "alloy-primitives", "alloy-rlp", - "alloy-serde 2.0.4", + "alloy-serde", "alloy-sol-types", "arbitrary", "itertools 0.14.0", @@ -657,28 +635,28 @@ dependencies = [ [[package]] name = "alloy-rpc-types-mev" -version = "2.0.4" +version = "2.1.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7adc1243d55744a66b3a6cbbbba96436e8df5d248f2ee8186bef4238ef704ec7" +checksum = "54c635f0c45a871b55c276653e3838d1687d86282eeaf3d83a1914e02563b3f2" dependencies = [ "alloy-consensus", - "alloy-eips 2.0.4", + "alloy-eips", "alloy-primitives", "alloy-rpc-types-eth", - "alloy-serde 2.0.4", + "alloy-serde", "serde", "serde_json", ] [[package]] name = "alloy-rpc-types-trace" -version = "2.0.4" +version = "2.1.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6184b5d14152b68b0bb8beb621339d94f0b761a37958bb365fbf7c00922125c2" +checksum = "796729a4e1783904c6040f11a503c4d55ddb16f69dc199ad15e50c4ad039f371" dependencies = [ "alloy-primitives", "alloy-rpc-types-eth", - "alloy-serde 2.0.4", + "alloy-serde", "serde", "serde_json", "thiserror 2.0.18", @@ -686,32 +664,21 @@ dependencies = [ [[package]] name = "alloy-rpc-types-txpool" -version = "2.0.4" +version = "2.1.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f00b631c361e7c7baaf4f1f5a9877730f3507fed2acb9d4b34841b8184b2ec28" +checksum = "8533c450895de79eb581875bfc317d1a239ae71aba79e20ee158fa153268f163" dependencies = [ "alloy-primitives", "alloy-rpc-types-eth", - "alloy-serde 2.0.4", + "alloy-serde", "serde", ] [[package]] name = "alloy-serde" -version = "1.8.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "11ece63b89294b8614ab3f483560c08d016930f842bf36da56bf0b764a15c11e" -dependencies = [ - "alloy-primitives", - "serde", - "serde_json", -] - -[[package]] -name = "alloy-serde" -version = "2.0.4" +version = "2.1.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a0eada2558e921b39dfcead33c487364df9b31374f5733c1c9d2c891c4529933" +checksum = "c1e97b3e0b9f816b25083045dcfa69431bd059a078e828e4d82d296d1949b96c" dependencies = [ "alloy-primitives", "arbitrary", @@ -721,9 +688,9 @@ dependencies = [ [[package]] name = "alloy-signer" -version = "2.0.4" +version = "2.1.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "41eb29f7a8adcd8941fbb8e134022a133e6f8dfd345f2e3b7109599f8a7dca08" +checksum = "6913d06ccc0d9c6ab67e2f82e2fbe292706da1f91442f30cded2d04b56bf0d58" dependencies = [ "alloy-primitives", "async-trait", @@ -736,9 +703,9 @@ dependencies = [ [[package]] name = "alloy-signer-local" -version = "2.0.4" +version = "2.1.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bef839e7ce9b59aa60fa9a175e97986c6145c888d643b0f1fb0a3e7b8e56a2e2" +checksum = "c880813cd26bd1ddf8c2236e31295896efd1fcc5cc761d8894ae894503aeea53" dependencies = [ "alloy-consensus", "alloy-network", @@ -755,9 +722,9 @@ dependencies = [ [[package]] name = "alloy-sol-macro" -version = "1.5.7" +version = "1.6.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ab81bab693da9bb79f7a95b64b394718259fdd7e41dceeced4cad57cb71c4f6a" +checksum = "840128ed2b2971d6d4668a553fe403a82683d3acc646c73e75887e7157408033" dependencies = [ "alloy-sol-macro-expander", "alloy-sol-macro-input", @@ -769,9 +736,9 @@ dependencies = [ [[package]] name = "alloy-sol-macro-expander" -version = "1.5.7" +version = "1.6.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "489f1620bb7e2483fb5819ed01ab6edc1d2f93939dce35a5695085a1afd1d699" +checksum = "63ec265e5d65d725175f6ca7711c970824c90ef9c0d1f1973711d4150ee612dd" dependencies = [ "alloy-sol-macro-input", "const-hex", @@ -780,16 +747,16 @@ dependencies = [ "proc-macro-error2", "proc-macro2", "quote", - "sha3", + "sha3 0.11.0", "syn 2.0.117", "syn-solidity", ] [[package]] name = "alloy-sol-macro-input" -version = "1.5.7" +version = "1.6.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "56cef806ad22d4392c5fc83cf8f2089f988eb99c7067b4e0c6f1971fc1cca318" +checksum = "89bf01077f18650876cfa682eb1f949967b5cde03f1a51c955c469d2c9b4aa67" dependencies = [ "const-hex", "dunce", @@ -803,19 +770,19 @@ dependencies = [ [[package]] name = "alloy-sol-type-parser" -version = "1.5.7" +version = "1.6.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a6df77fea9d6a2a75c0ef8d2acbdfd92286cc599983d3175ccdc170d3433d249" +checksum = "857b470ecdd2ed38beaf82ad1a38c516a8ff75266750f38b9eeed001d575241b" dependencies = [ "serde", - "winnow 0.7.15", + "winnow 1.0.2", ] [[package]] name = "alloy-sol-types" -version = "1.5.7" +version = "1.6.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "64612d29379782a5dde6f4b6570d9c756d734d760c0c94c254d361e678a6591f" +checksum = "384cf252de0db2dec52821eac037a7f57e2aa33fe5b900ce6fe39973402341f1" dependencies = [ "alloy-json-abi", "alloy-primitives", @@ -825,9 +792,9 @@ dependencies = [ [[package]] name = "alloy-transport" -version = "2.0.4" +version = "2.1.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3ac7a80c0bac3e44559d53d002e34c461dc2f23262b42cafec019bc70551abbe" +checksum = "999adfe5c91035c6bf4c4210e0eb8d0caed79d76fbf5e1b70d78721f6097ac04" dependencies = [ "alloy-json-rpc", "auto_impl", @@ -848,14 +815,14 @@ dependencies = [ [[package]] name = "alloy-transport-http" -version = "2.0.4" +version = "2.1.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "eed3ed3300a998f88639ed619fdbbd88bd82865e00c6a8ecb796c99eb12358f6" +checksum = "0e79a2c1793afc61eed9ca0963da370a988b27d2bbfa67c78013e262d47424c4" dependencies = [ "alloy-json-rpc", "alloy-transport", "itertools 0.14.0", - "reqwest 0.13.3", + "reqwest", "serde_json", "tower", "tracing", @@ -864,9 +831,9 @@ dependencies = [ [[package]] name = "alloy-transport-ipc" -version = "2.0.4" +version = "2.1.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1075d9d30fd4d71e50000fd4afb19ed2664ceab20c2a29f3889a6e988329e02d" +checksum = "c24422d5159996688b0c094babf1969cb0197d453902da483711c92c1624fea2" dependencies = [ "alloy-json-rpc", "alloy-pubsub", @@ -884,9 +851,9 @@ dependencies = [ [[package]] name = "alloy-transport-ws" -version = "2.0.4" +version = "2.1.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0e3bff84b2b2a46eb34cc522dc3f889a2867c70be90a377421429b662b3ec4ce" +checksum = "1bea36621cd4e4f06c1243e556b32fdc9c4db7b9b09b72a95a87383c0ffcc625" dependencies = [ "alloy-pubsub", "alloy-transport", @@ -914,7 +881,7 @@ dependencies = [ "derive_more", "nybbles", "proptest", - "proptest-derive", + "proptest-derive 0.7.0", "serde", "smallvec", "thiserror 2.0.18", @@ -923,11 +890,11 @@ dependencies = [ [[package]] name = "alloy-tx-macros" -version = "2.0.4" +version = "2.1.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "99fce0350197dcd4ba4e9a7dd43915d908c0eb0e7352755791709a705e1c76b6" +checksum = "406bc1183f6843e0aba09f7b3365e828b597213d60793ba5cb41befc863e3a78" dependencies = [ - "darling 0.23.0", + "darling", "proc-macro2", "quote", "syn 2.0.117", @@ -978,7 +945,7 @@ version = "1.1.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "40c48f72fd53cd289104fc64099abca73db4166ad86ea0b4341abe65af83dadc" dependencies = [ - "windows-sys 0.60.2", + "windows-sys 0.61.2", ] [[package]] @@ -989,7 +956,7 @@ checksum = "291e6a250ff86cd4a820112fb8898808a366d8f9f58ce16d1f538353ad55747d" dependencies = [ "anstyle", "once_cell_polyfill", - "windows-sys 0.60.2", + "windows-sys 0.61.2", ] [[package]] @@ -1622,12 +1589,12 @@ dependencies = [ ] [[package]] -name = "block-padding" -version = "0.3.3" +name = "block-buffer" +version = "0.12.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a8894febbff9f758034a5b8e12d87918f56dfc64a8e1fe757d65e29041538d93" +checksum = "d2f6c7dbe95a6ed67ad9f18e57daf93a2f034c524b99fd2b76d18fdfeb6660aa" dependencies = [ - "generic-array", + "hybrid-array", ] [[package]] @@ -1642,6 +1609,31 @@ dependencies = [ "zeroize", ] +[[package]] +name = "bon" +version = "3.9.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a602c73c7b0148ec6d12af6fd5cc7a46e2eacc8878271a999abac56eed12f561" +dependencies = [ + "bon-macros", + "rustversion", +] + +[[package]] +name = "bon-macros" +version = "3.9.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6dee98b0db6a962de883bf5d20362dee4d7ca0d12fe39a7c6c73c844e1cd7c1f" +dependencies = [ + "darling", + "ident_case", + "prettyplease", + "proc-macro2", + "quote", + "rustversion", + "syn 2.0.117", +] + [[package]] name = "borsh" version = "1.6.1" @@ -1765,39 +1757,6 @@ dependencies = [ "serde", ] -[[package]] -name = "camino" -version = "1.2.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e629a66d692cb9ff1a1c664e41771b3dcaf961985a9774c0eb0bd1b51cf60a48" -dependencies = [ - "serde_core", -] - -[[package]] -name = "cargo-platform" -version = "0.3.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "dd0061da739915fae12ea00e16397555ed4371a6bb285431aab930f61b0aa4ba" -dependencies = [ - "serde", - "serde_core", -] - -[[package]] -name = "cargo_metadata" -version = "0.23.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ef987d17b0a113becdd19d3d0022d04d7ef41f9efe4f3fb63ac44ba61df3ade9" -dependencies = [ - "camino", - "cargo-platform", - "semver 1.0.28", - "serde", - "serde_json", - "thiserror 2.0.18", -] - [[package]] name = "castaway" version = "0.2.4" @@ -1846,6 +1805,17 @@ version = "0.2.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "613afe47fcd5fac7ccf1db93babcb082c5994d996f20b8b159f2ad1658eb5724" +[[package]] +name = "chacha20" +version = "0.10.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d524456ba66e72eb8b115ff89e01e497f8e6d11d78b70b1aa13c0fbd97540a81" +dependencies = [ + "cfg-if", + "cpufeatures 0.3.0", + "rand_core 0.10.1", +] + [[package]] name = "chrono" version = "0.4.44" @@ -1866,7 +1836,7 @@ version = "0.4.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "773f3b9af64447d2ce9850330c473515014aa235e6a783b02db81ff39e4a3dad" dependencies = [ - "crypto-common", + "crypto-common 0.1.7", "inout", ] @@ -1977,7 +1947,7 @@ dependencies = [ "ripemd", "serde", "sha2", - "sha3", + "sha3 0.10.9", "thiserror 1.0.69", ] @@ -2111,6 +2081,16 @@ dependencies = [ "unicode-segmentation", ] +[[package]] +name = "core-foundation" +version = "0.9.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "91e195e091a93c46f7102ec7818a2aa394e1e1771c3ab4825963fa03e45afb8f" +dependencies = [ + "core-foundation-sys", + "libc", +] + [[package]] name = "core-foundation" version = "0.10.1" @@ -2274,6 +2254,15 @@ dependencies = [ "typenum", ] +[[package]] +name = "crypto-common" +version = "0.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ce6e4c961d6cd6c9a86db418387425e8bdeaf05b3c8bc1411e6dca4c252f1453" +dependencies = [ + "hybrid-array", +] + [[package]] name = "ctr" version = "0.9.2" @@ -2310,38 +2299,14 @@ dependencies = [ "syn 2.0.117", ] -[[package]] -name = "darling" -version = "0.20.11" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "fc7f46116c46ff9ab3eb1597a45688b6715c6e628b5c133e288e709a29bcb4ee" -dependencies = [ - "darling_core 0.20.11", - "darling_macro 0.20.11", -] - [[package]] name = "darling" version = "0.23.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "25ae13da2f202d56bd7f91c25fba009e7717a1e4a1cc98a76d844b65ae912e9d" dependencies = [ - "darling_core 0.23.0", - "darling_macro 0.23.0", -] - -[[package]] -name = "darling_core" -version = "0.20.11" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0d00b9596d185e565c2207a0b01f8bd1a135483d02d9b7b0a54b11da8d53412e" -dependencies = [ - "fnv", - "ident_case", - "proc-macro2", - "quote", - "strsim", - "syn 2.0.117", + "darling_core", + "darling_macro", ] [[package]] @@ -2358,33 +2323,22 @@ dependencies = [ "syn 2.0.117", ] -[[package]] -name = "darling_macro" -version = "0.20.11" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "fc34b93ccb385b40dc71c6fceac4b2ad23662c7eeb248cf10d529b7e055b6ead" -dependencies = [ - "darling_core 0.20.11", - "quote", - "syn 2.0.117", -] - [[package]] name = "darling_macro" version = "0.23.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "ac3984ec7bd6cfa798e62b4a642426a5be0e68f9401cfc2a01e3fa9ea2fcdb8d" dependencies = [ - "darling_core 0.23.0", + "darling_core", "quote", "syn 2.0.117", ] [[package]] name = "dashmap" -version = "6.1.0" +version = "6.2.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5041cc499144891f3790297212f32a74fb938e5136a14943f338ef9e0ae276cf" +checksum = "e6361d5c062261c78a176addb82d4c821ae42bed6089de0e12603cd25de2059c" dependencies = [ "arbitrary", "cfg-if", @@ -2419,7 +2373,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "ccc2776f0c61eca1ca32528f85548abd1a4be8fb53d1b21c013e4f18da1e7090" dependencies = [ "data-encoding", - "syn 1.0.109", + "syn 2.0.117", ] [[package]] @@ -2455,7 +2409,6 @@ version = "0.5.8" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "7cd812cc2bc1d69d4764bd80df88b4317eaef9e773c75226407d9bc0876b211c" dependencies = [ - "powerfmt", "serde_core", ] @@ -2492,37 +2445,6 @@ dependencies = [ "syn 2.0.117", ] -[[package]] -name = "derive_builder" -version = "0.20.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "507dfb09ea8b7fa618fcf76e953f4f5e192547945816d5358edffe39f6f94947" -dependencies = [ - "derive_builder_macro", -] - -[[package]] -name = "derive_builder_core" -version = "0.20.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2d5bcf7b024d6835cfb3d473887cd966994907effbe9227e8c8219824d06c4e8" -dependencies = [ - "darling 0.20.11", - "proc-macro2", - "quote", - "syn 2.0.117", -] - -[[package]] -name = "derive_builder_macro" -version = "0.20.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ab63b0e2bf4d5928aff72e83a7dace85d7bba5fe12dcc3c5a572d78caffd3f3c" -dependencies = [ - "derive_builder_core", - "syn 2.0.117", -] - [[package]] name = "derive_more" version = "2.1.1" @@ -2567,12 +2489,22 @@ version = "0.10.7" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "9ed9a281f7bc9b7576e61468ba615a66a5c8cfdff42420a70aa82701a3b1e292" dependencies = [ - "block-buffer", + "block-buffer 0.10.4", "const-oid", - "crypto-common", + "crypto-common 0.1.7", "subtle", ] +[[package]] +name = "digest" +version = "0.11.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f1dd6dbb5841937940781866fa1281a1ff7bd3bf827091440879f9994983d5c2" +dependencies = [ + "block-buffer 0.12.1", + "crypto-common 0.2.2", +] + [[package]] name = "dirs-next" version = "2.0.0" @@ -2717,9 +2649,9 @@ dependencies = [ [[package]] name = "either" -version = "1.15.0" +version = "1.16.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "48c757948c5ede0e46177b7add2e67155f70e33c07fea8284df6576da70b3719" +checksum = "91622ff5e7162018101f2fea40d6ebf4a78bbe5a49736a2020649edf9693679e" dependencies = [ "serde", ] @@ -2760,22 +2692,10 @@ dependencies = [ "rand 0.8.6", "secp256k1 0.30.0", "serde", - "sha3", + "sha3 0.10.9", "zeroize", ] -[[package]] -name = "enum-as-inner" -version = "0.6.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a1e6a265c649f3f5979b601d26f1d05ada116434c87741c9493cb56218f76cbc" -dependencies = [ - "heck", - "proc-macro2", - "quote", - "syn 2.0.117", -] - [[package]] name = "enum-ordinalize" version = "4.3.2" @@ -2809,7 +2729,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "39cab71617ae0d63f51a36d69f866391735b51691dbda63cf6f96d042b63efeb" dependencies = [ "libc", - "windows-sys 0.59.0", + "windows-sys 0.61.2", ] [[package]] @@ -2838,9 +2758,9 @@ dependencies = [ [[package]] name = "ethereum_ssz" -version = "0.10.3" +version = "0.10.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "368a4a4e4273b0135111fe9464e35465067766a8f664615b5a86338b73864407" +checksum = "e462875ad8693755ea8913d6e905715c76ea4836e2254e18c9cf0f7a8f8c2a13" dependencies = [ "alloy-primitives", "ethereum_serde_utils", @@ -2853,11 +2773,11 @@ dependencies = [ [[package]] name = "ethereum_ssz_derive" -version = "0.10.3" +version = "0.10.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f2cd82c68120c89361e1a457245cf212f7d9f541bffaffed530c8f2d54a160b2" +checksum = "daf022360bdbe9456eda5f35718a50476d5b2a0d51a97ed4eae27420737a6fba" dependencies = [ - "darling 0.23.0", + "darling", "proc-macro2", "quote", "syn 2.0.117", @@ -2957,9 +2877,9 @@ checksum = "5baebc0774151f905a1a2cc41989300b1e6fbb29aff0ceffa1064fdd3088d582" [[package]] name = "fixed-cache" -version = "0.1.8" +version = "0.1.10" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c41c7aa69c00ebccf06c3fa7ffe2a6cf26a58b5fe4deabfe646285ff48136a8f" +checksum = "2fe63500644ef0269fe6b744e7e5dc5c20b5eebf3d881bc2be53f194636f6583" dependencies = [ "equivalent", "rapidhash", @@ -3172,8 +3092,8 @@ dependencies = [ "libc", "log", "rustversion", - "windows-link 0.1.3", - "windows-result 0.3.4", + "windows-link 0.2.1", + "windows-result 0.4.1", ] [[package]] @@ -3223,6 +3143,7 @@ dependencies = [ "cfg-if", "libc", "r-efi 6.0.0", + "rand_core 0.10.1", "wasip2", "wasip3", ] @@ -3239,15 +3160,14 @@ dependencies = [ [[package]] name = "git2" -version = "0.20.4" +version = "0.21.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7b88256088d75a56f8ecfa070513a775dd9107f6530ef14919dac831af9cfe2b" +checksum = "ddddbf932745a6be37109b6112d3ee09696106f848449069d3a57bba937ab82e" dependencies = [ "bitflags", "libc", "libgit2-sys", "log", - "url", ] [[package]] @@ -3381,8 +3301,6 @@ dependencies = [ "allocator-api2", "equivalent", "foldhash 0.2.0", - "serde", - "serde_core", ] [[package]] @@ -3390,6 +3308,11 @@ name = "hashbrown" version = "0.17.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "4f467dd6dccf739c208452f8014c75c18bb8301b050ad1cfb27153803edb0f51" +dependencies = [ + "foldhash 0.2.0", + "serde", + "serde_core", +] [[package]] name = "hashlink" @@ -3438,48 +3361,72 @@ dependencies = [ ] [[package]] -name = "hickory-proto" -version = "0.25.2" +name = "hickory-net" +version = "0.26.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f8a6fe56c0038198998a6f217ca4e7ef3a5e51f46163bd6dd60b5c71ca6c6502" +checksum = "e2295ed2f9c31e471e1428a8f88a3f0e1f4b27c15049592138d1eebe9c35b183" dependencies = [ "async-trait", "cfg-if", "data-encoding", - "enum-as-inner", "futures-channel", "futures-io", "futures-util", + "hickory-proto", + "idna", + "ipnet", + "jni 0.22.4", + "rand 0.10.2", + "thiserror 2.0.18", + "tinyvec", + "tokio", + "tracing", + "url", +] + +[[package]] +name = "hickory-proto" +version = "0.26.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0bab31817bfb44672a252e97fe81cd0c18d1b2cf892108922f6818820df8c643" +dependencies = [ + "data-encoding", "idna", "ipnet", + "jni 0.22.4", "once_cell", - "rand 0.9.4", + "prefix-trie", + "rand 0.10.2", "ring", "serde", "thiserror 2.0.18", "tinyvec", - "tokio", "tracing", "url", ] [[package]] name = "hickory-resolver" -version = "0.25.2" +version = "0.26.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "dc62a9a99b0bfb44d2ab95a7208ac952d31060efc16241c87eaf36406fecf87a" +checksum = "f0d58d28879ceecde6607729660c2667a081ccdc082e082675042793960f178c" dependencies = [ "cfg-if", "futures-util", + "hickory-net", "hickory-proto", "ipconfig", + "ipnet", + "jni 0.22.4", "moka", + "ndk-context", "once_cell", "parking_lot", - "rand 0.9.4", + "rand 0.10.2", "resolv-conf", "serde", "smallvec", + "system-configuration", "thiserror 2.0.18", "tokio", "tracing", @@ -3576,6 +3523,15 @@ dependencies = [ "serde", ] +[[package]] +name = "hybrid-array" +version = "0.4.13" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "818356c5132c1fede50f837ca96afbe78ff42413047f4abb886217845e1b6c8c" +dependencies = [ + "typenum", +] + [[package]] name = "hyper" version = "1.9.0" @@ -3609,7 +3565,6 @@ dependencies = [ "hyper-util", "log", "rustls", - "rustls-native-certs", "tokio", "tokio-rustls", "tower-service", @@ -3663,7 +3618,7 @@ dependencies = [ "js-sys", "log", "wasm-bindgen", - "windows-core 0.61.2", + "windows-core 0.62.2", ] [[package]] @@ -3929,7 +3884,6 @@ version = "0.1.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "879f10e63c20629ecabbb64a8010319738c66a5cd0c29b02d63d272b03751d01" dependencies = [ - "block-padding", "generic-array", ] @@ -3939,7 +3893,7 @@ version = "0.3.12" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "5eb2d60ef19920a3a9193c3e371f726ec1dafc045dac788d0fb3704272458971" dependencies = [ - "darling 0.23.0", + "darling", "indoc", "proc-macro2", "quote", @@ -3979,6 +3933,9 @@ name = "ipnet" version = "2.12.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d98f6fed1fde3f8c21bc40a1abb88dd75e67924f9cffc3ef95607bad8017f8e2" +dependencies = [ + "serde", +] [[package]] name = "is_terminal_polyfill" @@ -4356,6 +4313,16 @@ dependencies = [ "cpufeatures 0.2.17", ] +[[package]] +name = "keccak" +version = "0.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9e24a010dd405bd7ed803e5253182815b41bf2e6a80cc3bfc066658e03a198aa" +dependencies = [ + "cfg-if", + "cpufeatures 0.3.0", +] + [[package]] name = "keccak-asm" version = "0.1.6" @@ -4656,9 +4623,9 @@ checksum = "dae608c151f68243f2b000364e1f7b186d9c29845f7d2d85bd31b9ad77ad552b" [[package]] name = "macro-string" -version = "0.1.4" +version = "0.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1b27834086c65ec3f9387b096d66e99f221cf081c2b738042aa252bcd41204e3" +checksum = "59a9dbbfc75d2688ed057456ce8a3ee3f48d12eec09229f560f3643b9f275653" dependencies = [ "proc-macro2", "quote", @@ -4706,9 +4673,9 @@ checksum = "f8ca58f447f06ed17d5fc4043ce1b10dd205e060fb3ce5b979b8ed8e59ff3f79" [[package]] name = "memmap2" -version = "0.9.10" +version = "0.9.11" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "714098028fe011992e1c3962653c96b2d578c4b4bce9036e15ff220319b1e0e3" +checksum = "d1219ed1b7f229ee7104d281dd01d6802fe28bb6e95d292942c4daacdeb798c0" dependencies = [ "libc", ] @@ -4876,12 +4843,12 @@ version = "1.0.0" dependencies = [ "alloy-chains", "alloy-consensus", - "alloy-eips 2.0.4", + "alloy-eips", "alloy-evm", "alloy-genesis", "alloy-hardforks", "alloy-primitives", - "alloy-serde 2.0.4", + "alloy-serde", "auto_impl", "eyre", "morph-primitives", @@ -4916,7 +4883,7 @@ name = "morph-engine-api" version = "1.0.0" dependencies = [ "alloy-consensus", - "alloy-eips 2.0.4", + "alloy-eips", "alloy-genesis", "alloy-primitives", "alloy-rpc-types-engine", @@ -4939,52 +4906,6 @@ dependencies = [ "tracing", ] -[[package]] -name = "morph-engine-tree-ext" -version = "1.0.0" -dependencies = [ - "alloy-consensus", - "alloy-eip7928", - "alloy-eips 2.0.4", - "alloy-evm", - "alloy-genesis", - "alloy-primitives", - "alloy-rlp", - "alloy-rpc-types-engine", - "alloy-rpc-types-eth", - "crossbeam-channel", - "derive_more", - "eyre", - "morph-chainspec", - "morph-node", - "morph-payload-types", - "morph-primitives", - "reth-chain-state", - "reth-consensus", - "reth-db", - "reth-engine-primitives", - "reth-engine-tree", - "reth-errors", - "reth-evm", - "reth-execution-cache", - "reth-node-api", - "reth-payload-builder", - "reth-payload-primitives", - "reth-primitives-traits", - "reth-provider", - "reth-revm", - "reth-tasks", - "reth-tracing", - "reth-trie", - "reth-trie-db", - "reth-trie-parallel", - "reth-trie-sparse", - "revm-primitives", - "serde_json", - "tokio", - "tracing", -] - [[package]] name = "morph-evm" version = "1.0.0" @@ -4993,6 +4914,7 @@ dependencies = [ "alloy-evm", "alloy-genesis", "alloy-primitives", + "alloy-rpc-types-eth", "derive_more", "morph-chainspec", "morph-payload-types", @@ -5016,7 +4938,7 @@ name = "morph-node" version = "1.0.0" dependencies = [ "alloy-consensus", - "alloy-eips 2.0.4", + "alloy-eips", "alloy-genesis", "alloy-hardforks", "alloy-primitives", @@ -5033,7 +4955,6 @@ dependencies = [ "morph-chainspec", "morph-consensus", "morph-engine-api", - "morph-engine-tree-ext", "morph-evm", "morph-payload-builder", "morph-payload-types", @@ -5042,6 +4963,7 @@ dependencies = [ "morph-rpc", "morph-txpool", "parking_lot", + "reth-chain-state", "reth-chainspec", "reth-db", "reth-db-api", @@ -5049,6 +4971,8 @@ dependencies = [ "reth-engine-local", "reth-engine-tree", "reth-errors", + "reth-evm", + "reth-execution-cache", "reth-exex", "reth-node-api", "reth-node-builder", @@ -5081,7 +5005,7 @@ name = "morph-payload-builder" version = "1.0.0" dependencies = [ "alloy-consensus", - "alloy-eips 2.0.4", + "alloy-eips", "alloy-primitives", "alloy-rlp", "metrics", @@ -5113,11 +5037,11 @@ name = "morph-payload-types" version = "1.0.0" dependencies = [ "alloy-consensus", - "alloy-eips 2.0.4", + "alloy-eips", "alloy-primitives", "alloy-rlp", "alloy-rpc-types-engine", - "alloy-serde 2.0.4", + "alloy-serde", "morph-primitives", "rand 0.8.6", "reth-ethereum-primitives", @@ -5133,10 +5057,10 @@ name = "morph-primitives" version = "1.0.0" dependencies = [ "alloy-consensus", - "alloy-eips 2.0.4", + "alloy-eips", "alloy-primitives", "alloy-rlp", - "alloy-serde 2.0.4", + "alloy-serde", "bytes", "modular-bitfield", "reth-codecs", @@ -5199,7 +5123,7 @@ name = "morph-revm" version = "1.0.0" dependencies = [ "alloy-consensus", - "alloy-eips 2.0.4", + "alloy-eips", "alloy-evm", "alloy-primitives", "alloy-sol-types", @@ -5222,12 +5146,12 @@ name = "morph-rpc" version = "1.0.0" dependencies = [ "alloy-consensus", - "alloy-eips 2.0.4", + "alloy-eips", "alloy-evm", "alloy-network", "alloy-primitives", "alloy-rpc-types-eth", - "alloy-serde 2.0.4", + "alloy-serde", "derive_more", "eyre", "jsonrpsee", @@ -5284,7 +5208,7 @@ name = "morph-txpool" version = "1.0.0" dependencies = [ "alloy-consensus", - "alloy-eips 2.0.4", + "alloy-eips", "alloy-evm", "alloy-primitives", "c-kzg", @@ -5346,6 +5270,12 @@ dependencies = [ "unsigned-varint", ] +[[package]] +name = "ndk-context" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "27b02d87554356db9e9a873add8782d4ea6e3e58ea071a9adb9a2e8ddb884a8b" + [[package]] name = "nom" version = "7.1.3" @@ -5356,6 +5286,15 @@ dependencies = [ "minimal-lexical", ] +[[package]] +name = "nonmax" +version = "0.5.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "610a5acd306ec67f907abe5567859a3c693fb9886eb1f012ab8f2a47bef3db51" +dependencies = [ + "serde", +] + [[package]] name = "notify" version = "8.2.0" @@ -5398,7 +5337,7 @@ version = "0.50.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "7957b9740744892f114936ab4a57b3f487491bbeafaf8083688b16841a4240e5" dependencies = [ - "windows-sys 0.59.0", + "windows-sys 0.61.2", ] [[package]] @@ -5436,9 +5375,9 @@ dependencies = [ [[package]] name = "num-conv" -version = "0.2.1" +version = "0.2.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c6673768db2d862beb9b39a78fdcb1a69439615d5794a1be50caa9bc92c81967" +checksum = "521739c6d2bac4aa25192232afe6841231376b2b26d4d9fae5ecf8ca5772e441" [[package]] name = "num-integer" @@ -5507,7 +5446,6 @@ version = "0.7.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "680998035259dcfcafe653688bf2aa6d3e2dc05e98be6ab46afb089dc84f1df8" dependencies = [ - "proc-macro-crate", "proc-macro2", "quote", "syn 2.0.117", @@ -5586,9 +5524,9 @@ checksum = "7c87def4c32ab89d880effc9e097653c8da5d6ef28e6b539d313baaacfbafcbe" [[package]] name = "opentelemetry" -version = "0.31.0" +version = "0.32.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b84bcd6ae87133e903af7ef497404dda70c60d0ea14895fc8a5e6722754fc2a0" +checksum = "b0142c63252a9e054e68a4c61a5778f7b14f576274d593f8ce883d191a099682" dependencies = [ "futures-core", "futures-sink", @@ -5600,22 +5538,22 @@ dependencies = [ [[package]] name = "opentelemetry-http" -version = "0.31.0" +version = "0.32.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d7a6d09a73194e6b66df7c8f1b680f156d916a1a942abf2de06823dd02b7855d" +checksum = "5683015d09e2df236ef005b17f6f196f0d5f6313c4fa43a7b6a53b52776e4331" dependencies = [ "async-trait", "bytes", "http", "opentelemetry", - "reqwest 0.12.28", + "reqwest", ] [[package]] name = "opentelemetry-otlp" -version = "0.31.1" +version = "0.32.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1f69cd6acbb9af919df949cd1ec9e5e7fdc2ef15d234b6b795aaa525cc02f71f" +checksum = "9966929966d17620d7c316c643ba62631826e10021409357772d5eea84f62c35" dependencies = [ "http", "opentelemetry", @@ -5623,18 +5561,18 @@ dependencies = [ "opentelemetry-proto", "opentelemetry_sdk", "prost", - "reqwest 0.12.28", + "reqwest", "thiserror 2.0.18", "tokio", "tonic", - "tracing", + "tonic-types", ] [[package]] name = "opentelemetry-proto" -version = "0.31.0" +version = "0.32.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a7175df06de5eaee9909d4805a3d07e28bb752c34cab57fa9cff549da596b30f" +checksum = "56d658ba1faf63f7b9c492cfbe6e0ec365440a16132d3270c1065f7b33f1b638" dependencies = [ "opentelemetry", "opentelemetry_sdk", @@ -5645,21 +5583,22 @@ dependencies = [ [[package]] name = "opentelemetry-semantic-conventions" -version = "0.31.0" +version = "0.32.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e62e29dfe041afb8ed2a6c9737ab57db4907285d999ef8ad3a59092a36bdc846" +checksum = "c913ac17a6c451661ee255f4625d143e51647ae78ebd969b75e41c4442f4fe47" [[package]] name = "opentelemetry_sdk" -version = "0.31.0" +version = "0.32.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e14ae4f5991976fd48df6d843de219ca6d31b01daaab2dad5af2badeded372bd" +checksum = "9b59f80e1ac4d5ff7a2db8fb6c80badb7f0f3f858211fba08dd9aaec750894f9" dependencies = [ "futures-channel", "futures-executor", "futures-util", "opentelemetry", "percent-encoding", + "portable-atomic", "rand 0.9.4", "thiserror 2.0.18", ] @@ -5951,6 +5890,17 @@ dependencies = [ "zerocopy", ] +[[package]] +name = "prefix-trie" +version = "0.8.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4cf6e3177f0684016a5c209b00882e15f8bdd3f3bb48f0491df10cd102d0c6e7" +dependencies = [ + "either", + "ipnet", + "num-traits", +] + [[package]] name = "pretty_assertions" version = "1.4.1" @@ -6095,6 +6045,17 @@ dependencies = [ "syn 2.0.117", ] +[[package]] +name = "proptest-derive" +version = "0.8.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c57924a81864dddafba92e1bf92f9bf82f97096c44489548a60e888e1547549b" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.117", +] + [[package]] name = "prost" version = "0.14.3" @@ -6118,6 +6079,15 @@ dependencies = [ "syn 2.0.117", ] +[[package]] +name = "prost-types" +version = "0.14.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8991c4cbdb8bc5b11f0b074ffe286c30e523de90fee5ba8132f1399f23cb3dd7" +dependencies = [ + "prost", +] + [[package]] name = "quanta" version = "0.12.6" @@ -6254,6 +6224,17 @@ dependencies = [ "serde", ] +[[package]] +name = "rand" +version = "0.10.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c7f5fa3a058cd35567ef9bfa5e75732bee0f9e4c55fa90477bef2dfcdbc4be80" +dependencies = [ + "chacha20", + "getrandom 0.4.2", + "rand_core 0.10.1", +] + [[package]] name = "rand_chacha" version = "0.3.1" @@ -6293,6 +6274,12 @@ dependencies = [ "serde", ] +[[package]] +name = "rand_core" +version = "0.10.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "63b8176103e19a2643978565ca18b50549f6101881c443590420e4dc998a3c69" + [[package]] name = "rand_xorshift" version = "0.4.0" @@ -6497,46 +6484,6 @@ version = "0.8.10" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "dc897dd8d9e8bd1ed8cdad82b5966c3e0ecae09fb1907d58efaa013543185d0a" -[[package]] -name = "reqwest" -version = "0.12.28" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "eddd3ca559203180a307f12d114c268abf583f59b03cb906fd0b3ff8646c1147" -dependencies = [ - "base64 0.22.1", - "bytes", - "futures-channel", - "futures-core", - "futures-util", - "http", - "http-body", - "http-body-util", - "hyper", - "hyper-rustls", - "hyper-util", - "js-sys", - "log", - "percent-encoding", - "pin-project-lite", - "quinn", - "rustls", - "rustls-native-certs", - "rustls-pki-types", - "serde", - "serde_json", - "serde_urlencoded", - "sync_wrapper", - "tokio", - "tokio-rustls", - "tower", - "tower-http", - "tower-service", - "url", - "wasm-bindgen", - "wasm-bindgen-futures", - "web-sys", -] - [[package]] name = "reqwest" version = "0.13.3" @@ -6587,11 +6534,11 @@ checksum = "1e061d1b48cb8d38042de4ae0a7a6401009d6143dc80d2e2d6f31f0bdd6470c7" [[package]] name = "reth-basic-payload-builder" -version = "2.2.0" -source = "git+https://github.com/paradigmxyz/reth?tag=v2.2.0#88505c7fcbfdebfd3b56d88c86b62e950043c6c4" +version = "2.3.0" +source = "git+https://github.com/paradigmxyz/reth?rev=f0a41795d72ae5bb0b6c3661ac42ffb7b78ef105#f0a41795d72ae5bb0b6c3661ac42ffb7b78ef105" dependencies = [ "alloy-consensus", - "alloy-eips 2.0.4", + "alloy-eips", "alloy-primitives", "futures-core", "futures-util", @@ -6614,11 +6561,11 @@ dependencies = [ [[package]] name = "reth-chain-state" -version = "2.2.0" -source = "git+https://github.com/paradigmxyz/reth?tag=v2.2.0#88505c7fcbfdebfd3b56d88c86b62e950043c6c4" +version = "2.3.0" +source = "git+https://github.com/paradigmxyz/reth?rev=f0a41795d72ae5bb0b6c3661ac42ffb7b78ef105#f0a41795d72ae5bb0b6c3661ac42ffb7b78ef105" dependencies = [ "alloy-consensus", - "alloy-eips 2.0.4", + "alloy-eips", "alloy-primitives", "alloy-signer", "alloy-signer-local", @@ -6635,9 +6582,10 @@ dependencies = [ "reth-metrics", "reth-primitives-traits", "reth-storage-api", + "reth-tasks", "reth-trie", - "revm-database", - "revm-state", + "reth-trie-sparse", + "revm", "serde", "tokio", "tokio-stream", @@ -6646,12 +6594,12 @@ dependencies = [ [[package]] name = "reth-chainspec" -version = "2.2.0" -source = "git+https://github.com/paradigmxyz/reth?tag=v2.2.0#88505c7fcbfdebfd3b56d88c86b62e950043c6c4" +version = "2.3.0" +source = "git+https://github.com/paradigmxyz/reth?rev=f0a41795d72ae5bb0b6c3661ac42ffb7b78ef105#f0a41795d72ae5bb0b6c3661ac42ffb7b78ef105" dependencies = [ "alloy-chains", "alloy-consensus", - "alloy-eips 2.0.4", + "alloy-eips", "alloy-evm", "alloy-genesis", "alloy-primitives", @@ -6666,8 +6614,8 @@ dependencies = [ [[package]] name = "reth-cli" -version = "2.2.0" -source = "git+https://github.com/paradigmxyz/reth?tag=v2.2.0#88505c7fcbfdebfd3b56d88c86b62e950043c6c4" +version = "2.3.0" +source = "git+https://github.com/paradigmxyz/reth?rev=f0a41795d72ae5bb0b6c3661ac42ffb7b78ef105#f0a41795d72ae5bb0b6c3661ac42ffb7b78ef105" dependencies = [ "alloy-genesis", "clap", @@ -6679,12 +6627,12 @@ dependencies = [ [[package]] name = "reth-cli-commands" -version = "2.2.0" -source = "git+https://github.com/paradigmxyz/reth?tag=v2.2.0#88505c7fcbfdebfd3b56d88c86b62e950043c6c4" +version = "2.3.0" +source = "git+https://github.com/paradigmxyz/reth?rev=f0a41795d72ae5bb0b6c3661ac42ffb7b78ef105#f0a41795d72ae5bb0b6c3661ac42ffb7b78ef105" dependencies = [ "alloy-chains", "alloy-consensus", - "alloy-eips 2.0.4", + "alloy-eips", "alloy-primitives", "alloy-rlp", "backon", @@ -6703,7 +6651,7 @@ dependencies = [ "parking_lot", "ratatui", "rayon", - "reqwest 0.13.3", + "reqwest", "reth-chainspec", "reth-cli", "reth-cli-runner", @@ -6751,6 +6699,7 @@ dependencies = [ "secp256k1 0.30.0", "serde", "serde_json", + "socket2", "tar", "tokio", "tokio-stream", @@ -6762,8 +6711,8 @@ dependencies = [ [[package]] name = "reth-cli-runner" -version = "2.2.0" -source = "git+https://github.com/paradigmxyz/reth?tag=v2.2.0#88505c7fcbfdebfd3b56d88c86b62e950043c6c4" +version = "2.3.0" +source = "git+https://github.com/paradigmxyz/reth?rev=f0a41795d72ae5bb0b6c3661ac42ffb7b78ef105#f0a41795d72ae5bb0b6c3661ac42ffb7b78ef105" dependencies = [ "reth-tasks", "tokio", @@ -6772,10 +6721,10 @@ dependencies = [ [[package]] name = "reth-cli-util" -version = "2.2.0" -source = "git+https://github.com/paradigmxyz/reth?tag=v2.2.0#88505c7fcbfdebfd3b56d88c86b62e950043c6c4" +version = "2.3.0" +source = "git+https://github.com/paradigmxyz/reth?rev=f0a41795d72ae5bb0b6c3661ac42ffb7b78ef105#f0a41795d72ae5bb0b6c3661ac42ffb7b78ef105" dependencies = [ - "alloy-eips 2.0.4", + "alloy-eips", "alloy-primitives", "cfg-if", "eyre", @@ -6791,12 +6740,12 @@ dependencies = [ [[package]] name = "reth-codecs" -version = "0.3.1" +version = "0.5.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "fce542a96bf888f31854803e80b3340bc233927743aa580838014e8a88fe0d66" +checksum = "eb43f4858fef4e3b42b80954d2edf1ff67c8ad7498347083cdc7d0f6eff2f081" dependencies = [ "alloy-consensus", - "alloy-eips 2.0.4", + "alloy-eips", "alloy-genesis", "alloy-primitives", "alloy-trie", @@ -6812,9 +6761,9 @@ dependencies = [ [[package]] name = "reth-codecs-derive" -version = "0.3.1" +version = "0.5.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "634c90f1cc0f9887680ca785b0b21aa961070b9465917bf65afaec56a6d005bb" +checksum = "5798ae4d6b264764bc0d742468bd32c7fb515e774645d4930f95ff6311f3ae14" dependencies = [ "proc-macro2", "quote", @@ -6823,8 +6772,8 @@ dependencies = [ [[package]] name = "reth-config" -version = "2.2.0" -source = "git+https://github.com/paradigmxyz/reth?tag=v2.2.0#88505c7fcbfdebfd3b56d88c86b62e950043c6c4" +version = "2.3.0" +source = "git+https://github.com/paradigmxyz/reth?rev=f0a41795d72ae5bb0b6c3661ac42ffb7b78ef105#f0a41795d72ae5bb0b6c3661ac42ffb7b78ef105" dependencies = [ "eyre", "humantime-serde", @@ -6839,10 +6788,11 @@ dependencies = [ [[package]] name = "reth-consensus" -version = "2.2.0" -source = "git+https://github.com/paradigmxyz/reth?tag=v2.2.0#88505c7fcbfdebfd3b56d88c86b62e950043c6c4" +version = "2.3.0" +source = "git+https://github.com/paradigmxyz/reth?rev=f0a41795d72ae5bb0b6c3661ac42ffb7b78ef105#f0a41795d72ae5bb0b6c3661ac42ffb7b78ef105" dependencies = [ "alloy-consensus", + "alloy-eip7928", "alloy-primitives", "auto_impl", "reth-execution-types", @@ -6852,11 +6802,11 @@ dependencies = [ [[package]] name = "reth-consensus-common" -version = "2.2.0" -source = "git+https://github.com/paradigmxyz/reth?tag=v2.2.0#88505c7fcbfdebfd3b56d88c86b62e950043c6c4" +version = "2.3.0" +source = "git+https://github.com/paradigmxyz/reth?rev=f0a41795d72ae5bb0b6c3661ac42ffb7b78ef105#f0a41795d72ae5bb0b6c3661ac42ffb7b78ef105" dependencies = [ "alloy-consensus", - "alloy-eips 2.0.4", + "alloy-eips", "alloy-primitives", "reth-chainspec", "reth-consensus", @@ -6865,11 +6815,11 @@ dependencies = [ [[package]] name = "reth-consensus-debug-client" -version = "2.2.0" -source = "git+https://github.com/paradigmxyz/reth?tag=v2.2.0#88505c7fcbfdebfd3b56d88c86b62e950043c6c4" +version = "2.3.0" +source = "git+https://github.com/paradigmxyz/reth?rev=f0a41795d72ae5bb0b6c3661ac42ffb7b78ef105#f0a41795d72ae5bb0b6c3661ac42ffb7b78ef105" dependencies = [ "alloy-consensus", - "alloy-eips 2.0.4", + "alloy-eips", "alloy-json-rpc", "alloy-primitives", "alloy-provider", @@ -6879,9 +6829,8 @@ dependencies = [ "derive_more", "eyre", "futures", - "reqwest 0.13.3", + "reqwest", "reth-node-api", - "reth-primitives-traits", "reth-tracing", "ringbuffer", "serde", @@ -6891,8 +6840,8 @@ dependencies = [ [[package]] name = "reth-db" -version = "2.2.0" -source = "git+https://github.com/paradigmxyz/reth?tag=v2.2.0#88505c7fcbfdebfd3b56d88c86b62e950043c6c4" +version = "2.3.0" +source = "git+https://github.com/paradigmxyz/reth?rev=f0a41795d72ae5bb0b6c3661ac42ffb7b78ef105#f0a41795d72ae5bb0b6c3661ac42ffb7b78ef105" dependencies = [ "alloy-primitives", "derive_more", @@ -6920,8 +6869,8 @@ dependencies = [ [[package]] name = "reth-db-api" -version = "2.2.0" -source = "git+https://github.com/paradigmxyz/reth?tag=v2.2.0#88505c7fcbfdebfd3b56d88c86b62e950043c6c4" +version = "2.3.0" +source = "git+https://github.com/paradigmxyz/reth?rev=f0a41795d72ae5bb0b6c3661ac42ffb7b78ef105#f0a41795d72ae5bb0b6c3661ac42ffb7b78ef105" dependencies = [ "alloy-consensus", "alloy-primitives", @@ -6946,8 +6895,8 @@ dependencies = [ [[package]] name = "reth-db-common" -version = "2.2.0" -source = "git+https://github.com/paradigmxyz/reth?tag=v2.2.0#88505c7fcbfdebfd3b56d88c86b62e950043c6c4" +version = "2.3.0" +source = "git+https://github.com/paradigmxyz/reth?rev=f0a41795d72ae5bb0b6c3661ac42ffb7b78ef105#f0a41795d72ae5bb0b6c3661ac42ffb7b78ef105" dependencies = [ "alloy-consensus", "alloy-genesis", @@ -6976,10 +6925,10 @@ dependencies = [ [[package]] name = "reth-db-models" -version = "2.2.0" -source = "git+https://github.com/paradigmxyz/reth?tag=v2.2.0#88505c7fcbfdebfd3b56d88c86b62e950043c6c4" +version = "2.3.0" +source = "git+https://github.com/paradigmxyz/reth?rev=f0a41795d72ae5bb0b6c3661ac42ffb7b78ef105#f0a41795d72ae5bb0b6c3661ac42ffb7b78ef105" dependencies = [ - "alloy-eips 2.0.4", + "alloy-eips", "alloy-primitives", "arbitrary", "bytes", @@ -6991,8 +6940,8 @@ dependencies = [ [[package]] name = "reth-discv4" -version = "2.2.0" -source = "git+https://github.com/paradigmxyz/reth?tag=v2.2.0#88505c7fcbfdebfd3b56d88c86b62e950043c6c4" +version = "2.3.0" +source = "git+https://github.com/paradigmxyz/reth?rev=f0a41795d72ae5bb0b6c3661ac42ffb7b78ef105#f0a41795d72ae5bb0b6c3661ac42ffb7b78ef105" dependencies = [ "alloy-primitives", "alloy-rlp", @@ -7016,8 +6965,8 @@ dependencies = [ [[package]] name = "reth-discv5" -version = "2.2.0" -source = "git+https://github.com/paradigmxyz/reth?tag=v2.2.0#88505c7fcbfdebfd3b56d88c86b62e950043c6c4" +version = "2.3.0" +source = "git+https://github.com/paradigmxyz/reth?rev=f0a41795d72ae5bb0b6c3661ac42ffb7b78ef105#f0a41795d72ae5bb0b6c3661ac42ffb7b78ef105" dependencies = [ "alloy-primitives", "alloy-rlp", @@ -7040,8 +6989,8 @@ dependencies = [ [[package]] name = "reth-dns-discovery" -version = "2.2.0" -source = "git+https://github.com/paradigmxyz/reth?tag=v2.2.0#88505c7fcbfdebfd3b56d88c86b62e950043c6c4" +version = "2.3.0" +source = "git+https://github.com/paradigmxyz/reth?rev=f0a41795d72ae5bb0b6c3661ac42ffb7b78ef105#f0a41795d72ae5bb0b6c3661ac42ffb7b78ef105" dependencies = [ "alloy-primitives", "dashmap", @@ -7064,11 +7013,11 @@ dependencies = [ [[package]] name = "reth-downloaders" -version = "2.2.0" -source = "git+https://github.com/paradigmxyz/reth?tag=v2.2.0#88505c7fcbfdebfd3b56d88c86b62e950043c6c4" +version = "2.3.0" +source = "git+https://github.com/paradigmxyz/reth?rev=f0a41795d72ae5bb0b6c3661ac42ffb7b78ef105#f0a41795d72ae5bb0b6c3661ac42ffb7b78ef105" dependencies = [ "alloy-consensus", - "alloy-eips 2.0.4", + "alloy-eips", "alloy-primitives", "alloy-rlp", "async-compression", @@ -7099,11 +7048,11 @@ dependencies = [ [[package]] name = "reth-e2e-test-utils" -version = "2.2.0" -source = "git+https://github.com/paradigmxyz/reth?tag=v2.2.0#88505c7fcbfdebfd3b56d88c86b62e950043c6c4" +version = "2.3.0" +source = "git+https://github.com/paradigmxyz/reth?rev=f0a41795d72ae5bb0b6c3661ac42ffb7b78ef105#f0a41795d72ae5bb0b6c3661ac42ffb7b78ef105" dependencies = [ "alloy-consensus", - "alloy-eips 2.0.4", + "alloy-eips", "alloy-network", "alloy-primitives", "alloy-provider", @@ -7156,13 +7105,12 @@ dependencies = [ [[package]] name = "reth-ecies" -version = "2.2.0" -source = "git+https://github.com/paradigmxyz/reth?tag=v2.2.0#88505c7fcbfdebfd3b56d88c86b62e950043c6c4" +version = "2.3.0" +source = "git+https://github.com/paradigmxyz/reth?rev=f0a41795d72ae5bb0b6c3661ac42ffb7b78ef105#f0a41795d72ae5bb0b6c3661ac42ffb7b78ef105" dependencies = [ "aes", "alloy-primitives", "alloy-rlp", - "block-padding", "byteorder", "cipher", "concat-kdf", @@ -7184,8 +7132,8 @@ dependencies = [ [[package]] name = "reth-engine-local" -version = "2.2.0" -source = "git+https://github.com/paradigmxyz/reth?tag=v2.2.0#88505c7fcbfdebfd3b56d88c86b62e950043c6c4" +version = "2.3.0" +source = "git+https://github.com/paradigmxyz/reth?rev=f0a41795d72ae5bb0b6c3661ac42ffb7b78ef105#f0a41795d72ae5bb0b6c3661ac42ffb7b78ef105" dependencies = [ "alloy-consensus", "alloy-primitives", @@ -7207,11 +7155,11 @@ dependencies = [ [[package]] name = "reth-engine-primitives" -version = "2.2.0" -source = "git+https://github.com/paradigmxyz/reth?tag=v2.2.0#88505c7fcbfdebfd3b56d88c86b62e950043c6c4" +version = "2.3.0" +source = "git+https://github.com/paradigmxyz/reth?rev=f0a41795d72ae5bb0b6c3661ac42ffb7b78ef105#f0a41795d72ae5bb0b6c3661ac42ffb7b78ef105" dependencies = [ "alloy-consensus", - "alloy-eips 2.0.4", + "alloy-eips", "alloy-primitives", "alloy-rpc-types-engine", "auto_impl", @@ -7232,12 +7180,12 @@ dependencies = [ [[package]] name = "reth-engine-tree" -version = "2.2.0" -source = "git+https://github.com/paradigmxyz/reth?tag=v2.2.0#88505c7fcbfdebfd3b56d88c86b62e950043c6c4" +version = "2.3.0" +source = "git+https://github.com/paradigmxyz/reth?rev=f0a41795d72ae5bb0b6c3661ac42ffb7b78ef105#f0a41795d72ae5bb0b6c3661ac42ffb7b78ef105" dependencies = [ "alloy-consensus", "alloy-eip7928", - "alloy-eips 2.0.4", + "alloy-eips", "alloy-evm", "alloy-primitives", "alloy-rlp", @@ -7248,7 +7196,6 @@ dependencies = [ "indexmap 2.14.0", "metrics", "moka", - "parking_lot", "rayon", "reth-chain-state", "reth-chainspec", @@ -7280,7 +7227,6 @@ dependencies = [ "reth-trie-parallel", "reth-trie-sparse", "revm", - "revm-primitives", "schnellru", "thiserror 2.0.18", "tokio", @@ -7289,10 +7235,12 @@ dependencies = [ [[package]] name = "reth-engine-util" -version = "2.2.0" -source = "git+https://github.com/paradigmxyz/reth?tag=v2.2.0#88505c7fcbfdebfd3b56d88c86b62e950043c6c4" +version = "2.3.0" +source = "git+https://github.com/paradigmxyz/reth?rev=f0a41795d72ae5bb0b6c3661ac42ffb7b78ef105#f0a41795d72ae5bb0b6c3661ac42ffb7b78ef105" dependencies = [ "alloy-consensus", + "alloy-primitives", + "alloy-rlp", "alloy-rpc-types-engine", "eyre", "futures", @@ -7317,13 +7265,15 @@ dependencies = [ [[package]] name = "reth-era" -version = "2.2.0" -source = "git+https://github.com/paradigmxyz/reth?tag=v2.2.0#88505c7fcbfdebfd3b56d88c86b62e950043c6c4" +version = "2.3.0" +source = "git+https://github.com/paradigmxyz/reth?rev=f0a41795d72ae5bb0b6c3661ac42ffb7b78ef105#f0a41795d72ae5bb0b6c3661ac42ffb7b78ef105" dependencies = [ "alloy-consensus", - "alloy-eips 2.0.4", + "alloy-eips", "alloy-primitives", "alloy-rlp", + "alloy-rpc-types-beacon", + "alloy-rpc-types-engine", "ethereum_ssz", "ethereum_ssz_derive", "sha2", @@ -7333,14 +7283,14 @@ dependencies = [ [[package]] name = "reth-era-downloader" -version = "2.2.0" -source = "git+https://github.com/paradigmxyz/reth?tag=v2.2.0#88505c7fcbfdebfd3b56d88c86b62e950043c6c4" +version = "2.3.0" +source = "git+https://github.com/paradigmxyz/reth?rev=f0a41795d72ae5bb0b6c3661ac42ffb7b78ef105#f0a41795d72ae5bb0b6c3661ac42ffb7b78ef105" dependencies = [ "alloy-primitives", "bytes", "eyre", "futures-util", - "reqwest 0.13.3", + "reqwest", "reth-era", "reth-fs-util", "sha2", @@ -7349,11 +7299,12 @@ dependencies = [ [[package]] name = "reth-era-utils" -version = "2.2.0" -source = "git+https://github.com/paradigmxyz/reth?tag=v2.2.0#88505c7fcbfdebfd3b56d88c86b62e950043c6c4" +version = "2.3.0" +source = "git+https://github.com/paradigmxyz/reth?rev=f0a41795d72ae5bb0b6c3661ac42ffb7b78ef105#f0a41795d72ae5bb0b6c3661ac42ffb7b78ef105" dependencies = [ "alloy-consensus", "alloy-primitives", + "alloy-rlp", "eyre", "futures-util", "reth-db-api", @@ -7371,8 +7322,8 @@ dependencies = [ [[package]] name = "reth-errors" -version = "2.2.0" -source = "git+https://github.com/paradigmxyz/reth?tag=v2.2.0#88505c7fcbfdebfd3b56d88c86b62e950043c6c4" +version = "2.3.0" +source = "git+https://github.com/paradigmxyz/reth?rev=f0a41795d72ae5bb0b6c3661ac42ffb7b78ef105#f0a41795d72ae5bb0b6c3661ac42ffb7b78ef105" dependencies = [ "reth-consensus", "reth-execution-errors", @@ -7382,8 +7333,8 @@ dependencies = [ [[package]] name = "reth-eth-wire" -version = "2.2.0" -source = "git+https://github.com/paradigmxyz/reth?tag=v2.2.0#88505c7fcbfdebfd3b56d88c86b62e950043c6c4" +version = "2.3.0" +source = "git+https://github.com/paradigmxyz/reth?rev=f0a41795d72ae5bb0b6c3661ac42ffb7b78ef105#f0a41795d72ae5bb0b6c3661ac42ffb7b78ef105" dependencies = [ "alloy-chains", "alloy-primitives", @@ -7410,13 +7361,13 @@ dependencies = [ [[package]] name = "reth-eth-wire-types" -version = "2.2.0" -source = "git+https://github.com/paradigmxyz/reth?tag=v2.2.0#88505c7fcbfdebfd3b56d88c86b62e950043c6c4" +version = "2.3.0" +source = "git+https://github.com/paradigmxyz/reth?rev=f0a41795d72ae5bb0b6c3661ac42ffb7b78ef105#f0a41795d72ae5bb0b6c3661ac42ffb7b78ef105" dependencies = [ "alloy-chains", "alloy-consensus", "alloy-eip7928", - "alloy-eips 2.0.4", + "alloy-eips", "alloy-hardforks", "alloy-primitives", "alloy-rlp", @@ -7432,8 +7383,8 @@ dependencies = [ [[package]] name = "reth-ethereum-cli" -version = "2.2.0" -source = "git+https://github.com/paradigmxyz/reth?tag=v2.2.0#88505c7fcbfdebfd3b56d88c86b62e950043c6c4" +version = "2.3.0" +source = "git+https://github.com/paradigmxyz/reth?rev=f0a41795d72ae5bb0b6c3661ac42ffb7b78ef105#f0a41795d72ae5bb0b6c3661ac42ffb7b78ef105" dependencies = [ "clap", "eyre", @@ -7455,11 +7406,11 @@ dependencies = [ [[package]] name = "reth-ethereum-consensus" -version = "2.2.0" -source = "git+https://github.com/paradigmxyz/reth?tag=v2.2.0#88505c7fcbfdebfd3b56d88c86b62e950043c6c4" +version = "2.3.0" +source = "git+https://github.com/paradigmxyz/reth?rev=f0a41795d72ae5bb0b6c3661ac42ffb7b78ef105#f0a41795d72ae5bb0b6c3661ac42ffb7b78ef105" dependencies = [ "alloy-consensus", - "alloy-eips 2.0.4", + "alloy-eips", "alloy-primitives", "reth-chainspec", "reth-consensus", @@ -7471,10 +7422,10 @@ dependencies = [ [[package]] name = "reth-ethereum-engine-primitives" -version = "2.2.0" -source = "git+https://github.com/paradigmxyz/reth?tag=v2.2.0#88505c7fcbfdebfd3b56d88c86b62e950043c6c4" +version = "2.3.0" +source = "git+https://github.com/paradigmxyz/reth?rev=f0a41795d72ae5bb0b6c3661ac42ffb7b78ef105#f0a41795d72ae5bb0b6c3661ac42ffb7b78ef105" dependencies = [ - "alloy-eips 2.0.4", + "alloy-eips", "alloy-primitives", "alloy-rpc-types-engine", "reth-engine-primitives", @@ -7487,8 +7438,8 @@ dependencies = [ [[package]] name = "reth-ethereum-forks" -version = "2.2.0" -source = "git+https://github.com/paradigmxyz/reth?tag=v2.2.0#88505c7fcbfdebfd3b56d88c86b62e950043c6c4" +version = "2.3.0" +source = "git+https://github.com/paradigmxyz/reth?rev=f0a41795d72ae5bb0b6c3661ac42ffb7b78ef105#f0a41795d72ae5bb0b6c3661ac42ffb7b78ef105" dependencies = [ "alloy-eip2124", "alloy-hardforks", @@ -7500,11 +7451,11 @@ dependencies = [ [[package]] name = "reth-ethereum-payload-builder" -version = "2.2.0" -source = "git+https://github.com/paradigmxyz/reth?tag=v2.2.0#88505c7fcbfdebfd3b56d88c86b62e950043c6c4" +version = "2.3.0" +source = "git+https://github.com/paradigmxyz/reth?rev=f0a41795d72ae5bb0b6c3661ac42ffb7b78ef105#f0a41795d72ae5bb0b6c3661ac42ffb7b78ef105" dependencies = [ "alloy-consensus", - "alloy-eips 2.0.4", + "alloy-eips", "alloy-primitives", "alloy-rlp", "alloy-rpc-types-engine", @@ -7530,11 +7481,11 @@ dependencies = [ [[package]] name = "reth-ethereum-primitives" -version = "2.2.0" -source = "git+https://github.com/paradigmxyz/reth?tag=v2.2.0#88505c7fcbfdebfd3b56d88c86b62e950043c6c4" +version = "2.3.0" +source = "git+https://github.com/paradigmxyz/reth?rev=f0a41795d72ae5bb0b6c3661ac42ffb7b78ef105#f0a41795d72ae5bb0b6c3661ac42ffb7b78ef105" dependencies = [ "alloy-consensus", - "alloy-eips 2.0.4", + "alloy-eips", "alloy-primitives", "alloy-rpc-types-eth", "reth-codecs", @@ -7544,8 +7495,8 @@ dependencies = [ [[package]] name = "reth-etl" -version = "2.2.0" -source = "git+https://github.com/paradigmxyz/reth?tag=v2.2.0#88505c7fcbfdebfd3b56d88c86b62e950043c6c4" +version = "2.3.0" +source = "git+https://github.com/paradigmxyz/reth?rev=f0a41795d72ae5bb0b6c3661ac42ffb7b78ef105#f0a41795d72ae5bb0b6c3661ac42ffb7b78ef105" dependencies = [ "rayon", "reth-db-api", @@ -7554,11 +7505,12 @@ dependencies = [ [[package]] name = "reth-evm" -version = "2.2.0" -source = "git+https://github.com/paradigmxyz/reth?tag=v2.2.0#88505c7fcbfdebfd3b56d88c86b62e950043c6c4" +version = "2.3.0" +source = "git+https://github.com/paradigmxyz/reth?rev=f0a41795d72ae5bb0b6c3661ac42ffb7b78ef105#f0a41795d72ae5bb0b6c3661ac42ffb7b78ef105" dependencies = [ "alloy-consensus", - "alloy-eips 2.0.4", + "alloy-eip7928", + "alloy-eips", "alloy-evm", "alloy-primitives", "auto_impl", @@ -7578,11 +7530,11 @@ dependencies = [ [[package]] name = "reth-evm-ethereum" -version = "2.2.0" -source = "git+https://github.com/paradigmxyz/reth?tag=v2.2.0#88505c7fcbfdebfd3b56d88c86b62e950043c6c4" +version = "2.3.0" +source = "git+https://github.com/paradigmxyz/reth?rev=f0a41795d72ae5bb0b6c3661ac42ffb7b78ef105#f0a41795d72ae5bb0b6c3661ac42ffb7b78ef105" dependencies = [ "alloy-consensus", - "alloy-eips 2.0.4", + "alloy-eips", "alloy-evm", "alloy-primitives", "alloy-rpc-types-engine", @@ -7598,8 +7550,8 @@ dependencies = [ [[package]] name = "reth-execution-cache" -version = "2.2.0" -source = "git+https://github.com/paradigmxyz/reth?tag=v2.2.0#88505c7fcbfdebfd3b56d88c86b62e950043c6c4" +version = "2.3.0" +source = "git+https://github.com/paradigmxyz/reth?rev=f0a41795d72ae5bb0b6c3661ac42ffb7b78ef105#f0a41795d72ae5bb0b6c3661ac42ffb7b78ef105" dependencies = [ "alloy-primitives", "fixed-cache", @@ -7616,8 +7568,8 @@ dependencies = [ [[package]] name = "reth-execution-errors" -version = "2.2.0" -source = "git+https://github.com/paradigmxyz/reth?tag=v2.2.0#88505c7fcbfdebfd3b56d88c86b62e950043c6c4" +version = "2.3.0" +source = "git+https://github.com/paradigmxyz/reth?rev=f0a41795d72ae5bb0b6c3661ac42ffb7b78ef105#f0a41795d72ae5bb0b6c3661ac42ffb7b78ef105" dependencies = [ "alloy-evm", "alloy-primitives", @@ -7629,11 +7581,11 @@ dependencies = [ [[package]] name = "reth-execution-types" -version = "2.2.0" -source = "git+https://github.com/paradigmxyz/reth?tag=v2.2.0#88505c7fcbfdebfd3b56d88c86b62e950043c6c4" +version = "2.3.0" +source = "git+https://github.com/paradigmxyz/reth?rev=f0a41795d72ae5bb0b6c3661ac42ffb7b78ef105#f0a41795d72ae5bb0b6c3661ac42ffb7b78ef105" dependencies = [ "alloy-consensus", - "alloy-eips 2.0.4", + "alloy-eips", "alloy-evm", "alloy-primitives", "alloy-rlp", @@ -7648,11 +7600,11 @@ dependencies = [ [[package]] name = "reth-exex" -version = "2.2.0" -source = "git+https://github.com/paradigmxyz/reth?tag=v2.2.0#88505c7fcbfdebfd3b56d88c86b62e950043c6c4" +version = "2.3.0" +source = "git+https://github.com/paradigmxyz/reth?rev=f0a41795d72ae5bb0b6c3661ac42ffb7b78ef105#f0a41795d72ae5bb0b6c3661ac42ffb7b78ef105" dependencies = [ "alloy-consensus", - "alloy-eips 2.0.4", + "alloy-eips", "alloy-primitives", "eyre", "futures", @@ -7686,10 +7638,10 @@ dependencies = [ [[package]] name = "reth-exex-types" -version = "2.2.0" -source = "git+https://github.com/paradigmxyz/reth?tag=v2.2.0#88505c7fcbfdebfd3b56d88c86b62e950043c6c4" +version = "2.3.0" +source = "git+https://github.com/paradigmxyz/reth?rev=f0a41795d72ae5bb0b6c3661ac42ffb7b78ef105#f0a41795d72ae5bb0b6c3661ac42ffb7b78ef105" dependencies = [ - "alloy-eips 2.0.4", + "alloy-eips", "alloy-primitives", "reth-chain-state", "reth-execution-types", @@ -7700,8 +7652,8 @@ dependencies = [ [[package]] name = "reth-fs-util" -version = "2.2.0" -source = "git+https://github.com/paradigmxyz/reth?tag=v2.2.0#88505c7fcbfdebfd3b56d88c86b62e950043c6c4" +version = "2.3.0" +source = "git+https://github.com/paradigmxyz/reth?rev=f0a41795d72ae5bb0b6c3661ac42ffb7b78ef105#f0a41795d72ae5bb0b6c3661ac42ffb7b78ef105" dependencies = [ "serde", "serde_json", @@ -7710,8 +7662,8 @@ dependencies = [ [[package]] name = "reth-invalid-block-hooks" -version = "2.2.0" -source = "git+https://github.com/paradigmxyz/reth?tag=v2.2.0#88505c7fcbfdebfd3b56d88c86b62e950043c6c4" +version = "2.3.0" +source = "git+https://github.com/paradigmxyz/reth?rev=f0a41795d72ae5bb0b6c3661ac42ffb7b78ef105#f0a41795d72ae5bb0b6c3661ac42ffb7b78ef105" dependencies = [ "alloy-consensus", "alloy-primitives", @@ -7730,16 +7682,14 @@ dependencies = [ "reth-tracing", "reth-trie", "revm", - "revm-bytecode", - "revm-database", "serde", "serde_json", ] [[package]] name = "reth-ipc" -version = "2.2.0" -source = "git+https://github.com/paradigmxyz/reth?tag=v2.2.0#88505c7fcbfdebfd3b56d88c86b62e950043c6c4" +version = "2.3.0" +source = "git+https://github.com/paradigmxyz/reth?rev=f0a41795d72ae5bb0b6c3661ac42ffb7b78ef105#f0a41795d72ae5bb0b6c3661ac42ffb7b78ef105" dependencies = [ "bytes", "futures", @@ -7758,8 +7708,8 @@ dependencies = [ [[package]] name = "reth-libmdbx" -version = "2.2.0" -source = "git+https://github.com/paradigmxyz/reth?tag=v2.2.0#88505c7fcbfdebfd3b56d88c86b62e950043c6c4" +version = "2.3.0" +source = "git+https://github.com/paradigmxyz/reth?rev=f0a41795d72ae5bb0b6c3661ac42ffb7b78ef105#f0a41795d72ae5bb0b6c3661ac42ffb7b78ef105" dependencies = [ "bitflags", "byteorder", @@ -7775,8 +7725,8 @@ dependencies = [ [[package]] name = "reth-mdbx-sys" -version = "2.2.0" -source = "git+https://github.com/paradigmxyz/reth?tag=v2.2.0#88505c7fcbfdebfd3b56d88c86b62e950043c6c4" +version = "2.3.0" +source = "git+https://github.com/paradigmxyz/reth?rev=f0a41795d72ae5bb0b6c3661ac42ffb7b78ef105#f0a41795d72ae5bb0b6c3661ac42ffb7b78ef105" dependencies = [ "bindgen", "cc", @@ -7784,8 +7734,8 @@ dependencies = [ [[package]] name = "reth-metrics" -version = "2.2.0" -source = "git+https://github.com/paradigmxyz/reth?tag=v2.2.0#88505c7fcbfdebfd3b56d88c86b62e950043c6c4" +version = "2.3.0" +source = "git+https://github.com/paradigmxyz/reth?rev=f0a41795d72ae5bb0b6c3661ac42ffb7b78ef105#f0a41795d72ae5bb0b6c3661ac42ffb7b78ef105" dependencies = [ "futures", "metrics", @@ -7797,8 +7747,8 @@ dependencies = [ [[package]] name = "reth-net-banlist" -version = "2.2.0" -source = "git+https://github.com/paradigmxyz/reth?tag=v2.2.0#88505c7fcbfdebfd3b56d88c86b62e950043c6c4" +version = "2.3.0" +source = "git+https://github.com/paradigmxyz/reth?rev=f0a41795d72ae5bb0b6c3661ac42ffb7b78ef105#f0a41795d72ae5bb0b6c3661ac42ffb7b78ef105" dependencies = [ "alloy-primitives", "ipnet", @@ -7806,12 +7756,12 @@ dependencies = [ [[package]] name = "reth-net-nat" -version = "2.2.0" -source = "git+https://github.com/paradigmxyz/reth?tag=v2.2.0#88505c7fcbfdebfd3b56d88c86b62e950043c6c4" +version = "2.3.0" +source = "git+https://github.com/paradigmxyz/reth?rev=f0a41795d72ae5bb0b6c3661ac42ffb7b78ef105#f0a41795d72ae5bb0b6c3661ac42ffb7b78ef105" dependencies = [ "futures-util", "if-addrs", - "reqwest 0.13.3", + "reqwest", "serde_with", "thiserror 2.0.18", "tokio", @@ -7820,11 +7770,11 @@ dependencies = [ [[package]] name = "reth-network" -version = "2.2.0" -source = "git+https://github.com/paradigmxyz/reth?tag=v2.2.0#88505c7fcbfdebfd3b56d88c86b62e950043c6c4" +version = "2.3.0" +source = "git+https://github.com/paradigmxyz/reth?rev=f0a41795d72ae5bb0b6c3661ac42ffb7b78ef105#f0a41795d72ae5bb0b6c3661ac42ffb7b78ef105" dependencies = [ "alloy-consensus", - "alloy-eips 2.0.4", + "alloy-eips", "alloy-primitives", "alloy-rlp", "aquamarine", @@ -7878,8 +7828,8 @@ dependencies = [ [[package]] name = "reth-network-api" -version = "2.2.0" -source = "git+https://github.com/paradigmxyz/reth?tag=v2.2.0#88505c7fcbfdebfd3b56d88c86b62e950043c6c4" +version = "2.3.0" +source = "git+https://github.com/paradigmxyz/reth?rev=f0a41795d72ae5bb0b6c3661ac42ffb7b78ef105#f0a41795d72ae5bb0b6c3661ac42ffb7b78ef105" dependencies = [ "alloy-consensus", "alloy-primitives", @@ -7903,11 +7853,12 @@ dependencies = [ [[package]] name = "reth-network-p2p" -version = "2.2.0" -source = "git+https://github.com/paradigmxyz/reth?tag=v2.2.0#88505c7fcbfdebfd3b56d88c86b62e950043c6c4" +version = "2.3.0" +source = "git+https://github.com/paradigmxyz/reth?rev=f0a41795d72ae5bb0b6c3661ac42ffb7b78ef105#f0a41795d72ae5bb0b6c3661ac42ffb7b78ef105" dependencies = [ "alloy-consensus", - "alloy-eips 2.0.4", + "alloy-eip7928", + "alloy-eips", "alloy-primitives", "auto_impl", "derive_more", @@ -7926,8 +7877,8 @@ dependencies = [ [[package]] name = "reth-network-peers" -version = "2.2.0" -source = "git+https://github.com/paradigmxyz/reth?tag=v2.2.0#88505c7fcbfdebfd3b56d88c86b62e950043c6c4" +version = "2.3.0" +source = "git+https://github.com/paradigmxyz/reth?rev=f0a41795d72ae5bb0b6c3661ac42ffb7b78ef105#f0a41795d72ae5bb0b6c3661ac42ffb7b78ef105" dependencies = [ "alloy-primitives", "alloy-rlp", @@ -7941,8 +7892,8 @@ dependencies = [ [[package]] name = "reth-network-types" -version = "2.2.0" -source = "git+https://github.com/paradigmxyz/reth?tag=v2.2.0#88505c7fcbfdebfd3b56d88c86b62e950043c6c4" +version = "2.3.0" +source = "git+https://github.com/paradigmxyz/reth?rev=f0a41795d72ae5bb0b6c3661ac42ffb7b78ef105#f0a41795d72ae5bb0b6c3661ac42ffb7b78ef105" dependencies = [ "alloy-eip2124", "humantime-serde", @@ -7955,8 +7906,8 @@ dependencies = [ [[package]] name = "reth-nippy-jar" -version = "2.2.0" -source = "git+https://github.com/paradigmxyz/reth?tag=v2.2.0#88505c7fcbfdebfd3b56d88c86b62e950043c6c4" +version = "2.3.0" +source = "git+https://github.com/paradigmxyz/reth?rev=f0a41795d72ae5bb0b6c3661ac42ffb7b78ef105#f0a41795d72ae5bb0b6c3661ac42ffb7b78ef105" dependencies = [ "anyhow", "bincode", @@ -7972,8 +7923,8 @@ dependencies = [ [[package]] name = "reth-node-api" -version = "2.2.0" -source = "git+https://github.com/paradigmxyz/reth?tag=v2.2.0#88505c7fcbfdebfd3b56d88c86b62e950043c6c4" +version = "2.3.0" +source = "git+https://github.com/paradigmxyz/reth?rev=f0a41795d72ae5bb0b6c3661ac42ffb7b78ef105#f0a41795d72ae5bb0b6c3661ac42ffb7b78ef105" dependencies = [ "alloy-rpc-types-engine", "eyre", @@ -7996,11 +7947,11 @@ dependencies = [ [[package]] name = "reth-node-builder" -version = "2.2.0" -source = "git+https://github.com/paradigmxyz/reth?tag=v2.2.0#88505c7fcbfdebfd3b56d88c86b62e950043c6c4" +version = "2.3.0" +source = "git+https://github.com/paradigmxyz/reth?rev=f0a41795d72ae5bb0b6c3661ac42ffb7b78ef105#f0a41795d72ae5bb0b6c3661ac42ffb7b78ef105" dependencies = [ "alloy-consensus", - "alloy-eips 2.0.4", + "alloy-eips", "alloy-primitives", "alloy-provider", "alloy-rpc-types", @@ -8064,11 +8015,11 @@ dependencies = [ [[package]] name = "reth-node-core" -version = "2.2.0" -source = "git+https://github.com/paradigmxyz/reth?tag=v2.2.0#88505c7fcbfdebfd3b56d88c86b62e950043c6c4" +version = "2.3.0" +source = "git+https://github.com/paradigmxyz/reth?rev=f0a41795d72ae5bb0b6c3661ac42ffb7b78ef105#f0a41795d72ae5bb0b6c3661ac42ffb7b78ef105" dependencies = [ "alloy-consensus", - "alloy-eips 2.0.4", + "alloy-eips", "alloy-primitives", "alloy-rpc-types-engine", "clap", @@ -8119,14 +8070,20 @@ dependencies = [ [[package]] name = "reth-node-ethereum" -version = "2.2.0" -source = "git+https://github.com/paradigmxyz/reth?tag=v2.2.0#88505c7fcbfdebfd3b56d88c86b62e950043c6c4" +version = "2.3.0" +source = "git+https://github.com/paradigmxyz/reth?rev=f0a41795d72ae5bb0b6c3661ac42ffb7b78ef105#f0a41795d72ae5bb0b6c3661ac42ffb7b78ef105" dependencies = [ - "alloy-eips 2.0.4", + "alloy-consensus", + "alloy-eips", "alloy-network", + "alloy-primitives", "alloy-rpc-types-engine", "alloy-rpc-types-eth", + "ethereum_ssz", + "ethereum_ssz_derive", "eyre", + "http-body-util", + "jsonrpsee", "reth-chainspec", "reth-engine-local", "reth-engine-primitives", @@ -8139,6 +8096,7 @@ dependencies = [ "reth-network", "reth-node-api", "reth-node-builder", + "reth-node-core", "reth-payload-primitives", "reth-primitives-traits", "reth-provider", @@ -8152,13 +8110,16 @@ dependencies = [ "reth-tracing", "reth-transaction-pool", "revm", + "serde", + "serde_json", "tokio", + "tower", ] [[package]] name = "reth-node-ethstats" -version = "2.2.0" -source = "git+https://github.com/paradigmxyz/reth?tag=v2.2.0#88505c7fcbfdebfd3b56d88c86b62e950043c6c4" +version = "2.3.0" +source = "git+https://github.com/paradigmxyz/reth?rev=f0a41795d72ae5bb0b6c3661ac42ffb7b78ef105#f0a41795d72ae5bb0b6c3661ac42ffb7b78ef105" dependencies = [ "alloy-consensus", "alloy-primitives", @@ -8181,11 +8142,11 @@ dependencies = [ [[package]] name = "reth-node-events" -version = "2.2.0" -source = "git+https://github.com/paradigmxyz/reth?tag=v2.2.0#88505c7fcbfdebfd3b56d88c86b62e950043c6c4" +version = "2.3.0" +source = "git+https://github.com/paradigmxyz/reth?rev=f0a41795d72ae5bb0b6c3661ac42ffb7b78ef105#f0a41795d72ae5bb0b6c3661ac42ffb7b78ef105" dependencies = [ "alloy-consensus", - "alloy-eips 2.0.4", + "alloy-eips", "alloy-primitives", "alloy-rpc-types-engine", "derive_more", @@ -8205,8 +8166,8 @@ dependencies = [ [[package]] name = "reth-node-metrics" -version = "2.2.0" -source = "git+https://github.com/paradigmxyz/reth?tag=v2.2.0#88505c7fcbfdebfd3b56d88c86b62e950043c6c4" +version = "2.3.0" +source = "git+https://github.com/paradigmxyz/reth?rev=f0a41795d72ae5bb0b6c3661ac42ffb7b78ef105#f0a41795d72ae5bb0b6c3661ac42ffb7b78ef105" dependencies = [ "bytes", "eyre", @@ -8221,7 +8182,7 @@ dependencies = [ "metrics-util", "pprof_util", "procfs", - "reqwest 0.13.3", + "reqwest", "reth-fs-util", "reth-metrics", "reth-tasks", @@ -8234,8 +8195,8 @@ dependencies = [ [[package]] name = "reth-node-types" -version = "2.2.0" -source = "git+https://github.com/paradigmxyz/reth?tag=v2.2.0#88505c7fcbfdebfd3b56d88c86b62e950043c6c4" +version = "2.3.0" +source = "git+https://github.com/paradigmxyz/reth?rev=f0a41795d72ae5bb0b6c3661ac42ffb7b78ef105#f0a41795d72ae5bb0b6c3661ac42ffb7b78ef105" dependencies = [ "reth-chainspec", "reth-db-api", @@ -8246,8 +8207,8 @@ dependencies = [ [[package]] name = "reth-payload-builder" -version = "2.2.0" -source = "git+https://github.com/paradigmxyz/reth?tag=v2.2.0#88505c7fcbfdebfd3b56d88c86b62e950043c6c4" +version = "2.3.0" +source = "git+https://github.com/paradigmxyz/reth?rev=f0a41795d72ae5bb0b6c3661ac42ffb7b78ef105#f0a41795d72ae5bb0b6c3661ac42ffb7b78ef105" dependencies = [ "alloy-consensus", "alloy-primitives", @@ -8270,8 +8231,8 @@ dependencies = [ [[package]] name = "reth-payload-builder-primitives" -version = "2.2.0" -source = "git+https://github.com/paradigmxyz/reth?tag=v2.2.0#88505c7fcbfdebfd3b56d88c86b62e950043c6c4" +version = "2.3.0" +source = "git+https://github.com/paradigmxyz/reth?rev=f0a41795d72ae5bb0b6c3661ac42ffb7b78ef105#f0a41795d72ae5bb0b6c3661ac42ffb7b78ef105" dependencies = [ "pin-project", "reth-payload-primitives", @@ -8282,17 +8243,16 @@ dependencies = [ [[package]] name = "reth-payload-primitives" -version = "2.2.0" -source = "git+https://github.com/paradigmxyz/reth?tag=v2.2.0#88505c7fcbfdebfd3b56d88c86b62e950043c6c4" +version = "2.3.0" +source = "git+https://github.com/paradigmxyz/reth?rev=f0a41795d72ae5bb0b6c3661ac42ffb7b78ef105#f0a41795d72ae5bb0b6c3661ac42ffb7b78ef105" dependencies = [ "alloy-consensus", - "alloy-eips 2.0.4", + "alloy-eips", "alloy-primitives", "alloy-rlp", "alloy-rpc-types-engine", "auto_impl", "either", - "reth-chain-state", "reth-chainspec", "reth-errors", "reth-execution-types", @@ -8306,8 +8266,8 @@ dependencies = [ [[package]] name = "reth-payload-util" -version = "2.2.0" -source = "git+https://github.com/paradigmxyz/reth?tag=v2.2.0#88505c7fcbfdebfd3b56d88c86b62e950043c6c4" +version = "2.3.0" +source = "git+https://github.com/paradigmxyz/reth?rev=f0a41795d72ae5bb0b6c3661ac42ffb7b78ef105#f0a41795d72ae5bb0b6c3661ac42ffb7b78ef105" dependencies = [ "alloy-consensus", "alloy-primitives", @@ -8316,8 +8276,8 @@ dependencies = [ [[package]] name = "reth-payload-validator" -version = "2.2.0" -source = "git+https://github.com/paradigmxyz/reth?tag=v2.2.0#88505c7fcbfdebfd3b56d88c86b62e950043c6c4" +version = "2.3.0" +source = "git+https://github.com/paradigmxyz/reth?rev=f0a41795d72ae5bb0b6c3661ac42ffb7b78ef105#f0a41795d72ae5bb0b6c3661ac42ffb7b78ef105" dependencies = [ "alloy-consensus", "alloy-rpc-types-engine", @@ -8326,12 +8286,12 @@ dependencies = [ [[package]] name = "reth-primitives-traits" -version = "0.3.1" +version = "0.5.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8ee12e304adbacbb32248c9806ebafbe1e2811fbfefe53c5e5b710a8438b7ec0" +checksum = "f59c928b2865af5bcdbce94800270b7f7798f27a22ca0aabdd2b7d3e6587c9ce" dependencies = [ "alloy-consensus", - "alloy-eips 2.0.4", + "alloy-eips", "alloy-genesis", "alloy-primitives", "alloy-rlp", @@ -8359,11 +8319,12 @@ dependencies = [ [[package]] name = "reth-provider" -version = "2.2.0" -source = "git+https://github.com/paradigmxyz/reth?tag=v2.2.0#88505c7fcbfdebfd3b56d88c86b62e950043c6c4" +version = "2.3.0" +source = "git+https://github.com/paradigmxyz/reth?rev=f0a41795d72ae5bb0b6c3661ac42ffb7b78ef105#f0a41795d72ae5bb0b6c3661ac42ffb7b78ef105" dependencies = [ "alloy-consensus", - "alloy-eips 2.0.4", + "alloy-eip7928", + "alloy-eips", "alloy-genesis", "alloy-primitives", "alloy-rpc-types-engine", @@ -8393,11 +8354,12 @@ dependencies = [ "reth-storage-api", "reth-storage-errors", "reth-tasks", + "reth-tokio-util", "reth-trie", "reth-trie-db", - "revm-database", - "revm-state", + "revm", "rocksdb", + "smallvec", "strum", "tokio", "tracing", @@ -8405,15 +8367,13 @@ dependencies = [ [[package]] name = "reth-prune" -version = "2.2.0" -source = "git+https://github.com/paradigmxyz/reth?tag=v2.2.0#88505c7fcbfdebfd3b56d88c86b62e950043c6c4" +version = "2.3.0" +source = "git+https://github.com/paradigmxyz/reth?rev=f0a41795d72ae5bb0b6c3661ac42ffb7b78ef105#f0a41795d72ae5bb0b6c3661ac42ffb7b78ef105" dependencies = [ "alloy-consensus", - "alloy-eips 2.0.4", "alloy-primitives", "itertools 0.14.0", "metrics", - "rayon", "reth-config", "reth-db-api", "reth-errors", @@ -8434,8 +8394,8 @@ dependencies = [ [[package]] name = "reth-prune-types" -version = "2.2.0" -source = "git+https://github.com/paradigmxyz/reth?tag=v2.2.0#88505c7fcbfdebfd3b56d88c86b62e950043c6c4" +version = "2.3.0" +source = "git+https://github.com/paradigmxyz/reth?rev=f0a41795d72ae5bb0b6c3661ac42ffb7b78ef105#f0a41795d72ae5bb0b6c3661ac42ffb7b78ef105" dependencies = [ "alloy-primitives", "arbitrary", @@ -8450,8 +8410,8 @@ dependencies = [ [[package]] name = "reth-revm" -version = "2.2.0" -source = "git+https://github.com/paradigmxyz/reth?tag=v2.2.0#88505c7fcbfdebfd3b56d88c86b62e950043c6c4" +version = "2.3.0" +source = "git+https://github.com/paradigmxyz/reth?rev=f0a41795d72ae5bb0b6c3661ac42ffb7b78ef105#f0a41795d72ae5bb0b6c3661ac42ffb7b78ef105" dependencies = [ "alloy-primitives", "alloy-rlp", @@ -8465,12 +8425,13 @@ dependencies = [ [[package]] name = "reth-rpc" -version = "2.2.0" -source = "git+https://github.com/paradigmxyz/reth?tag=v2.2.0#88505c7fcbfdebfd3b56d88c86b62e950043c6c4" +version = "2.3.0" +source = "git+https://github.com/paradigmxyz/reth?rev=f0a41795d72ae5bb0b6c3661ac42ffb7b78ef105#f0a41795d72ae5bb0b6c3661ac42ffb7b78ef105" dependencies = [ "alloy-consensus", "alloy-dyn-abi", - "alloy-eips 2.0.4", + "alloy-eip7928", + "alloy-eips", "alloy-evm", "alloy-genesis", "alloy-network", @@ -8486,7 +8447,7 @@ dependencies = [ "alloy-rpc-types-mev", "alloy-rpc-types-trace", "alloy-rpc-types-txpool", - "alloy-serde 2.0.4", + "alloy-serde", "alloy-signer", "alloy-signer-local", "async-trait", @@ -8514,6 +8475,7 @@ dependencies = [ "reth-network-peers", "reth-network-types", "reth-node-api", + "reth-payload-primitives", "reth-primitives-traits", "reth-revm", "reth-rpc-api", @@ -8524,12 +8486,10 @@ dependencies = [ "reth-rpc-server-types", "reth-storage-api", "reth-tasks", - "reth-tracing", "reth-transaction-pool", "reth-trie-common", "revm", "revm-inspectors", - "revm-primitives", "serde", "serde_json", "sha2", @@ -8542,10 +8502,10 @@ dependencies = [ [[package]] name = "reth-rpc-api" -version = "2.2.0" -source = "git+https://github.com/paradigmxyz/reth?tag=v2.2.0#88505c7fcbfdebfd3b56d88c86b62e950043c6c4" +version = "2.3.0" +source = "git+https://github.com/paradigmxyz/reth?rev=f0a41795d72ae5bb0b6c3661ac42ffb7b78ef105#f0a41795d72ae5bb0b6c3661ac42ffb7b78ef105" dependencies = [ - "alloy-eips 2.0.4", + "alloy-eips", "alloy-genesis", "alloy-json-rpc", "alloy-primitives", @@ -8559,7 +8519,7 @@ dependencies = [ "alloy-rpc-types-mev", "alloy-rpc-types-trace", "alloy-rpc-types-txpool", - "alloy-serde 2.0.4", + "alloy-serde", "jsonrpsee", "reth-chain-state", "reth-engine-primitives", @@ -8572,8 +8532,8 @@ dependencies = [ [[package]] name = "reth-rpc-builder" -version = "2.2.0" -source = "git+https://github.com/paradigmxyz/reth?tag=v2.2.0#88505c7fcbfdebfd3b56d88c86b62e950043c6c4" +version = "2.3.0" +source = "git+https://github.com/paradigmxyz/reth?rev=f0a41795d72ae5bb0b6c3661ac42ffb7b78ef105#f0a41795d72ae5bb0b6c3661ac42ffb7b78ef105" dependencies = [ "alloy-network", "alloy-provider", @@ -8615,8 +8575,8 @@ dependencies = [ [[package]] name = "reth-rpc-convert" -version = "2.2.0" -source = "git+https://github.com/paradigmxyz/reth?tag=v2.2.0#88505c7fcbfdebfd3b56d88c86b62e950043c6c4" +version = "2.3.0" +source = "git+https://github.com/paradigmxyz/reth?rev=f0a41795d72ae5bb0b6c3661ac42ffb7b78ef105#f0a41795d72ae5bb0b6c3661ac42ffb7b78ef105" dependencies = [ "alloy-consensus", "alloy-evm", @@ -8635,10 +8595,10 @@ dependencies = [ [[package]] name = "reth-rpc-engine-api" -version = "2.2.0" -source = "git+https://github.com/paradigmxyz/reth?tag=v2.2.0#88505c7fcbfdebfd3b56d88c86b62e950043c6c4" +version = "2.3.0" +source = "git+https://github.com/paradigmxyz/reth?rev=f0a41795d72ae5bb0b6c3661ac42ffb7b78ef105#f0a41795d72ae5bb0b6c3661ac42ffb7b78ef105" dependencies = [ - "alloy-eips 2.0.4", + "alloy-eips", "alloy-primitives", "alloy-rlp", "alloy-rpc-types-engine", @@ -8666,13 +8626,13 @@ dependencies = [ [[package]] name = "reth-rpc-eth-api" -version = "2.2.0" -source = "git+https://github.com/paradigmxyz/reth?tag=v2.2.0#88505c7fcbfdebfd3b56d88c86b62e950043c6c4" +version = "2.3.0" +source = "git+https://github.com/paradigmxyz/reth?rev=f0a41795d72ae5bb0b6c3661ac42ffb7b78ef105#f0a41795d72ae5bb0b6c3661ac42ffb7b78ef105" dependencies = [ "alloy-consensus", "alloy-dyn-abi", "alloy-eip7928", - "alloy-eips 2.0.4", + "alloy-eips", "alloy-evm", "alloy-json-rpc", "alloy-network", @@ -8680,7 +8640,7 @@ dependencies = [ "alloy-rlp", "alloy-rpc-types-eth", "alloy-rpc-types-mev", - "alloy-serde 2.0.4", + "alloy-serde", "async-trait", "auto_impl", "dyn-clone", @@ -8695,6 +8655,7 @@ dependencies = [ "reth-network-api", "reth-node-api", "reth-primitives-traits", + "reth-prune-types", "reth-revm", "reth-rpc-convert", "reth-rpc-eth-types", @@ -8712,16 +8673,19 @@ dependencies = [ [[package]] name = "reth-rpc-eth-types" -version = "2.2.0" -source = "git+https://github.com/paradigmxyz/reth?tag=v2.2.0#88505c7fcbfdebfd3b56d88c86b62e950043c6c4" +version = "2.3.0" +source = "git+https://github.com/paradigmxyz/reth?rev=f0a41795d72ae5bb0b6c3661ac42ffb7b78ef105#f0a41795d72ae5bb0b6c3661ac42ffb7b78ef105" dependencies = [ + "alloy-chains", "alloy-consensus", - "alloy-eips 2.0.4", + "alloy-eip7928", + "alloy-eips", "alloy-evm", "alloy-network", "alloy-primitives", "alloy-rpc-client", "alloy-rpc-types-eth", + "alloy-serde", "alloy-sol-types", "alloy-transport", "derive_more", @@ -8731,7 +8695,7 @@ dependencies = [ "jsonrpsee-types", "metrics", "rand 0.9.4", - "reqwest 0.13.3", + "reqwest", "reth-chain-state", "reth-chainspec", "reth-errors", @@ -8760,8 +8724,8 @@ dependencies = [ [[package]] name = "reth-rpc-layer" -version = "2.2.0" -source = "git+https://github.com/paradigmxyz/reth?tag=v2.2.0#88505c7fcbfdebfd3b56d88c86b62e950043c6c4" +version = "2.3.0" +source = "git+https://github.com/paradigmxyz/reth?rev=f0a41795d72ae5bb0b6c3661ac42ffb7b78ef105#f0a41795d72ae5bb0b6c3661ac42ffb7b78ef105" dependencies = [ "alloy-rpc-types-engine", "http", @@ -8774,10 +8738,10 @@ dependencies = [ [[package]] name = "reth-rpc-server-types" -version = "2.2.0" -source = "git+https://github.com/paradigmxyz/reth?tag=v2.2.0#88505c7fcbfdebfd3b56d88c86b62e950043c6c4" +version = "2.3.0" +source = "git+https://github.com/paradigmxyz/reth?rev=f0a41795d72ae5bb0b6c3661ac42ffb7b78ef105#f0a41795d72ae5bb0b6c3661ac42ffb7b78ef105" dependencies = [ - "alloy-eips 2.0.4", + "alloy-eips", "alloy-primitives", "alloy-rpc-types-engine", "jsonrpsee-core", @@ -8790,9 +8754,9 @@ dependencies = [ [[package]] name = "reth-rpc-traits" -version = "0.3.1" +version = "0.5.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "860fe223501a76ff14aa3bf164f739f31008c2a2905ac85708bfd88f042e6151" +checksum = "ace77dbcdd59c014fda4565ebfec76cd1fe6f5d98584b4655e8d22a947b9eee3" dependencies = [ "alloy-consensus", "alloy-network", @@ -8805,11 +8769,10 @@ dependencies = [ [[package]] name = "reth-stages" -version = "2.2.0" -source = "git+https://github.com/paradigmxyz/reth?tag=v2.2.0#88505c7fcbfdebfd3b56d88c86b62e950043c6c4" +version = "2.3.0" +source = "git+https://github.com/paradigmxyz/reth?rev=f0a41795d72ae5bb0b6c3661ac42ffb7b78ef105#f0a41795d72ae5bb0b6c3661ac42ffb7b78ef105" dependencies = [ "alloy-consensus", - "alloy-eips 2.0.4", "alloy-primitives", "alloy-rlp", "eyre", @@ -8818,7 +8781,7 @@ dependencies = [ "num-traits", "page_size", "rayon", - "reqwest 0.13.3", + "reqwest", "reth-chainspec", "reth-codecs", "reth-config", @@ -8857,10 +8820,10 @@ dependencies = [ [[package]] name = "reth-stages-api" -version = "2.2.0" -source = "git+https://github.com/paradigmxyz/reth?tag=v2.2.0#88505c7fcbfdebfd3b56d88c86b62e950043c6c4" +version = "2.3.0" +source = "git+https://github.com/paradigmxyz/reth?rev=f0a41795d72ae5bb0b6c3661ac42ffb7b78ef105#f0a41795d72ae5bb0b6c3661ac42ffb7b78ef105" dependencies = [ - "alloy-eips 2.0.4", + "alloy-eips", "alloy-primitives", "aquamarine", "auto_impl", @@ -8885,8 +8848,8 @@ dependencies = [ [[package]] name = "reth-stages-types" -version = "2.2.0" -source = "git+https://github.com/paradigmxyz/reth?tag=v2.2.0#88505c7fcbfdebfd3b56d88c86b62e950043c6c4" +version = "2.3.0" +source = "git+https://github.com/paradigmxyz/reth?rev=f0a41795d72ae5bb0b6c3661ac42ffb7b78ef105#f0a41795d72ae5bb0b6c3661ac42ffb7b78ef105" dependencies = [ "alloy-primitives", "arbitrary", @@ -8899,8 +8862,8 @@ dependencies = [ [[package]] name = "reth-static-file" -version = "2.2.0" -source = "git+https://github.com/paradigmxyz/reth?tag=v2.2.0#88505c7fcbfdebfd3b56d88c86b62e950043c6c4" +version = "2.3.0" +source = "git+https://github.com/paradigmxyz/reth?rev=f0a41795d72ae5bb0b6c3661ac42ffb7b78ef105#f0a41795d72ae5bb0b6c3661ac42ffb7b78ef105" dependencies = [ "alloy-primitives", "parking_lot", @@ -8919,8 +8882,8 @@ dependencies = [ [[package]] name = "reth-static-file-types" -version = "2.2.0" -source = "git+https://github.com/paradigmxyz/reth?tag=v2.2.0#88505c7fcbfdebfd3b56d88c86b62e950043c6c4" +version = "2.3.0" +source = "git+https://github.com/paradigmxyz/reth?rev=f0a41795d72ae5bb0b6c3661ac42ffb7b78ef105#f0a41795d72ae5bb0b6c3661ac42ffb7b78ef105" dependencies = [ "alloy-primitives", "clap", @@ -8934,11 +8897,12 @@ dependencies = [ [[package]] name = "reth-storage-api" -version = "2.2.0" -source = "git+https://github.com/paradigmxyz/reth?tag=v2.2.0#88505c7fcbfdebfd3b56d88c86b62e950043c6c4" +version = "2.3.0" +source = "git+https://github.com/paradigmxyz/reth?rev=f0a41795d72ae5bb0b6c3661ac42ffb7b78ef105#f0a41795d72ae5bb0b6c3661ac42ffb7b78ef105" dependencies = [ "alloy-consensus", - "alloy-eips 2.0.4", + "alloy-eip7928", + "alloy-eips", "alloy-primitives", "alloy-rpc-types-engine", "auto_impl", @@ -8951,17 +8915,18 @@ dependencies = [ "reth-prune-types", "reth-stages-types", "reth-storage-errors", + "reth-tokio-util", "reth-trie-common", - "revm-database", + "revm", "serde_json", ] [[package]] name = "reth-storage-errors" -version = "2.2.0" -source = "git+https://github.com/paradigmxyz/reth?tag=v2.2.0#88505c7fcbfdebfd3b56d88c86b62e950043c6c4" +version = "2.3.0" +source = "git+https://github.com/paradigmxyz/reth?rev=f0a41795d72ae5bb0b6c3661ac42ffb7b78ef105#f0a41795d72ae5bb0b6c3661ac42ffb7b78ef105" dependencies = [ - "alloy-eips 2.0.4", + "alloy-eips", "alloy-primitives", "alloy-rlp", "derive_more", @@ -8969,15 +8934,14 @@ dependencies = [ "reth-primitives-traits", "reth-prune-types", "reth-static-file-types", - "revm-database-interface", - "revm-state", + "revm", "thiserror 2.0.18", ] [[package]] name = "reth-tasks" -version = "2.2.0" -source = "git+https://github.com/paradigmxyz/reth?tag=v2.2.0#88505c7fcbfdebfd3b56d88c86b62e950043c6c4" +version = "2.3.0" +source = "git+https://github.com/paradigmxyz/reth?rev=f0a41795d72ae5bb0b6c3661ac42ffb7b78ef105#f0a41795d72ae5bb0b6c3661ac42ffb7b78ef105" dependencies = [ "crossbeam-utils", "dashmap", @@ -8997,11 +8961,11 @@ dependencies = [ [[package]] name = "reth-testing-utils" -version = "2.2.0" -source = "git+https://github.com/paradigmxyz/reth?tag=v2.2.0#88505c7fcbfdebfd3b56d88c86b62e950043c6c4" +version = "2.3.0" +source = "git+https://github.com/paradigmxyz/reth?rev=f0a41795d72ae5bb0b6c3661ac42ffb7b78ef105#f0a41795d72ae5bb0b6c3661ac42ffb7b78ef105" dependencies = [ "alloy-consensus", - "alloy-eips 2.0.4", + "alloy-eips", "alloy-genesis", "alloy-primitives", "rand 0.8.6", @@ -9013,8 +8977,8 @@ dependencies = [ [[package]] name = "reth-tokio-util" -version = "2.2.0" -source = "git+https://github.com/paradigmxyz/reth?tag=v2.2.0#88505c7fcbfdebfd3b56d88c86b62e950043c6c4" +version = "2.3.0" +source = "git+https://github.com/paradigmxyz/reth?rev=f0a41795d72ae5bb0b6c3661ac42ffb7b78ef105#f0a41795d72ae5bb0b6c3661ac42ffb7b78ef105" dependencies = [ "tokio", "tokio-stream", @@ -9023,8 +8987,8 @@ dependencies = [ [[package]] name = "reth-tracing" -version = "2.2.0" -source = "git+https://github.com/paradigmxyz/reth?tag=v2.2.0#88505c7fcbfdebfd3b56d88c86b62e950043c6c4" +version = "2.3.0" +source = "git+https://github.com/paradigmxyz/reth?rev=f0a41795d72ae5bb0b6c3661ac42ffb7b78ef105#f0a41795d72ae5bb0b6c3661ac42ffb7b78ef105" dependencies = [ "clap", "eyre", @@ -9032,6 +8996,7 @@ dependencies = [ "rolling-file", "tracing", "tracing-appender", + "tracing-chrome", "tracing-journald", "tracing-logfmt", "tracing-samply", @@ -9040,9 +9005,10 @@ dependencies = [ [[package]] name = "reth-tracing-otlp" -version = "2.2.0" -source = "git+https://github.com/paradigmxyz/reth?tag=v2.2.0#88505c7fcbfdebfd3b56d88c86b62e950043c6c4" +version = "2.3.0" +source = "git+https://github.com/paradigmxyz/reth?rev=f0a41795d72ae5bb0b6c3661ac42ffb7b78ef105#f0a41795d72ae5bb0b6c3661ac42ffb7b78ef105" dependencies = [ + "base64 0.22.1", "clap", "eyre", "opentelemetry", @@ -9057,11 +9023,11 @@ dependencies = [ [[package]] name = "reth-transaction-pool" -version = "2.2.0" -source = "git+https://github.com/paradigmxyz/reth?tag=v2.2.0#88505c7fcbfdebfd3b56d88c86b62e950043c6c4" +version = "2.3.0" +source = "git+https://github.com/paradigmxyz/reth?rev=f0a41795d72ae5bb0b6c3661ac42ffb7b78ef105#f0a41795d72ae5bb0b6c3661ac42ffb7b78ef105" dependencies = [ "alloy-consensus", - "alloy-eips 2.0.4", + "alloy-eips", "alloy-primitives", "alloy-rlp", "aquamarine", @@ -9087,8 +9053,6 @@ dependencies = [ "reth-storage-api", "reth-tasks", "revm", - "revm-interpreter", - "revm-primitives", "rustc-hash", "schnellru", "serde", @@ -9102,11 +9066,11 @@ dependencies = [ [[package]] name = "reth-trie" -version = "2.2.0" -source = "git+https://github.com/paradigmxyz/reth?tag=v2.2.0#88505c7fcbfdebfd3b56d88c86b62e950043c6c4" +version = "2.3.0" +source = "git+https://github.com/paradigmxyz/reth?rev=f0a41795d72ae5bb0b6c3661ac42ffb7b78ef105#f0a41795d72ae5bb0b6c3661ac42ffb7b78ef105" dependencies = [ "alloy-consensus", - "alloy-eips 2.0.4", + "alloy-eips", "alloy-primitives", "alloy-rlp", "alloy-trie", @@ -9121,21 +9085,20 @@ dependencies = [ "reth-storage-errors", "reth-trie-common", "reth-trie-sparse", - "revm-database", "tracing", "triehash", ] [[package]] name = "reth-trie-common" -version = "2.2.0" -source = "git+https://github.com/paradigmxyz/reth?tag=v2.2.0#88505c7fcbfdebfd3b56d88c86b62e950043c6c4" +version = "2.3.0" +source = "git+https://github.com/paradigmxyz/reth?rev=f0a41795d72ae5bb0b6c3661ac42ffb7b78ef105#f0a41795d72ae5bb0b6c3661ac42ffb7b78ef105" dependencies = [ "alloy-consensus", "alloy-primitives", "alloy-rlp", "alloy-rpc-types-eth", - "alloy-serde 2.0.4", + "alloy-serde", "alloy-trie", "arbitrary", "arrayvec", @@ -9148,15 +9111,15 @@ dependencies = [ "rayon", "reth-codecs", "reth-primitives-traits", - "revm-database", + "revm", "serde", "serde_with", ] [[package]] name = "reth-trie-db" -version = "2.2.0" -source = "git+https://github.com/paradigmxyz/reth?tag=v2.2.0#88505c7fcbfdebfd3b56d88c86b62e950043c6c4" +version = "2.3.0" +source = "git+https://github.com/paradigmxyz/reth?rev=f0a41795d72ae5bb0b6c3661ac42ffb7b78ef105#f0a41795d72ae5bb0b6c3661ac42ffb7b78ef105" dependencies = [ "alloy-primitives", "metrics", @@ -9165,7 +9128,6 @@ dependencies = [ "reth-execution-errors", "reth-metrics", "reth-primitives-traits", - "reth-stages-types", "reth-storage-api", "reth-storage-errors", "reth-trie", @@ -9175,19 +9137,15 @@ dependencies = [ [[package]] name = "reth-trie-parallel" -version = "2.2.0" -source = "git+https://github.com/paradigmxyz/reth?tag=v2.2.0#88505c7fcbfdebfd3b56d88c86b62e950043c6c4" +version = "2.3.0" +source = "git+https://github.com/paradigmxyz/reth?rev=f0a41795d72ae5bb0b6c3661ac42ffb7b78ef105#f0a41795d72ae5bb0b6c3661ac42ffb7b78ef105" dependencies = [ - "alloy-eip7928", "alloy-evm", "alloy-primitives", "alloy-rlp", "crossbeam-channel", "crossbeam-utils", - "derive_more", - "itertools 0.14.0", "metrics", - "rayon", "reth-execution-errors", "reth-metrics", "reth-primitives-traits", @@ -9196,19 +9154,19 @@ dependencies = [ "reth-tasks", "reth-trie", "reth-trie-sparse", - "revm-state", + "revm", "thiserror 2.0.18", "tracing", ] [[package]] name = "reth-trie-sparse" -version = "2.2.0" -source = "git+https://github.com/paradigmxyz/reth?tag=v2.2.0#88505c7fcbfdebfd3b56d88c86b62e950043c6c4" +version = "2.3.0" +source = "git+https://github.com/paradigmxyz/reth?rev=f0a41795d72ae5bb0b6c3661ac42ffb7b78ef105#f0a41795d72ae5bb0b6c3661ac42ffb7b78ef105" dependencies = [ "alloy-primitives", - "alloy-rlp", "alloy-trie", + "either", "metrics", "rayon", "reth-execution-errors", @@ -9219,23 +9177,24 @@ dependencies = [ "serde_json", "slotmap", "smallvec", + "strum", "tracing", ] [[package]] name = "reth-zstd-compressors" -version = "0.3.1" +version = "0.5.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c12fafa33d2f420a9d39249a3e0357b1928d09429f30758b85280409092873b2" +checksum = "e1bd4ae4f54a2ba813eef082910bc646bd31cab8ed134308fa71f1068331fb84" dependencies = [ "zstd", ] [[package]] name = "revm" -version = "38.0.0" +version = "41.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "91202d39dbe8e8d10e9e8f2b76c30da68ecd1d25be69ba6d853ad0d03a3a398a" +checksum = "9d51c254839fb36357d0b02d02e46ba57e683d7b017e2da33b47ae74685a9e59" dependencies = [ "revm-bytecode", "revm-context", @@ -9252,9 +9211,9 @@ dependencies = [ [[package]] name = "revm-bytecode" -version = "10.0.0" +version = "41.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bdbb3a3d735efa94c91f2ef6bf20a35f99a77bc78f3e25bd758336901bdf9661" +checksum = "e33493cf6d996e9242db4f2002caef48ec9aa8c4f3fff3b18aed10ef3942eec7" dependencies = [ "bitvec", "phf", @@ -9264,9 +9223,9 @@ dependencies = [ [[package]] name = "revm-context" -version = "16.0.1" +version = "41.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c5f68d928d8b228e0faeb1c6ed75c4fde7d124f1ddf9119b67e7a0ad4041237d" +checksum = "86e48fc736d9542c082ea5305be44b6a14b53a615f0147ad57f62189fd813068" dependencies = [ "bitvec", "cfg-if", @@ -9281,9 +9240,9 @@ dependencies = [ [[package]] name = "revm-context-interface" -version = "17.0.1" +version = "41.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1f3758e6167c4ba7a59a689c519a047edaefcd4c37d74f279b93ed87bc8aece4" +checksum = "ac68282a454318246817486835a446519291dd0a8167d09de14c7e96f2147696" dependencies = [ "alloy-eip2930", "alloy-eip7702", @@ -9297,11 +9256,13 @@ dependencies = [ [[package]] name = "revm-database" -version = "13.0.1" +version = "41.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c281a1f11d3bcb8c0bba1199ed6bcb001d1aeb3d4fb366819e14f88723989a4e" +checksum = "d761681a43e48408ea23f7f66ff56ff7b6128cadb8f9135d1b94787d0c0fc6b4" dependencies = [ - "alloy-eips 1.8.3", + "alloy-eips", + "derive_more", + "either", "revm-bytecode", "revm-database-interface", "revm-primitives", @@ -9311,9 +9272,9 @@ dependencies = [ [[package]] name = "revm-database-interface" -version = "11.0.1" +version = "41.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d89efb9832a4e3742bb4ded5f7fe5bf905e8860e69427d4dfec153484fc6d304" +checksum = "b96e37d62fa60b45987b408486c84ebef4af4e7ddf1b3b96b4955a7263fd4e92" dependencies = [ "auto_impl", "either", @@ -9325,9 +9286,9 @@ dependencies = [ [[package]] name = "revm-handler" -version = "18.1.0" +version = "41.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "783e903d6922b7f5f9a940d1bb229530502d2924b1aed9d5ca5a94ebf065d460" +checksum = "f5e043ebfc0899c435e44475796285898e4f822e637bd67856da79cc170bd9df" dependencies = [ "auto_impl", "derive-where", @@ -9344,9 +9305,9 @@ dependencies = [ [[package]] name = "revm-inspector" -version = "19.0.0" +version = "41.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8216ad58422090d0daa9eb430e0a081f7ad07e7fd30681dee71f8420c99624e0" +checksum = "2e470b2a5e177918944d8cd26174455b61b8fbfbbe7fe411d28b4097e410d11e" dependencies = [ "auto_impl", "either", @@ -9362,9 +9323,9 @@ dependencies = [ [[package]] name = "revm-inspectors" -version = "0.39.0" +version = "0.41.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "731b682530a732ef9c189ef831589128e2ce34d4a306c956322ae2dffe009715" +checksum = "54e22e1aa328f78e8c3dea6a3a860a3a3c3a77c4a7e775e3fdd3dcbb24a152ac" dependencies = [ "alloy-primitives", "alloy-rpc-types-eth", @@ -9380,9 +9341,9 @@ dependencies = [ [[package]] name = "revm-interpreter" -version = "35.0.1" +version = "41.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1ece9f41b69658c15d748288a4dbdfc06a63f3ce93d983af440de3f1631dce6a" +checksum = "fa5547f653f9e4c47b4f9a9075a0b7c0faea5fa29b7873f392733a943837825d" dependencies = [ "revm-bytecode", "revm-context-interface", @@ -9393,9 +9354,9 @@ dependencies = [ [[package]] name = "revm-precompile" -version = "34.0.0" +version = "41.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a346a8cc6c8c39bd65306641c692191299c0a7b63d38810e39e8fe9b92378660" +checksum = "6d785931e4b8750cf9d79c808c22f05979c8d191baafc8badd4651db82584c1c" dependencies = [ "ark-bls12-381", "ark-bn254", @@ -9419,24 +9380,24 @@ dependencies = [ [[package]] name = "revm-primitives" -version = "23.0.0" +version = "41.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0c99bda77d9661521ba0b4bc04558c6692074f01e65dd420fa3b893033d9b8a2" +checksum = "66f39151a31afe93d97f7ac894dd9dcc989368d5ccba19ce079bfc1d73b77452" dependencies = [ "alloy-primitives", - "num_enum", "once_cell", "serde", ] [[package]] name = "revm-state" -version = "11.0.1" +version = "41.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c32490ed687dba31c3c882beb8c20408bdd30ef96690d8f145b0ee9a87040bfe" +checksum = "73ef2fffc28acc41e8ef7525f2b1d16288a10f80579a1b33e7e62abc9892314a" dependencies = [ "alloy-eip7928", "bitflags", + "nonmax", "revm-bytecode", "revm-primitives", "serde", @@ -9444,9 +9405,9 @@ dependencies = [ [[package]] name = "revm-statetest-types" -version = "17.0.1" +version = "41.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "94743de1e54a812077b79dd3a87b4059175a804650b21582fb6feae96ef2be9c" +checksum = "d9ced634104a9857185f02a7a3a5dd4a503590fababafbb295ad213df6dbf302" dependencies = [ "alloy-eip7928", "k256", @@ -9650,7 +9611,7 @@ dependencies = [ "errno", "libc", "linux-raw-sys", - "windows-sys 0.59.0", + "windows-sys 0.61.2", ] [[package]] @@ -9697,7 +9658,7 @@ version = "0.5.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "19787cda76408ec5404443dc8b31795c87cd8fec49762dc75fa727740d34acc1" dependencies = [ - "core-foundation", + "core-foundation 0.10.1", "core-foundation-sys", "jni 0.21.1", "log", @@ -9718,7 +9679,7 @@ version = "0.7.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "26d1e2536ce4f35f4846aa13bff16bd0ff40157cdb14cc056c7b14ba41233ba0" dependencies = [ - "core-foundation", + "core-foundation 0.10.1", "core-foundation-sys", "jni 0.22.4", "log", @@ -9730,7 +9691,7 @@ dependencies = [ "security-framework", "security-framework-sys", "webpki-root-certs 1.0.7", - "windows-sys 0.59.0", + "windows-sys 0.61.2", ] [[package]] @@ -9912,7 +9873,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b7f4bc775c73d9a02cde8bf7b2ec4c9d12743edf609006c7facc23998404cd1d" dependencies = [ "bitflags", - "core-foundation", + "core-foundation 0.10.1", "core-foundation-sys", "libc", "security-framework-sys", @@ -9942,10 +9903,6 @@ name = "semver" version = "1.0.28" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "8a7852d02fc848982e0c167ef163aaff9cd91dc640ba85e263cb1ce46fae51cd" -dependencies = [ - "serde", - "serde_core", -] [[package]] name = "semver-parser" @@ -10058,7 +10015,7 @@ version = "3.19.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "cf2ebbe86054f9b45bc3881e865683ccfaccce97b9b4cb53f3039d67f355a334" dependencies = [ - "darling 0.23.0", + "darling", "proc-macro2", "quote", "syn 2.0.117", @@ -10103,7 +10060,17 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "77fd7028345d415a4034cf8777cd4f8ab1851274233b45f84e3d955502d93874" dependencies = [ "digest 0.10.7", - "keccak", + "keccak 0.1.6", +] + +[[package]] +name = "sha3" +version = "0.11.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "be176f1a57ce4e3d31c1a166222d9768de5954f811601fb7ca06fc8203905ce1" +dependencies = [ + "digest 0.11.3", + "keccak 0.2.0", ] [[package]] @@ -10256,7 +10223,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "3a766e1110788c36f4fa1c2b71b387a7815aa65f88ce0229841826633d93723e" dependencies = [ "libc", - "windows-sys 0.60.2", + "windows-sys 0.61.2", ] [[package]] @@ -10360,9 +10327,9 @@ dependencies = [ [[package]] name = "syn-solidity" -version = "1.5.7" +version = "1.6.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "53f425ae0b12e2f5ae65542e00898d500d4d318b4baf09f40fd0d410454e9947" +checksum = "ec005042c7d952febc1a3ef5b0f6674e9054aa836877a31c90b20e25b3d31744" dependencies = [ "paste", "proc-macro2", @@ -10404,6 +10371,27 @@ dependencies = [ "windows 0.62.2", ] +[[package]] +name = "system-configuration" +version = "0.7.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a13f3d0daba03132c0aa9767f98351b3488edc2c100cda2d2ec2b04f3d8d3c8b" +dependencies = [ + "bitflags", + "core-foundation 0.9.4", + "system-configuration-sys", +] + +[[package]] +name = "system-configuration-sys" +version = "0.6.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8e1d1b10ced5ca923a1fcb8d03e96b8d3268065d724548c0211415ff6ac6bac4" +dependencies = [ + "core-foundation-sys", + "libc", +] + [[package]] name = "tagptr" version = "0.2.0" @@ -10437,7 +10425,7 @@ dependencies = [ "getrandom 0.4.2", "once_cell", "rustix", - "windows-sys 0.59.0", + "windows-sys 0.61.2", ] [[package]] @@ -10545,12 +10533,11 @@ dependencies = [ [[package]] name = "time" -version = "0.3.47" +version = "0.3.53" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "743bd48c283afc0388f9b8827b976905fb217ad9e647fae3a379a9283c4def2c" +checksum = "18dfaaeddcb932337b5e7866ee7d0ce9b76d2fd092997146f187ec09b4558a50" dependencies = [ "deranged", - "itoa", "libc", "num-conv", "num_threads", @@ -10562,15 +10549,15 @@ dependencies = [ [[package]] name = "time-core" -version = "0.1.8" +version = "0.1.9" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7694e1cfe791f8d31026952abf09c69ca6f6fa4e1a1229e18988f06a04a12dca" +checksum = "9e1c906769ad99c88eaa54e728060edef082f8e358ff32030cb7c7d315e81109" [[package]] name = "time-macros" -version = "0.2.27" +version = "0.2.31" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2e70e4c5a0e0a8a4823ad65dfe1a6930e4f4d756dcd9dd7939022b5e8c501215" +checksum = "c431b87111666e491a90baa837f914fb45cd5dc3c268591b0220ff5057f2085f" dependencies = [ "num-conv", "time-core", @@ -10603,9 +10590,9 @@ checksum = "1f3ccbac311fea05f86f61904b462b55fb3df8837a366dfc601a0161d0532f20" [[package]] name = "tokio" -version = "1.52.2" +version = "1.52.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "110a78583f19d5cdb2c5ccf321d1290344e71313c6c37d43520d386027d18386" +checksum = "8fc7f01b389ac15039e4dc9531aa973a135d7a4135281b12d7c1bc79fd57fffe" dependencies = [ "bytes", "libc", @@ -10780,6 +10767,17 @@ dependencies = [ "tonic", ] +[[package]] +name = "tonic-types" +version = "0.14.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "73ab1b02061f83d519bba3caa167f88f261ef05720ab8ebc954ade70de3348e8" +dependencies = [ + "prost", + "prost-types", + "tonic", +] + [[package]] name = "tower" version = "0.5.3" @@ -10879,6 +10877,17 @@ dependencies = [ "syn 2.0.117", ] +[[package]] +name = "tracing-chrome" +version = "0.7.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bf0a738ed5d6450a9fb96e86a23ad808de2b727fd1394585da5cdd6788ffe724" +dependencies = [ + "serde_json", + "tracing-core", + "tracing-subscriber 0.3.23", +] + [[package]] name = "tracing-core" version = "0.1.36" @@ -10935,9 +10944,9 @@ dependencies = [ [[package]] name = "tracing-opentelemetry" -version = "0.32.1" +version = "0.33.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1ac28f2d093c6c477eaa76b23525478f38de514fa9aeb1285738d4b97a9552fc" +checksum = "adbc64cba7137545b8044cb1fe9814f7aacf3c6b5f9b45be8bb5db538befdb26" dependencies = [ "js-sys", "opentelemetry", @@ -11024,7 +11033,7 @@ version = "0.12.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "8840ad4d852e325d3afa7fde8a50b2412f89dce47d7eb291c0cc7f87cd040f38" dependencies = [ - "darling 0.23.0", + "darling", "proc-macro2", "quote", "syn 2.0.117", @@ -11160,7 +11169,7 @@ version = "0.5.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "fc1de2c688dc15305988b563c3854064043356019f97a4b46276fe734c4f07ea" dependencies = [ - "crypto-common", + "crypto-common 0.1.7", "subtle", ] @@ -11238,14 +11247,12 @@ checksum = "accd4ea62f7bb7a82fe23066fb0957d48ef677f6eeb8215f372f52e48bb32426" [[package]] name = "vergen" -version = "9.1.0" +version = "10.0.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b849a1f6d8639e8de261e81ee0fc881e3e3620db1af9f2e0da015d4382ceaf75" +checksum = "b5574dd2f922b1a46a06a4b1dc11193a4012108fd54cf725e1816cb8183d8778" dependencies = [ "anyhow", - "cargo_metadata", - "derive_builder", - "regex", + "bon", "rustversion", "time", "vergen-lib", @@ -11253,12 +11260,12 @@ dependencies = [ [[package]] name = "vergen-git2" -version = "9.1.0" +version = "10.0.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d51ab55ddf1188c8d679f349775362b0fa9e90bd7a4ac69838b2a087623f0d57" +checksum = "410e2f72acce10471037150cd9c585ee7b54560f348bf70cd5fc8e87d3433e45" dependencies = [ "anyhow", - "derive_builder", + "bon", "git2", "rustversion", "time", @@ -11268,12 +11275,12 @@ dependencies = [ [[package]] name = "vergen-lib" -version = "9.1.0" +version = "10.0.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b34a29ba7e9c59e62f229ae1932fb1b8fb8a6fdcc99215a641913f5f5a59a569" +checksum = "8cd42fd155c2c2971f6d00face12ec245fbb604fce011ccaf2306d014c2e97ca" dependencies = [ "anyhow", - "derive_builder", + "bon", "rustversion", ] @@ -11556,7 +11563,7 @@ version = "0.1.11" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "c2a7b1c03c876122aa43f3020e6c3c3ee5c05081c9a00739faf7503aeba10d22" dependencies = [ - "windows-sys 0.59.0", + "windows-sys 0.61.2", ] [[package]] @@ -12011,9 +12018,6 @@ name = "winnow" version = "0.7.15" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "df79d97927682d2fd8adb29682d1140b343be4ac0f08fd68b7765d9c059d3945" -dependencies = [ - "memchr", -] [[package]] name = "winnow" diff --git a/Cargo.toml b/Cargo.toml index a9b4e0f3..f30e05fc 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -1,7 +1,7 @@ [workspace.package] version = "1.0.0" edition = "2024" -rust-version = "1.93" +rust-version = "1.95" license = "MIT OR Apache-2.0" publish = false @@ -13,7 +13,6 @@ members = [ "crates/chainspec", "crates/consensus", "crates/engine-api", - "crates/engine-tree-ext", "crates/evm", "crates/node", "crates/rpc", @@ -90,7 +89,6 @@ codegen-units = 1 morph-chainspec = { path = "crates/chainspec", default-features = false } morph-consensus = { path = "crates/consensus", default-features = false } morph-engine-api = { path = "crates/engine-api", default-features = false } -morph-engine-tree-ext = { path = "crates/engine-tree-ext", default-features = false } morph-evm = { path = "crates/evm", default-features = false } morph-node = { path = "crates/node"} morph-payload-builder = { path = "crates/payload/builder", default-features = false } @@ -101,98 +99,98 @@ morph-rpc = { path = "crates/rpc" } morph-revm = { path = "crates/revm", default-features = false } morph-txpool = { path = "crates/txpool", default-features = false } -reth-basic-payload-builder = { git = "https://github.com/paradigmxyz/reth", tag = "v2.2.0" } -reth-chain-state = { git = "https://github.com/paradigmxyz/reth", tag = "v2.2.0" } -reth-chainspec = { git = "https://github.com/paradigmxyz/reth", tag = "v2.2.0" } -reth-cli = { git = "https://github.com/paradigmxyz/reth", tag = "v2.2.0" } -reth-cli-commands = { git = "https://github.com/paradigmxyz/reth", tag = "v2.2.0" } -reth-cli-util = { git = "https://github.com/paradigmxyz/reth", tag = "v2.2.0" } -reth-codecs = { version = "0.3.1", default-features = false } -reth-codecs-derive = "0.3.1" -reth-consensus = { git = "https://github.com/paradigmxyz/reth", tag = "v2.2.0" } -reth-consensus-common = { git = "https://github.com/paradigmxyz/reth", tag = "v2.2.0" } -reth-db = { git = "https://github.com/paradigmxyz/reth", tag = "v2.2.0" } -reth-db-api = { git = "https://github.com/paradigmxyz/reth", tag = "v2.2.0" } -reth-e2e-test-utils = { git = "https://github.com/paradigmxyz/reth", tag = "v2.2.0" } -reth-engine-local = { git = "https://github.com/paradigmxyz/reth", tag = "v2.2.0" } -reth-engine-primitives = { git = "https://github.com/paradigmxyz/reth", tag = "v2.2.0" } -reth-engine-tree = { git = "https://github.com/paradigmxyz/reth", tag = "v2.2.0" } -reth-errors = { git = "https://github.com/paradigmxyz/reth", tag = "v2.2.0" } -reth-eth-wire-types = { git = "https://github.com/paradigmxyz/reth", tag = "v2.2.0" } -reth-exex = { git = "https://github.com/paradigmxyz/reth", tag = "v2.2.0" } -reth-exex-test-utils = { git = "https://github.com/paradigmxyz/reth", tag = "v2.2.0" } -reth-exex-types = { git = "https://github.com/paradigmxyz/reth", tag = "v2.2.0" } -reth-ethereum = { git = "https://github.com/paradigmxyz/reth", tag = "v2.2.0" } -reth-ethereum-cli = { git = "https://github.com/paradigmxyz/reth", tag = "v2.2.0" } -reth-ethereum-consensus = { git = "https://github.com/paradigmxyz/reth", tag = "v2.2.0" } -reth-ethereum-engine-primitives = { git = "https://github.com/paradigmxyz/reth", tag = "v2.2.0" } -reth-ethereum-primitives = { git = "https://github.com/paradigmxyz/reth", tag = "v2.2.0", default-features = false } -reth-evm = { git = "https://github.com/paradigmxyz/reth", tag = "v2.2.0" } -reth-evm-ethereum = { git = "https://github.com/paradigmxyz/reth", tag = "v2.2.0" } -reth-execution-types = { git = "https://github.com/paradigmxyz/reth", tag = "v2.2.0" } -reth-metrics = { git = "https://github.com/paradigmxyz/reth", tag = "v2.2.0" } -reth-network-peers = { git = "https://github.com/paradigmxyz/reth", tag = "v2.2.0" } -reth-node-api = { git = "https://github.com/paradigmxyz/reth", tag = "v2.2.0" } -reth-node-builder = { git = "https://github.com/paradigmxyz/reth", tag = "v2.2.0" } -reth-node-core = { git = "https://github.com/paradigmxyz/reth", tag = "v2.2.0" } -reth-node-ethereum = { git = "https://github.com/paradigmxyz/reth", tag = "v2.2.0" } -reth-node-metrics = { git = "https://github.com/paradigmxyz/reth", tag = "v2.2.0" } -reth-payload-builder = { git = "https://github.com/paradigmxyz/reth", tag = "v2.2.0" } -reth-payload-primitives = { git = "https://github.com/paradigmxyz/reth", tag = "v2.2.0" } -reth-payload-util = { git = "https://github.com/paradigmxyz/reth", tag = "v2.2.0" } -reth-primitives-traits = { version = "0.3.1", default-features = false } -reth-provider = { git = "https://github.com/paradigmxyz/reth", tag = "v2.2.0" } -reth-rpc = { git = "https://github.com/paradigmxyz/reth", tag = "v2.2.0" } -reth-rpc-api = { git = "https://github.com/paradigmxyz/reth", tag = "v2.2.0" } -reth-rpc-builder = { git = "https://github.com/paradigmxyz/reth", tag = "v2.2.0" } -reth-rpc-convert = { git = "https://github.com/paradigmxyz/reth", tag = "v2.2.0" } -reth-rpc-eth-api = { git = "https://github.com/paradigmxyz/reth", tag = "v2.2.0" } -reth-rpc-eth-types = { git = "https://github.com/paradigmxyz/reth", tag = "v2.2.0" } -reth-rpc-server-types = { git = "https://github.com/paradigmxyz/reth", tag = "v2.2.0" } -reth-storage-api = { git = "https://github.com/paradigmxyz/reth", tag = "v2.2.0" } -reth-tasks = { git = "https://github.com/paradigmxyz/reth", tag = "v2.2.0" } -reth-tracing = { git = "https://github.com/paradigmxyz/reth", tag = "v2.2.0" } -reth-trie = { git = "https://github.com/paradigmxyz/reth", tag = "v2.2.0" } -reth-transaction-pool = { git = "https://github.com/paradigmxyz/reth", tag = "v2.2.0" } -reth-zstd-compressors = { version = "0.3.1", default-features = false } -reth-execution-cache = { git = "https://github.com/paradigmxyz/reth", tag = "v2.2.0" } -reth-trie-db = { git = "https://github.com/paradigmxyz/reth", tag = "v2.2.0" } -reth-trie-parallel = { git = "https://github.com/paradigmxyz/reth", tag = "v2.2.0" } -reth-trie-sparse = { git = "https://github.com/paradigmxyz/reth", tag = "v2.2.0", default-features = false } +reth-basic-payload-builder = { git = "https://github.com/paradigmxyz/reth", rev = "f0a41795d72ae5bb0b6c3661ac42ffb7b78ef105" } +reth-chain-state = { git = "https://github.com/paradigmxyz/reth", rev = "f0a41795d72ae5bb0b6c3661ac42ffb7b78ef105" } +reth-chainspec = { git = "https://github.com/paradigmxyz/reth", rev = "f0a41795d72ae5bb0b6c3661ac42ffb7b78ef105" } +reth-cli = { git = "https://github.com/paradigmxyz/reth", rev = "f0a41795d72ae5bb0b6c3661ac42ffb7b78ef105" } +reth-cli-commands = { git = "https://github.com/paradigmxyz/reth", rev = "f0a41795d72ae5bb0b6c3661ac42ffb7b78ef105" } +reth-cli-util = { git = "https://github.com/paradigmxyz/reth", rev = "f0a41795d72ae5bb0b6c3661ac42ffb7b78ef105" } +reth-codecs = { version = "0.5.0", default-features = false } +reth-codecs-derive = "0.5.0" +reth-consensus = { git = "https://github.com/paradigmxyz/reth", rev = "f0a41795d72ae5bb0b6c3661ac42ffb7b78ef105" } +reth-consensus-common = { git = "https://github.com/paradigmxyz/reth", rev = "f0a41795d72ae5bb0b6c3661ac42ffb7b78ef105" } +reth-db = { git = "https://github.com/paradigmxyz/reth", rev = "f0a41795d72ae5bb0b6c3661ac42ffb7b78ef105" } +reth-db-api = { git = "https://github.com/paradigmxyz/reth", rev = "f0a41795d72ae5bb0b6c3661ac42ffb7b78ef105" } +reth-e2e-test-utils = { git = "https://github.com/paradigmxyz/reth", rev = "f0a41795d72ae5bb0b6c3661ac42ffb7b78ef105" } +reth-engine-local = { git = "https://github.com/paradigmxyz/reth", rev = "f0a41795d72ae5bb0b6c3661ac42ffb7b78ef105" } +reth-engine-primitives = { git = "https://github.com/paradigmxyz/reth", rev = "f0a41795d72ae5bb0b6c3661ac42ffb7b78ef105" } +reth-engine-tree = { git = "https://github.com/paradigmxyz/reth", rev = "f0a41795d72ae5bb0b6c3661ac42ffb7b78ef105" } +reth-errors = { git = "https://github.com/paradigmxyz/reth", rev = "f0a41795d72ae5bb0b6c3661ac42ffb7b78ef105" } +reth-eth-wire-types = { git = "https://github.com/paradigmxyz/reth", rev = "f0a41795d72ae5bb0b6c3661ac42ffb7b78ef105" } +reth-exex = { git = "https://github.com/paradigmxyz/reth", rev = "f0a41795d72ae5bb0b6c3661ac42ffb7b78ef105" } +reth-exex-test-utils = { git = "https://github.com/paradigmxyz/reth", rev = "f0a41795d72ae5bb0b6c3661ac42ffb7b78ef105" } +reth-exex-types = { git = "https://github.com/paradigmxyz/reth", rev = "f0a41795d72ae5bb0b6c3661ac42ffb7b78ef105" } +reth-ethereum = { git = "https://github.com/paradigmxyz/reth", rev = "f0a41795d72ae5bb0b6c3661ac42ffb7b78ef105" } +reth-ethereum-cli = { git = "https://github.com/paradigmxyz/reth", rev = "f0a41795d72ae5bb0b6c3661ac42ffb7b78ef105" } +reth-ethereum-consensus = { git = "https://github.com/paradigmxyz/reth", rev = "f0a41795d72ae5bb0b6c3661ac42ffb7b78ef105" } +reth-ethereum-engine-primitives = { git = "https://github.com/paradigmxyz/reth", rev = "f0a41795d72ae5bb0b6c3661ac42ffb7b78ef105" } +reth-ethereum-primitives = { git = "https://github.com/paradigmxyz/reth", rev = "f0a41795d72ae5bb0b6c3661ac42ffb7b78ef105", default-features = false } +reth-evm = { git = "https://github.com/paradigmxyz/reth", rev = "f0a41795d72ae5bb0b6c3661ac42ffb7b78ef105" } +reth-evm-ethereum = { git = "https://github.com/paradigmxyz/reth", rev = "f0a41795d72ae5bb0b6c3661ac42ffb7b78ef105" } +reth-execution-types = { git = "https://github.com/paradigmxyz/reth", rev = "f0a41795d72ae5bb0b6c3661ac42ffb7b78ef105" } +reth-metrics = { git = "https://github.com/paradigmxyz/reth", rev = "f0a41795d72ae5bb0b6c3661ac42ffb7b78ef105" } +reth-network-peers = { git = "https://github.com/paradigmxyz/reth", rev = "f0a41795d72ae5bb0b6c3661ac42ffb7b78ef105" } +reth-node-api = { git = "https://github.com/paradigmxyz/reth", rev = "f0a41795d72ae5bb0b6c3661ac42ffb7b78ef105" } +reth-node-builder = { git = "https://github.com/paradigmxyz/reth", rev = "f0a41795d72ae5bb0b6c3661ac42ffb7b78ef105" } +reth-node-core = { git = "https://github.com/paradigmxyz/reth", rev = "f0a41795d72ae5bb0b6c3661ac42ffb7b78ef105" } +reth-node-ethereum = { git = "https://github.com/paradigmxyz/reth", rev = "f0a41795d72ae5bb0b6c3661ac42ffb7b78ef105" } +reth-node-metrics = { git = "https://github.com/paradigmxyz/reth", rev = "f0a41795d72ae5bb0b6c3661ac42ffb7b78ef105" } +reth-payload-builder = { git = "https://github.com/paradigmxyz/reth", rev = "f0a41795d72ae5bb0b6c3661ac42ffb7b78ef105" } +reth-payload-primitives = { git = "https://github.com/paradigmxyz/reth", rev = "f0a41795d72ae5bb0b6c3661ac42ffb7b78ef105" } +reth-payload-util = { git = "https://github.com/paradigmxyz/reth", rev = "f0a41795d72ae5bb0b6c3661ac42ffb7b78ef105" } +reth-primitives-traits = { version = "0.5.0", default-features = false } +reth-provider = { git = "https://github.com/paradigmxyz/reth", rev = "f0a41795d72ae5bb0b6c3661ac42ffb7b78ef105" } +reth-rpc = { git = "https://github.com/paradigmxyz/reth", rev = "f0a41795d72ae5bb0b6c3661ac42ffb7b78ef105" } +reth-rpc-api = { git = "https://github.com/paradigmxyz/reth", rev = "f0a41795d72ae5bb0b6c3661ac42ffb7b78ef105" } +reth-rpc-builder = { git = "https://github.com/paradigmxyz/reth", rev = "f0a41795d72ae5bb0b6c3661ac42ffb7b78ef105" } +reth-rpc-convert = { git = "https://github.com/paradigmxyz/reth", rev = "f0a41795d72ae5bb0b6c3661ac42ffb7b78ef105" } +reth-rpc-eth-api = { git = "https://github.com/paradigmxyz/reth", rev = "f0a41795d72ae5bb0b6c3661ac42ffb7b78ef105" } +reth-rpc-eth-types = { git = "https://github.com/paradigmxyz/reth", rev = "f0a41795d72ae5bb0b6c3661ac42ffb7b78ef105" } +reth-rpc-server-types = { git = "https://github.com/paradigmxyz/reth", rev = "f0a41795d72ae5bb0b6c3661ac42ffb7b78ef105" } +reth-storage-api = { git = "https://github.com/paradigmxyz/reth", rev = "f0a41795d72ae5bb0b6c3661ac42ffb7b78ef105" } +reth-tasks = { git = "https://github.com/paradigmxyz/reth", rev = "f0a41795d72ae5bb0b6c3661ac42ffb7b78ef105" } +reth-tracing = { git = "https://github.com/paradigmxyz/reth", rev = "f0a41795d72ae5bb0b6c3661ac42ffb7b78ef105" } +reth-trie = { git = "https://github.com/paradigmxyz/reth", rev = "f0a41795d72ae5bb0b6c3661ac42ffb7b78ef105" } +reth-transaction-pool = { git = "https://github.com/paradigmxyz/reth", rev = "f0a41795d72ae5bb0b6c3661ac42ffb7b78ef105" } +reth-zstd-compressors = { version = "0.5.0", default-features = false } +reth-execution-cache = { git = "https://github.com/paradigmxyz/reth", rev = "f0a41795d72ae5bb0b6c3661ac42ffb7b78ef105" } +reth-trie-db = { git = "https://github.com/paradigmxyz/reth", rev = "f0a41795d72ae5bb0b6c3661ac42ffb7b78ef105" } +reth-trie-parallel = { git = "https://github.com/paradigmxyz/reth", rev = "f0a41795d72ae5bb0b6c3661ac42ffb7b78ef105" } +reth-trie-sparse = { git = "https://github.com/paradigmxyz/reth", rev = "f0a41795d72ae5bb0b6c3661ac42ffb7b78ef105", default-features = false } -reth-revm = { git = "https://github.com/paradigmxyz/reth", tag = "v2.2.0", features = [ +reth-revm = { git = "https://github.com/paradigmxyz/reth", rev = "f0a41795d72ae5bb0b6c3661ac42ffb7b78ef105", features = [ "std", "optional-checks", ] } -revm = { version = "38.0.0", features = [ +revm = { version = "41.0.0", features = [ "optional_fee_charge", "optional_eip7623", ], default-features = false } -alloy = { version = "2.0.4", default-features = false } -alloy-consensus = { version = "2.0.4", default-features = false } -alloy-contract = { version = "2.0.4", default-features = false } -alloy-eip7928 = { version = "0.3.4", default-features = false } -alloy-eips = { version = "2.0.4", default-features = false } -alloy-evm = { version = "0.34.0", default-features = false } -alloy-genesis = "2.0.4" +alloy = { version = "2.1.1", default-features = false } +alloy-consensus = { version = "2.1.1", default-features = false } +alloy-contract = { version = "2.1.1", default-features = false } +alloy-eip7928 = { version = "0.4.5", default-features = false } +alloy-eips = { version = "2.1.1", default-features = false } +alloy-evm = { version = "0.37.0", default-features = false } +alloy-genesis = "2.1.1" alloy-hardforks = "0.4.7" -alloy-network = { version = "2.0.4", default-features = false } -alloy-primitives = { version = "1.5.6", default-features = false } -alloy-provider = { version = "2.0.4", default-features = false } -alloy-rlp = "0.3.13" -alloy-trie = "0.9.5" -alloy-rpc-types-engine = "2.0.4" -alloy-rpc-types-eth = { version = "2.0.4" } -alloy-serde = "2.0.4" -alloy-signer = "2.0.4" -alloy-signer-local = "2.0.4" -alloy-sol-types = { version = "1.5.6", default-features = false } -alloy-transport = "2.0.4" +alloy-network = { version = "2.1.1", default-features = false } +alloy-primitives = { version = "1.6.0", default-features = false } +alloy-provider = { version = "2.1.1", default-features = false } +alloy-rlp = "0.3.16" +alloy-trie = "0.9.4" +alloy-rpc-types-engine = "2.1.1" +alloy-rpc-types-eth = { version = "2.1.1" } +alloy-serde = "2.1.1" +alloy-signer = "2.1.1" +alloy-signer-local = "2.1.1" +alloy-sol-types = { version = "1.6.0", default-features = false } +alloy-transport = "2.1.1" alloy-chains = { version = "0.2.33", default-features = false } crossbeam-channel = "0.5.13" -revm-primitives = { version = "23.0.0", default-features = false } -revm-statetest-types = { version = "17.0.1", default-features = false } +revm-primitives = { version = "41.0.0", default-features = false } +revm-statetest-types = { version = "41.0.0", default-features = false } arbitrary = { version = "1.3", features = ["derive"] } async-lock = "3.4.1" async-trait = "0.1" @@ -234,8 +232,8 @@ tokio-stream = "0.1.17" tokio-util = "0.7.16" tracing = "0.1.41" tracing-subscriber = "0.3.22" -vergen = "9.1.0" -vergen-git2 = "9.1.0" +vergen = "10.0.1" +vergen-git2 = "10.0.1" criterion = "0.7.0" test-case = "3" pyroscope = "0.5.8" diff --git a/crates/chainspec/src/hardfork.rs b/crates/chainspec/src/hardfork.rs index 42d75185..63b005bf 100644 --- a/crates/chainspec/src/hardfork.rs +++ b/crates/chainspec/src/hardfork.rs @@ -234,7 +234,7 @@ mod tests { let params = GasParams::new_spec(spec); assert_eq!( - params.tx_eip7702_per_auth_state_gas(), + params.tx_eip7702_state_gas(), 0, "MorphHardfork {fork:?} must not enable EIP-8037 state gas" ); @@ -245,15 +245,14 @@ mod tests { fn test_eip7702_refund_stays_regular_for_morph_specs() { for spec in [SpecId::CANCUN, SpecId::PRAGUE, SpecId::OSAKA] { let params = GasParams::new_spec(spec); - let total_refund = 25_000; - let (state_refund, regular_refund) = params.split_eip7702_refund(total_refund); - assert_eq!( - state_refund, 0, + params.tx_eip7702_state_refund(2, 2), + 0, "spec={spec:?}: Morph must not route EIP-7702 refunds to state gas" ); assert_eq!( - regular_refund, total_refund, + params.tx_eip7702_auth_refund_regular(), + params.tx_eip7702_auth_refund(), "spec={spec:?}: Morph EIP-7702 refunds must remain subject to the regular refund cap" ); } diff --git a/crates/consensus/src/validation.rs b/crates/consensus/src/validation.rs index 3b32297a..cd12183c 100644 --- a/crates/consensus/src/validation.rs +++ b/crates/consensus/src/validation.rs @@ -89,7 +89,7 @@ const GAS_LIMIT_BOUND_DIVISOR: u64 = 1024; /// start, have sequential queue indices, and are consistent with `header.next_l1_msg_index`. /// 2. **Cross-block monotonicity** (`validate_header_against_parent`): `header.next_l1_msg_index` /// is monotonically non-decreasing relative to the parent. -/// 3. **Parent-aware exactness** (`MorphBasicEngineValidator`): once the engine tree has +/// 3. **Parent-aware exactness** (`MorphTreeEngineValidator`): once the engine tree has /// both parent header and block body, Jade blocks must exactly match the value derived /// from `parent.next_l1_msg_index` and the block's leading L1 messages. /// @@ -356,6 +356,7 @@ impl FullConsensus for MorphConsensus { block: &RecoveredBlock, result: &BlockExecutionResult, receipt_root_bloom: Option, + _block_access_list_hash: Option, ) -> Result<(), ConsensusError> { // Verify the block gas used let cumulative_gas_used = result @@ -2070,7 +2071,7 @@ mod tests { let recovered = reth_primitives_traits::RecoveredBlock::new_unhashed(block, vec![Address::ZERO]); - let post_result = consensus.validate_block_post_execution(&recovered, &result, None); + let post_result = consensus.validate_block_post_execution(&recovered, &result, None, None); assert!(matches!( post_result, Err(ConsensusError::BlockGasUsed { .. }) diff --git a/crates/engine-api/src/builder.rs b/crates/engine-api/src/builder.rs index 390e5a10..26aa0e3b 100644 --- a/crates/engine-api/src/builder.rs +++ b/crates/engine-api/src/builder.rs @@ -725,6 +725,7 @@ impl RealMorphL2EngineApi { parent_beacon_block_root: None, // Morph L2 has no PoS slot semantics; introduced in alloy 2.0. slot_number: None, + target_gas_limit: None, }, transactions: Some(params.transactions), gas_limit: gas_limit_override, @@ -737,7 +738,7 @@ impl RealMorphL2EngineApi { attributes: rpc_attributes, parent_hash, cache: None, - trie_handle: None, + state_root_handle: None, }; let _ = self .payload_builder diff --git a/crates/engine-tree-ext/Cargo.toml b/crates/engine-tree-ext/Cargo.toml deleted file mode 100644 index 255e6c88..00000000 --- a/crates/engine-tree-ext/Cargo.toml +++ /dev/null @@ -1,80 +0,0 @@ -[package] -name = "morph-engine-tree-ext" -version.workspace = true -edition.workspace = true -rust-version.workspace = true -license.workspace = true -publish.workspace = true - -[lints] -workspace = true - -[dependencies] -# morph local crates -morph-chainspec = { workspace = true } -# morph-primitives brought in transitively via morph-chainspec; direct dep here -# enables the reth-codec feature so cargo test -p morph-engine-tree-ext satisfies -# NodePrimitives::Receipt: FullReceipt (requires Compact impl). -morph-primitives = { workspace = true, features = ["reth-codec"] } - -# reth (workspace = true after Task 4 flips root Cargo.toml to paradigmxyz/reth v2.0.0) -reth-chain-state = { workspace = true } -reth-consensus = { workspace = true } -reth-db = { workspace = true } -reth-engine-primitives = { workspace = true } -reth-engine-tree = { workspace = true } -reth-errors = { workspace = true } -reth-evm = { workspace = true } -reth-execution-cache = { workspace = true } -reth-payload-primitives = { workspace = true } -reth-primitives-traits = { workspace = true } -reth-provider = { workspace = true } -reth-revm = { workspace = true } -reth-tasks = { workspace = true } -reth-trie = { workspace = true } -reth-trie-db = { workspace = true } -reth-trie-parallel = { workspace = true } - -# alloy / revm — workspace versions from reth v2.0.0 -alloy-consensus = { workspace = true } -alloy-eip7928 = { workspace = true } -alloy-eips = { workspace = true } -alloy-evm = { workspace = true } -alloy-primitives = { workspace = true } -alloy-rlp = { workspace = true } -revm-primitives = { workspace = true } - -# utilities -crossbeam-channel = { workspace = true } -derive_more = { workspace = true } -tokio = { workspace = true } -tracing = { workspace = true } - -[features] -default = ["std"] -std = [] -trie-debug = ["dep:reth-trie-sparse"] -# Forwards `morph-node`'s `test-utils` feature, which provides `MorphTestNode`, -# `TestNodeBuilder`, `HardforkSchedule`, etc. used by the Jade-boundary -# integration test. -test-utils = ["morph-node/test-utils"] - -[dependencies.reth-trie-sparse] -workspace = true -default-features = false -optional = true - -[dev-dependencies] -alloy-genesis = { workspace = true } -serde_json = { workspace = true } -# Jade boundary integration test — exercises MorphBasicEngineValidator through -# the real MorphNode stack via morph-node's test-utils feature. -morph-node = { workspace = true, features = ["test-utils"] } -morph-payload-types = { workspace = true } -reth-node-api = { workspace = true } -reth-payload-builder = { workspace = true } -reth-tracing = { workspace = true } -alloy-rpc-types-engine = { workspace = true } -alloy-rpc-types-eth = { workspace = true } -eyre = { workspace = true } -tokio = { workspace = true, features = ["macros", "rt-multi-thread"] } diff --git a/crates/engine-tree-ext/src/gate.rs b/crates/engine-tree-ext/src/gate.rs deleted file mode 100644 index e3dbd9db..00000000 --- a/crates/engine-tree-ext/src/gate.rs +++ /dev/null @@ -1,67 +0,0 @@ -//! Decides whether strict state-root equality must be enforced for a block. -//! -//! Pre-Jade Morph blocks store ZK-trie roots in `header.state_root`, while reth -//! computes MPT roots. Skipping the strict check pre-Jade is safe because the -//! first post-Jade block's successful strict check retroactively anchors the -//! MPT state accumulated across the pre-Jade range (transitivity of EVM -//! execution; see the design spec). - -use morph_chainspec::{MorphChainSpec, MorphHardforks}; - -/// Returns `true` iff reth must enforce the strict -/// `computed_state_root == header.state_root()` check at the given block -/// timestamp. Post-Jade: true. Pre-Jade: false. -pub fn state_root_enforced_at(chain_spec: &MorphChainSpec, timestamp: u64) -> bool { - chain_spec.is_jade_active_at_timestamp(timestamp) -} - -#[cfg(test)] -mod tests { - use super::*; - use alloy_genesis::Genesis; - use serde_json::json; - use std::sync::Arc; - - fn chain_spec_with_jade_at(jade_time: u64) -> Arc { - let genesis: Genesis = serde_json::from_value(json!({ - "config": { - "chainId": 1337, - "bernoulliBlock": 0, - "curieBlock": 0, - "morph": {}, - "jadeForkTime": jade_time - }, - "alloc": {} - })) - .expect("valid test genesis"); - Arc::new(MorphChainSpec::from(genesis)) - } - - #[test] - fn skipped_before_jade() { - let cs = chain_spec_with_jade_at(1_000); - assert!(!state_root_enforced_at(&cs, 0)); - assert!(!state_root_enforced_at(&cs, 500)); - assert!(!state_root_enforced_at(&cs, 999)); - } - - #[test] - fn strict_at_and_after_jade() { - let cs = chain_spec_with_jade_at(1_000); - assert!(state_root_enforced_at(&cs, 1_000)); - assert!(state_root_enforced_at(&cs, 1_001)); - assert!(state_root_enforced_at(&cs, u64::MAX)); - } - - #[test] - fn skipped_when_jade_unset() { - let genesis: Genesis = serde_json::from_value(json!({ - "config": { "chainId": 1337, "bernoulliBlock": 0, "curieBlock": 0, "morph": {} }, - "alloc": {} - })) - .unwrap(); - let cs = Arc::new(MorphChainSpec::from(genesis)); - assert!(!state_root_enforced_at(&cs, 0)); - assert!(!state_root_enforced_at(&cs, 1_000_000)); - } -} diff --git a/crates/engine-tree-ext/src/lib.rs b/crates/engine-tree-ext/src/lib.rs deleted file mode 100644 index fc277227..00000000 --- a/crates/engine-tree-ext/src/lib.rs +++ /dev/null @@ -1,13 +0,0 @@ -//! Morph extension of reth's engine tree. -//! -//! Verbatim copy of `reth_engine_tree::tree::payload_validator` from reth v2.0.0, -//! used as the foundation for a Jade-gated state-root skip. -//! -//! `trie_updates` is a verbatim copy of the private sibling module from the same -//! source tree, required by `payload_validator::compare_trie_updates_with_serial`. - -pub mod gate; -pub mod payload_validator; -pub(crate) mod trie_updates; - -pub use payload_validator::MorphBasicEngineValidator; diff --git a/crates/engine-tree-ext/src/payload_validator.rs b/crates/engine-tree-ext/src/payload_validator.rs deleted file mode 100644 index 5d52e082..00000000 --- a/crates/engine-tree-ext/src/payload_validator.rs +++ /dev/null @@ -1,2193 +0,0 @@ -//! Types and traits for validating blocks and payloads. -//! -//! # Validation pipeline -//! -//! When the engine processes a new payload (`engine_newPayload`), validation happens in phases: -//! -//! ## Phase 1 – Payload conversion -//! [`PayloadValidator::convert_payload_to_block`] decodes the execution payload (RLP, hashing) -//! into a [`SealedBlock`]. This runs on a background thread concurrently with state setup. -//! -//! ## Phase 2 – Pre-execution consensus -//! - [`HeaderValidator::validate_header`] — standalone header checks (hash, gas, base fee, -//! fork-specific fields) -//! - [`Consensus::validate_block_pre_execution`] — body vs header (tx root, ommer hash, withdrawals -//! root) -//! - [`HeaderValidator::validate_header_against_parent`] — sequential checks (block number, -//! timestamp, gas limit, base fee vs parent) -//! -//! ## Phase 3 – Execution -//! Block transactions are executed via the EVM. Receipt roots are computed incrementally. -//! -//! ## Phase 4 – Post-execution consensus -//! - [`FullConsensus::validate_block_post_execution`] — gas used, receipt root, logs bloom, -//! requests hash -//! - [`PayloadValidator::validate_block_post_execution_with_hashed_state`] — network-specific -//! (no-op on L1, used by OP Stack) -//! -//! ## Payload attributes validation (`engine_forkchoiceUpdated`) -//! When the CL provides payload attributes to start building a block: -//! - [`PayloadValidator::validate_payload_attributes_against_header`] — ensures timestamp ordering -//! -//! If validation passes, a payload build job is started. If it fails, -//! `INVALID_PAYLOAD_ATTRIBUTES` is returned without rolling back the forkchoice update. -//! -//! [`HeaderValidator::validate_header`]: reth_consensus::HeaderValidator::validate_header -//! [`Consensus::validate_block_pre_execution`]: reth_consensus::Consensus::validate_block_pre_execution -//! [`HeaderValidator::validate_header_against_parent`]: reth_consensus::HeaderValidator::validate_header_against_parent -//! [`FullConsensus::validate_block_post_execution`]: reth_consensus::FullConsensus::validate_block_post_execution -//! [`SealedBlock`]: reth_primitives_traits::SealedBlock - -use alloy_consensus::transaction::{Either, TxHashRef}; -use alloy_eip7928::BlockAccessList; -use alloy_eips::{NumHash, eip1898::BlockWithParent, eip4895::Withdrawal}; -use alloy_evm::Evm; -use alloy_primitives::{B256, map::B256Set}; -use reth_engine_tree::tree::{ - CacheWaitDurations, CachedStateProvider, EngineApiMetrics, EngineApiTreeState, EngineValidator, - ExecutionEnv, PayloadHandle, StateProviderBuilder, TreeConfig, WaitForCaches, - error::{InsertBlockError, InsertBlockErrorKind, InsertPayloadError}, - instrumented_state::{InstrumentedStateProvider, StateProviderStats}, - payload_processor::{ - PayloadProcessor, - multiproof::{StateRootComputeOutcome, StateRootHandle}, - }, - payload_validator::TreeCtx, - precompile_cache::{CachedPrecompile, CachedPrecompileMetrics, PrecompileCacheMap}, -}; -use reth_revm::database::StateProviderDatabase; -#[cfg(feature = "trie-debug")] -use reth_trie_sparse::debug_recorder::TrieDebugRecorder; - -use crate::gate::state_root_enforced_at; -use morph_chainspec::{MorphChainSpec, MorphHardforks}; -use morph_primitives::{MorphHeader, MorphTxEnvelope}; -use reth_chain_state::{DeferredTrieData, ExecutedBlock, ExecutionTimingStats, LazyOverlay}; -use reth_consensus::{ConsensusError, FullConsensus, ReceiptRootBloom}; -use reth_engine_primitives::{ - ConfigureEngineEvm, ExecutableTxIterator, ExecutionPayload, InvalidBlockHook, PayloadValidator, -}; -use reth_engine_tree::tree::payload_processor::receipt_root_task::{ - IndexedReceipt, ReceiptRootTaskHandle, -}; -use reth_errors::{BlockExecutionError, ProviderResult}; -use reth_evm::{ - ConfigureEvm, EvmEnvFor, ExecutionCtxFor, OnStateHook, SpecFor, block::BlockExecutor, - execute::ExecutableTxFor, -}; -use reth_execution_cache::{CacheStats, SavedCache}; -use reth_payload_primitives::{ - BuiltPayload, InvalidPayloadAttributesError, NewPayloadError, PayloadTypes, -}; -use reth_primitives_traits::{ - AlloyBlockHeader, BlockBody, BlockTy, FastInstant as Instant, GotExpected, NodePrimitives, - RecoveredBlock, SealedBlock, SealedHeader, SignerRecoverable, -}; -use reth_provider::{ - BlockExecutionOutput, BlockNumReader, BlockReader, ChangeSetReader, DatabaseProviderFactory, - DatabaseProviderROFactory, HashedPostStateProvider, ProviderError, PruneCheckpointReader, - StageCheckpointReader, StateProvider, StateProviderFactory, StateReader, - StorageChangeSetReader, StorageSettingsCache, - providers::{OverlayBuilder, OverlayStateProviderFactory}, -}; -use reth_revm::db::{BundleAccount, State, states::bundle_state::BundleRetention}; -use reth_trie::{HashedPostState, StateRoot, trie_cursor::TrieCursorFactory, updates::TrieUpdates}; -use reth_trie_db::ChangesetCache; -use reth_trie_parallel::root::{ParallelStateRoot, ParallelStateRootError}; -use revm_primitives::{Address, KECCAK_EMPTY}; -use std::{ - collections::HashMap, - panic::{self, AssertUnwindSafe}, - sync::{ - Arc, - atomic::{AtomicUsize, Ordering}, - mpsc::RecvTimeoutError, - }, - time::Duration, -}; -use tracing::{Span, debug, debug_span, error, info, instrument, trace, warn}; - -/// Output of block or payload validation. -pub type ValidationOutcome>> = - Result<(ExecutedBlock, Option>), E>; - -/// Handle to a [`HashedPostState`] computed on a background thread. -type LazyHashedPostState = reth_tasks::LazyHandle; - -/// Result type for block validation with optional timing stats. -type InsertPayloadResult = Result< - (ExecutedBlock, Option>), - InsertPayloadError<::Block>, ->; - -/// Context providing access to tree state during validation. -/// -/// A helper type that provides reusable payload validation logic for network-specific validators. -/// -/// This type satisfies [`EngineValidator`] and is responsible for executing blocks/payloads. -/// -/// This type contains common validation, execution, and state root computation logic that can be -/// used by network-specific payload validators (e.g., Ethereum, Optimism). It is not meant to be -/// used as a standalone component, but rather as a building block for concrete implementations. -#[derive(derive_more::Debug)] -pub struct MorphBasicEngineValidator -where - Evm: ConfigureEvm, -{ - /// Provider for database access. - provider: P, - /// Consensus implementation for validation. - consensus: Arc>, - /// EVM configuration. - evm_config: Evm, - /// Configuration for the tree. - config: TreeConfig, - /// Payload processor for state root computation. - payload_processor: PayloadProcessor, - /// Precompile cache map. - precompile_cache_map: PrecompileCacheMap>, - /// Precompile cache metrics. - precompile_cache_metrics: HashMap, - /// Hook to call when invalid blocks are encountered. - #[debug(skip)] - invalid_block_hook: Box>, - /// Metrics for the engine api. - metrics: EngineApiMetrics, - /// Validator for the payload. - validator: V, - /// Changeset cache for in-memory trie changesets - changeset_cache: ChangesetCache, - /// Task runtime for spawning parallel work. - runtime: reth_tasks::Runtime, - /// Chain spec, used to gate pre-Jade state-root skipping. - chain_spec: Arc, -} - -impl MorphBasicEngineValidator -where - N: NodePrimitives< - Block = morph_primitives::Block, - BlockHeader = MorphHeader, - SignedTx = MorphTxEnvelope, - >, - P: DatabaseProviderFactory< - Provider: BlockReader - + StageCheckpointReader - + PruneCheckpointReader - + ChangeSetReader - + StorageChangeSetReader - + BlockNumReader - + StorageSettingsCache, - > + BlockReader
- + ChangeSetReader - + BlockNumReader - + StateProviderFactory - + StateReader - + HashedPostStateProvider - + Clone - + 'static, - Evm: ConfigureEvm + 'static, -{ - /// Creates a new `TreePayloadValidator`. - #[expect(clippy::too_many_arguments)] - pub fn new( - provider: P, - consensus: Arc>, - evm_config: Evm, - validator: V, - config: TreeConfig, - invalid_block_hook: Box>, - changeset_cache: ChangesetCache, - runtime: reth_tasks::Runtime, - chain_spec: Arc, - ) -> Self { - let precompile_cache_map = PrecompileCacheMap::default(); - let payload_processor = PayloadProcessor::new( - runtime.clone(), - evm_config.clone(), - &config, - precompile_cache_map.clone(), - ); - Self { - provider, - consensus, - evm_config, - payload_processor, - precompile_cache_map, - precompile_cache_metrics: HashMap::new(), - config, - invalid_block_hook, - metrics: EngineApiMetrics::default(), - validator, - changeset_cache, - runtime, - chain_spec, - } - } - - /// Converts a [`BlockOrPayload`] to a recovered block. - #[instrument(level = "debug", target = "engine::tree::payload_validator", skip_all)] - pub fn convert_to_block>>( - &self, - input: BlockOrPayload, - ) -> Result, NewPayloadError> - where - V: PayloadValidator, - { - match input { - BlockOrPayload::Payload(payload) => self.validator.convert_payload_to_block(payload), - BlockOrPayload::Block(block) => Ok(block), - } - } - - /// Returns EVM environment for the given payload or block. - pub fn evm_env_for>>( - &self, - input: &BlockOrPayload, - ) -> Result, Evm::Error> - where - V: PayloadValidator, - Evm: ConfigureEngineEvm, - { - match input { - BlockOrPayload::Payload(payload) => Ok(self.evm_config.evm_env_for_payload(payload)?), - BlockOrPayload::Block(block) => Ok(self.evm_config.evm_env(block.header())?), - } - } - - /// Returns [`ExecutableTxIterator`] for the given payload or block. - pub fn tx_iterator_for<'a, T: PayloadTypes>>( - &'a self, - input: &'a BlockOrPayload, - ) -> Result, NewPayloadError> - where - V: PayloadValidator, - Evm: ConfigureEngineEvm, - { - Ok(match input { - BlockOrPayload::Payload(payload) => { - let iter = self - .evm_config - .tx_iterator_for_payload(payload) - .map_err(NewPayloadError::other)?; - Either::Left(iter) - } - BlockOrPayload::Block(block) => { - let txs = block.body().clone_transactions(); - let convert = |tx: N::SignedTx| tx.try_into_recovered(); - Either::Right((txs, convert)) - } - }) - } - - /// Returns a [`ExecutionCtxFor`] for the given payload or block. - pub fn execution_ctx_for<'a, T: PayloadTypes>>( - &self, - input: &'a BlockOrPayload, - ) -> Result, Evm::Error> - where - V: PayloadValidator, - Evm: ConfigureEngineEvm, - { - match input { - BlockOrPayload::Payload(payload) => Ok(self.evm_config.context_for_payload(payload)?), - BlockOrPayload::Block(block) => Ok(self.evm_config.context_for_block(block)?), - } - } - - /// Handles execution errors by checking if header validation errors should take precedence. - /// - /// When an execution error occurs, this function checks if there are any header validation - /// errors that should be reported instead, as header validation errors have higher priority. - fn handle_execution_error>>( - &self, - input: BlockOrPayload, - execution_err: InsertBlockErrorKind, - parent_block: &SealedHeader, - ) -> InsertPayloadResult - where - V: PayloadValidator, - { - debug!( - target: "engine::tree::payload_validator", - ?execution_err, - block = ?input.num_hash(), - "Block execution failed, checking for header validation errors" - ); - - // If execution failed, we should first check if there are any header validation - // errors that take precedence over the execution error - let block = self.convert_to_block(input)?; - - // Validate block consensus rules which includes header validation - if let Err(consensus_err) = self.validate_block_inner(&block, None) { - // Header validation error takes precedence over execution error - return Err(InsertBlockError::new(block, consensus_err.into()).into()); - } - - // Also validate against the parent - if let Err(consensus_err) = self - .consensus - .validate_header_against_parent(block.sealed_header(), parent_block) - { - // Parent validation error takes precedence over execution error - return Err(InsertBlockError::new(block, consensus_err.into()).into()); - } - - // No header validation errors, return the original execution error - Err(InsertBlockError::new(block, execution_err).into()) - } - - /// Validates a block that has already been converted from a payload. - /// - /// This method performs: - /// - Consensus validation - /// - Block execution - /// - State root computation - /// - Fork detection - #[instrument( - level = "debug", - target = "engine::tree::payload_validator", - skip_all, - fields( - parent = ?input.parent_hash(), - type_name = ?input.type_name(), - ) - )] - pub fn validate_block_with_state>>( - &mut self, - input: BlockOrPayload, - mut ctx: TreeCtx<'_, N>, - ) -> InsertPayloadResult - where - V: PayloadValidator + Clone, - Evm: ConfigureEngineEvm, - { - // Spawn payload conversion on a background thread so it runs concurrently with the - // rest of the function (setup + execution). For payloads this overlaps the cost of - // RLP decoding + header hashing. - let is_payload = matches!(&input, BlockOrPayload::Payload(_)); - let convert_to_block = match &input { - BlockOrPayload::Payload(_) => { - let payload_clone = input.clone(); - let validator = self.validator.clone(); - let handle = self.payload_processor.executor().spawn_blocking_named( - "payload-convert", - move || { - let BlockOrPayload::Payload(payload) = payload_clone else { - unreachable!() - }; - validator.convert_payload_to_block(payload) - }, - ); - Either::Left(handle) - } - BlockOrPayload::Block(_) => Either::Right(()), - }; - - // Returns the sealed block, either by awaiting the background conversion task (for - // payloads) or by extracting the already-converted block directly. - let convert_to_block = - move |input: BlockOrPayload| -> Result, NewPayloadError> { - match convert_to_block { - Either::Left(handle) => handle.try_into_inner().expect("sole handle"), - Either::Right(()) => { - let BlockOrPayload::Block(block) = input else { - unreachable!() - }; - Ok(block) - } - } - }; - - /// A helper macro that returns the block in case there was an error - /// This macro is used for early returns before block conversion - macro_rules! ensure_ok { - ($expr:expr) => { - match $expr { - Ok(val) => val, - Err(e) => { - let block = convert_to_block(input)?; - return Err(InsertBlockError::new(block, e.into()).into()); - } - } - }; - } - - /// A helper macro for handling errors after the input has been converted to a block - macro_rules! ensure_ok_post_block { - ($expr:expr, $block:expr) => { - match $expr { - Ok(val) => val, - Err(e) => { - return Err( - InsertBlockError::new($block.into_sealed_block(), e.into()).into() - ) - } - } - }; - } - - let parent_hash = input.parent_hash(); - - trace!(target: "engine::tree::payload_validator", "Fetching block state provider"); - let _enter = - debug_span!(target: "engine::tree::payload_validator", "state_provider").entered(); - let Some(provider_builder) = - ensure_ok!(self.state_provider_builder(parent_hash, ctx.state())) - else { - // this is pre-validated in the tree - return Err(InsertBlockError::new( - convert_to_block(input)?, - ProviderError::HeaderNotFound(parent_hash.into()).into(), - ) - .into()); - }; - let mut state_provider = ensure_ok!(provider_builder.build()); - drop(_enter); - - // Fetch parent block. This goes to memory most of the time unless the parent block is - // beyond the in-memory buffer. - let Some(parent_block) = ensure_ok!(self.sealed_header_by_hash(parent_hash, ctx.state())) - else { - return Err(InsertBlockError::new( - convert_to_block(input)?, - ProviderError::HeaderNotFound(parent_hash.into()).into(), - ) - .into()); - }; - - let evm_env = debug_span!(target: "engine::tree::payload_validator", "evm_env") - .in_scope(|| self.evm_env_for(&input)) - .map_err(NewPayloadError::other)?; - - let env = ExecutionEnv { - evm_env, - hash: input.hash(), - parent_hash: input.parent_hash(), - parent_state_root: parent_block.state_root(), - transaction_count: input.transaction_count(), - gas_used: input.gas_used(), - withdrawals: input.withdrawals().map(|w| w.to_vec()), - // Morph blocks (pre-Amsterdam) carry no BAL. Skip BAL-driven prewarming - // and let the payload processor install its own state hook for prewarm - // updates instead. - decoded_bal: None, - }; - - // Plan the strategy used for state root computation. - let strategy = self.plan_state_root_computation(); - - debug!( - target: "engine::tree::payload_validator", - ?strategy, - "Decided which state root algorithm to run" - ); - - // Get an iterator over the transactions in the payload - let txs = self.tx_iterator_for(&input)?; - - // Morph blocks (pre-Amsterdam) carry no BAL, so we deliberately drop the - // upstream BAL extraction. The header validator returns `None` for - // `block_access_list_hash()`, so any BAL bytes carried on the wire are - // ignored. - - // Create lazy overlay from ancestors - this doesn't block, allowing execution to start - // before the trie data is ready. The overlay will be computed on first access. - let (lazy_overlay, anchor_hash) = Self::get_parent_lazy_overlay(parent_hash, ctx.state()); - - // Create overlay factory for payload processor (StateRootTask path needs it for - // multiproofs). v2.2.0 moved `with_block_hash` / `with_lazy_overlay` onto - // `OverlayBuilder`, so we configure the builder first and pass it to the factory. - let overlay_builder = OverlayBuilder::::new(anchor_hash, self.changeset_cache.clone()) - .with_lazy_overlay(lazy_overlay); - let overlay_factory = - OverlayStateProviderFactory::new(self.provider.clone(), overlay_builder); - - // Spawn the appropriate processor based on strategy - let mut handle = ensure_ok!(self.spawn_payload_processor( - env.clone(), - txs, - provider_builder, - overlay_factory.clone(), - strategy, - )); - - // Create optional cache stats for detailed block logging - let slow_block_enabled = self.config.slow_block_threshold().is_some(); - let cache_stats = slow_block_enabled.then(|| Arc::new(CacheStats::default())); - - // Use cached state provider before executing, used in execution after prewarming threads - // complete - if let Some((caches, cache_metrics)) = handle.caches().zip(handle.cache_metrics()) { - state_provider = Box::new( - CachedStateProvider::new(state_provider, caches, cache_metrics) - .with_cache_stats(cache_stats.clone()), - ); - }; - - let state_provider_stats = if slow_block_enabled || self.config.state_provider_metrics() { - let instrumented_state_provider = - InstrumentedStateProvider::new(state_provider, "engine"); - let stats = slow_block_enabled.then(|| instrumented_state_provider.stats()); - state_provider = Box::new(instrumented_state_provider); - stats - } else { - None - }; - - // Execute the block and handle any execution errors. - // The receipt root task is spawned before execution and receives receipts incrementally - // as transactions complete, allowing parallel computation during execution. - let execute_block_start = Instant::now(); - let (output, senders, receipt_root_rx) = - match self.execute_block(state_provider, env, &input, &mut handle) { - Ok(output) => output, - Err(err) => return self.handle_execution_error(input, err, &parent_block), - }; - let execution_duration = execute_block_start.elapsed(); - - // After executing the block we can stop prewarming transactions - handle.stop_prewarming_execution(); - - // Create ExecutionOutcome early so we can terminate caching before validation and state - // root computation. Using Arc allows sharing with both the caching task and the deferred - // trie task without cloning the expensive BundleState. - let output = Arc::new(output); - - // Terminate caching task early since execution is complete and caching is no longer - // needed. This frees up resources while state root computation continues. - let valid_block_tx = handle.terminate_caching(Some(output.clone())); - - // Spawn hashed post state computation in background so it runs concurrently with - // block conversion and receipt root computation. This is a pure CPU-bound task - // (keccak256 hashing of all changed addresses and storage slots). - let hashed_state_output = output.clone(); - let hashed_state_provider = self.provider.clone(); - let hashed_state: LazyHashedPostState = self - .payload_processor - .executor() - .spawn_blocking_named("hash-post-state", move || { - let _span = debug_span!( - target: "engine::tree::payload_validator", - "hashed_post_state", - ) - .entered(); - hashed_state_provider.hashed_post_state(&hashed_state_output.state) - }); - - let block = convert_to_block(input)?; - let transaction_root = is_payload.then(|| { - let body = block.body().clone(); - let parent_span = Span::current(); - let num_hash = block.num_hash(); - self.payload_processor.executor().spawn_blocking_named("payload-tx-root", move || { - let _span = - debug_span!(target: "engine::tree::payload_validator", parent: parent_span, "payload_tx_root", block = ?num_hash) - .entered(); - body.calculate_tx_root() - }) - }); - let block = block.with_senders(senders); - - // Wait for the receipt root computation to complete. - let receipt_root_bloom = { - let _enter = debug_span!( - target: "engine::tree::payload_validator", - "wait_receipt_root", - ) - .entered(); - - receipt_root_rx - .blocking_recv() - .inspect_err(|_| { - tracing::error!( - target: "engine::tree::payload_validator", - "Receipt root task dropped sender without result, receipt root calculation likely aborted" - ); - }) - .ok() - }; - let transaction_root = transaction_root.map(|handle| { - let _span = - debug_span!(target: "engine::tree::payload_validator", "wait_payload_tx_root") - .entered(); - handle.try_into_inner().expect("sole handle") - }); - - let hashed_state = ensure_ok_post_block!( - self.validate_post_execution( - &block, - &parent_block, - &output, - &mut ctx, - transaction_root, - receipt_root_bloom, - hashed_state, - ), - block - ); - - let root_time = Instant::now(); - let mut maybe_state_root = None; - let mut state_root_task_failed = false; - #[cfg(feature = "trie-debug")] - let mut trie_debug_recorders = Vec::new(); - - match strategy { - StateRootStrategy::StateRootTask => { - debug!(target: "engine::tree::payload_validator", "Using sparse trie state root algorithm"); - - let task_result = ensure_ok_post_block!( - self.await_state_root_with_timeout( - &mut handle, - overlay_factory.clone(), - &hashed_state, - ), - block - ); - - match task_result { - Ok(StateRootComputeOutcome { - state_root, - trie_updates, - #[cfg(feature = "trie-debug")] - debug_recorders, - }) => { - let elapsed = root_time.elapsed(); - info!(target: "engine::tree::payload_validator", ?state_root, ?elapsed, "State root task finished"); - - #[cfg(feature = "trie-debug")] - { - trie_debug_recorders = debug_recorders; - } - - // Compare trie updates with serial computation if configured - if self.config.always_compare_trie_updates() { - let _has_diff = self.compare_trie_updates_with_serial( - overlay_factory.clone(), - &hashed_state, - trie_updates.as_ref().clone(), - ); - #[cfg(feature = "trie-debug")] - if _has_diff { - Self::write_trie_debug_recorders( - block.header().number(), - &trie_debug_recorders, - ); - } - } - - // Pre-Jade Morph blocks store a ZK-trie root in header.state_root; - // reth computes an MPT root which will never match. The strict check - // is only enforced post-Jade. See gate::state_root_enforced_at and - // the design spec for transitivity of EVM execution. - let strict_enforced = - state_root_enforced_at(&self.chain_spec, block.header().timestamp()); - - // we double check the state root here for good measure - if !strict_enforced || state_root == block.header().state_root() { - maybe_state_root = Some((state_root, trie_updates, elapsed)) - } else { - warn!( - target: "engine::tree::payload_validator", - ?state_root, - block_state_root = ?block.header().state_root(), - "State root task returned incorrect state root" - ); - #[cfg(feature = "trie-debug")] - Self::write_trie_debug_recorders( - block.header().number(), - &trie_debug_recorders, - ); - state_root_task_failed = true; - } - } - Err(error) => { - debug!(target: "engine::tree::payload_validator", %error, "State root task failed"); - state_root_task_failed = true; - } - } - } - StateRootStrategy::Parallel => { - debug!(target: "engine::tree::payload_validator", "Using parallel state root algorithm"); - match self.compute_state_root_parallel(overlay_factory.clone(), &hashed_state) { - Ok(result) => { - let elapsed = root_time.elapsed(); - info!( - target: "engine::tree::payload_validator", - regular_state_root = ?result.0, - ?elapsed, - "Regular root task finished" - ); - maybe_state_root = Some((result.0, Arc::new(result.1), elapsed)); - } - Err(error) => { - debug!(target: "engine::tree::payload_validator", %error, "Parallel state root computation failed"); - } - } - } - StateRootStrategy::Synchronous => {} - } - - // Determine the state root. - // If the state root was computed in parallel, we use it. - // Otherwise, we fall back to computing it synchronously. - let (state_root, trie_output, root_elapsed) = if let Some(maybe_state_root) = - maybe_state_root - { - maybe_state_root - } else { - // fallback is to compute the state root regularly in sync - if self.config.state_root_fallback() { - debug!(target: "engine::tree::payload_validator", "Using state root fallback for testing"); - } else { - warn!(target: "engine::tree::payload_validator", "Failed to compute state root in parallel"); - self.metrics - .block_validation - .state_root_parallel_fallback_total - .increment(1); - } - - let (root, updates) = ensure_ok_post_block!( - Self::compute_state_root_serial(overlay_factory.clone(), &hashed_state), - block - ); - - if state_root_task_failed { - self.metrics - .block_validation - .state_root_task_fallback_success_total - .increment(1); - } - - (root, Arc::new(updates), root_time.elapsed()) - }; - - self.metrics - .block_validation - .record_state_root(&trie_output, root_elapsed.as_secs_f64()); - self.metrics - .record_state_root_gas_bucket(block.header().gas_used(), root_elapsed.as_secs_f64()); - debug!(target: "engine::tree::payload_validator", ?root_elapsed, "Calculated state root"); - - // ensure state root matches — post-Jade only. Pre-Jade blocks carry a - // ZK-trie root in the header which will never equal reth's MPT root. - // Correctness is proven transitively by the first post-Jade block's - // successful strict check (see design spec). - let strict_enforced = state_root_enforced_at(&self.chain_spec, block.header().timestamp()); - - if strict_enforced && state_root != block.header().state_root() { - #[cfg(feature = "trie-debug")] - Self::write_trie_debug_recorders(block.header().number(), &trie_debug_recorders); - - // call post-block hook - self.on_invalid_block( - &parent_block, - &block, - &output, - Some((&trie_output, state_root)), - ctx.state_mut(), - ); - let block_state_root = block.header().state_root(); - return Err(InsertBlockError::new( - block.into_sealed_block(), - ConsensusError::BodyStateRootDiff( - GotExpected { - got: state_root, - expected: block_state_root, - } - .into(), - ) - .into(), - ) - .into()); - } - - let timing_stats = state_provider_stats.map(|stats| { - self.calculate_timing_stats( - &block, - stats, - cache_stats, - &output, - execution_duration, - root_elapsed, - ) - }); - - if let Some(valid_block_tx) = valid_block_tx { - let _ = valid_block_tx.send(()); - } - - // Create the overlay provider NOW, while we're on the engine loop thread and trie changeset - // eviction cannot race with us. If we deferred this to the background task, persistence - // could advance and evict changeset cache entries between factory creation and the task - // actually running, causing expensive DB fallback computations when building the overlay. - let changeset_provider = - ensure_ok_post_block!(overlay_factory.database_provider_ro(), block); - - let executed_block = self.spawn_deferred_trie_task( - block, - output, - &ctx, - hashed_state, - trie_output, - changeset_provider, - ); - Ok((executed_block, timing_stats)) - } - - /// Return sealed block header from database or in-memory state by hash. - fn sealed_header_by_hash( - &self, - hash: B256, - state: &EngineApiTreeState, - ) -> ProviderResult>> { - // check memory first - let header = state.tree_state().sealed_header_by_hash(&hash); - - if header.is_some() { - Ok(header) - } else { - self.provider.sealed_header_by_hash(hash) - } - } - - /// Validate if block is correct and satisfies all the consensus rules that concern the header - /// and block body itself. - #[instrument(level = "debug", target = "engine::tree::payload_validator", skip_all)] - fn validate_block_inner( - &self, - block: &SealedBlock, - transaction_root: Option, - ) -> Result<(), ConsensusError> { - if let Err(e) = self.consensus.validate_header(block.sealed_header()) { - error!(target: "engine::tree::payload_validator", ?block, "Failed to validate header {}: {e}", block.hash()); - return Err(e); - } - - if let Err(e) = self - .consensus - .validate_block_pre_execution_with_tx_root(block, transaction_root) - { - error!(target: "engine::tree::payload_validator", ?block, "Failed to validate block {}: {e}", block.hash()); - return Err(e); - } - - Ok(()) - } - - /// Validates Morph's parent-aware L1 message index invariant. - /// - /// go-ethereum derives `NextL1MsgIndex` from the parent queue index plus the - /// L1 messages processed by this block. Since `next_l1_msg_index` is not part - /// of the block hash, Jade and later must reject any mismatch before accepting - /// the payload. - fn validate_next_l1_msg_index_against_parent( - &self, - block: &RecoveredBlock, - parent_block: &SealedHeader, - ) -> Result<(), ConsensusError> { - if !self - .chain_spec - .is_jade_active_at_timestamp(block.header().timestamp()) - { - return Ok(()); - } - - let mut expected = parent_block.next_l1_msg_index; - for tx in block.body().transactions() { - if !tx.is_l1_msg() { - break; - } - - let queue_index = tx.queue_index().ok_or_else(|| { - ConsensusError::msg("L1 message transaction is missing queue index") - })?; - // Match geth's `NumL1MessagesProcessed(parent_queue_index)`: forward skips - // are counted as processed, so the derived next index is last_queue + 1. - expected = queue_index.checked_add(1).ok_or_else(|| { - ConsensusError::msg(format!( - "invalid block.NextL1MsgIndex: expected {}, got {}", - u64::MAX, - block.header().next_l1_msg_index - )) - })?; - } - - let actual = block.header().next_l1_msg_index; - if actual != expected { - return Err(ConsensusError::msg(format!( - "invalid block.NextL1MsgIndex: expected {expected}, got {actual}" - ))); - } - - Ok(()) - } - - /// Executes a block with the given state provider. - /// - /// This method orchestrates block execution: - /// 1. Sets up the EVM with state database and precompile caching - /// 2. Spawns a background task for incremental receipt root computation - /// 3. Executes transactions with metrics collection via state hooks - /// 4. Merges state transitions and records execution metrics - #[instrument(level = "debug", target = "engine::tree::payload_validator", skip_all)] - #[expect(clippy::type_complexity)] - fn execute_block( - &mut self, - state_provider: S, - env: ExecutionEnv, - input: &BlockOrPayload, - handle: &mut PayloadHandle, Err, N::Receipt>, - ) -> Result< - ( - BlockExecutionOutput, - Vec
, - tokio::sync::oneshot::Receiver<(B256, alloy_primitives::Bloom)>, - ), - InsertBlockErrorKind, - > - where - S: StateProvider + Send, - Err: core::error::Error + Send + Sync + 'static, - V: PayloadValidator, - T: PayloadTypes>, - Evm: ConfigureEngineEvm, - { - debug!(target: "engine::tree::payload_validator", "Executing block"); - - let mut db = debug_span!(target: "engine::tree", "build_state_db").in_scope(|| { - State::builder() - .with_database(StateProviderDatabase::new(state_provider)) - .with_bundle_update() - .build() - }); - - let (spec_id, mut executor) = { - let _span = debug_span!(target: "engine::tree", "create_evm").entered(); - let spec_id = *env.evm_env.spec_id(); - let evm = self.evm_config.evm_with_env(&mut db, env.evm_env); - let ctx = self - .execution_ctx_for(input) - .map_err(|e| InsertBlockErrorKind::Other(Box::new(e)))?; - let executor = self.evm_config.create_executor(evm, ctx); - (spec_id, executor) - }; - - if !self.config.precompile_cache_disabled() { - let _span = debug_span!(target: "engine::tree", "setup_precompile_cache").entered(); - executor - .evm_mut() - .precompiles_mut() - .map_cacheable_precompiles(|address, precompile| { - let metrics = self - .precompile_cache_metrics - .entry(*address) - .or_insert_with(|| CachedPrecompileMetrics::new_with_address(*address)) - .clone(); - CachedPrecompile::wrap( - precompile, - self.precompile_cache_map.cache_for_address(*address), - spec_id, - Some(metrics), - ) - }); - } - - // Spawn background task to compute receipt root and logs bloom incrementally. - // Unbounded channel is used since tx count bounds capacity anyway (max ~30k txs per block). - let receipts_len = input.transaction_count(); - let (receipt_tx, receipt_rx) = crossbeam_channel::unbounded(); - let (result_tx, result_rx) = tokio::sync::oneshot::channel(); - let task_handle = ReceiptRootTaskHandle::new(receipt_rx, result_tx); - self.payload_processor - .executor() - .spawn_blocking_named("receipt-root", move || task_handle.run(receipts_len)); - - let transaction_count = input.transaction_count(); - let executed_tx_index = Arc::clone(handle.executed_tx_index()); - let executor = executor.with_state_hook( - handle - .state_hook() - .map(|hook| Box::new(hook) as Box), - ); - - let execution_start = Instant::now(); - - // Execute all transactions and finalize - let (executor, senders) = self.execute_transactions( - executor, - transaction_count, - handle.iter_transactions(), - &receipt_tx, - &executed_tx_index, - )?; - drop(receipt_tx); - - // Finish execution and get the result - let post_exec_start = Instant::now(); - let (_evm, result) = debug_span!(target: "engine::tree", "BlockExecutor::finish") - .in_scope(|| executor.finish()) - .map(|(evm, result)| (evm.into_db(), result))?; - self.metrics - .record_post_execution(post_exec_start.elapsed()); - - // Merge transitions into bundle state - debug_span!(target: "engine::tree", "merge_transitions") - .in_scope(|| db.merge_transitions(BundleRetention::Reverts)); - - let output = BlockExecutionOutput { - result, - state: db.take_bundle(), - }; - - let execution_duration = execution_start.elapsed(); - self.metrics - .record_block_execution(&output, execution_duration); - self.metrics - .record_block_execution_gas_bucket(output.result.gas_used, execution_duration); - debug!(target: "engine::tree::payload_validator", elapsed = ?execution_duration, "Executed block"); - - Ok((output, senders, result_rx)) - } - - /// Executes transactions and collects senders, streaming receipts to a background task. - /// - /// This method handles: - /// - Applying pre-execution changes (e.g., beacon root updates) - /// - Executing each transaction with timing metrics - /// - Streaming receipts to the receipt root computation task - /// - Collecting transaction senders for later use - /// - /// Returns the executor (for finalization) and the collected senders. - fn execute_transactions( - &self, - mut executor: E, - transaction_count: usize, - transactions: impl Iterator>, - receipt_tx: &crossbeam_channel::Sender>, - executed_tx_index: &AtomicUsize, - ) -> Result<(E, Vec
), BlockExecutionError> - where - E: BlockExecutor, - Tx: alloy_evm::block::ExecutableTx + alloy_evm::RecoveredTx, - InnerTx: TxHashRef, - Err: core::error::Error + Send + Sync + 'static, - { - let mut senders = Vec::with_capacity(transaction_count); - - // Apply pre-execution changes (e.g., beacon root update) - let pre_exec_start = Instant::now(); - debug_span!(target: "engine::tree", "pre_execution") - .in_scope(|| executor.apply_pre_execution_changes())?; - self.metrics.record_pre_execution(pre_exec_start.elapsed()); - - // Execute transactions - let exec_span = debug_span!(target: "engine::tree", "execution").entered(); - let mut transactions = transactions.into_iter(); - // Some executors may execute transactions that do not append receipts during the - // main loop (e.g., system transactions whose receipts are added during finalization). - // In that case, invoking the callback on every transaction would resend the previous - // receipt with the same index and can panic the ordered root builder. - let mut last_sent_len = 0usize; - loop { - // Measure time spent waiting for next transaction from iterator - // (e.g., parallel signature recovery) - let wait_start = Instant::now(); - let Some(tx_result) = transactions.next() else { - break; - }; - self.metrics.record_transaction_wait(wait_start.elapsed()); - - let tx = tx_result.map_err(BlockExecutionError::other)?; - let tx_signer = *>::signer(&tx); - - senders.push(tx_signer); - - let _enter = debug_span!( - target: "engine::tree", - "execute tx", - tx_index = senders.len() - 1, - ) - .entered(); - trace!(target: "engine::tree", "Executing transaction"); - - let tx_start = Instant::now(); - executor.execute_transaction(tx)?; - self.metrics - .record_transaction_execution(tx_start.elapsed()); - - // advance the shared counter so prewarm workers skip already-executed txs - executed_tx_index.store(senders.len(), Ordering::Relaxed); - - let current_len = executor.receipts().len(); - if current_len > last_sent_len { - last_sent_len = current_len; - // Send the latest receipt to the background task for incremental root computation. - if let Some(receipt) = executor.receipts().last() { - let tx_index = current_len - 1; - let _ = receipt_tx.send(IndexedReceipt::new(tx_index, receipt.clone())); - } - } - } - drop(exec_span); - - Ok((executor, senders)) - } - - /// Compute state root for the given hashed post state in parallel. - /// - /// Uses an overlay factory which provides the state of the parent block, along with the - /// [`HashedPostState`] containing the changes of this block, to compute the state root and - /// trie updates for this block. - /// - /// # Returns - /// - /// Returns `Ok(_)` if computed successfully. - /// Returns `Err(_)` if error was encountered during computation. - #[instrument(level = "debug", target = "engine::tree::payload_validator", skip_all)] - fn compute_state_root_parallel( - &self, - overlay_factory: OverlayStateProviderFactory, - hashed_state: &LazyHashedPostState, - ) -> Result<(B256, TrieUpdates), ParallelStateRootError> { - let hashed_state = hashed_state.get(); - // The `hashed_state` argument will be taken into account as part of the overlay, but we - // need to use the prefix sets which were generated from it to indicate to the - // ParallelStateRoot which parts of the trie need to be recomputed. - let prefix_sets = hashed_state.construct_prefix_sets().freeze(); - let overlay_factory = - overlay_factory.with_extended_hashed_state_overlay(hashed_state.clone_into_sorted()); - ParallelStateRoot::new(overlay_factory, prefix_sets, self.runtime.clone()) - .incremental_root_with_updates() - } - - /// Compute state root for the given hashed post state in serial. - /// - /// Uses an overlay factory which provides the state of the parent block, along with the - /// [`HashedPostState`] containing the changes of this block, to compute the state root and - /// trie updates for this block. - fn compute_state_root_serial( - overlay_factory: OverlayStateProviderFactory, - hashed_state: &LazyHashedPostState, - ) -> ProviderResult<(B256, TrieUpdates)> { - let hashed_state = hashed_state.get(); - // The `hashed_state` argument will be taken into account as part of the overlay, but we - // need to use the prefix sets which were generated from it to indicate to the - // StateRoot which parts of the trie need to be recomputed. - let prefix_sets = hashed_state.construct_prefix_sets().freeze(); - let overlay_factory = - overlay_factory.with_extended_hashed_state_overlay(hashed_state.clone_into_sorted()); - - let provider = overlay_factory.database_provider_ro()?; - - Ok(StateRoot::new(&provider, &provider) - .with_prefix_sets(prefix_sets) - .root_with_updates()?) - } - - /// Awaits the state root from the background task, with an optional timeout fallback. - /// - /// If a timeout is configured (`state_root_task_timeout`), this method first waits for the - /// state root task up to the timeout duration. If the task doesn't complete in time, a - /// sequential state root computation is spawned via `spawn_blocking`. Both computations - /// then race: the main thread polls the task receiver and the sequential result channel - /// in a loop, returning whichever finishes first. - /// - /// If no timeout is configured, this simply awaits the state root task without any fallback. - /// - /// Returns `ProviderResult>` where the outer `ProviderResult` captures - /// unrecoverable errors from the sequential fallback (e.g. DB errors), while the inner - /// `Result` captures parallel state root task errors that can still fall back to serial. - #[instrument( - level = "debug", - target = "engine::tree::payload_validator", - name = "await_state_root", - skip_all - )] - fn await_state_root_with_timeout( - &self, - handle: &mut PayloadHandle, - overlay_factory: OverlayStateProviderFactory, - hashed_state: &LazyHashedPostState, - ) -> ProviderResult> { - let Some(timeout) = self.config.state_root_task_timeout() else { - return Ok(handle.state_root()); - }; - - let task_rx = handle.take_state_root_rx(); - - match task_rx.recv_timeout(timeout) { - Ok(result) => Ok(result), - Err(RecvTimeoutError::Disconnected) => Ok(Err(ParallelStateRootError::Other( - "sparse trie task dropped".to_string(), - ))), - Err(RecvTimeoutError::Timeout) => { - warn!( - target: "engine::tree::payload_validator", - ?timeout, - "State root task timed out, spawning sequential fallback" - ); - self.metrics - .block_validation - .state_root_task_timeout_total - .increment(1); - - let (seq_tx, seq_rx) = - std::sync::mpsc::channel::>(); - - let seq_overlay = overlay_factory; - let seq_hashed_state = hashed_state.clone(); - self.payload_processor - .executor() - .spawn_blocking_named("serial-root", move || { - let result = - Self::compute_state_root_serial(seq_overlay, &seq_hashed_state); - let _ = seq_tx.send(result); - }); - - const POLL_INTERVAL: std::time::Duration = std::time::Duration::from_millis(10); - - loop { - match task_rx.recv_timeout(POLL_INTERVAL) { - Ok(result) => { - debug!( - target: "engine::tree::payload_validator", - source = "task", - "State root timeout race won" - ); - return Ok(result); - } - Err(RecvTimeoutError::Disconnected) => { - debug!( - target: "engine::tree::payload_validator", - "State root task dropped, waiting for sequential fallback" - ); - let result = seq_rx.recv().map_err(|_| { - ProviderError::other(std::io::Error::other( - "both state root computations failed", - )) - })?; - let (state_root, trie_updates) = result?; - return Ok(Ok(StateRootComputeOutcome { - state_root, - trie_updates: Arc::new(trie_updates), - #[cfg(feature = "trie-debug")] - debug_recorders: Vec::new(), - })); - } - Err(RecvTimeoutError::Timeout) => {} - } - - if let Ok(result) = seq_rx.try_recv() { - debug!( - target: "engine::tree::payload_validator", - source = "sequential", - "State root timeout race won" - ); - let (state_root, trie_updates) = result?; - return Ok(Ok(StateRootComputeOutcome { - state_root, - trie_updates: Arc::new(trie_updates), - #[cfg(feature = "trie-debug")] - debug_recorders: Vec::new(), - })); - } - } - } - } - } - - /// Compares trie updates from the state root task with serial state root computation. - /// - /// This is used for debugging and validating the correctness of the parallel state root - /// task implementation. When enabled via `--engine.state-root-task-compare-updates`, this - /// method runs a separate serial state root computation and compares the resulting trie - /// updates. - fn compare_trie_updates_with_serial( - &self, - overlay_factory: OverlayStateProviderFactory, - hashed_state: &LazyHashedPostState, - task_trie_updates: TrieUpdates, - ) -> bool { - debug!(target: "engine::tree::payload_validator", "Comparing trie updates with serial computation"); - - match Self::compute_state_root_serial(overlay_factory.clone(), hashed_state) { - Ok((serial_root, serial_trie_updates)) => { - debug!( - target: "engine::tree::payload_validator", - ?serial_root, - "Serial state root computation finished for comparison" - ); - - // Get a database provider to use as trie cursor factory - match overlay_factory.database_provider_ro() { - Ok(provider) => { - match crate::trie_updates::compare_trie_updates( - &provider, - task_trie_updates, - serial_trie_updates, - ) { - Ok(has_diff) => return has_diff, - Err(err) => { - warn!( - target: "engine::tree::payload_validator", - %err, - "Error comparing trie updates" - ); - return true; - } - } - } - Err(err) => { - warn!( - target: "engine::tree::payload_validator", - %err, - "Failed to get database provider for trie update comparison" - ); - } - } - } - Err(err) => { - warn!( - target: "engine::tree::payload_validator", - %err, - "Failed to compute serial state root for comparison" - ); - } - } - false - } - - /// Writes trie debug recorders to a JSON file for the given block number. - /// - /// The file is written to the current working directory as - /// `trie_debug_block_{block_number}.json`. - #[cfg(feature = "trie-debug")] - fn write_trie_debug_recorders( - block_number: u64, - recorders: &[(Option, TrieDebugRecorder)], - ) { - let path = format!("trie_debug_block_{block_number}.json"); - match serde_json::to_string_pretty(recorders) { - Ok(json) => match std::fs::write(&path, json) { - Ok(()) => { - warn!( - target: "engine::tree::payload_validator", - %path, - "Wrote trie debug recorders to file" - ); - } - Err(err) => { - warn!( - target: "engine::tree::payload_validator", - %err, - %path, - "Failed to write trie debug recorders" - ); - } - }, - Err(err) => { - warn!( - target: "engine::tree::payload_validator", - %err, - "Failed to serialize trie debug recorders" - ); - } - } - } - - /// Validates the block after execution. - /// - /// This performs: - /// - parent header validation - /// - post-execution consensus validation - /// - state-root based post-execution validation - /// - /// If `receipt_root_bloom` is provided, it will be used instead of computing the receipt root - /// and logs bloom from the receipts. - /// - /// The `hashed_state` handle wraps the background hashed post state computation. - #[instrument(level = "debug", target = "engine::tree::payload_validator", skip_all)] - #[expect(clippy::too_many_arguments)] - fn validate_post_execution>>( - &self, - block: &RecoveredBlock, - parent_block: &SealedHeader, - output: &BlockExecutionOutput, - ctx: &mut TreeCtx<'_, N>, - transaction_root: Option, - receipt_root_bloom: Option, - hashed_state: LazyHashedPostState, - ) -> Result - where - V: PayloadValidator, - { - let start = Instant::now(); - - trace!(target: "engine::tree::payload_validator", block=?block.num_hash(), "Validating block consensus"); - // validate block consensus rules - if let Err(e) = self.validate_block_inner(block, transaction_root) { - return Err(e.into()); - } - - // now validate against the parent - let _enter = debug_span!(target: "engine::tree::payload_validator", "validate_header_against_parent").entered(); - if let Err(e) = self - .consensus - .validate_header_against_parent(block.sealed_header(), parent_block) - { - warn!(target: "engine::tree::payload_validator", ?block, "Failed to validate header {} against parent: {e}", block.hash()); - return Err(e.into()); - } - drop(_enter); - - if let Err(e) = self.validate_next_l1_msg_index_against_parent(block, parent_block) { - warn!( - target: "engine::tree::payload_validator", - ?block, - "Failed to validate NextL1MsgIndex against parent for block {}: {e}", - block.hash() - ); - return Err(e.into()); - } - - // Validate block post-execution rules - let _enter = - debug_span!(target: "engine::tree::payload_validator", "validate_block_post_execution") - .entered(); - if let Err(err) = - self.consensus - .validate_block_post_execution(block, output, receipt_root_bloom) - { - // call post-block hook - self.on_invalid_block(parent_block, block, output, None, ctx.state_mut()); - return Err(err.into()); - } - drop(_enter); - - // Wait for the background keccak256 hashing task to complete. This blocks until - // all changed addresses and storage slots have been hashed. - let hashed_state_ref = - debug_span!(target: "engine::tree::payload_validator", "wait_hashed_post_state") - .in_scope(|| hashed_state.get()); - - let _enter = debug_span!(target: "engine::tree::payload_validator", "validate_block_post_execution_with_hashed_state").entered(); - if let Err(err) = self - .validator - .validate_block_post_execution_with_hashed_state(hashed_state_ref, block) - { - // call post-block hook - self.on_invalid_block(parent_block, block, output, None, ctx.state_mut()); - return Err(err.into()); - } - - // record post-execution validation duration - self.metrics - .block_validation - .post_execution_validation_duration - .record(start.elapsed().as_secs_f64()); - - Ok(hashed_state) - } - - /// Spawns a payload processor task based on the state root strategy. - /// - /// This method determines how to execute the block and compute its state root based on - /// the selected strategy: - /// - `StateRootTask`: Uses a dedicated task for state root computation with proof generation - /// - `Parallel`: Computes state root in parallel with block execution - /// - `Synchronous`: Falls back to sequential execution and state root computation - /// - /// The method handles strategy fallbacks if the preferred approach fails, ensuring - /// block execution always completes with a valid state root. - /// - /// # Arguments - /// - /// * `overlay_factory` - Pre-computed overlay factory for multiproof generation - /// (`StateRootTask`) - #[instrument( - level = "debug", - target = "engine::tree::payload_validator", - skip_all, - fields(?strategy) - )] - fn spawn_payload_processor>( - &mut self, - env: ExecutionEnv, - txs: T, - provider_builder: StateProviderBuilder, - overlay_factory: OverlayStateProviderFactory, - strategy: StateRootStrategy, - ) -> Result< - PayloadHandle< - impl ExecutableTxFor + use, - impl core::error::Error + Send + Sync + 'static + use, - N::Receipt, - >, - InsertBlockErrorKind, - > { - match strategy { - StateRootStrategy::StateRootTask => { - let spawn_start = Instant::now(); - - let handle = self.payload_processor.spawn( - env, - txs, - provider_builder, - overlay_factory, - &self.config, - ); - - // record prewarming initialization duration - self.metrics - .block_validation - .spawn_payload_processor - .record(spawn_start.elapsed().as_secs_f64()); - - Ok(handle) - } - StateRootStrategy::Parallel | StateRootStrategy::Synchronous => { - let start = Instant::now(); - let handle = - self.payload_processor - .spawn_cache_exclusive(env, txs, provider_builder); - - // Record prewarming initialization duration - self.metrics - .block_validation - .spawn_payload_processor - .record(start.elapsed().as_secs_f64()); - - Ok(handle) - } - } - } - - /// Creates a `StateProviderBuilder` for the given parent hash. - /// - /// This method checks if the parent is in the tree state (in-memory) or persisted to disk, - /// and creates the appropriate provider builder. - fn state_provider_builder( - &self, - hash: B256, - state: &EngineApiTreeState, - ) -> ProviderResult>> { - if let Some((historical, blocks)) = state.tree_state().blocks_by_hash(hash) { - debug!(target: "engine::tree::payload_validator", %hash, %historical, "found canonical state for block in memory, creating provider builder"); - // the block leads back to the canonical chain - return Ok(Some(StateProviderBuilder::new( - self.provider.clone(), - historical, - Some(blocks), - ))); - } - - // Check if the block is persisted - if let Some(header) = self.provider.header(hash)? { - debug!(target: "engine::tree::payload_validator", %hash, number = %header.number(), "found canonical state for block in database, creating provider builder"); - // For persisted blocks, we create a builder that will fetch state directly from the - // database - return Ok(Some(StateProviderBuilder::new( - self.provider.clone(), - hash, - None, - ))); - } - - debug!(target: "engine::tree::payload_validator", %hash, "no canonical state found for block"); - Ok(None) - } - - /// Determines the state root computation strategy based on configuration. - /// - /// Note: Use state root task only if prefix sets are empty, otherwise proof generation is - /// too expensive because it requires walking all paths in every proof. - const fn plan_state_root_computation(&self) -> StateRootStrategy { - if self.config.state_root_fallback() { - StateRootStrategy::Synchronous - } else if self.config.use_state_root_task() { - StateRootStrategy::StateRootTask - } else { - StateRootStrategy::Parallel - } - } - - /// Called when an invalid block is encountered during validation. - fn on_invalid_block( - &self, - parent_header: &SealedHeader, - block: &RecoveredBlock, - output: &BlockExecutionOutput, - trie_updates: Option<(&TrieUpdates, B256)>, - state: &mut EngineApiTreeState, - ) { - if state.has_invalid_header(&block.hash()) { - // we already marked this block as invalid - return; - } - self.invalid_block_hook - .on_invalid_block(parent_header, block, output, trie_updates); - } - - /// Creates a [`LazyOverlay`] for the parent block without blocking. - /// - /// Returns a lazy overlay that will compute the trie input on first access, and the anchor - /// block hash (the highest persisted ancestor). This allows execution to start immediately - /// while the trie input computation is deferred until the overlay is actually needed. - /// - /// If parent is on disk (no in-memory blocks), returns `None` for the lazy overlay. - /// - /// Uses a cached overlay if available for the canonical head (the common case). - fn get_parent_lazy_overlay( - parent_hash: B256, - state: &EngineApiTreeState, - ) -> (Option>, B256) { - // Get blocks leading to the parent to determine the anchor - let (anchor_hash, blocks) = state - .tree_state() - .blocks_by_hash(parent_hash) - .unwrap_or_else(|| (parent_hash, vec![])); - - if blocks.is_empty() { - debug!(target: "engine::tree::payload_validator", "Parent found on disk, no lazy overlay needed"); - return (None, anchor_hash); - } - - // Try to use the cached overlay if it matches both parent hash and anchor - if let Some(cached) = state - .tree_state() - .get_cached_overlay(parent_hash, anchor_hash) - { - debug!( - target: "engine::tree::payload_validator", - %parent_hash, - %anchor_hash, - "Using cached canonical overlay" - ); - return (Some(cached.overlay.clone()), cached.anchor_hash); - } - - debug!( - target: "engine::tree::payload_validator", - %anchor_hash, - num_blocks = blocks.len(), - "Creating lazy overlay for in-memory blocks" - ); - - // v2.2.0 ingests the executed blocks themselves; per-block - // `DeferredTrieData` extraction is no longer required. - (Some(LazyOverlay::new(blocks)), anchor_hash) - } - - /// Spawns a background task to compute and sort trie data for the executed block. - /// - /// This function creates a [`DeferredTrieData`] handle with fallback inputs and spawns a - /// blocking task that calls `wait_cloned()` to: - /// 1. Sort the block's hashed state and trie updates - /// 2. Merge ancestor overlays and extend with the sorted data - /// 3. Create an [`AnchoredTrieInput`](reth_chain_state::AnchoredTrieInput) for efficient future - /// trie computations - /// 4. Cache the result so subsequent calls return immediately - /// - /// If the background task hasn't completed when `trie_data()` is called, `wait_cloned()` - /// computes from the stored inputs, eliminating deadlock risk and duplicate computation. - /// - /// The validation hot path can return immediately after state root verification, - /// while consumers (DB writes, overlay providers, proofs) get trie data either - /// from the completed task or via fallback computation. - fn spawn_deferred_trie_task( - &self, - block: RecoveredBlock, - execution_outcome: Arc>, - ctx: &TreeCtx<'_, N>, - hashed_state: LazyHashedPostState, - trie_output: Arc, - changeset_provider: impl TrieCursorFactory + Send + 'static, - ) -> ExecutedBlock { - // Capture parent hash and ancestor overlays for deferred trie input construction. - let (anchor_hash, overlay_blocks) = ctx - .state() - .tree_state() - .blocks_by_hash(block.parent_hash()) - .unwrap_or_else(|| (block.parent_hash(), Vec::new())); - - // Collect lightweight ancestor trie data handles. We don't call trie_data() here; - // the merge and any fallback sorting happens in the compute_trie_input_task. - let ancestors: Vec = overlay_blocks - .iter() - .rev() - .map(|b| b.trie_data_handle()) - .collect(); - - // Create deferred handle with fallback inputs in case the background task hasn't completed. - // Resolve the lazy handle into Arc. By this point the hashed state has - // already been computed and used for state root verification, so .get() returns instantly. - let hashed_state = match hashed_state.try_into_inner() { - Ok(state) => Arc::new(state), - Err(handle) => Arc::new(handle.get().clone()), - }; - let deferred_trie_data = - DeferredTrieData::pending(hashed_state, trie_output, anchor_hash, ancestors); - let deferred_handle_task = deferred_trie_data.clone(); - let block_validation_metrics = self.metrics.block_validation.clone(); - - // Capture block info and cache handle for changeset computation - let block_hash = block.hash(); - let block_number = block.number(); - - // Register a pending changeset entry so that concurrent readers will wait for - // this computation to finish rather than falling back to the expensive DB path. - // The guard ensures the pending entry is cancelled if the task panics. - let pending_changeset_guard = self.changeset_cache.register_pending(block_hash); - - // Spawn background task to compute trie data. Calling `wait_cloned` will compute from - // the stored inputs and cache the result, so subsequent calls return immediately. - let compute_trie_input_task = move || { - let _span = debug_span!( - target: "engine::tree::payload_validator", - "compute_trie_input_task", - block_number - ) - .entered(); - - let result = panic::catch_unwind(AssertUnwindSafe(|| { - let compute_start = Instant::now(); - let computed = deferred_handle_task.wait_cloned(); - block_validation_metrics - .deferred_trie_compute_duration - .record(compute_start.elapsed().as_secs_f64()); - - // Record sizes of the computed trie data - block_validation_metrics - .hashed_post_state_size - .record(computed.hashed_state.total_len() as f64); - block_validation_metrics - .trie_updates_sorted_size - .record(computed.trie_updates.total_len() as f64); - if let Some(anchored) = &computed.anchored_trie_input { - block_validation_metrics - .anchored_overlay_trie_updates_size - .record(anchored.trie_input.nodes.total_len() as f64); - block_validation_metrics - .anchored_overlay_hashed_state_size - .record(anchored.trie_input.state.total_len() as f64); - } - - // Compute and cache changesets using the computed trie_updates. - // Use the pre-created provider to avoid races with changeset cache - // eviction that can happen between task spawn and execution. - let changeset_start = Instant::now(); - - match reth_trie::changesets::compute_trie_changesets( - &changeset_provider, - &computed.trie_updates, - ) { - Ok(changesets) => { - debug!( - target: "engine::tree::changeset", - ?block_number, - elapsed = ?changeset_start.elapsed(), - "Computed and caching changesets" - ); - - pending_changeset_guard.resolve(block_number, Arc::new(changesets)); - } - Err(e) => { - warn!( - target: "engine::tree::changeset", - ?block_number, - ?e, - "Failed to compute changesets in deferred trie task" - ); - } - } - })); - - if result.is_err() { - error!( - target: "engine::tree::payload_validator", - "Deferred trie task panicked; fallback computation will be used when trie data is accessed" - ); - } - }; - - // Spawn task that computes trie data asynchronously. - self.payload_processor - .executor() - .spawn_blocking_named("trie-input", compute_trie_input_task); - - ExecutedBlock::with_deferred_trie_data( - Arc::new(block), - execution_outcome, - deferred_trie_data, - ) - } - - fn calculate_timing_stats( - &self, - block: &RecoveredBlock, - provider_stats: Arc, - cache_stats: Option>, - output: &BlockExecutionOutput, - execution_duration: Duration, - state_hash_duration: Duration, - ) -> Box { - let accounts_read = provider_stats.total_account_fetches(); - let storage_read = provider_stats.total_storage_fetches(); - let code_read = provider_stats.total_code_fetches(); - let code_bytes_read = provider_stats.total_code_fetched_bytes(); - - // Write stats from BundleState (final state changes) - let accounts_changed = output.state.state.len(); - let accounts_deleted = output - .state - .state - .values() - .filter(|acc| acc.was_destroyed()) - .count(); - let storage_slots_changed = output - .state - .state - .values() - .map(|account| account.storage.len()) - .sum::(); - let storage_slots_deleted = output - .state - .state - .values() - .flat_map(|account| account.storage.values()) - .filter(|slot| { - slot.present_value.is_zero() && !slot.previous_or_original_value.is_zero() - }) - .count(); - - // Helper: check if account represents a new contract deployment - let is_new_deployment = |acc: &BundleAccount| -> bool { - let has_code_now = acc - .info - .as_ref() - .is_some_and(|info| info.code_hash != KECCAK_EMPTY); - let had_no_code_before = acc - .original_info - .as_ref() - .map(|info| info.code_hash == KECCAK_EMPTY) - .unwrap_or(true); - has_code_now && had_no_code_before - }; - - let bytecodes_changed = output - .state - .state - .values() - .filter(|acc| is_new_deployment(acc)) - .count(); - - // Unique new code hashes to count actual bytes persisted (deduplicated) - let unique_new_code_hashes: B256Set = output - .state - .state - .values() - .filter(|acc| is_new_deployment(acc)) - .filter_map(|acc| acc.info.as_ref().map(|info| info.code_hash)) - .collect(); - let code_bytes_written: usize = unique_new_code_hashes - .iter() - .filter_map(|hash| { - output - .state - .contracts - .get(hash) - .map(|bytecode| bytecode.original_bytes().len()) - }) - .sum(); - - // Total time spent fetching state during execution - let state_read_duration = provider_stats.total_account_fetch_latency() - + provider_stats.total_storage_fetch_latency() - + provider_stats.total_code_fetch_latency(); - - // EIP-7702 delegation tracking from bytecode changes - // Count new EIP-7702 bytecodes as delegations set - let eip7702_delegations_set = output - .state - .contracts - .values() - .filter(|bytecode| bytecode.is_eip7702()) - .count(); - // Delegations cleared: accounts where bytecode changed FROM EIP-7702 TO empty - // This detects when an EIP-7702 delegation is removed by setting code to empty - // Note: Clearing a delegation does NOT destroy the account - it just empties the - // bytecode - let eip7702_delegations_cleared = output - .state - .state - .values() - .filter(|acc| { - // Check if original bytecode was EIP-7702 - let original_was_eip7702 = acc - .original_info - .as_ref() - .and_then(|info| info.code.as_ref()) - .map(|bytecode| bytecode.is_eip7702()) - .unwrap_or(false); - - // Check if current code is empty (delegation cleared) - let code_now_empty = acc - .info - .as_ref() - .map(|info| info.code_hash == KECCAK_EMPTY) - .unwrap_or(false); - - original_was_eip7702 && code_now_empty - }) - .count(); - - // Get cache statistics for detailed block logging - let (account_cache_hits, account_cache_misses) = cache_stats - .as_ref() - .map(|s| (s.account_hits(), s.account_misses())) - .unwrap_or_default(); - let (storage_cache_hits, storage_cache_misses) = cache_stats - .as_ref() - .map(|s| (s.storage_hits(), s.storage_misses())) - .unwrap_or_default(); - let (code_cache_hits, code_cache_misses) = cache_stats - .as_ref() - .map(|s| (s.code_hits(), s.code_misses())) - .unwrap_or_default(); - - // Build execution timing stats for detailed block logging - Box::new(ExecutionTimingStats { - block_number: block.number(), - block_hash: block.hash(), - gas_used: output.result.gas_used, - tx_count: block.transaction_count(), - execution_duration, - state_read_duration, - state_hash_duration, - accounts_read, - storage_read, - code_read, - code_bytes_read, - accounts_changed, - accounts_deleted, - storage_slots_changed, - storage_slots_deleted, - bytecodes_changed, - code_bytes_written, - eip7702_delegations_set, - eip7702_delegations_cleared, - account_cache_hits, - account_cache_misses, - storage_cache_hits, - storage_cache_misses, - code_cache_hits, - code_cache_misses, - }) - } -} - -/// Strategy describing how to compute the state root. -#[derive(Debug, Clone, Copy, PartialEq, Eq)] -enum StateRootStrategy { - /// Use the state root task (background sparse trie computation). - StateRootTask, - /// Run the parallel state root computation on the calling thread. - Parallel, - /// Fall back to synchronous computation via the state provider. - Synchronous, -} - -impl EngineValidator for MorphBasicEngineValidator -where - P: DatabaseProviderFactory< - Provider: BlockReader - + StageCheckpointReader - + PruneCheckpointReader - + ChangeSetReader - + StorageChangeSetReader - + BlockNumReader - + StorageSettingsCache, - > + BlockReader
- + StateProviderFactory - + StateReader - + ChangeSetReader - + BlockNumReader - + HashedPostStateProvider - + Clone - + 'static, - N: NodePrimitives< - Block = morph_primitives::Block, - BlockHeader = MorphHeader, - SignedTx = MorphTxEnvelope, - >, - V: PayloadValidator + Clone, - Evm: ConfigureEngineEvm + 'static, - Types: PayloadTypes>, -{ - fn validate_payload_attributes_against_header( - &self, - attr: &Types::PayloadAttributes, - header: &N::BlockHeader, - ) -> Result<(), InvalidPayloadAttributesError> { - self.validator - .validate_payload_attributes_against_header(attr, header) - } - - fn convert_payload_to_block( - &self, - payload: Types::ExecutionData, - ) -> Result, NewPayloadError> { - let block = self.validator.convert_payload_to_block(payload)?; - Ok(block) - } - - fn validate_payload( - &mut self, - payload: Types::ExecutionData, - ctx: TreeCtx<'_, N>, - ) -> ValidationOutcome { - self.validate_block_with_state(BlockOrPayload::Payload(payload), ctx) - } - - fn validate_block( - &mut self, - block: SealedBlock, - ctx: TreeCtx<'_, N>, - ) -> ValidationOutcome { - self.validate_block_with_state(BlockOrPayload::Block(block), ctx) - } - - fn on_inserted_executed_block(&self, block: ExecutedBlock) { - self.payload_processor.on_inserted_executed_block( - block.recovered_block.block_with_parent(), - &block.execution_output.state, - ); - } - - fn cache_for(&self, block_hash: B256) -> Option { - Some(self.payload_processor.cache_for(block_hash)) - } - - fn sparse_trie_handle_for( - &self, - parent_hash: B256, - parent_state_root: B256, - state: &EngineApiTreeState, - ) -> Option { - let (lazy_overlay, anchor_hash) = Self::get_parent_lazy_overlay(parent_hash, state); - let overlay_builder = OverlayBuilder::::new(anchor_hash, self.changeset_cache.clone()) - .with_lazy_overlay(lazy_overlay); - let overlay_factory = - OverlayStateProviderFactory::new(self.provider.clone(), overlay_builder); - - Some(self.payload_processor.spawn_state_root( - overlay_factory, - parent_state_root, - // Full proof workers — tx count unknown at FCU time (block built incrementally) - false, - &self.config, - )) - } -} - -impl WaitForCaches for MorphBasicEngineValidator -where - Evm: ConfigureEvm, -{ - fn wait_for_caches(&self) -> CacheWaitDurations { - self.payload_processor.wait_for_caches() - } -} - -/// Enum representing either block or payload being validated. -#[derive(Debug, Clone)] -pub enum BlockOrPayload { - /// Payload. - Payload(T::ExecutionData), - /// Block. - Block(SealedBlock::Primitives>>), -} - -impl BlockOrPayload { - /// Returns the hash of the block. - pub fn hash(&self) -> B256 { - match self { - Self::Payload(payload) => payload.block_hash(), - Self::Block(block) => block.hash(), - } - } - - /// Returns the number and hash of the block. - pub fn num_hash(&self) -> NumHash { - match self { - Self::Payload(payload) => payload.num_hash(), - Self::Block(block) => block.num_hash(), - } - } - - /// Returns the parent hash of the block. - pub fn parent_hash(&self) -> B256 { - match self { - Self::Payload(payload) => payload.parent_hash(), - Self::Block(block) => block.parent_hash(), - } - } - - /// Returns [`BlockWithParent`] for the block. - pub fn block_with_parent(&self) -> BlockWithParent { - match self { - Self::Payload(payload) => payload.block_with_parent(), - Self::Block(block) => block.block_with_parent(), - } - } - - /// Returns a string showing whether or not this is a block or payload. - pub const fn type_name(&self) -> &'static str { - match self { - Self::Payload(_) => "payload", - Self::Block(_) => "block", - } - } - - /// Returns the block access list if available. - pub const fn block_access_list(&self) -> Option> { - // TODO decode and return `BlockAccessList` - None - } - - /// Returns the number of transactions in the payload or block. - pub fn transaction_count(&self) -> usize - where - T::ExecutionData: ExecutionPayload, - { - match self { - Self::Payload(payload) => payload.transaction_count(), - Self::Block(block) => block.transaction_count(), - } - } - - /// Returns the withdrawals from the payload or block. - pub fn withdrawals(&self) -> Option<&[Withdrawal]> - where - T::ExecutionData: ExecutionPayload, - { - match self { - Self::Payload(payload) => payload.withdrawals().map(|w| w.as_slice()), - Self::Block(block) => block.body().withdrawals().map(|w| w.as_slice()), - } - } - - /// Returns the total gas used by the block. - pub fn gas_used(&self) -> u64 - where - T::ExecutionData: ExecutionPayload, - { - match self { - Self::Payload(payload) => payload.gas_used(), - Self::Block(block) => block.gas_used(), - } - } -} diff --git a/crates/engine-tree-ext/src/trie_updates.rs b/crates/engine-tree-ext/src/trie_updates.rs deleted file mode 100644 index 3ac7b523..00000000 --- a/crates/engine-tree-ext/src/trie_updates.rs +++ /dev/null @@ -1,357 +0,0 @@ -use alloy_primitives::{ - B256, - map::{B256Map, HashMap}, -}; -use reth_db::DatabaseError; -use reth_trie::{ - BranchNodeCompact, Nibbles, - trie_cursor::{TrieCursor, TrieCursorFactory}, - updates::{StorageTrieUpdates, TrieUpdates}, -}; -use std::collections::BTreeSet; -use tracing::warn; - -#[derive(Debug)] -struct EntryDiff { - task: T, - regular: T, - database: T, -} - -#[derive(Debug, Default)] -struct TrieUpdatesDiff { - account_nodes: HashMap>>, - removed_nodes: HashMap>, - storage_tries: B256Map, -} - -impl TrieUpdatesDiff { - fn has_differences(&self) -> bool { - !self.account_nodes.is_empty() - || !self.removed_nodes.is_empty() - || !self.storage_tries.is_empty() - } - - pub(super) fn log_differences(mut self) { - if self.has_differences() { - for ( - path, - EntryDiff { - task, - regular, - database, - }, - ) in &mut self.account_nodes - { - warn!(target: "engine::tree", ?path, ?task, ?regular, ?database, "Difference in account trie updates"); - } - - for ( - path, - EntryDiff { - task: task_removed, - regular: regular_removed, - database: database_not_exists, - }, - ) in &self.removed_nodes - { - warn!(target: "engine::tree", ?path, ?task_removed, ?regular_removed, ?database_not_exists, "Difference in removed account trie nodes"); - } - - for (address, storage_diff) in self.storage_tries { - storage_diff.log_differences(address); - } - } - } -} - -#[derive(Debug, Default)] -struct StorageTrieUpdatesDiff { - is_deleted: Option>, - storage_nodes: HashMap>>, - removed_nodes: HashMap>, -} - -impl StorageTrieUpdatesDiff { - fn has_differences(&self) -> bool { - self.is_deleted.is_some() - || !self.storage_nodes.is_empty() - || !self.removed_nodes.is_empty() - } - - fn log_differences(&self, address: B256) { - if let Some(EntryDiff { - task: task_deleted, - regular: regular_deleted, - database: database_not_exists, - }) = self.is_deleted - { - warn!(target: "engine::tree", ?address, ?task_deleted, ?regular_deleted, ?database_not_exists, "Difference in storage trie deletion"); - } - - for ( - path, - EntryDiff { - task, - regular, - database, - }, - ) in &self.storage_nodes - { - warn!(target: "engine::tree", ?address, ?path, ?task, ?regular, ?database, "Difference in storage trie updates"); - } - - for ( - path, - EntryDiff { - task: task_removed, - regular: regular_removed, - database: database_not_exists, - }, - ) in &self.removed_nodes - { - warn!(target: "engine::tree", ?address, ?path, ?task_removed, ?regular_removed, ?database_not_exists, "Difference in removed storage trie nodes"); - } - } -} - -/// Compares the trie updates from state root task, regular state root calculation and database, -/// and logs the differences if there's any. -/// -/// Returns `true` if there are differences. -pub(crate) fn compare_trie_updates( - trie_cursor_factory: impl TrieCursorFactory, - task: TrieUpdates, - regular: TrieUpdates, -) -> Result { - let mut task = adjust_trie_updates(task); - let mut regular = adjust_trie_updates(regular); - - let mut diff = TrieUpdatesDiff::default(); - - // compare account nodes - let mut account_trie_cursor = trie_cursor_factory.account_trie_cursor()?; - for key in task - .account_nodes - .keys() - .chain(regular.account_nodes.keys()) - .copied() - .collect::>() - { - let (task, regular) = ( - task.account_nodes.remove(&key), - regular.account_nodes.remove(&key), - ); - let database = account_trie_cursor.seek_exact(key)?.map(|x| x.1); - - if !branch_nodes_equal(task.as_ref(), regular.as_ref(), database.as_ref())? { - diff.account_nodes.insert( - key, - EntryDiff { - task, - regular, - database, - }, - ); - } - } - - // compare removed nodes - let mut account_trie_cursor = trie_cursor_factory.account_trie_cursor()?; - for key in task - .removed_nodes - .iter() - .chain(regular.removed_nodes.iter()) - .copied() - .collect::>() - { - let (task_removed, regular_removed) = ( - task.removed_nodes.contains(&key), - regular.removed_nodes.contains(&key), - ); - let database_not_exists = account_trie_cursor.seek_exact(key)?.is_none(); - // If the deletion is a no-op, meaning that the entry is not in the - // database, do not add it to the diff. - if task_removed != regular_removed && !database_not_exists { - diff.removed_nodes.insert( - key, - EntryDiff { - task: task_removed, - regular: regular_removed, - database: database_not_exists, - }, - ); - } - } - - // compare storage tries - for key in task - .storage_tries - .keys() - .chain(regular.storage_tries.keys()) - .copied() - .collect::>() - { - let (mut task, mut regular) = ( - task.storage_tries.remove(&key), - regular.storage_tries.remove(&key), - ); - if task != regular { - #[expect(clippy::or_fun_call)] - let storage_diff = compare_storage_trie_updates( - || trie_cursor_factory.storage_trie_cursor(key), - // Compare non-existent storage tries as empty. - task.as_mut().unwrap_or(&mut Default::default()), - regular.as_mut().unwrap_or(&mut Default::default()), - )?; - if storage_diff.has_differences() { - diff.storage_tries.insert(key, storage_diff); - } - } - } - - // log differences - let has_differences = diff.has_differences(); - diff.log_differences(); - - Ok(has_differences) -} - -fn compare_storage_trie_updates( - trie_cursor: impl Fn() -> Result, - task: &mut StorageTrieUpdates, - regular: &mut StorageTrieUpdates, -) -> Result { - // Check if the storage trie exists by seeking to the first entry - let database_not_exists = trie_cursor()?.seek(Nibbles::default())?.is_none(); - let mut diff = StorageTrieUpdatesDiff { - // If the deletion is a no-op, meaning that the entry is not in the - // database, do not add it to the diff. - is_deleted: (task.is_deleted != regular.is_deleted && !database_not_exists).then_some( - EntryDiff { - task: task.is_deleted, - regular: regular.is_deleted, - database: database_not_exists, - }, - ), - ..Default::default() - }; - - // compare storage nodes - let mut storage_trie_cursor = trie_cursor()?; - for key in task - .storage_nodes - .keys() - .chain(regular.storage_nodes.keys()) - .copied() - .collect::>() - { - let (task, regular) = ( - task.storage_nodes.remove(&key), - regular.storage_nodes.remove(&key), - ); - let database = storage_trie_cursor.seek_exact(key)?.map(|x| x.1); - if !branch_nodes_equal(task.as_ref(), regular.as_ref(), database.as_ref())? { - diff.storage_nodes.insert( - key, - EntryDiff { - task, - regular, - database, - }, - ); - } - } - - // compare removed nodes - let mut storage_trie_cursor = trie_cursor()?; - for key in task - .removed_nodes - .iter() - .chain(regular.removed_nodes.iter()) - .collect::>() - { - let (task_removed, regular_removed) = ( - task.removed_nodes.contains(key), - regular.removed_nodes.contains(key), - ); - if task_removed == regular_removed { - continue; - } - let database_not_exists = storage_trie_cursor.seek_exact(*key)?.map(|x| x.1).is_none(); - // If the deletion is a no-op, meaning that the entry is not in the - // database, do not add it to the diff. - if !database_not_exists { - diff.removed_nodes.insert( - *key, - EntryDiff { - task: task_removed, - regular: regular_removed, - database: database_not_exists, - }, - ); - } - } - - Ok(diff) -} - -/// Filters the removed nodes of both account trie updates and storage trie updates, so that they -/// don't include those nodes that were also updated. -fn adjust_trie_updates(trie_updates: TrieUpdates) -> TrieUpdates { - TrieUpdates { - removed_nodes: trie_updates - .removed_nodes - .into_iter() - .filter(|key| !trie_updates.account_nodes.contains_key(key)) - .collect(), - storage_tries: trie_updates - .storage_tries - .into_iter() - .map(|(address, updates)| { - ( - address, - StorageTrieUpdates { - removed_nodes: updates - .removed_nodes - .into_iter() - .filter(|key| !updates.storage_nodes.contains_key(key)) - .collect(), - ..updates - }, - ) - }) - .collect(), - ..trie_updates - } -} - -/// Compares the branch nodes from state root task and regular state root calculation. -/// -/// If one of the branch nodes is [`None`], it means it's not updated and the other is compared to -/// the branch node from the database. -/// -/// Returns `true` if they are equal. -fn branch_nodes_equal( - task: Option<&BranchNodeCompact>, - regular: Option<&BranchNodeCompact>, - database: Option<&BranchNodeCompact>, -) -> Result { - Ok(match (task, regular) { - (Some(task), Some(regular)) => { - task.state_mask == regular.state_mask - && task.tree_mask == regular.tree_mask - && task.hash_mask == regular.hash_mask - && task.hashes == regular.hashes - && task.root_hash == regular.root_hash - } - (None, None) => true, - _ => { - if task.is_some() { - task == database - } else { - regular == database - } - } - }) -} diff --git a/crates/engine-tree-ext/tests/jade_boundary.rs b/crates/engine-tree-ext/tests/jade_boundary.rs deleted file mode 100644 index 7f5d8469..00000000 --- a/crates/engine-tree-ext/tests/jade_boundary.rs +++ /dev/null @@ -1,212 +0,0 @@ -//! End-to-end verification of retroactive trust across the Jade hardfork -//! boundary. -//! -//! These tests exercise the `MorphBasicEngineValidator` through the real -//! `MorphNode` stack: they spin up an ephemeral node, build a valid block, -//! tamper with the header's `state_root`, and re-import via the Engine API. -//! -//! Retroactive-trust invariants: -//! * Pre-Jade: header state root is NOT compared against the MPT root the -//! validator computes — a mismatched `state_root` must still import. -//! * Post-Jade: the validator enforces strict MPT equality — the same -//! tampered `state_root` must be rejected as INVALID. -//! -//! Two existing sibling tests verify these invariants at the crate-of-use -//! level (`crates/node/tests/it/engine.rs::state_root_validation_skipped_pre_jade` -//! and `crates/node/tests/it/consensus.rs::post_jade_state_root_mismatch_is_rejected`). -//! The tests in this file pin the contract to the engine-tree-ext crate: if -//! someone tweaks `MorphBasicEngineValidator` in a way that breaks the -//! boundary, `cargo test -p morph-engine-tree-ext` is expected to catch it. - -#![cfg(feature = "test-utils")] - -use alloy_consensus::{BlockHeader, proofs}; -use alloy_primitives::{Address, B256}; -use alloy_rpc_types_engine::PayloadAttributes; -use morph_node::test_utils::{HardforkSchedule, MorphTestNode, TestNodeBuilder}; -use morph_payload_types::{MorphBuiltPayload, MorphPayloadAttributes, MorphPayloadTypes}; -use reth_node_api::PayloadTypes; -use reth_payload_builder::BuildNewPayload; -use reth_payload_primitives::BuiltPayload; -use reth_primitives_traits::SealedBlock; -use reth_provider::BlockReaderIdExt; - -/// Build an empty block through the payload builder without submitting it. -/// -/// `node.advance_block()` would time out waiting for a non-empty payload since -/// the pool is empty — instead, drive the builder directly with empty L1 -/// messages and resolve the payload synchronously via -/// `PayloadKind::WaitForPending`. This avoids fixed sleeps + polling, which -/// were flake-prone on loaded CI runners. -async fn build_candidate_block(node: &mut MorphTestNode) -> eyre::Result { - const BUILD_TIMEOUT: std::time::Duration = std::time::Duration::from_secs(30); - - let head = node - .inner - .provider - .sealed_header_by_number_or_tag(alloy_rpc_types_eth::BlockNumberOrTag::Latest)?; - let (head_hash, head_ts) = head - .map(|h| (h.hash(), h.timestamp())) - .unwrap_or((B256::ZERO, 0)); - - let rpc_attrs = MorphPayloadAttributes { - inner: PayloadAttributes { - timestamp: head_ts + 1, - prev_randao: B256::ZERO, - suggested_fee_recipient: Address::ZERO, - withdrawals: Some(vec![]), - parent_beacon_block_root: Some(B256::ZERO), - }, - transactions: Some(vec![]), - gas_limit: None, - base_fee_per_gas: None, - }; - - let payload_id = node - .inner - .payload_builder_handle - .send_new_payload(BuildNewPayload { - attributes: rpc_attrs, - parent_hash: head_hash, - cache: None, - trie_handle: None, - }) - .await? - .map_err(|e| eyre::eyre!("payload build failed: {e}"))?; - - tokio::time::timeout( - BUILD_TIMEOUT, - node.inner - .payload_builder_handle - .resolve_kind(payload_id, reth_node_api::PayloadKind::WaitForPending), - ) - .await - .map_err(|_| eyre::eyre!("payload build timed out after {:?}", BUILD_TIMEOUT))? - .ok_or_else(|| eyre::eyre!("no payload response for id {payload_id:?}"))? - .map_err(|e| eyre::eyre!("payload build error: {e}")) -} - -/// Tamper with a payload's header and ask the engine to import the result. -/// -/// Returns `true` if the engine accepted the block (VALID), `false` otherwise. -async fn try_import_with_tampered_state_root( - node: &mut MorphTestNode, - base: &MorphBuiltPayload, - bogus_state_root: B256, -) -> eyre::Result { - let sealed = base.block(); - let morph_header: morph_primitives::MorphHeader = sealed.header().inner.clone().into(); - let body = sealed.body().clone(); - let mut block = morph_primitives::Block::new(morph_header, body); - - block.header.inner.state_root = bogus_state_root; - block.header.inner.transactions_root = - proofs::calculate_transaction_root(&block.body.transactions); - - let modified_sealed = SealedBlock::seal_slow(block); - let execution_data = MorphPayloadTypes::block_to_payload(modified_sealed); - let status = node - .inner - .add_ons_handle - .beacon_engine_handle - .new_payload(execution_data) - .await?; - - Ok(status.is_valid()) -} - -#[tokio::test(flavor = "multi_thread")] -async fn pre_jade_block_with_tampered_state_root_imports() -> eyre::Result<()> { - reth_tracing::init_test_tracing(); - - let (mut nodes, _wallet) = TestNodeBuilder::new() - .with_schedule(HardforkSchedule::PreJade) - .build() - .await?; - let mut node = nodes.pop().unwrap(); - - let base_payload = build_candidate_block(&mut node).await?; - let accepted = - try_import_with_tampered_state_root(&mut node, &base_payload, B256::from([0xFF; 32])) - .await?; - - assert!( - accepted, - "pre-Jade block with tampered state_root must be accepted — retroactive trust skips \ - state-root validation before Jade" - ); - Ok(()) -} - -#[tokio::test(flavor = "multi_thread")] -async fn post_jade_block_with_tampered_state_root_is_rejected() -> eyre::Result<()> { - reth_tracing::init_test_tracing(); - - let (mut nodes, _wallet) = TestNodeBuilder::new() - .with_schedule(HardforkSchedule::AllActive) - .build() - .await?; - let mut node = nodes.pop().unwrap(); - - let base_payload = build_candidate_block(&mut node).await?; - let accepted = - try_import_with_tampered_state_root(&mut node, &base_payload, B256::from([0xFF; 32])) - .await?; - - assert!( - !accepted, - "post-Jade block with tampered state_root must be rejected — MorphBasicEngineValidator \ - enforces strict MPT root equality after Jade" - ); - Ok(()) -} - -/// Regression test: P2P-downloaded blocks enter the engine tree via the -/// Block-input path (`insert_block`), which never invokes -/// `convert_payload_to_block` and therefore registers no withdraw-trie-root -/// expectation. The validator must treat the missing entry as -/// `SkipValidation` so the downloaded block is accepted; otherwise sync -/// stalls indefinitely with `"missing withdraw trie root expectation -/// cache entry"`. -/// -/// The test runs two interconnected nodes: node[0] builds and imports a -/// block via the Engine API (Payload path → expectation registered on -/// node[0] only), then node[1] points its forkchoice at the new head so -/// reth's downloader fetches the block from node[0] over P2P. The download -/// lands in node[1]'s engine tree as a `Block` input. -#[tokio::test(flavor = "multi_thread")] -async fn p2p_downloaded_block_imports_without_registered_expectation() -> eyre::Result<()> { - reth_tracing::init_test_tracing(); - - let (mut nodes, _wallet) = TestNodeBuilder::new() - .with_schedule(HardforkSchedule::AllActive) - .with_num_nodes(2) - .build() - .await?; - let node1 = nodes.pop().expect("two nodes requested"); - let mut node0 = nodes.pop().expect("two nodes requested"); - - let payload = build_candidate_block(&mut node0).await?; - let head_hash = payload.block().hash(); - - let exec_data = MorphPayloadTypes::block_to_payload(payload.block().clone()); - let status = node0 - .inner - .add_ons_handle - .beacon_engine_handle - .new_payload(exec_data) - .await?; - assert!( - status.is_valid(), - "node[0] must accept its own freshly-built block, got {status:?}" - ); - node0.update_forkchoice(head_hash, head_hash).await?; - - // sync_to keeps issuing FCU until node[1]'s head matches `head_hash`. - // Since node[1] does not have the block, reth's engine tree will trigger - // its block downloader against the connected peer (node[0]) — downloaded - // blocks enter via the Block input path, exercising the bug. - node1.sync_to(head_hash).await?; - - Ok(()) -} diff --git a/crates/evm/Cargo.toml b/crates/evm/Cargo.toml index a9f9cf1a..282f0989 100644 --- a/crates/evm/Cargo.toml +++ b/crates/evm/Cargo.toml @@ -27,6 +27,7 @@ reth-rpc-eth-api = { workspace = true, optional = true } alloy-evm.workspace = true alloy-consensus.workspace = true alloy-primitives.workspace = true +alloy-rpc-types-eth.workspace = true derive_more.workspace = true thiserror.workspace = true diff --git a/crates/evm/src/block/mod.rs b/crates/evm/src/block/mod.rs index 6292f88a..42ce3269 100644 --- a/crates/evm/src/block/mod.rs +++ b/crates/evm/src/block/mod.rs @@ -19,8 +19,7 @@ use alloy_consensus::transaction::TxHashRef; use alloy_evm::{ Database, Evm, RecoveredTx, block::{ - BlockExecutionError, BlockExecutionResult, BlockExecutor, ExecutableTx, GasOutput, - OnStateHook, StateChangeSource, TxResult, + BlockExecutionError, BlockExecutionResult, BlockExecutor, ExecutableTx, GasOutput, TxResult, }, }; use alloy_primitives::{Address, Log, U256}; @@ -96,9 +95,6 @@ pub struct MorphBlockExecutor { /// Cached hardfork for this block (constant across all transactions). /// Set in `apply_pre_execution_changes`, reused in `commit_transaction`. hardfork: MorphHardfork, - /// Hook for notifying the engine about per-transaction state changes. - /// Used by `StateRootTask` for incremental sparse trie computation. - state_hook: Option>, } impl MorphBlockExecutor @@ -124,7 +120,6 @@ where receipts: Vec::new(), gas_used: 0, hardfork: MorphHardfork::default(), - state_hook: None, } } @@ -275,11 +270,6 @@ where post_fee_logs, } = output; - // Notify the state hook (e.g. StateRootTask) BEFORE committing. - if let Some(hook) = &mut self.state_hook { - hook.on_state(StateChangeSource::Transaction(self.receipts.len()), &state); - } - // EIP-8037 separates regular and state gas; pre-Amsterdam morph treats // them as a single number, so use the unified `tx_gas_used` getter. let gas_used = result.gas().tx_gas_used(); @@ -334,10 +324,6 @@ where )) } - fn set_state_hook(&mut self, hook: Option>) { - self.state_hook = hook; - } - fn evm_mut(&mut self) -> &mut Self::Evm { &mut self.evm } diff --git a/crates/evm/src/context.rs b/crates/evm/src/context.rs index 083a0302..de01f683 100644 --- a/crates/evm/src/context.rs +++ b/crates/evm/src/context.rs @@ -17,9 +17,10 @@ impl reth_rpc_eth_api::helpers::pending_block::BuildPendingEnv, + block_overrides: Option<&alloy_rpc_types_eth::BlockOverrides>, ) -> Self { Self { - inner: NextBlockEnvAttributes::build_pending_env(parent), + inner: NextBlockEnvAttributes::build_pending_env(parent, block_overrides), base_fee_per_gas: None, } } diff --git a/crates/evm/src/evm.rs b/crates/evm/src/evm.rs index 96005018..e2454e4d 100644 --- a/crates/evm/src/evm.rs +++ b/crates/evm/src/evm.rs @@ -4,7 +4,7 @@ use alloy_evm::{ revm::{ Context, ExecuteEvm, InspectEvm, Inspector, SystemCallEvm, context::{ - CfgEnv, + CfgEnv, DBErrorMarker, result::{EVMError, ResultAndState}, }, inspector::NoOpInspector, @@ -26,8 +26,7 @@ impl EvmFactory for MorphEvmFactory { type Evm>> = MorphEvm; type Context = MorphContext; type Tx = MorphTxEnv; - type Error = - EVMError; + type Error = EVMError; type HaltReason = MorphHaltReason; type Spec = MorphHardfork; type BlockEnv = MorphBlockEnv; diff --git a/crates/node/Cargo.toml b/crates/node/Cargo.toml index 3c6eca37..53257bb2 100644 --- a/crates/node/Cargo.toml +++ b/crates/node/Cargo.toml @@ -14,7 +14,6 @@ workspace = true morph-chainspec.workspace = true morph-consensus.workspace = true morph-engine-api.workspace = true -morph-engine-tree-ext.workspace = true morph-evm.workspace = true morph-payload-builder.workspace = true morph-payload-types.workspace = true @@ -26,11 +25,14 @@ morph-txpool.workspace = true # Reth dependencies reth-db.workspace = true reth-db-api.workspace = true +reth-chain-state.workspace = true reth-node-core.workspace = true reth-chainspec.workspace = true reth-engine-local.workspace = true reth-engine-tree.workspace = true reth-errors.workspace = true +reth-evm.workspace = true +reth-execution-cache.workspace = true reth-exex.workspace = true reth-node-api.workspace = true reth-node-builder.workspace = true @@ -77,7 +79,7 @@ alloy-signer = { workspace = true, optional = true } alloy-signer-local = { workspace = true, optional = true } [build-dependencies] -vergen.workspace = true +vergen = { workspace = true, features = ["build", "cargo", "emit_and_set"] } vergen-git2.workspace = true [dev-dependencies] diff --git a/crates/node/build.rs b/crates/node/build.rs index d068046d..6a83410c 100644 --- a/crates/node/build.rs +++ b/crates/node/build.rs @@ -1,26 +1,23 @@ #![allow(missing_docs)] use std::{env, error::Error}; -use vergen::{BuildBuilder, CargoBuilder, Emitter}; -use vergen_git2::Git2Builder; +use vergen::{Build, Cargo, Emitter}; +use vergen_git2::Git2; fn main() -> Result<(), Box> { let mut emitter = Emitter::default(); - let build_builder = BuildBuilder::default().build_timestamp(true).build()?; + let build_builder = Build::builder().build_timestamp(true).build(); emitter.add_instructions(&build_builder)?; - let cargo_builder = CargoBuilder::default() - .features(true) - .target_triple(true) - .build()?; + let cargo_builder = Cargo::builder().features(true).target_triple(true).build(); emitter.add_instructions(&cargo_builder)?; - let git_builder = Git2Builder::default() + let git_builder = Git2::builder() .describe(false, true, None) .dirty(true) .sha(false) - .build()?; + .build(); emitter.add_instructions(&git_builder)?; emitter.emit_and_set()?; diff --git a/crates/node/src/node.rs b/crates/node/src/node.rs index 5256862c..347cef93 100644 --- a/crates/node/src/node.rs +++ b/crates/node/src/node.rs @@ -213,6 +213,7 @@ impl PayloadAttributesBuilder .then(B256::random), // Morph L2 has no PoS slot semantics; field added in alloy 2.0. slot_number: None, + target_gas_limit: None, }, // No L1 transactions in local mining mode transactions: None, diff --git a/crates/node/src/test_utils.rs b/crates/node/src/test_utils.rs index d8f50d00..d58cfe5e 100644 --- a/crates/node/src/test_utils.rs +++ b/crates/node/src/test_utils.rs @@ -354,6 +354,7 @@ pub async fn advance_empty_block(node: &mut MorphTestNode) -> eyre::Result eyre::Result morph_payload_types::MorphPay withdrawals: Some(vec![]), parent_beacon_block_root: Some(B256::ZERO), slot_number: None, + target_gas_limit: None, }, transactions: None, gas_limit: None, diff --git a/crates/node/src/validator.rs b/crates/node/src/validator.rs index b43e9c3d..977a4e6d 100644 --- a/crates/node/src/validator.rs +++ b/crates/node/src/validator.rs @@ -6,12 +6,26 @@ use alloy_primitives::{B256, keccak256}; use dashmap::DashMap; use morph_chainspec::{ L2_MESSAGE_QUEUE_ADDRESS, L2_MESSAGE_QUEUE_WITHDRAW_TRIE_ROOT_SLOT, MorphChainSpec, + MorphHardforks, }; use morph_payload_types::{MorphExecutionData, MorphPayloadTypes}; -use morph_primitives::MorphHeader; +use morph_primitives::{MorphHeader, MorphPrimitives}; use parking_lot::Mutex; +use reth_chain_state::{ExecutedBlock, StateTrieOverlayManager}; use reth_chainspec::EthChainSpec; -use reth_errors::ConsensusError; +use reth_engine_tree::tree::payload_validator::TreeCtx; +use reth_engine_tree::tree::{ + BasicEngineValidator, CacheWaitDurations, EngineApiTreeState, EngineValidator, WaitForCaches, + error::{InsertBlockError, InsertBlockErrorKind}, + state_root_strategy::{ + DefaultStateRootStrategy, LazyHashedPostState, PayloadStateRootHandle, + PayloadStateRootJobContext, PreparedStateRootJob, StateRootJob, StateRootJobContext, + StateRootJobOutcome, StateRootStrategy, + }, +}; +use reth_errors::{ConsensusError, ProviderResult}; +use reth_evm::{ConfigureEvm, revm::context::Block as _}; +use reth_execution_cache::SavedCache; use reth_node_api::{ AddOnsContext, FullNodeComponents, InvalidPayloadAttributesError, NewPayloadError, NodeTypes, PayloadAttributes, PayloadTypes, PayloadValidator, @@ -20,8 +34,12 @@ use reth_node_builder::{ invalid_block_hook::InvalidBlockHookExt, rpc::{EngineValidatorBuilder, PayloadValidatorBuilder}, }; +use reth_payload_primitives::BuiltPayloadExecutedBlock; use reth_primitives_traits::{RecoveredBlock, SealedBlock}; -use reth_provider::ChainSpecProvider; +use reth_provider::{ + BlockExecutionOutput, BlockReader, ChainSpecProvider, StateProviderFactory, StateReader, + StateRootProvider, +}; use reth_tracing::tracing; use std::{collections::VecDeque, sync::Arc}; @@ -80,20 +98,16 @@ where >, >, Node::Provider: ChainSpecProvider, - PVB: PayloadValidatorBuilder, - PVB::Validator: reth_node_api::PayloadValidator< - ::Payload, - Block = reth_node_api::BlockTy, - > + Clone, + PVB: PayloadValidatorBuilder, { - type EngineValidator = - morph_engine_tree_ext::MorphBasicEngineValidator; + type EngineValidator = MorphTreeEngineValidator; async fn build_tree_validator( self, ctx: &AddOnsContext<'_, Node>, tree_config: reth_node_api::TreeConfig, changeset_cache: reth_trie_db::ChangesetCache, + state_trie_overlays: StateTrieOverlayManager, ) -> eyre::Result { let validator = self.payload_validator_builder.build(ctx).await?; let data_dir = ctx @@ -103,17 +117,336 @@ where .resolve_datadir(ctx.config.chain.chain()); let invalid_block_hook = ctx.create_invalid_block_hook(&data_dir).await?; let chain_spec = ctx.node.provider().chain_spec(); - - Ok(morph_engine_tree_ext::MorphBasicEngineValidator::new( - ctx.node.provider().clone(), + let post_execution_validator = validator.clone(); + + let provider = ctx.node.provider().clone(); + let state_root_strategy: Arc< + dyn StateRootStrategy, + > = Arc::new(MorphStateRootStrategy::new(chain_spec.clone())); + let tree_config = strict_morph_tree_config(tree_config); + let validator = BasicEngineValidator::new( + provider.clone(), Arc::new(ctx.node.consensus().clone()), ctx.node.evm_config().clone(), validator, tree_config, invalid_block_hook, changeset_cache, + state_trie_overlays, ctx.node.task_executor().clone(), + ) + .with_state_root_strategy(state_root_strategy); + + Ok(MorphTreeEngineValidator::new( + validator, + provider, + chain_spec, + post_execution_validator, + )) + } +} + +/// Morph only relaxes the header-root equality check before Jade, through +/// [`MorphStateRootStrategy`]. The upstream debug skip also suppresses trie +/// updates, so allowing it here would make both pre-Jade persistence and +/// post-Jade validation unsafe. +fn strict_morph_tree_config(tree_config: reth_node_api::TreeConfig) -> reth_node_api::TreeConfig { + if tree_config.skip_state_root() { + tracing::warn!( + target: "reth::cli", + "ignoring --debug.skip-state-root: Morph requires trie updates before Jade and strict state-root validation from Jade onward" + ); + } + tree_config.with_skip_state_root(false) +} + +/// Upstream engine validator plus Morph-specific validation hooks. +/// +/// State execution, caching and trie maintenance remain entirely upstream. This +/// wrapper preserves the parent-aware L1 queue invariant and the optional +/// consensus-layer withdraw-trie-root cross-check. +pub struct MorphTreeEngineValidator +where + Evm: ConfigureEvm, +{ + inner: BasicEngineValidator, + provider: P, + chain_spec: Arc, + post_execution_validator: MorphEngineValidator, +} + +impl MorphTreeEngineValidator +where + Evm: ConfigureEvm, +{ + const fn new( + inner: BasicEngineValidator, + provider: P, + chain_spec: Arc, + post_execution_validator: MorphEngineValidator, + ) -> Self { + Self { + inner, + provider, + chain_spec, + post_execution_validator, + } + } +} + +impl MorphTreeEngineValidator +where + Evm: ConfigureEvm, + P: BlockReader
, +{ + fn validate_next_l1_msg_index( + &self, + block: &SealedBlock, + ctx: &TreeCtx<'_, MorphPrimitives>, + ) -> Result<(), InsertBlockErrorKind> { + if !self + .chain_spec + .is_jade_active_at_timestamp(block.header().timestamp()) + { + return Ok(()); + } + + let parent_hash = block.parent_hash(); + let parent = match ctx.state().tree_state().sealed_header_by_hash(&parent_hash) { + Some(parent) => Some(parent), + None => self.provider.sealed_header_by_hash(parent_hash)?, + }; + let Some(parent) = parent else { + // The upstream validator reports the missing parent with the + // canonical provider error before executing the block. + return Ok(()); + }; + + let mut expected = parent.next_l1_msg_index; + for tx in block.body().transactions() { + if !tx.is_l1_msg() { + break; + } + let queue_index = tx.queue_index().ok_or_else(|| { + ConsensusError::msg("L1 message transaction is missing queue index") + })?; + expected = queue_index.checked_add(1).ok_or_else(|| { + ConsensusError::msg(format!( + "invalid block.NextL1MsgIndex: expected {}, got {}", + u64::MAX, + block.header().next_l1_msg_index + )) + })?; + } + + let actual = block.header().next_l1_msg_index; + if actual != expected { + return Err(ConsensusError::msg(format!( + "invalid block.NextL1MsgIndex: expected {expected}, got {actual}" + )) + .into()); + } + Ok(()) + } + + fn validate_next_l1_msg_index_for_block( + &self, + block: &SealedBlock, + ctx: &TreeCtx<'_, MorphPrimitives>, + ) -> Result<(), reth_engine_tree::tree::error::InsertPayloadError> + { + self.validate_next_l1_msg_index(block, ctx) + .map_err(|kind| InsertBlockError::new(block.clone(), kind).into()) + } + + fn validate_withdraw_trie_root( + &self, + output: reth_engine_tree::tree::ValidationOutput, + ) -> reth_engine_tree::tree::ValidationOutcome { + let block = output.executed_block.recovered_block(); + if let Err(err) = self + .post_execution_validator + .validate_withdraw_trie_root_update(block.hash(), || { + output.executed_block.hashed_state() + }) + { + return Err(InsertBlockError::consensus_error( + err, + output.executed_block.sealed_block().clone(), + ) + .into()); + } + + Ok(output) + } +} + +impl EngineValidator + for MorphTreeEngineValidator +where + P: BlockReader
+ Send + Sync + 'static, + Evm: ConfigureEvm + 'static, + BasicEngineValidator: + EngineValidator, +{ + fn validate_payload_attributes_against_header( + &self, + attr: &::PayloadAttributes, + header: &MorphHeader, + ) -> Result<(), InvalidPayloadAttributesError> { + self.inner + .validate_payload_attributes_against_header(attr, header) + } + + fn convert_payload_to_block( + &self, + payload: MorphExecutionData, + ) -> Result, NewPayloadError> { + self.inner.convert_payload_to_block(payload) + } + + fn validate_payload( + &mut self, + payload: MorphExecutionData, + ctx: TreeCtx<'_, MorphPrimitives>, + ) -> reth_engine_tree::tree::ValidationOutcome { + let block_hash = payload.block.hash(); + self.validate_next_l1_msg_index_for_block(payload.block.as_ref(), &ctx)?; + match self.inner.validate_payload(payload, ctx) { + Ok(output) => self.validate_withdraw_trie_root(output), + Err(err) => { + self.post_execution_validator + .take_withdraw_trie_root_expectation(block_hash); + Err(err) + } + } + } + + fn validate_block( + &mut self, + block: SealedBlock, + ctx: TreeCtx<'_, MorphPrimitives>, + ) -> reth_engine_tree::tree::ValidationOutcome { + self.validate_next_l1_msg_index_for_block(&block, &ctx)?; + let output = self.inner.validate_block(block, ctx)?; + self.validate_withdraw_trie_root(output) + } + + fn on_inserted_executed_block( + &self, + block: BuiltPayloadExecutedBlock, + ) -> ProviderResult> { + self.inner.on_inserted_executed_block(block) + } + + fn cache_for(&self, block_hash: B256) -> Option { + self.inner.cache_for(block_hash) + } + + fn payload_state_root_handle_for( + &self, + parent_hash: B256, + parent_header: &MorphHeader, + timestamp: u64, + state: &mut EngineApiTreeState, + ) -> Option { + self.inner + .payload_state_root_handle_for(parent_hash, parent_header, timestamp, state) + } +} + +impl WaitForCaches for MorphTreeEngineValidator +where + Evm: ConfigureEvm, + BasicEngineValidator: WaitForCaches, +{ + fn wait_for_caches(&self) -> CacheWaitDurations { + self.inner.wait_for_caches() + } +} + +/// Uses upstream's optimized state-root path from Jade onward. Before Jade it +/// computes and persists the real MPT updates but trusts the historical header +/// root, which is a ZK-trie commitment and cannot be compared to the MPT root. +#[derive(Debug)] +struct MorphStateRootStrategy { + chain_spec: Arc, + default: DefaultStateRootStrategy, +} + +impl MorphStateRootStrategy { + fn new(chain_spec: Arc) -> Self { + Self { chain_spec, + default: DefaultStateRootStrategy::default(), + } + } +} + +impl StateRootStrategy for MorphStateRootStrategy +where + P: BlockReader
+ + StateProviderFactory + + StateReader + + Clone + + Send + + Sync + + 'static, + Evm: ConfigureEvm + 'static, + DefaultStateRootStrategy: StateRootStrategy, +{ + fn prepare( + &self, + ctx: StateRootJobContext<'_, MorphPrimitives, P, Evm>, + ) -> ProviderResult> { + let timestamp: u64 = ctx.env().evm_env.block_env.timestamp().saturating_to(); + if self.chain_spec.is_jade_active_at_timestamp(timestamp) { + return self.default.prepare(ctx); + } + + Ok(PreparedStateRootJob::new( + Box::new(PreJadeStateRootJob { + provider_builder: ctx.provider_builder(), + }), + None, + )) + } + + fn prepare_payload_builder( + &self, + ctx: PayloadStateRootJobContext<'_, MorphPrimitives, P>, + ) -> ProviderResult> { + if self.chain_spec.is_jade_active_at_timestamp(ctx.timestamp()) { + return self.default.prepare_payload_builder(ctx); + } + Ok(None) + } +} + +struct PreJadeStateRootJob

{ + provider_builder: reth_engine_tree::tree::StateProviderBuilder, +} + +impl

StateRootJob for PreJadeStateRootJob

+where + P: BlockReader + StateProviderFactory + StateReader + Clone + Send + Sync + 'static, +{ + fn name(&self) -> &'static str { + "morph-pre-jade-trusted-header" + } + + fn finish( + &mut self, + block: &RecoveredBlock, + _output: Arc>, + hashed_state: &LazyHashedPostState, + ) -> ProviderResult { + let provider = self.provider_builder.build()?; + let (_mpt_root, trie_updates) = + provider.state_root_with_updates(hashed_state.get().as_ref().clone())?; + Ok(StateRootJobOutcome::new( + block.header().state_root(), + Arc::new(trie_updates), )) } } @@ -184,8 +517,8 @@ impl MorphEngineValidator { removed } - fn updated_withdraw_trie_root_from_hashed_state( - state_updates: &reth_trie::HashedPostState, + fn updated_withdraw_trie_root_from_sorted_hashed_state( + state_updates: &reth_trie::HashedPostStateSorted, ) -> Option { let hashed_address = keccak256(L2_MESSAGE_QUEUE_ADDRESS); let hashed_slot = keccak256(B256::from(L2_MESSAGE_QUEUE_WITHDRAW_TRIE_ROOT_SLOT)); @@ -193,8 +526,44 @@ impl MorphEngineValidator { state_updates .storages .get(&hashed_address) - .and_then(|storage| storage.storage.get(&hashed_slot).copied()) - .map(B256::from) + .and_then(|storage| { + storage + .storage_slots + .binary_search_by_key(&hashed_slot, |(slot, _)| *slot) + .ok() + .map(|index| B256::from(storage.storage_slots[index].1)) + }) + } + + fn validate_withdraw_trie_root_update( + &self, + block_hash: B256, + state_updates: impl FnOnce() -> Arc, + ) -> Result<(), ConsensusError> { + let Some(expectation) = self.take_withdraw_trie_root_expectation(block_hash) else { + tracing::debug!( + target: "morph::engine_validator", + %block_hash, + "no withdraw trie root expectation registered; skipping CL cross-check" + ); + return Ok(()); + }; + let WithdrawTrieRootExpectation::Verify(expected) = expectation else { + return Ok(()); + }; + + let Some(actual) = + Self::updated_withdraw_trie_root_from_sorted_hashed_state(state_updates().as_ref()) + else { + // The slot was not touched, so its value is unchanged from the parent. + return Ok(()); + }; + if actual != expected { + return Err(ConsensusError::msg(format!( + "withdraw trie root mismatch: expected {expected}, got {actual}" + ))); + } + Ok(()) } } @@ -219,51 +588,6 @@ impl PayloadValidator for MorphEngineValidator { Ok(sealed_block) } - fn validate_block_post_execution_with_hashed_state( - &self, - state_updates: &reth_trie::HashedPostState, - block: &RecoveredBlock, - ) -> Result<(), ConsensusError> { - let Some(expectation) = self.take_withdraw_trie_root_expectation(block.hash()) else { - // No CL-supplied expectation. Reachable on the Block-input path - // (P2P-downloaded blocks, pipeline backfill, and buffered blocks - // whose expectation was evicted from the bounded LRU before - // reattach) — `convert_payload_to_block` was never invoked to - // register one. Treat as SkipValidation so sync isn't stalled; - // the strict post-Jade state-root check upstream still covers - // withdraw-trie consistency through state-root equality. - tracing::debug!( - target: "morph::engine_validator", - block_hash = %block.hash(), - "no withdraw trie root expectation registered; skipping CL cross-check" - ); - return Ok(()); - }; - let WithdrawTrieRootExpectation::Verify(expected_withdraw_trie_root) = expectation else { - return Ok(()); - }; - - // Only validate if the withdraw trie root slot was actually updated in this block. - // If the slot is absent from hashed_state, the root is unchanged from the parent — - // the consensus layer guarantees the expected value is correct in that case. - // Doing a DB read for the parent state here would be expensive (history_by_block_hash - // + storage lookup) and would occur while holding the execution cache write lock, - // causing lock contention with the next block's cache lookup. - let Some(actual_withdraw_trie_root) = - Self::updated_withdraw_trie_root_from_hashed_state(state_updates) - else { - return Ok(()); - }; - - if actual_withdraw_trie_root != expected_withdraw_trie_root { - return Err(ConsensusError::msg(format!( - "withdraw trie root mismatch: expected {expected_withdraw_trie_root}, got {actual_withdraw_trie_root}" - ))); - } - - Ok(()) - } - fn validate_payload_attributes_against_header( &self, attr: &::PayloadAttributes, @@ -283,6 +607,13 @@ mod tests { use alloy_primitives::U256; use reth_trie::{HashedPostState, HashedStorage}; + #[test] + fn morph_tree_config_never_uses_upstream_empty_update_state_root_skip() { + let config = reth_node_api::TreeConfig::default().with_skip_state_root(true); + + assert!(!strict_morph_tree_config(config).skip_state_root()); + } + #[test] fn test_extract_updated_withdraw_trie_root_from_hashed_state() { let expected = B256::from([0x11; 32]); @@ -295,7 +626,9 @@ mod tests { ); assert_eq!( - MorphEngineValidator::updated_withdraw_trie_root_from_hashed_state(&state), + MorphEngineValidator::updated_withdraw_trie_root_from_sorted_hashed_state( + &state.into_sorted(), + ), Some(expected) ); } @@ -304,7 +637,9 @@ mod tests { fn test_extract_updated_withdraw_trie_root_from_hashed_state_missing_slot() { let state = HashedPostState::default(); assert_eq!( - MorphEngineValidator::updated_withdraw_trie_root_from_hashed_state(&state), + MorphEngineValidator::updated_withdraw_trie_root_from_sorted_hashed_state( + &state.into_sorted(), + ), None ); } @@ -436,7 +771,10 @@ mod tests { HashedStorage::from_iter(false, [(hashed_slot, U256::from_be_bytes([0x11; 32]))]), ); assert!( - MorphEngineValidator::updated_withdraw_trie_root_from_hashed_state(&state).is_none() + MorphEngineValidator::updated_withdraw_trie_root_from_sorted_hashed_state( + &state.into_sorted(), + ) + .is_none() ); } @@ -450,7 +788,10 @@ mod tests { HashedStorage::from_iter(false, [(wrong_slot, U256::from_be_bytes([0x22; 32]))]), ); assert!( - MorphEngineValidator::updated_withdraw_trie_root_from_hashed_state(&state).is_none() + MorphEngineValidator::updated_withdraw_trie_root_from_sorted_hashed_state( + &state.into_sorted(), + ) + .is_none() ); } @@ -475,8 +816,10 @@ mod tests { let validator = MorphEngineValidator::new(); let block = empty_recovered_block_with_hash(B256::from([0xab; 32])); - let result = validator - .validate_block_post_execution_with_hashed_state(&HashedPostState::default(), &block); + let state = HashedPostState::default(); + let result = validator.validate_withdraw_trie_root_update(block.hash(), || { + Arc::new(state.clone().into_sorted()) + }); assert!( result.is_ok(), @@ -496,8 +839,10 @@ mod tests { ); let block = empty_recovered_block_with_hash(hash); - let result = validator - .validate_block_post_execution_with_hashed_state(&HashedPostState::default(), &block); + let state = HashedPostState::default(); + let result = validator.validate_withdraw_trie_root_update(block.hash(), || { + Arc::new(state.clone().into_sorted()) + }); assert!(result.is_ok()); // expectation must be consumed. @@ -522,8 +867,10 @@ mod tests { // empty hashed state → no withdraw-slot diff → skip per the doc-comment // explanation in the validator. - let result = validator - .validate_block_post_execution_with_hashed_state(&HashedPostState::default(), &block); + let state = HashedPostState::default(); + let result = validator.validate_withdraw_trie_root_update(block.hash(), || { + Arc::new(state.clone().into_sorted()) + }); assert!(result.is_ok()); } @@ -552,7 +899,9 @@ mod tests { HashedStorage::from_iter(false, [(hashed_slot, U256::from_be_bytes(actual.0))]), ); - let result = validator.validate_block_post_execution_with_hashed_state(&state, &block); + let result = validator.validate_withdraw_trie_root_update(block.hash(), || { + Arc::new(state.clone().into_sorted()) + }); assert!( result.is_ok(), @@ -582,7 +931,9 @@ mod tests { let block = empty_recovered_block_with_hash(hash); let err = validator - .validate_block_post_execution_with_hashed_state(&state, &block) + .validate_withdraw_trie_root_update(block.hash(), || { + Arc::new(state.clone().into_sorted()) + }) .expect_err("mismatched root must fail"); assert!( err.to_string().contains("withdraw trie root mismatch"), @@ -617,6 +968,7 @@ mod tests { withdrawals: None, parent_beacon_block_root: None, slot_number: None, + target_gas_limit: None, }, transactions: None, gas_limit: None, @@ -637,6 +989,7 @@ mod tests { withdrawals: None, parent_beacon_block_root: None, slot_number: None, + target_gas_limit: None, }, transactions: None, gas_limit: None, @@ -657,6 +1010,7 @@ mod tests { withdrawals: None, parent_beacon_block_root: None, slot_number: None, + target_gas_limit: None, }, transactions: None, gas_limit: None, diff --git a/crates/node/tests/it/engine.rs b/crates/node/tests/it/engine.rs index 7404758b..90be9bb0 100644 --- a/crates/node/tests/it/engine.rs +++ b/crates/node/tests/it/engine.rs @@ -10,9 +10,11 @@ use alloy_rpc_types_engine::PayloadAttributes; use jsonrpsee::core::client::ClientT; use morph_node::test_utils::{HardforkSchedule, TestNodeBuilder}; use morph_payload_types::{ - AssembleL2BlockParams, ExecutableL2Data, GenericResponse, MorphPayloadAttributes, SafeL2Data, + AssembleL2BlockParams, ExecutableL2Data, GenericResponse, MorphPayloadAttributes, + MorphPayloadTypes, SafeL2Data, }; use morph_primitives::MorphHeader; +use reth_node_api::PayloadTypes; use reth_payload_builder::BuildNewPayload; use reth_payload_primitives::BuiltPayload; use reth_provider::{BlockIdReader, BlockReaderIdExt}; @@ -55,6 +57,39 @@ async fn state_root_validation_skipped_pre_jade() -> eyre::Result<()> { Ok(()) } +/// P2P-downloaded blocks enter through the block-input path, where no +/// Engine-API withdraw-root expectation has been registered locally. +#[tokio::test(flavor = "multi_thread")] +async fn p2p_downloaded_block_imports_without_registered_expectation() -> eyre::Result<()> { + reth_tracing::init_test_tracing(); + + let (mut nodes, _wallet) = TestNodeBuilder::new() + .with_schedule(HardforkSchedule::AllActive) + .with_num_nodes(2) + .build() + .await?; + let node1 = nodes.pop().expect("two nodes requested"); + let mut node0 = nodes.pop().expect("two nodes requested"); + + let payload = build_block_no_submit(&mut node0, vec![]).await?; + let head_hash = payload.block().hash(); + let execution_data = MorphPayloadTypes::block_to_payload(payload.block().clone(), None); + let status = node0 + .inner + .add_ons_handle + .beacon_engine_handle + .new_payload(execution_data) + .await?; + assert!( + status.is_valid(), + "source node rejected its block: {status:?}" + ); + node0.update_forkchoice(head_hash, head_hash).await?; + + node1.sync_to(head_hash).await?; + Ok(()) +} + /// `engine_newL2Block` can import consecutive blocks assembled over the authenticated RPC. #[tokio::test(flavor = "multi_thread")] async fn new_l2_block_imports_consecutive_assembled_blocks_over_rpc() -> eyre::Result<()> { @@ -580,6 +615,7 @@ async fn payload_builder_hash_matches_block_hash_with_nonzero_prev_randao() -> e withdrawals: Some(vec![]), parent_beacon_block_root: Some(B256::ZERO), slot_number: None, + target_gas_limit: None, }, transactions: Some(vec![]), gas_limit: None, @@ -593,7 +629,7 @@ async fn payload_builder_hash_matches_block_hash_with_nonzero_prev_randao() -> e attributes: rpc_attrs, parent_hash: head_hash, cache: None, - trie_handle: None, + state_root_handle: None, }) .await? .map_err(|e| eyre::eyre!("payload build failed: {e}"))?; diff --git a/crates/node/tests/it/helpers.rs b/crates/node/tests/it/helpers.rs index 808e9ed2..b08a17b3 100644 --- a/crates/node/tests/it/helpers.rs +++ b/crates/node/tests/it/helpers.rs @@ -48,6 +48,7 @@ pub(crate) async fn advance_block_with_l1_messages( withdrawals: Some(vec![]), parent_beacon_block_root: Some(B256::ZERO), slot_number: None, + target_gas_limit: None, }, transactions: Some(l1_messages), gas_limit: None, @@ -61,7 +62,7 @@ pub(crate) async fn advance_block_with_l1_messages( attributes: rpc_attrs, parent_hash: head_hash, cache: None, - trie_handle: None, + state_root_handle: None, }) .await? .map_err(|e| eyre::eyre!("payload build failed: {e}"))?; @@ -122,6 +123,7 @@ pub(crate) async fn build_block_no_submit( withdrawals: Some(vec![]), parent_beacon_block_root: Some(B256::ZERO), slot_number: None, + target_gas_limit: None, }, transactions: Some(l1_messages), gas_limit: None, @@ -135,7 +137,7 @@ pub(crate) async fn build_block_no_submit( attributes: rpc_attrs, parent_hash: head_hash, cache: None, - trie_handle: None, + state_root_handle: None, }) .await? .map_err(|e| eyre::eyre!("payload build failed: {e}"))?; @@ -192,7 +194,7 @@ pub(crate) async fn craft_and_try_import_block( let modified_sealed = SealedBlock::seal_slow(block); // Convert to execution payload and try to import - let execution_data = MorphPayloadTypes::block_to_payload(modified_sealed); + let execution_data = MorphPayloadTypes::block_to_payload(modified_sealed, None); let status = node .inner .add_ons_handle @@ -230,6 +232,7 @@ pub(crate) async fn expect_payload_build_failure( withdrawals: Some(vec![]), parent_beacon_block_root: Some(B256::ZERO), slot_number: None, + target_gas_limit: None, }, transactions: Some(l1_messages), gas_limit: None, @@ -243,7 +246,7 @@ pub(crate) async fn expect_payload_build_failure( attributes: rpc_attrs, parent_hash: head_hash, cache: None, - trie_handle: None, + state_root_handle: None, }) .await? { diff --git a/crates/payload/builder/src/builder.rs b/crates/payload/builder/src/builder.rs index 2f35359d..e2dc38b4 100644 --- a/crates/payload/builder/src/builder.rs +++ b/crates/payload/builder/src/builder.rs @@ -21,7 +21,7 @@ use reth_chainspec::ChainSpecProvider; use reth_evm::{ ConfigureEvm, Database, Evm, NextBlockEnvAttributes, block::{BlockExecutionError, BlockValidationError}, - execute::{BlockBuilder, BlockBuilderOutcome, BlockExecutor}, + execute::{BlockBuilder, BlockBuilderOutcome}, }; use reth_execution_cache::{CachedStateMetrics, CachedStateMetricsSource, CachedStateProvider}; use reth_execution_types::BlockExecutionOutput; @@ -169,7 +169,7 @@ where let BuildArguments { mut cached_reads, execution_cache, - trie_handle, + state_root_handle, config, cancel, best_payload, @@ -179,6 +179,7 @@ where let parent_hash = config.parent_header.hash(); let payload_id = config.payload_id; let parent_header = config.parent_header.clone(); + let parent_block_info = config.parent_block_info; let builder_attrs = MorphPayloadBuilderAttributes::try_new( parent_hash, config.attributes, @@ -187,6 +188,7 @@ where .map_err(|e| PayloadBuilderError::Other(e.into()))?; let builder_config = PayloadConfig { parent_header, + parent_block_info, attributes: builder_attrs, payload_id, }; @@ -215,7 +217,9 @@ where state_provider = Box::new(CachedStateProvider::new( state_provider, execution_cache.cache().clone(), - CachedStateMetrics::zeroed(CachedStateMetricsSource::Builder), + Some(CachedStateMetrics::zeroed( + CachedStateMetricsSource::Builder, + )), )); } let state = StateProviderDatabase::new(state_provider.as_ref()); @@ -226,7 +230,7 @@ where state_provider.as_ref(), ctx, best, - trie_handle, + state_root_handle, ) .map(|out| out.with_cached_reads(cached_reads)) } @@ -268,7 +272,7 @@ where config, cached_reads: Default::default(), execution_cache: None, - trie_handle: None, + state_root_handle: None, cancel: Default::default(), best_payload: None, }; @@ -686,7 +690,7 @@ fn build_payload_inner<'a, DB, BestTxs>( state_provider: &(impl StateProvider + ?Sized), ctx: MorphPayloadBuilderCtx, best: impl FnOnce(BestTransactionsAttributes) -> BestTxs + Send + Sync + 'a, - trie_handle: Option, + mut state_root_handle: Option, ) -> Result, PayloadBuilderError> where DB: Database, @@ -734,10 +738,11 @@ where // state hook so per-tx state diffs stream to the background trie task // during execution. The final `state_root()` recv() at finish time will // return quickly since most work is done concurrently. - if let Some(ref handle) = trie_handle { + if let Some(handle) = state_root_handle.as_mut() { builder - .executor_mut() - .set_state_hook(Some(Box::new(handle.state_hook()))); + .evm_mut() + .db_mut() + .set_state_hook(Some(Box::new(handle.take_state_hook()))); } // 1. Apply pre-execution changes (system contracts, etc.) @@ -815,8 +820,9 @@ where hashed_state, trie_updates, mut block, - } = if let Some(mut handle) = trie_handle { - builder.executor_mut().set_state_hook(None); + block_access_list: _, + } = if let Some(mut handle) = state_root_handle { + builder.evm_mut().db_mut().set_state_hook(None); match handle.state_root() { Ok(outcome) => builder.finish( state_provider, @@ -890,8 +896,9 @@ where recovered_block: Arc::new(block), execution_output: Arc::new(execution_output), // Keep unsorted; conversion to sorted is deferred until required. - hashed_state: Err(Arc::new(hashed_state)).into(), - trie_updates: Err(Arc::new(trie_updates)).into(), + hashed_state: Arc::new(hashed_state), + trie_updates: Arc::new(trie_updates), + changed_paths: None, }; let payload = MorphBuiltPayload::new( diff --git a/crates/payload/types/src/attributes.rs b/crates/payload/types/src/attributes.rs index 3aaea3ae..86d72131 100644 --- a/crates/payload/types/src/attributes.rs +++ b/crates/payload/types/src/attributes.rs @@ -334,6 +334,7 @@ mod tests { parent_beacon_block_root: None, // Morph L2 has no PoS slot semantics; field added in alloy 2.0. slot_number: None, + target_gas_limit: None, }, transactions: None, gas_limit: None, diff --git a/crates/payload/types/src/lib.rs b/crates/payload/types/src/lib.rs index 9ded5aae..f940589b 100644 --- a/crates/payload/types/src/lib.rs +++ b/crates/payload/types/src/lib.rs @@ -83,6 +83,12 @@ impl MorphExecutionData { } } +impl From for MorphExecutionData { + fn from(value: MorphBuiltPayload) -> Self { + Self::new(value.block) + } +} + impl ExecutionPayload for MorphExecutionData { fn parent_hash(&self) -> B256 { self.block.parent_hash() @@ -140,6 +146,7 @@ impl PayloadTypes for MorphPayloadTypes { block: SealedBlock< <::Primitives as NodePrimitives>::Block, >, + _bal: Option, ) -> Self::ExecutionData { MorphExecutionData::new(Arc::new(block)) } @@ -237,7 +244,7 @@ mod tests { fn test_block_to_payload() { let block = create_test_block(); let hash = block.hash(); - let data = MorphPayloadTypes::block_to_payload(block); + let data = MorphPayloadTypes::block_to_payload(block, None); assert_eq!(data.block_hash(), hash); assert!(data.expected_withdraw_trie_root.is_none()); } diff --git a/crates/revm/src/evm.rs b/crates/revm/src/evm.rs index d8ba55ba..10d7fe5a 100644 --- a/crates/revm/src/evm.rs +++ b/crates/revm/src/evm.rs @@ -14,7 +14,7 @@ use revm::{ }, inspector::InspectorEvmTr, interpreter::{ - Host, Instruction, InstructionContext, InstructionResult, + Host, Instruction, InstructionContext, InstructionExecResult, InstructionResult, gas::{BLOCKHASH, WARM_STORAGE_READ_COST}, interpreter::EthInterpreter, interpreter_types::{RuntimeFlag, StackTr}, @@ -70,10 +70,9 @@ fn morph_blockhash_result(chain_id: u64, current_number: u64, requested_number: /// for numbers within the 256-block lookup window. fn blockhash_morph( context: InstructionContext<'_, MorphContext, EthInterpreter>, -) { +) -> InstructionExecResult { let Some(([], number)) = StackTr::popn_top::<0>(&mut context.interpreter.stack) else { - context.interpreter.halt_underflow(); - return; + return Err(InstructionResult::StackUnderflow); }; let requested_number_u64 = as_u64_saturated(*number); @@ -81,6 +80,7 @@ fn blockhash_morph( let chain_id_u64 = as_u64_saturated(context.host.chain_id()); *number = morph_blockhash_result(chain_id_u64, current_number_u64, requested_number_u64); + Ok(()) } /// Morph custom SLOAD opcode. @@ -93,10 +93,11 @@ fn blockhash_morph( /// instead of "dirty" (100 gas), causing a 2800 gas mismatch vs go-eth. /// /// The DB read hits the State cache (O(1)) and only triggers on cold SLOADs. -fn sload_morph(context: InstructionContext<'_, MorphContext, EthInterpreter>) { +fn sload_morph( + context: InstructionContext<'_, MorphContext, EthInterpreter>, +) -> InstructionExecResult { let Some(([], index)) = StackTr::popn_top::<0>(&mut context.interpreter.stack) else { - context.interpreter.halt_underflow(); - return; + return Err(InstructionResult::StackUnderflow); }; let target = context.interpreter.input.target_address; @@ -125,16 +126,16 @@ fn sload_morph(context: InstructionContext<'_, MorphContext, E .gas .record_regular_cost(additional_cold_cost) { - context.interpreter.halt_oog(); - return; + return Err(InstructionResult::OutOfGas); } } *index = storage.data; } - Err(LoadError::ColdLoadSkipped) => context.interpreter.halt_oog(), - Err(LoadError::DBError) => context.interpreter.halt_fatal(), + Err(LoadError::ColdLoadSkipped) => return Err(InstructionResult::OutOfGas), + Err(LoadError::DBError) => return Err(InstructionResult::FatalExternalError), } + Ok(()) } /// Morph custom SSTORE opcode. @@ -150,17 +151,15 @@ fn sload_morph(context: InstructionContext<'_, MorphContext, E /// same 2800-gas-per-write divergence vs go-eth that `sload_morph` fixes. /// /// Uses DB-direct lookup (no per-tx runtime map needed). -fn sstore_morph(context: InstructionContext<'_, MorphContext, EthInterpreter>) { +fn sstore_morph( + context: InstructionContext<'_, MorphContext, EthInterpreter>, +) -> InstructionExecResult { if context.interpreter.runtime_flag.is_static() { - context - .interpreter - .halt(InstructionResult::StateChangeDuringStaticCall); - return; + return Err(InstructionResult::StateChangeDuringStaticCall); } let Some([index, value]) = StackTr::popn::<2>(&mut context.interpreter.stack) else { - context.interpreter.halt_underflow(); - return; + return Err(InstructionResult::StackUnderflow); }; let target = context.interpreter.input.target_address; @@ -169,10 +168,7 @@ fn sstore_morph(context: InstructionContext<'_, MorphContext, if spec_id.is_enabled_in(ISTANBUL) && context.interpreter.gas.remaining() <= context.host.gas_params().call_stipend() { - context - .interpreter - .halt(InstructionResult::ReentrancySentryOOG); - return; + return Err(InstructionResult::ReentrancySentryOOG); } if !context @@ -180,8 +176,7 @@ fn sstore_morph(context: InstructionContext<'_, MorphContext, .gas .record_regular_cost(context.host.gas_params().sstore_static_gas()) { - context.interpreter.halt_oog(); - return; + return Err(InstructionResult::OutOfGas); } let mut state_load = if spec_id.is_enabled_in(BERLIN) { @@ -193,18 +188,15 @@ fn sstore_morph(context: InstructionContext<'_, MorphContext, { Ok(load) => load, Err(LoadError::ColdLoadSkipped) => { - context.interpreter.halt_oog(); - return; + return Err(InstructionResult::OutOfGas); } Err(LoadError::DBError) => { - context.interpreter.halt_fatal(); - return; + return Err(InstructionResult::FatalExternalError); } } } else { let Some(load) = context.host.sstore(target, index, value) else { - context.interpreter.halt_fatal(); - return; + return Err(InstructionResult::FatalExternalError); }; load }; @@ -232,8 +224,7 @@ fn sstore_morph(context: InstructionContext<'_, MorphContext, state_load.is_cold, ); if !context.interpreter.gas.record_regular_cost(dynamic_gas) { - context.interpreter.halt_oog(); - return; + return Err(InstructionResult::OutOfGas); } context.interpreter.gas.record_refund( @@ -242,6 +233,7 @@ fn sstore_morph(context: InstructionContext<'_, MorphContext, .gas_params() .sstore_refund(is_istanbul, &state_load.data), ); + Ok(()) } /// MorphEvm extends the Evm with Morph specific types and logic. @@ -291,22 +283,27 @@ impl MorphEvm { let mut instructions = EthInstructions::new_mainnet_with_spec(spec.into()); // Morph custom BLOCKHASH implementation (matches Morph geth). - instructions.insert_instruction(0x40, Instruction::new(blockhash_morph::, BLOCKHASH)); + instructions.insert_instruction( + 0x40, + Instruction::new(blockhash_morph::), + BLOCKHASH as u16, + ); // Morph custom SLOAD: fixes original_value corruption from token fee deduction. instructions.insert_instruction( 0x54, - Instruction::new(sload_morph::, WARM_STORAGE_READ_COST), + Instruction::new(sload_morph::), + WARM_STORAGE_READ_COST as u16, ); // Morph custom SSTORE: same original_value fix on the SSTORE cold path. // Static gas = 0 because sstore_morph manages all gas accounting itself // (static + dynamic + refund). - instructions.insert_instruction(0x55, Instruction::new(sstore_morph::, 0)); + instructions.insert_instruction(0x55, Instruction::new(sstore_morph::), 0); // SELFDESTRUCT is disabled in Morph - instructions.insert_instruction(0xff, Instruction::unknown()); + instructions.insert_instruction(0xff, Instruction::unknown(), 0); // BLOBHASH is disabled in Morph - instructions.insert_instruction(0x49, Instruction::unknown()); + instructions.insert_instruction(0x49, Instruction::unknown(), 0); // BLOBBASEFEE is disabled in Morph - instructions.insert_instruction(0x4a, Instruction::unknown()); + instructions.insert_instruction(0x4a, Instruction::unknown(), 0); Self::new_inner(Evm { ctx, inspector, diff --git a/crates/revm/src/precompiles.rs b/crates/revm/src/precompiles.rs index 02d0cf32..cce783fa 100644 --- a/crates/revm/src/precompiles.rs +++ b/crates/revm/src/precompiles.rs @@ -43,9 +43,8 @@ use revm::{ precompile::{ Precompile, PrecompileHalt, PrecompileId, PrecompileOutput, PrecompileResult, Precompiles, }, - primitives::{OnceLock, hardfork::SpecId}, + primitives::{AddressSet, OnceLock, hardfork::SpecId}, }; -use std::boxed::Box; use std::string::String; /// Standard precompile addresses @@ -395,7 +394,7 @@ where } #[inline] - fn warm_addresses(&self) -> Box> { + fn warm_addresses(&self) -> &AddressSet { self.inner.warm_addresses() } diff --git a/crates/rpc/src/eth/call.rs b/crates/rpc/src/eth/call.rs index f4a5e7f2..a1718bbf 100644 --- a/crates/rpc/src/eth/call.rs +++ b/crates/rpc/src/eth/call.rs @@ -57,6 +57,10 @@ where self.eth_api().max_simulate_blocks() } + fn compute_state_root_for_eth_simulate(&self) -> bool { + self.eth_api().compute_state_root_for_eth_simulate() + } + fn evm_memory_limit(&self) -> u64 { self.eth_api().evm_memory_limit() } diff --git a/crates/rpc/src/eth/mod.rs b/crates/rpc/src/eth/mod.rs index 96a08d78..cb7331a1 100644 --- a/crates/rpc/src/eth/mod.rs +++ b/crates/rpc/src/eth/mod.rs @@ -17,8 +17,8 @@ use reth_rpc_convert::{RpcConvert, RpcConverter, RpcTypes}; use reth_rpc_eth_api::{ EthApiTypes, RpcNodeCore, RpcNodeCoreExt, helpers::{ - EthApiSpec, EthBlocks, EthFees, EthState, EthTransactions, LoadBlock, LoadFee, - LoadPendingBlock, LoadState, LoadTransaction, SpawnBlocking, Trace, + EthApiSpec, EthBlocks, EthFees, EthState, EthSubscriptions, EthTransactions, LoadBlock, + LoadFee, LoadPendingBlock, LoadState, LoadTransaction, SpawnBlocking, Trace, bal::GetBlockAccessList, pending_block::PendingEnvBuilder, }, }; @@ -320,6 +320,20 @@ where self.inner.eth_api.send_raw_transaction_sync_timeout() } + async fn send_pool_transaction( + &self, + origin: reth_transaction_pool::TransactionOrigin, + tx: reth_primitives_traits::WithEncoded>, + ) -> Result { + reth_rpc_eth_api::helpers::EthTransactions::send_pool_transaction( + &self.inner.eth_api, + origin, + tx, + ) + .await + .map_err(Into::into) + } + async fn send_transaction( &self, origin: reth_transaction_pool::TransactionOrigin, @@ -337,6 +351,13 @@ where } } +impl EthSubscriptions for MorphEthApi +where + N: MorphNodeCore, + Rpc: RpcConvert, +{ +} + impl EthFees for MorphEthApi where N: MorphNodeCore, From 204c2ce1dff20bbee9f0ff81a54dd8072f9411ce Mon Sep 17 00:00:00 2001 From: panos Date: Tue, 14 Jul 2026 10:11:52 +0800 Subject: [PATCH 2/6] fix(node): clear withdraw expectation on L1 index pre-check failure validate_payload already registered an expectation in convert_payload_to_block; clearing it on the early L1-index reject path matches the inner-failure cleanup and avoids a stale cache entry until LRU eviction. --- crates/node/src/validator.rs | 11 ++++++++++- 1 file changed, 10 insertions(+), 1 deletion(-) diff --git a/crates/node/src/validator.rs b/crates/node/src/validator.rs index 977a4e6d..463824cc 100644 --- a/crates/node/src/validator.rs +++ b/crates/node/src/validator.rs @@ -311,7 +311,16 @@ where ctx: TreeCtx<'_, MorphPrimitives>, ) -> reth_engine_tree::tree::ValidationOutcome { let block_hash = payload.block.hash(); - self.validate_next_l1_msg_index_for_block(payload.block.as_ref(), &ctx)?; + // `convert_payload_to_block` may already have registered a withdraw-root + // expectation for this hash. Clear it on every early reject path so a + // later re-import of the same hash does not observe a stale entry. + if let Err(err) = + self.validate_next_l1_msg_index_for_block(payload.block.as_ref(), &ctx) + { + self.post_execution_validator + .take_withdraw_trie_root_expectation(block_hash); + return Err(err); + } match self.inner.validate_payload(payload, ctx) { Ok(output) => self.validate_withdraw_trie_root(output), Err(err) => { From b33856fff1c4f0cf0342a366fdabe86f0fca7ddd Mon Sep 17 00:00:00 2001 From: panos Date: Tue, 14 Jul 2026 10:18:32 +0800 Subject: [PATCH 3/6] chore: fix rustfmt and bump crossbeam-epoch for RUSTSEC-2026-0204 Format the L1-index cleanup path, upgrade crossbeam-epoch to 0.9.20, and drop advisory ignores that no longer match after the reth main bump. --- Cargo.lock | 4 ++-- crates/node/src/validator.rs | 4 +--- deny.toml | 14 -------------- 3 files changed, 3 insertions(+), 19 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 52398dd5..da7e94b3 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -2176,9 +2176,9 @@ dependencies = [ [[package]] name = "crossbeam-epoch" -version = "0.9.18" +version = "0.9.20" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5b82ac4a3c2ca9c3460964f020e1402edd5753411d7737aa39c3714ad1b5420e" +checksum = "2d6914041f254d6e9176c01941b21115dcfb7089e55135a35411081bd106ef3f" dependencies = [ "crossbeam-utils", ] diff --git a/crates/node/src/validator.rs b/crates/node/src/validator.rs index 463824cc..33057571 100644 --- a/crates/node/src/validator.rs +++ b/crates/node/src/validator.rs @@ -314,9 +314,7 @@ where // `convert_payload_to_block` may already have registered a withdraw-root // expectation for this hash. Clear it on every early reject path so a // later re-import of the same hash does not observe a stale entry. - if let Err(err) = - self.validate_next_l1_msg_index_for_block(payload.block.as_ref(), &ctx) - { + if let Err(err) = self.validate_next_l1_msg_index_for_block(payload.block.as_ref(), &ctx) { self.post_execution_validator .take_withdraw_trie_root_expectation(block_hash); return Err(err); diff --git a/deny.toml b/deny.toml index 26053b0c..0b19c29f 100644 --- a/deny.toml +++ b/deny.toml @@ -8,20 +8,6 @@ ignore = [ "RUSTSEC-2024-0436", # https://rustsec.org/advisories/RUSTSEC-2025-0141 bincode is unmaintained "RUSTSEC-2025-0141", - # https://rustsec.org/advisories/RUSTSEC-2026-0097 rand unsound with custom logger - # pinned transitively via reth; we don't install a custom logger so this is - # a false positive in our usage, but cargo-deny can't see usage - "RUSTSEC-2026-0097", - # https://rustsec.org/advisories/RUSTSEC-2026-0098 rustls-webpki URI name constraints - "RUSTSEC-2026-0098", - # https://rustsec.org/advisories/RUSTSEC-2026-0099 rustls-webpki wildcard name constraints - "RUSTSEC-2026-0099", - # https://rustsec.org/advisories/RUSTSEC-2026-0118 hickory-proto NSEC3 validation loop - # pinned transitively via reth-dns-discovery -> hickory-resolver ^0.25; no 0.25 fix is available - "RUSTSEC-2026-0118", - # https://rustsec.org/advisories/RUSTSEC-2026-0119 hickory-proto O(n^2) name compression - # fixed in hickory-proto 0.26.1, but hickory-resolver ^0.25 cannot select it - "RUSTSEC-2026-0119", # https://rustsec.org/advisories/RUSTSEC-2026-0173 proc-macro-error2 is unmaintained # build-time proc-macro pulled transitively via alloy-sol-macro; no fix available "RUSTSEC-2026-0173", From c38f45fbe4934fbe2eee042fd503486d6d53afa4 Mon Sep 17 00:00:00 2001 From: panos-xyz Date: Tue, 14 Jul 2026 18:03:21 +0800 Subject: [PATCH 4/6] refactor(reference-index): replace ExEx with canonical cursor runtime --- Cargo.lock | 15 +- Cargo.toml | 3 - bin/morph-reth/Cargo.toml | 3 - bin/morph-reth/src/main.rs | 30 +- crates/node/Cargo.toml | 5 - crates/node/src/add_ons.rs | 133 ++-- crates/node/src/args.rs | 12 + crates/node/src/exex/mod.rs | 5 - crates/node/src/exex/reference_index.rs | 569 ----------------- crates/node/src/lib.rs | 1 - crates/node/tests/it/reference_index.rs | 151 ++--- crates/reference-index/Cargo.toml | 7 +- crates/reference-index/src/backfill.rs | 353 ++--------- crates/reference-index/src/db.rs | 271 ++++---- crates/reference-index/src/lib.rs | 37 +- crates/reference-index/src/metrics.rs | 33 + crates/reference-index/src/reader.rs | 344 +++++++++-- crates/reference-index/src/reconcile.rs | 142 +---- crates/reference-index/src/runtime.rs | 787 ++++++++++++++++++++++++ crates/reference-index/src/source.rs | 86 +++ crates/reference-index/src/tables.rs | 2 + crates/reference-index/src/types.rs | 70 ++- crates/reference-index/src/writer.rs | 83 +-- crates/rpc/Cargo.toml | 1 + crates/rpc/src/morph/handler.rs | 317 +++++++++- crates/rpc/src/morph/rpc.rs | 11 +- 26 files changed, 1995 insertions(+), 1476 deletions(-) delete mode 100644 crates/node/src/exex/mod.rs delete mode 100644 crates/node/src/exex/reference_index.rs create mode 100644 crates/reference-index/src/metrics.rs create mode 100644 crates/reference-index/src/runtime.rs create mode 100644 crates/reference-index/src/source.rs diff --git a/Cargo.lock b/Cargo.lock index da7e94b3..16307f55 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -4966,14 +4966,12 @@ dependencies = [ "reth-chain-state", "reth-chainspec", "reth-db", - "reth-db-api", "reth-e2e-test-utils", "reth-engine-local", "reth-engine-tree", "reth-errors", "reth-evm", "reth-execution-cache", - "reth-exex", "reth-node-api", "reth-node-builder", "reth-node-core", @@ -4984,7 +4982,6 @@ dependencies = [ "reth-provider", "reth-rpc-builder", "reth-rpc-eth-api", - "reth-storage-api", "reth-tasks", "reth-tracing", "reth-transaction-pool", @@ -4992,7 +4989,6 @@ dependencies = [ "reth-trie-db", "serde", "serde_json", - "tempfile", "tokio", "tokio-stream", "tracing", @@ -5080,16 +5076,19 @@ dependencies = [ "alloy-primitives", "alloy-rlp", "eyre", - "morph-chainspec", + "metrics", "morph-primitives", - "reth-chainspec", + "parking_lot", + "reth-chain-state", "reth-codecs", "reth-db", "reth-db-api", "reth-errors", + "reth-metrics", "reth-primitives-traits", "reth-provider", "reth-storage-api", + "reth-tasks", "serde", "tempfile", "thiserror 2.0.18", @@ -5103,13 +5102,10 @@ version = "1.0.0" dependencies = [ "alloy-primitives", "clap", - "eyre", "morph-chainspec", "morph-consensus", "morph-evm", "morph-node", - "morph-reference-index", - "reth-chainspec", "reth-cli", "reth-cli-util", "reth-ethereum-cli", @@ -5178,6 +5174,7 @@ dependencies = [ "revm", "serde", "serde_json", + "tempfile", "thiserror 2.0.18", "tokio", "tracing", diff --git a/Cargo.toml b/Cargo.toml index f30e05fc..cd966912 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -117,9 +117,6 @@ reth-engine-primitives = { git = "https://github.com/paradigmxyz/reth", rev = "f reth-engine-tree = { git = "https://github.com/paradigmxyz/reth", rev = "f0a41795d72ae5bb0b6c3661ac42ffb7b78ef105" } reth-errors = { git = "https://github.com/paradigmxyz/reth", rev = "f0a41795d72ae5bb0b6c3661ac42ffb7b78ef105" } reth-eth-wire-types = { git = "https://github.com/paradigmxyz/reth", rev = "f0a41795d72ae5bb0b6c3661ac42ffb7b78ef105" } -reth-exex = { git = "https://github.com/paradigmxyz/reth", rev = "f0a41795d72ae5bb0b6c3661ac42ffb7b78ef105" } -reth-exex-test-utils = { git = "https://github.com/paradigmxyz/reth", rev = "f0a41795d72ae5bb0b6c3661ac42ffb7b78ef105" } -reth-exex-types = { git = "https://github.com/paradigmxyz/reth", rev = "f0a41795d72ae5bb0b6c3661ac42ffb7b78ef105" } reth-ethereum = { git = "https://github.com/paradigmxyz/reth", rev = "f0a41795d72ae5bb0b6c3661ac42ffb7b78ef105" } reth-ethereum-cli = { git = "https://github.com/paradigmxyz/reth", rev = "f0a41795d72ae5bb0b6c3661ac42ffb7b78ef105" } reth-ethereum-consensus = { git = "https://github.com/paradigmxyz/reth", rev = "f0a41795d72ae5bb0b6c3661ac42ffb7b78ef105" } diff --git a/bin/morph-reth/Cargo.toml b/bin/morph-reth/Cargo.toml index b953ca54..485617c2 100644 --- a/bin/morph-reth/Cargo.toml +++ b/bin/morph-reth/Cargo.toml @@ -20,10 +20,8 @@ morph-chainspec = { workspace = true, features = ["cli"] } morph-consensus.workspace = true morph-evm.workspace = true morph-node.workspace = true -morph-reference-index.workspace = true # Reth CLI -reth-chainspec.workspace = true reth-cli.workspace = true reth-cli-util.workspace = true reth-ethereum-cli.workspace = true @@ -33,7 +31,6 @@ reth-rpc-server-types.workspace = true # Other alloy-primitives.workspace = true clap.workspace = true -eyre.workspace = true tracing.workspace = true [features] diff --git a/bin/morph-reth/src/main.rs b/bin/morph-reth/src/main.rs index 45b7d24f..a39d8adf 100644 --- a/bin/morph-reth/src/main.rs +++ b/bin/morph-reth/src/main.rs @@ -20,12 +20,7 @@ use clap::Parser; use morph_chainspec::{MORPH_DEFAULT_PRIORITY_FEE, MorphChainSpec, MorphChainSpecParser}; use morph_consensus::MorphConsensus; use morph_evm::{MorphEvmConfig, evm::MorphEvmFactory}; -use morph_node::{ - MorphAddOns, MorphArgs, MorphNode, - exex::{ReferenceIndexControl, reference_index_exex}, -}; -use morph_reference_index::ReferenceIndexDb; -use reth_chainspec::EthChainSpec; +use morph_node::{MorphAddOns, MorphArgs, MorphNode}; use reth_cli_util::sigsegv_handler; use reth_ethereum_cli::{Cli, Commands}; use reth_node_builder::Node; @@ -80,33 +75,12 @@ fn main() { cli.run_with_components::(components, async move |builder, morph_args| { info!(target: "morph::cli", "Starting Morph-Reth node"); - // Open the reference index DB before launching the node so we - // can wire it into both the ExEx and the add-ons. - let chain_spec = builder.config().chain.clone(); - let datadir = builder.config().datadir(); - let reference_index_path = datadir.data_dir().join("morph").join("reference_index"); - let chain_id = chain_spec.chain().id(); - let genesis_hash = chain_spec.genesis_hash(); // from EthChainSpec trait - - info!( - target: "morph::reference_index", - path = %reference_index_path.display(), - chain_id, - "opening Morph reference index database" - ); - let db = ReferenceIndexDb::open(&reference_index_path, chain_id, genesis_hash)?; - let (control, startup_rx) = ReferenceIndexControl::new(db); - - let exex_control = control.clone(); let node = MorphNode::new(morph_args); let handle = builder .with_types::() .with_components(node.components_builder()) - .with_add_ons(MorphAddOns::new().with_reference_index(control)) - .install_exex("morph-reference-index", async move |ctx| { - Ok(reference_index_exex(ctx, exex_control, startup_rx)) - }) + .with_add_ons(MorphAddOns::new()) .launch_with_debug_capabilities() .await?; diff --git a/crates/node/Cargo.toml b/crates/node/Cargo.toml index 53257bb2..bb6427b6 100644 --- a/crates/node/Cargo.toml +++ b/crates/node/Cargo.toml @@ -24,7 +24,6 @@ morph-txpool.workspace = true # Reth dependencies reth-db.workspace = true -reth-db-api.workspace = true reth-chain-state.workspace = true reth-node-core.workspace = true reth-chainspec.workspace = true @@ -33,7 +32,6 @@ reth-engine-tree.workspace = true reth-errors.workspace = true reth-evm.workspace = true reth-execution-cache.workspace = true -reth-exex.workspace = true reth-node-api.workspace = true reth-node-builder.workspace = true reth-node-ethereum.workspace = true @@ -43,8 +41,6 @@ reth-primitives-traits.workspace = true reth-provider.workspace = true reth-rpc-builder.workspace = true reth-rpc-eth-api.workspace = true -reth-storage-api.workspace = true -reth-tasks.workspace = true reth-transaction-pool.workspace = true reth-tracing.workspace = true reth-trie.workspace = true @@ -99,7 +95,6 @@ morph-chainspec.workspace = true morph-payload-types.workspace = true morph-primitives.workspace = true serde_json.workspace = true -tempfile.workspace = true [[test]] name = "it" diff --git a/crates/node/src/add_ons.rs b/crates/node/src/add_ons.rs index 2a1583bf..65249d57 100644 --- a/crates/node/src/add_ons.rs +++ b/crates/node/src/add_ons.rs @@ -2,16 +2,19 @@ use crate::{ MorphNode, - exex::ReferenceIndexControl, validator::{MorphEngineValidatorBuilder, MorphTreeEngineValidatorBuilder}, }; +use alloy_hardforks::ForkCondition; +use morph_chainspec::{MorphHardfork, MorphHardforks}; use morph_evm::MorphEvmConfig; use morph_primitives::{Block, MorphHeader, MorphReceipt}; -use morph_reference_index::{DEFAULT_LAG_THRESHOLD, ReferenceIndexReader}; +use morph_reference_index::{ReferenceIndexConfig, ReferenceIndexRuntime}; use morph_rpc::{ MorphEthApiBuilder, MorphEthConfigApiServer, MorphEthConfigHandler, morph::{MorphRpc, MorphRpcHandler, MorphRpcServer}, }; +use reth_chain_state::CanonStateSubscriptions; +use reth_chainspec::EthChainSpec; use reth_node_api::{AddOnsContext, FullNodeComponents, FullNodeTypes, NodeAddOns, NodePrimitives}; use reth_node_builder::{ NodeAdapter, @@ -44,10 +47,6 @@ pub struct MorphAddOns< > { /// Inner RPC add-ons from reth. inner: RpcAddOns, - /// Optional reference-index control injected by `main.rs`. When present - /// the add-on spawns startup indexing on launch and registers the - /// `morph_` RPC namespace. - reference_index: Option, } impl MorphAddOns, MorphEthApiBuilder> @@ -69,16 +68,19 @@ where Identity::default(), Identity::default(), ), - reference_index: None, } } +} - /// Attach a reference index control so the add-on can spawn startup - /// indexing and register the `morph_` RPC namespace on launch. - pub fn with_reference_index(mut self, control: ReferenceIndexControl) -> Self { - self.reference_index = Some(control); - self +fn ensure_reference_index_pruning_compatible( + bodies_history_pruning_enabled: bool, +) -> eyre::Result<()> { + if bodies_history_pruning_enabled { + eyre::bail!( + "Morph reference index requires canonical block bodies from Jade onward; disable bodies-history pruning" + ) } + Ok(()) } impl Default for MorphAddOns, MorphEthApiBuilder> @@ -119,40 +121,56 @@ where let task_executor = ctx.node.task_executor().clone(); let block_tag_tracker = std::sync::Arc::new(morph_engine_api::BlockTagTracker::default()); + // The reference index backfills directly from canonical block bodies. Read the + // effective modes from the provider so both CLI arguments and reth.toml are covered. + let bodies_history_pruning_enabled = provider + .database_provider_ro()? + .prune_modes_ref() + .bodies_history + .is_some(); + ensure_reference_index_pruning_compatible(bodies_history_pruning_enabled)?; + + let canonical_notifications = provider.subscribe_to_canonical_state(); + let jade_timestamp = match chain_spec.morph_fork_activation(MorphHardfork::Jade) { + ForkCondition::Timestamp(timestamp) => timestamp, + ForkCondition::Never => u64::MAX, + condition => eyre::bail!( + "Morph reference index requires timestamp-based Jade activation, got {condition:?}" + ), + }; + let reference_index_path = ctx + .config + .datadir() + .data_dir() + .join("morph") + .join("reference_index"); + let reference_index_config = ReferenceIndexConfig::new( + &reference_index_path, + chain_spec.chain().id(), + chain_spec.genesis_hash(), + jade_timestamp, + ); + let (reference_index_runtime, reference_index_handle) = + ReferenceIndexRuntime::new(reference_index_config, provider.clone()); + let runtime_executor = task_executor.clone(); + task_executor.spawn_task(async move { + reference_index_runtime + .run(runtime_executor, canonical_notifications) + .await; + }); + + tracing::info!( + target: "morph::reference_index", + path = %reference_index_path.display(), + "Morph reference index background runtime started" + ); + // Create Morph eth_config handler (EIP-7910 + morph extension) let eth_config_handler = MorphEthConfigHandler::new(ctx.node.provider().clone(), ctx.node.evm_config().clone()); - // Spawn reference index startup indexing (Task A) if configured. - let reference_rpc_handler = if let Some(control) = self.reference_index { - let startup_control = control.clone(); - let startup_node = ctx.node.clone(); - // spawn_critical_task causes node shutdown on panic/error, matching the spec - // requirement that reference index startup failures are fatal. - task_executor.spawn_critical_task("morph reference index startup", async move { - let result = tokio::task::spawn_blocking(move || { - crate::exex::run_startup_indexing(&startup_node, &startup_control) - }) - .await - .unwrap_or_else(|e| Err(eyre::eyre!("reference index startup panicked: {e}"))); - - match result { - Ok(()) => {} - Err(err) => { - // Propagate to spawn_critical_task which will shut down the node. - panic!("reference index startup failed: {err:?}"); - } - } - }); - - let morph_rpc_ctx = MorphRpc::new( - ReferenceIndexReader::new(control.db, DEFAULT_LAG_THRESHOLD), - provider.clone(), - ); - Some(MorphRpcHandler::new(morph_rpc_ctx)) - } else { - None - }; + let morph_rpc_ctx = MorphRpc::new(reference_index_handle, provider.clone()); + let reference_rpc_handler = MorphRpcHandler::new(morph_rpc_ctx); // Use launch_add_ons_with to register custom Engine API and eth_config self.inner @@ -171,14 +189,13 @@ where .map_err(|e| eyre::eyre!("Failed to register eth_config handler: {}", e))?; tracing::info!(target: "morph::node", "Morph eth_config handler registered successfully"); - // Register morph_ RPC namespace (if reference index was configured). - if let Some(handler) = reference_rpc_handler { - tracing::debug!(target: "morph::node", "Registering morph_ RPC namespace"); - modules - .merge_configured(handler.into_rpc()) - .map_err(|e| eyre::eyre!("Failed to register morph_ RPC: {}", e))?; - tracing::info!(target: "morph::node", "morph_ RPC namespace registered"); - } + // The namespace remains registered while the index catches up; handlers return + // a structured unavailable/behind error until the durable cursor is live. + tracing::debug!(target: "morph::node", "Registering morph_ RPC namespace"); + modules + .merge_configured(reference_rpc_handler.into_rpc()) + .map_err(|e| eyre::eyre!("Failed to register morph_ RPC: {}", e))?; + tracing::info!(target: "morph::node", "morph_ RPC namespace registered"); // Create and register Morph L2 Engine API tracing::debug!(target: "morph::node", "Registering Morph L2 Engine API"); @@ -242,3 +259,19 @@ where self.inner.engine_validator_builder() } } + +#[cfg(test)] +mod tests { + use super::ensure_reference_index_pruning_compatible; + + #[test] + fn reference_index_accepts_retained_block_bodies() { + assert!(ensure_reference_index_pruning_compatible(false).is_ok()); + } + + #[test] + fn reference_index_rejects_bodies_history_pruning() { + let error = ensure_reference_index_pruning_compatible(true).unwrap_err(); + assert!(error.to_string().contains("bodies-history pruning")); + } +} diff --git a/crates/node/src/args.rs b/crates/node/src/args.rs index 333e77d3..7e5e9233 100644 --- a/crates/node/src/args.rs +++ b/crates/node/src/args.rs @@ -93,6 +93,18 @@ mod tests { assert_eq!(args.max_tx_per_block, Some(1000)); } + #[test] + fn reference_index_disable_flag_is_not_supported() { + assert!( + CommandParser::::try_parse_from(["test", "--morph.disable-reference-index"]) + .is_err() + ); + assert!( + CommandParser::::try_parse_from(["test", "--disable-reference-index"]) + .is_err() + ); + } + #[test] fn test_default_trait_impl() { let args = MorphArgs::default(); diff --git a/crates/node/src/exex/mod.rs b/crates/node/src/exex/mod.rs deleted file mode 100644 index b6e8c227..00000000 --- a/crates/node/src/exex/mod.rs +++ /dev/null @@ -1,5 +0,0 @@ -//! Morph-specific Execution Extensions (ExEx). - -pub mod reference_index; - -pub use reference_index::{ReferenceIndexControl, reference_index_exex, run_startup_indexing}; diff --git a/crates/node/src/exex/reference_index.rs b/crates/node/src/exex/reference_index.rs deleted file mode 100644 index dc25835b..00000000 --- a/crates/node/src/exex/reference_index.rs +++ /dev/null @@ -1,569 +0,0 @@ -//! Reference index ExEx: drains canonical chain notifications and keeps the -//! index incrementally up to date. -//! -//! ## Lifecycle -//! -//! **Task A** (`run_startup_indexing`): runs in a spawned task, executes -//! backfill → reconcile, then sets `is_ready = true` and signals the ExEx with -//! `FinishedHeight(indexed_to)`. -//! -//! **Task B** (`reference_index_exex`): registered via `install_exex` and -//! started by reth's framework at node launch. It drains all notifications -//! immediately to avoid backpressure; writes are gated behind `is_ready`. - -use alloy_consensus::BlockHeader; -use alloy_eips::BlockNumHash; -use futures::FutureExt; -use morph_chainspec::spec::MorphChainSpec; -use morph_primitives::MorphPrimitives; -use morph_reference_index::{ - DEFAULT_BACKFILL_BATCH_BLOCKS, DEFAULT_MAX_REORG_DEPTH, ReferenceIndexDb, - backfill::{maybe_reset_jade_sentinel, run_backfill}, - reconcile::run_startup_reconcile, - writer::{delete_block, update_indexed_to, write_block}, -}; -use reth_db_api::transaction::{DbTx, DbTxMut}; -use reth_exex::{ExExContext, ExExEvent, ExExNotification}; -use reth_node_api::{FullNodeComponents, NodeTypes}; -use reth_provider::{ - BlockHashReader, BlockNumReader, BlockReader, ChainSpecProvider, HeaderProvider, -}; -use reth_storage_api::TransactionVariant; -use tokio::sync::watch; -use tokio_stream::{Stream, StreamExt}; -use tracing::{debug, error, info}; - -const TARGET: &str = "morph::reference_index"; -const MAX_LIVE_NOTIFICATION_BATCH: usize = 1024; - -// ── shared control ──────────────────────────────────────────────────────────── - -/// Shared handle that connects Task A (startup indexing) with Task B (ExEx). -/// -/// Task A completes, sets `is_ready`, then sends the startup -/// `FinishedHeight` through the watch channel. Task B receives this and -/// forwards it to reth's ExEx event bus. -#[derive(Clone, Debug)] -pub struct ReferenceIndexControl { - pub db: ReferenceIndexDb, - startup_tx: watch::Sender>, -} - -impl ReferenceIndexControl { - /// Create a new control pair. - /// - /// Returns `(control, receiver)`. The receiver must be passed to - /// [`reference_index_exex`] so the ExEx knows when startup has finished. - pub fn new(db: ReferenceIndexDb) -> (Self, watch::Receiver>) { - let (startup_tx, startup_rx) = watch::channel(None); - (Self { db, startup_tx }, startup_rx) - } - - /// Called by Task A after backfill + reconcile complete. - pub fn mark_startup_finished(&self, block: BlockNumHash) -> eyre::Result<()> { - self.startup_tx.send(Some(block))?; - Ok(()) - } -} - -// ── Task B: ExEx ────────────────────────────────────────────────────────────── - -/// Main ExEx loop. -/// -/// Drains notifications from node launch to avoid backpressure. While -/// `is_ready = false` each notification is discarded. After `is_ready` -/// the first notification triggers a gap check before normal processing. -pub async fn reference_index_exex( - mut ctx: ExExContext, - control: ReferenceIndexControl, - mut startup_rx: watch::Receiver>, -) -> eyre::Result<()> -where - Node: FullNodeComponents< - Types: NodeTypes, - >, - Node::Provider: BlockReader - + BlockNumReader - + HeaderProvider

- + BlockHashReader, -{ - let mut first_ready = true; - // Track last sent FinishedHeight to keep progress monotonic. Without this, - // the `tokio::select!` arms can race: a live commit may send `FinishedHeight(H+k)` - // before the startup watch channel is observed, which would otherwise - // regress reth's pruning marker back to `H`. - let mut last_finished: Option = None; - - loop { - tokio::select! { - // Forward the startup FinishedHeight when Task A finishes. - changed = startup_rx.changed() => { - if changed.is_ok() && let Some(block) = *startup_rx.borrow_and_update() { - debug!( - target: TARGET, - block_number = block.number, - "startup complete; forwarding initial FinishedHeight" - ); - send_finished_height_monotonic(&ctx.events, block, &mut last_finished)?; - } - } - - maybe_notification = ctx.notifications.try_next() => { - let Some(notification) = maybe_notification? else { break; }; - - if !control.db.is_ready() { - // Drain without writing to avoid backpressure. - if let Some(chain) = notification.committed_chain() { - debug!( - target: TARGET, - tip = chain.tip().number(), - "drained notification while index initializing" - ); - } - continue; - } - - // On the first is_ready notification: fill any gap that opened - // between when Task A finished reconcile and when this notification - // arrived. The goal is that after this branch runs, the index - // covers exactly the canonical range that is not going to be - // touched by the upcoming handle_notification call. - // - // fill_gap_idempotent delete-then-writes so stale entries from - // possible mini-reorgs during the drain window are cleaned up. - if first_ready { - first_ready = false; - let indexed_to = control.db.indexed_to()?; - if let Some(chain) = notification.committed_chain() { - // ChainCommitted / ChainReorged: blocks to index start at - // chain.first(). Anything in (indexed_to, notif_start-1] - // must be backfilled from main DB so handle_notification - // can pick up from notif_start. - let notif_start = chain.first().number(); - if notif_start > indexed_to + 1 { - fill_gap_idempotent( - &control.db, - &ctx.components.provider().clone(), - indexed_to + 1, - notif_start - 1, - )?; - } - } else if let Some(old) = notification.reverted_chain() { - // ChainReverted: handle_notification will delete old.blocks - // and set indexed_to = parent (= revert_start - 1). - // If parent > indexed_to, the drain window committed and - // never wrote canonical blocks (indexed_to+1..=parent) that - // will SURVIVE this revert. We must fill them now so the - // post-revert indexed_to = parent is consistent. - let revert_start = old.first().number(); - let parent = revert_start.saturating_sub(1); - if parent > indexed_to { - fill_gap_idempotent( - &control.db, - &ctx.components.provider().clone(), - indexed_to + 1, - parent, - )?; - } - // parent <= indexed_to: revert overlaps already-indexed - // range; handle_notification's delete_block + rollback of - // indexed_to takes care of it, no backfill needed. - } - } - - let notifications = drain_ready_notifications(&mut ctx.notifications, notification)?; - match handle_notifications_batch(&ctx.events, &control.db, notifications, &mut last_finished) { - Ok(()) => {} - Err(e) => { - error!(target: TARGET, ?e, "error processing notification"); - return Err(e); - } - } - } - } - } - - Ok(()) -} - -/// Fill (or repair) index entries for blocks `[from, to]` using canonical main DB data. -/// -/// Uses delete-then-write per block to stay idempotent even if some entries -/// already exist (e.g. partial prior write or mini-reorg during drain window). -fn fill_gap_idempotent( - db: &ReferenceIndexDb, - provider: &Provider, - from: u64, - to: u64, -) -> eyre::Result<()> -where - Provider: BlockReader, -{ - if from > to { - return Ok(()); - } - info!( - target: TARGET, - from, to, - "idempotent gap fill between startup reconcile and first ExEx notification" - ); - let tx = db.tx_mut()?; - for number in from..=to { - // Delete any stale entries before writing canonical ones. - delete_block(&tx, number)?; - // `WithHash` is required: write_block stores tx hashes in the index keys. - let block = provider - .sealed_block_with_senders(number.into(), TransactionVariant::WithHash)? - .ok_or_else(|| eyre::eyre!("missing block {number} during gap fill"))?; - write_block( - &tx, - block.number(), - block.hash(), - block.timestamp(), - &block.body().transactions, - )?; - } - update_indexed_to(&tx, to)?; - tx.commit()?; - Ok(()) -} - -/// Drain already-ready notifications so fast catch-up can be indexed in one DB transaction. -fn drain_ready_notifications( - notifications: &mut S, - first: ExExNotification, -) -> eyre::Result>> -where - S: Stream>> + Unpin, -{ - let mut batch = Vec::with_capacity(32); - batch.push(first); - - while batch.len() < MAX_LIVE_NOTIFICATION_BATCH { - match notifications.try_next().now_or_never() { - Some(Ok(Some(notification))) => batch.push(notification), - Some(Ok(None)) | None => break, - Some(Err(err)) => return Err(err), - } - } - - Ok(batch) -} - -/// Process ready ExEx notifications in one atomic commit. -fn handle_notifications_batch( - events: &tokio::sync::mpsc::UnboundedSender, - db: &ReferenceIndexDb, - notifications: Vec>, - last_finished: &mut Option, -) -> eyre::Result<()> { - if notifications.is_empty() { - return Ok(()); - } - - let tx = db.tx_mut()?; - let mut finished_height = None; - let mut allow_regression = false; - for notification in notifications { - let effect = apply_notification_to_tx(&tx, notification)?; - allow_regression |= effect.allow_regression; - finished_height = effect.finished_height; - } - tx.commit()?; - - if let Some(block) = finished_height { - if allow_regression { - events.send(ExExEvent::FinishedHeight(block))?; - *last_finished = Some(block); - } else { - send_finished_height_monotonic(events, block, last_finished)?; - } - } - - Ok(()) -} - -#[derive(Clone, Copy, Debug, Default, PartialEq, Eq)] -struct NotificationEffect { - finished_height: Option, - allow_regression: bool, -} - -fn apply_notification_to_tx( - tx: &Tx, - notification: ExExNotification, -) -> eyre::Result { - match notification { - ExExNotification::ChainCommitted { new } => { - for block in new.blocks_iter() { - write_block( - tx, - block.number(), - block.hash(), - block.timestamp(), - &block.body().transactions, - )?; - } - update_indexed_to(tx, new.tip().number())?; - Ok(NotificationEffect { - finished_height: Some(new.tip().num_hash()), - allow_regression: false, - }) - } - ExExNotification::ChainReverted { old } => { - let parent = old.first().number().saturating_sub(1); - for block in old.blocks_iter() { - delete_block(tx, block.number())?; - } - update_indexed_to(tx, parent)?; - // FinishedHeight not sent on revert per spec. - Ok(NotificationEffect { - finished_height: None, - allow_regression: true, - }) - } - ExExNotification::ChainReorged { old, new } => { - for block in old.blocks_iter() { - delete_block(tx, block.number())?; - } - for block in new.blocks_iter() { - write_block( - tx, - block.number(), - block.hash(), - block.timestamp(), - &block.body().transactions, - )?; - } - update_indexed_to(tx, new.tip().number())?; - // On reorg to a shorter chain the tip may go down; that is the one - // case we ALLOW FinishedHeight to regress (reth's ExExEvent doc - // explicitly permits "on reorgs, height may go down"). - Ok(NotificationEffect { - finished_height: Some(new.tip().num_hash()), - allow_regression: true, - }) - } - } -} - -/// Send a FinishedHeight only if it strictly advances `last_finished`. -/// -/// Used for ChainCommitted and the startup watch channel, where progress -/// must only go forward. ChainReorged has its own allow-regress path. -fn send_finished_height_monotonic( - events: &tokio::sync::mpsc::UnboundedSender, - block: BlockNumHash, - last_finished: &mut Option, -) -> eyre::Result<()> { - if last_finished.is_none_or(|last| block.number > last.number) { - events.send(ExExEvent::FinishedHeight(block))?; - *last_finished = Some(block); - } - Ok(()) -} - -// ── Task A: startup indexing ────────────────────────────────────────────────── - -/// Execute backfill → reconcile, set `is_ready = true`, then send the startup -/// `FinishedHeight` through `control`. -/// -/// Call once from a spawned task after the node's provider is available. -pub fn run_startup_indexing(node: &Node, control: &ReferenceIndexControl) -> eyre::Result<()> -where - Node: FullNodeComponents< - Types: NodeTypes, - >, - Node::Provider: BlockReader - + BlockNumReader - + HeaderProvider
- + BlockHashReader - + ChainSpecProvider, -{ - let provider = node.provider().clone(); - let chain_spec = provider.chain_spec(); - let head = provider.best_block_number()?; - - // Re-resolve jade sentinel if Jade has since activated. - maybe_reset_jade_sentinel(&control.db, &provider, chain_spec.as_ref(), head)?; - - // Run backfill (no-op if already Complete). - run_backfill( - &control.db, - &provider, - chain_spec.as_ref(), - head, - DEFAULT_BACKFILL_BATCH_BLOCKS, - )?; - - // Re-read head in case new blocks arrived during backfill. - let current_head = provider.best_block_number()?; - - // Startup reconcile: canonical hash check + suffix gap. - run_startup_reconcile( - &control.db, - &provider, - current_head, - DEFAULT_MAX_REORG_DEPTH, - )?; - - // Atomically mark ready and signal the ExEx. - control.db.set_ready(true); - - let indexed_to = control.db.indexed_to()?; - let hash = provider - .block_hash(indexed_to)? - .ok_or_else(|| eyre::eyre!("missing canonical hash for block {indexed_to}"))?; - - control.mark_startup_finished(BlockNumHash { - number: indexed_to, - hash, - })?; - - info!( - target: TARGET, - indexed_to, - "reference index ready" - ); - - Ok(()) -} - -#[cfg(test)] -mod tests { - use super::*; - use alloy_consensus::Header; - use alloy_primitives::B256; - use morph_primitives::{Block, BlockBody, MorphHeader, MorphPrimitives}; - use reth_primitives_traits::SealedBlock; - use reth_provider::{Chain, ExecutionOutcome}; - use std::{collections::BTreeMap, sync::Arc}; - - #[test] - fn batched_commits_emit_finished_height_once_at_tip() -> eyre::Result<()> { - let dir = tempfile::tempdir()?; - let db = ReferenceIndexDb::open(dir.path(), 2910, B256::ZERO)?; - let (events, mut events_rx) = tokio::sync::mpsc::unbounded_channel(); - let mut last_finished = None; - - let block1 = recovered_block(1, B256::ZERO); - let block1_hash = block1.hash(); - let block2 = recovered_block(2, block1_hash); - let block2_num_hash = block2.num_hash(); - - handle_notifications_batch( - &events, - &db, - vec![ - committed_notification(vec![block1]), - committed_notification(vec![block2]), - ], - &mut last_finished, - )?; - - assert_eq!(db.indexed_to()?, 2); - assert!(db.indexed_block_hash(1)?.is_some()); - assert!(db.indexed_block_hash(2)?.is_some()); - assert_eq!(last_finished, Some(block2_num_hash)); - assert_eq!( - events_rx.try_recv()?, - ExExEvent::FinishedHeight(block2_num_hash) - ); - assert!(events_rx.try_recv().is_err()); - - Ok(()) - } - - #[test] - fn batch_with_reorg_allows_final_finished_height_to_regress() -> eyre::Result<()> { - let dir = tempfile::tempdir()?; - let db = ReferenceIndexDb::open(dir.path(), 2910, B256::ZERO)?; - let (events, mut events_rx) = tokio::sync::mpsc::unbounded_channel(); - let mut last_finished = Some(BlockNumHash { - number: 12, - hash: B256::repeat_byte(0x12), - }); - - let old10 = recovered_block(10, B256::ZERO); - let old11 = recovered_block(11, old10.hash()); - let old12 = recovered_block(12, old11.hash()); - let new10 = recovered_block(10, B256::repeat_byte(0x99)); - let new11 = recovered_block(11, new10.hash()); - let new11_num_hash = new11.num_hash(); - - handle_notifications_batch( - &events, - &db, - vec![ - reorg_notification(vec![old10, old11, old12], vec![new10]), - committed_notification(vec![new11]), - ], - &mut last_finished, - )?; - - assert_eq!(db.indexed_to()?, 11); - assert_eq!(last_finished, Some(new11_num_hash)); - assert_eq!( - events_rx.try_recv()?, - ExExEvent::FinishedHeight(new11_num_hash) - ); - assert!(events_rx.try_recv().is_err()); - - Ok(()) - } - - fn committed_notification( - blocks: Vec>, - ) -> ExExNotification { - ExExNotification::ChainCommitted { - new: Arc::new(Chain::new( - blocks, - ExecutionOutcome::default(), - BTreeMap::new(), - )), - } - } - - fn reorg_notification( - old: Vec>, - new: Vec>, - ) -> ExExNotification { - ExExNotification::ChainReorged { - old: Arc::new(Chain::new( - old, - ExecutionOutcome::default(), - BTreeMap::new(), - )), - new: Arc::new(Chain::new( - new, - ExecutionOutcome::default(), - BTreeMap::new(), - )), - } - } - - fn recovered_block( - number: u64, - parent_hash: B256, - ) -> reth_primitives_traits::RecoveredBlock { - let header = MorphHeader { - inner: Header { - number, - parent_hash, - timestamp: 1_000 + number, - gas_limit: 30_000_000, - parent_beacon_block_root: Some(B256::ZERO), - ..Default::default() - }, - next_l1_msg_index: 0, - }; - let body = BlockBody { - transactions: vec![], - ommers: vec![], - withdrawals: None, - }; - SealedBlock::seal_slow(Block { header, body }) - .try_recover() - .expect("empty test block should recover") - } -} diff --git a/crates/node/src/lib.rs b/crates/node/src/lib.rs index 70a01590..399c8a57 100644 --- a/crates/node/src/lib.rs +++ b/crates/node/src/lib.rs @@ -22,7 +22,6 @@ pub mod add_ons; pub mod args; pub mod components; -pub mod exex; pub mod node; #[cfg(feature = "test-utils")] pub mod test_utils; diff --git a/crates/node/tests/it/reference_index.rs b/crates/node/tests/it/reference_index.rs index 5a19f94f..7d9afaa4 100644 --- a/crates/node/tests/it/reference_index.rs +++ b/crates/node/tests/it/reference_index.rs @@ -1,53 +1,81 @@ -//! Integration tests for `morph_getTransactionHashesByReference`. +//! End-to-end tests for `morph_getTransactionHashesByReference`. //! -//! These tests spin up a real Morph test node, produce blocks with reference- -//! carrying `MorphTx` transactions, then run the reference index backfill + -//! reconcile against the node's provider, and verify the query results. -//! -//! Because `reth_e2e_test_utils::setup_engine` doesn't support ExEx injection, -//! we test the storage layer (backfill/reconcile/query) directly rather than -//! through the live ExEx. The ExEx logic is separately covered by the unit -//! tests in `morph-reference-index`. +//! The node add-on owns the reference-index runtime. These tests only produce +//! canonical blocks and call the public RPC, so they cover notification wakeups, +//! provider-backed backfill, cursor readiness, and RPC registration together. use alloy_consensus::transaction::TxHashRef; use alloy_primitives::{B256, U64}; -use morph_node::test_utils::{MorphTxBuilder, TEST_TOKEN_ID, TestNodeBuilder}; -use morph_reference_index::{ - DEFAULT_LAG_THRESHOLD, DEFAULT_MAX_REORG_DEPTH, ReferenceIndexDb, ReferenceIndexReader, - ReferenceQuery, backfill::run_backfill, reconcile::run_startup_reconcile, +use jsonrpsee::core::client::ClientT; +use morph_node::test_utils::{ + HardforkSchedule, MorphTestNode, MorphTxBuilder, TEST_TOKEN_ID, TestNodeBuilder, }; +use morph_reference_index::ReferenceTransactionResult; +use morph_rpc::ReferenceQueryArgs; use reth_payload_primitives::BuiltPayload; -use reth_provider::BlockNumReader; -use tempfile::TempDir; - -// ── helpers ─────────────────────────────────────────────────────────────────── - -async fn open_and_backfill_index

(provider: &P, dir: &TempDir) -> ReferenceIndexDb -where - P: reth_provider::BlockReader - + reth_provider::HeaderProvider

- + BlockNumReader - + reth_provider::BlockHashReader - + reth_provider::ChainSpecProvider, -{ - let chain_spec = provider.chain_spec(); - let chain_id = reth_chainspec::EthChainSpec::chain(chain_spec.as_ref()).id(); - let genesis_hash = reth_chainspec::EthChainSpec::genesis_hash(chain_spec.as_ref()); - - let db = ReferenceIndexDb::open(dir.path(), chain_id, genesis_hash).unwrap(); - - let head = provider.best_block_number().unwrap(); - run_backfill(&db, provider, chain_spec.as_ref(), head, 256).unwrap(); - run_startup_reconcile(&db, provider, head, DEFAULT_MAX_REORG_DEPTH).unwrap(); - - db.set_ready(true); - db +use std::time::Duration; + +async fn wait_for_reference_query( + node: &MorphTestNode, + args: ReferenceQueryArgs, +) -> eyre::Result> { + let client = node + .rpc_client() + .ok_or_else(|| eyre::eyre!("HTTP RPC client not available"))?; + let deadline = tokio::time::Instant::now() + Duration::from_secs(10); + let mut last_error = None; + + while tokio::time::Instant::now() < deadline { + match client + .request("morph_getTransactionHashesByReference", (args.clone(),)) + .await + { + Ok(results) => return Ok(results), + Err(error) => { + last_error = Some(error.to_string()); + tokio::time::sleep(Duration::from_millis(25)).await; + } + } + } + + Err(eyre::eyre!( + "reference index did not become queryable before timeout: {}", + last_error.as_deref().unwrap_or("no RPC response") + )) } -// ── tests ───────────────────────────────────────────────────────────────────── +fn query_args(reference: B256, offset: Option, limit: Option) -> ReferenceQueryArgs { + ReferenceQueryArgs { + reference, + offset, + limit, + } +} + +/// Before Jade, the namespace is available and returns an empty result without +/// requiring per-block index writes. +#[tokio::test(flavor = "multi_thread")] +async fn reference_index_is_empty_before_jade() -> eyre::Result<()> { + reth_tracing::init_test_tracing(); + + let (mut nodes, _wallet) = TestNodeBuilder::new() + .with_schedule(HardforkSchedule::PreJade) + .build() + .await?; + let node = nodes.pop().unwrap(); + assert!( + !node.inner.data_dir.exex_wal().exists(), + "reference index must not create an ExEx WAL" + ); + let results = + wait_for_reference_query(&node, query_args(B256::with_last_byte(0x98), None, None)).await?; + + assert!(results.is_empty()); + Ok(()) +} -/// Produce one block with a reference-carrying MorphTx and verify the index -/// returns it for the correct reference. +/// Produce one block with a reference-carrying MorphTx and verify the live +/// background index returns it through the public RPC. #[tokio::test(flavor = "multi_thread")] async fn reference_index_finds_single_morph_tx() -> eyre::Result<()> { reth_tracing::init_test_tracing(); @@ -70,13 +98,7 @@ async fn reference_index_finds_single_morph_tx() -> eyre::Result<()> { .unwrap() .tx_hash(); - let dir = TempDir::new()?; - let db = open_and_backfill_index(&node.inner.provider, &dir).await; - - let reader = ReferenceIndexReader::new(db, DEFAULT_LAG_THRESHOLD); - let canonical_tip = node.inner.provider.best_block_number()?; - let query = ReferenceQuery::new(reference, None, None).unwrap(); - let results = reader.query(query, canonical_tip)?; + let results = wait_for_reference_query(&node, query_args(reference, None, None)).await?; assert_eq!(results.len(), 1, "should find exactly one transaction"); assert_eq!(results[0].transaction_hash, tx_hash); @@ -85,7 +107,7 @@ async fn reference_index_finds_single_morph_tx() -> eyre::Result<()> { Ok(()) } -/// Produce multiple blocks with the same reference and verify pagination. +/// Produce multiple blocks with the same reference and verify RPC pagination. #[tokio::test(flavor = "multi_thread")] async fn reference_index_pagination() -> eyre::Result<()> { reth_tracing::init_test_tracing(); @@ -117,33 +139,20 @@ async fn reference_index_pagination() -> eyre::Result<()> { ); } - let dir = TempDir::new()?; - let db = open_and_backfill_index(&node.inner.provider, &dir).await; - - let reader = ReferenceIndexReader::new(db, DEFAULT_LAG_THRESHOLD); - let canonical_tip = node.inner.provider.best_block_number()?; - - // Page 1: offset=0, limit=2 → first two entries. - let page1 = reader.query( - ReferenceQuery::new(reference, Some(0), Some(2)).unwrap(), - canonical_tip, - )?; + let page1 = wait_for_reference_query(&node, query_args(reference, Some(0), Some(2))).await?; assert_eq!(page1.len(), 2); assert_eq!(page1[0].transaction_hash, tx_hashes[0]); assert_eq!(page1[1].transaction_hash, tx_hashes[1]); - // Page 2: offset=2, limit=2 → last entry only. - let page2 = reader.query( - ReferenceQuery::new(reference, Some(2), Some(2)).unwrap(), - canonical_tip, - )?; + let page2 = wait_for_reference_query(&node, query_args(reference, Some(2), Some(2))).await?; assert_eq!(page2.len(), 1); assert_eq!(page2[0].transaction_hash, tx_hashes[2]); Ok(()) } -/// A different reference key returns no results. +/// A different reference key returns an empty successful result once the index +/// is exactly at the canonical head. #[tokio::test(flavor = "multi_thread")] async fn reference_index_no_results_for_unrelated_reference() -> eyre::Result<()> { reth_tracing::init_test_tracing(); @@ -160,15 +169,7 @@ async fn reference_index_no_results_for_unrelated_reference() -> eyre::Result<() node.rpc.inject_tx(raw_tx).await?; node.advance_block().await?; - let dir = TempDir::new()?; - let db = open_and_backfill_index(&node.inner.provider, &dir).await; - let reader = ReferenceIndexReader::new(db, DEFAULT_LAG_THRESHOLD); - let canonical_tip = node.inner.provider.best_block_number()?; - - let results = reader.query( - ReferenceQuery::new(other_reference, None, None).unwrap(), - canonical_tip, - )?; + let results = wait_for_reference_query(&node, query_args(other_reference, None, None)).await?; assert!( results.is_empty(), "unrelated reference should return nothing" diff --git a/crates/reference-index/Cargo.toml b/crates/reference-index/Cargo.toml index fd2cc786..6afde198 100644 --- a/crates/reference-index/Cargo.toml +++ b/crates/reference-index/Cargo.toml @@ -14,16 +14,19 @@ alloy-consensus.workspace = true alloy-primitives.workspace = true alloy-rlp.workspace = true eyre.workspace = true -morph-chainspec.workspace = true morph-primitives = { workspace = true, features = ["reth-codec"] } -reth-chainspec.workspace = true +metrics.workspace = true +parking_lot.workspace = true reth-codecs.workspace = true +reth-chain-state.workspace = true reth-db.workspace = true reth-db-api.workspace = true reth-errors.workspace = true +reth-metrics.workspace = true reth-provider.workspace = true reth-primitives-traits.workspace = true reth-storage-api.workspace = true +reth-tasks.workspace = true serde.workspace = true thiserror.workspace = true tokio.workspace = true diff --git a/crates/reference-index/src/backfill.rs b/crates/reference-index/src/backfill.rs index 099cc33f..e7ac5dae 100644 --- a/crates/reference-index/src/backfill.rs +++ b/crates/reference-index/src/backfill.rs @@ -1,319 +1,76 @@ -//! Historical reference index backfill. -//! -//! Runs once at startup to index all MorphTx transactions from -//! `jade_first_block_number` up to the chain head at startup time. +//! Bounded historical reference-index backfill helpers. use crate::{ - JADE_NOT_ACTIVE_SENTINEL, ReferenceIndexDb, - types::{BackfillState, ReferenceIndexError}, - writer::{ - set_backfill_state, set_jade_first_block_number, update_backfill_current, - update_indexed_from, update_indexed_to, write_block, - }, + db::ReferenceIndexDb, + source::{CanonicalBlock, CanonicalChain}, + types::ReferenceIndexError, + writer::{set_jade_first_block_number, update_indexed_to, write_block}, }; -use alloy_consensus::BlockHeader; -use morph_chainspec::hardfork::{MorphHardfork, MorphHardforks}; -use morph_primitives::MorphHeader; -use reth_chainspec::ForkCondition; use reth_db_api::transaction::DbTx; -use reth_provider::{BlockReader, HeaderProvider}; -use reth_storage_api::{BlockNumReader, TransactionVariant}; -use tracing::{debug, info}; -const TARGET: &str = "morph::reference_index"; - -/// Determine the first block number at which the Jade hardfork is active. -/// -/// Returns `JADE_NOT_ACTIVE_SENTINEL` (`u64::MAX`) when: -/// - The chain spec has no Jade timestamp condition, or -/// - The current canonical head has not yet reached Jade activation. -/// -/// The dual-condition check (`block(N).timestamp >= jade_ts` AND -/// `block(N-1).timestamp < jade_ts`) guarantees we return the **first** -/// Jade block, not just any block after activation. -pub fn resolve_jade_first_block( - provider: &P, - chain_spec: &CS, - head: u64, -) -> Result -where - P: HeaderProvider
, - CS: MorphHardforks, -{ - let jade_ts = match chain_spec.morph_fork_activation(MorphHardfork::Jade) { - ForkCondition::Timestamp(ts) => ts, - _ => return Ok(JADE_NOT_ACTIVE_SENTINEL), - }; - - let head_header = provider - .header_by_number(head)? - .ok_or_else(|| ReferenceIndexError::Other(eyre::eyre!("missing canonical head header")))?; - - if head_header.timestamp() < jade_ts { - return Ok(JADE_NOT_ACTIVE_SENTINEL); +/// Find the first canonical block whose timestamp activates Jade. +pub(crate) fn resolve_jade_first_block( + chain: &C, + head: crate::CanonicalTip, + jade_timestamp: u64, +) -> Result, ReferenceIndexError> { + if head.timestamp < jade_timestamp { + return Ok(None); } - // Binary-search over [0, head] to find the lowest block whose timestamp - // is >= jade_ts. We still verify the prev block to guard against unusual - // timestamp monotonicity violations. - let mut lo = 0u64; - let mut hi = head; - while lo < hi { - let mid = lo + (hi - lo) / 2; - let mid_header = provider - .header_by_number(mid)? - .ok_or_else(|| ReferenceIndexError::Other(eyre::eyre!("missing header at {mid}")))?; - if mid_header.timestamp() < jade_ts { - lo = mid + 1; - } else { - hi = mid; - } - } - - // `lo` is now the first block whose timestamp >= jade_ts. - // Verify that the previous block (if any) has timestamp < jade_ts. - if lo > 0 { - let prev = provider.header_by_number(lo - 1)?.ok_or_else(|| { - ReferenceIndexError::Other(eyre::eyre!("missing header at {}", lo - 1)) + let mut low = 0u64; + let mut high = head.number; + while low < high { + let middle = low + (high - low) / 2; + let timestamp = chain.block_timestamp(middle)?.ok_or_else(|| { + ReferenceIndexError::Other(eyre::eyre!( + "missing canonical header {middle} while resolving Jade" + )) })?; - if prev.timestamp() >= jade_ts { - // Shouldn't happen on a well-formed chain, but fall back to sentinel. - return Ok(JADE_NOT_ACTIVE_SENTINEL); + if timestamp < jade_timestamp { + low = middle + 1; + } else { + high = middle; } } - Ok(lo) + Ok(Some(low)) } -/// Run (or resume) the historical backfill. -/// -/// This function is synchronous and blocks the calling task until backfill is -/// complete. Batches are committed atomically so a crash mid-run can be -/// safely resumed from the last persisted checkpoint. -pub fn run_backfill( +/// Commit one already-validated contiguous canonical block batch. +pub(crate) fn commit_canonical_batch( db: &ReferenceIndexDb, - provider: &P, - chain_spec: &CS, - head_at_startup: u64, - batch_size: u64, -) -> Result<(), ReferenceIndexError> -where - P: BlockReader - + BlockNumReader - + HeaderProvider
, - CS: MorphHardforks, -{ - validate_backfill_batch_size(batch_size)?; - - let state = db.backfill_state()?; - - // `jade_first_block_number` is our canonical lower bound; it's written as the - // very first step on the NotStarted path. InProgress/Complete resumes read - // it back from IndexMeta. - let start = match state { - BackfillState::Complete => return Ok(()), - BackfillState::InProgress => { - // Resume from `max(indexed_to + 1, jade_first_block_number)`: if the - // crash happened between InProgress write and the first batch commit, - // `indexed_to` is still 0 and must not be used as-is. - let jade_first_stored = db - .jade_first_block_number()? - .unwrap_or(JADE_NOT_ACTIVE_SENTINEL); - if jade_first_stored == JADE_NOT_ACTIVE_SENTINEL { - // Previously sentinel; crash may have happened before the follow-up - // txn that would have marked Complete. Re-resolve against the current - // head: Jade might have activated in the meantime. - let new_jade = resolve_jade_first_block(provider, chain_spec, head_at_startup)?; - if new_jade == JADE_NOT_ACTIVE_SENTINEL { - // Still not active; safe to mark complete immediately. - let tx = db.tx_mut()?; - update_indexed_from(&tx, head_at_startup)?; - update_indexed_to(&tx, head_at_startup)?; - set_backfill_state(&tx, BackfillState::Complete)?; - tx.commit()?; - return Ok(()); - } - // Jade now active; persist the real first block and continue backfill. - let tx = db.tx_mut()?; - set_jade_first_block_number(&tx, new_jade)?; - tx.commit()?; - db.indexed_to()?.saturating_add(1).max(new_jade) - } else { - db.indexed_to()?.saturating_add(1).max(jade_first_stored) - } - } - BackfillState::NotStarted => { - let jade_first = resolve_jade_first_block(provider, chain_spec, head_at_startup)?; - - let tx = db.tx_mut()?; - set_jade_first_block_number(&tx, jade_first)?; - set_backfill_state(&tx, BackfillState::InProgress)?; - tx.commit()?; - - if jade_first == JADE_NOT_ACTIVE_SENTINEL || jade_first > head_at_startup { - // Nothing to backfill; mark complete immediately. - let tx = db.tx_mut()?; - update_indexed_from(&tx, head_at_startup)?; - update_indexed_to(&tx, head_at_startup)?; - set_backfill_state(&tx, BackfillState::Complete)?; - tx.commit()?; - return Ok(()); - } - - jade_first - } + jade_first_block: u64, + expected_start: u64, + blocks: &[CanonicalBlock], +) -> Result { + let Some(last) = blocks.last() else { + return Ok(0); }; - if start > head_at_startup { - // Already up to date. - let tx = db.tx_mut()?; - set_backfill_state(&tx, BackfillState::Complete)?; - tx.commit()?; - return Ok(()); - } - - let jade_first = db.jade_first_block_number()?.unwrap_or(start); - - info!( - target: TARGET, - start, head_at_startup, - "starting reference index backfill" - ); - - let mut current = start; - - while current <= head_at_startup { - let batch_end = current.saturating_add(batch_size - 1).min(head_at_startup); - let is_last_batch = batch_end == head_at_startup; - - let tx = db.tx_mut()?; - for number in current..=batch_end { - // `WithHash` is required because write_block stores transaction hashes in the - // reference index keys. `NoHash` would leave tx hashes uninitialised per - // reth's BlockReader contract (see blockchain_provider.rs:309). - let block = provider - .sealed_block_with_senders(number.into(), TransactionVariant::WithHash)? - .ok_or_else(|| { - ReferenceIndexError::Other(eyre::eyre!( - "missing block {number} during backfill" - )) - })?; - - write_block( - &tx, - block.number(), - block.hash(), - block.timestamp(), - &block.body().transactions, - )?; - } - - if is_last_batch { - // Atomic: data + Complete + indexed_from + indexed_to in one commit. - update_indexed_from(&tx, jade_first)?; - update_indexed_to(&tx, head_at_startup)?; - set_backfill_state(&tx, BackfillState::Complete)?; - } else { - update_backfill_current(&tx, batch_end)?; - update_indexed_to(&tx, batch_end)?; - set_backfill_state(&tx, BackfillState::InProgress)?; - } - tx.commit()?; - - debug!( - target: TARGET, - batch_start = current, - batch_end, - is_last_batch, - "backfill batch committed" - ); - - current = batch_end + 1; - } - - info!(target: TARGET, "reference index backfill complete"); - Ok(()) -} - -fn validate_backfill_batch_size(batch_size: u64) -> Result<(), ReferenceIndexError> { - if batch_size == 0 { - return Err(ReferenceIndexError::InvalidBackfillBatchSize); - } - Ok(()) -} - -/// Re-validate and (if needed) reset `jade_first_block_number` when the DB -/// was opened with a sentinel value (`u64::MAX`) from a previous run where -/// Jade had not yet activated. -/// -/// Call this once at startup before `run_backfill`. -pub fn maybe_reset_jade_sentinel( - db: &ReferenceIndexDb, - provider: &P, - chain_spec: &CS, - head: u64, -) -> Result<(), ReferenceIndexError> -where - P: HeaderProvider
, - CS: MorphHardforks, -{ - if db.backfill_state()? != BackfillState::Complete { - return Ok(()); - } - match db.jade_first_block_number()? { - Some(n) if n == JADE_NOT_ACTIVE_SENTINEL => { - // Jade was not active when last resolved. Re-try now. - let new = resolve_jade_first_block(provider, chain_spec, head)?; - if new != JADE_NOT_ACTIVE_SENTINEL { - // Jade has since activated; reset backfill state so the - // next startup picks it up. - let tx = db.tx_mut()?; - set_jade_first_block_number(&tx, new)?; - set_backfill_state(&tx, BackfillState::NotStarted)?; - tx.commit()?; - info!( - target: TARGET, - jade_first_block = new, - "Jade has activated; resetting backfill to index from first Jade block" - ); - } + for (offset, block) in blocks.iter().enumerate() { + let expected = expected_start.saturating_add(offset as u64); + if block.number != expected { + return Err(ReferenceIndexError::Other(eyre::eyre!( + "non-contiguous reference-index batch: expected block {expected}, got {}", + block.number + ))); } - _ => {} - } - Ok(()) -} - -#[cfg(test)] -mod tests { - use super::*; - - #[test] - fn jade_not_active_sentinel_is_u64_max() { - assert_eq!(JADE_NOT_ACTIVE_SENTINEL, u64::MAX); - } - - #[test] - fn backfill_state_try_from_roundtrip() { - assert_eq!( - BackfillState::try_from(0u8).unwrap(), - BackfillState::NotStarted - ); - assert_eq!( - BackfillState::try_from(1u8).unwrap(), - BackfillState::InProgress - ); - assert_eq!( - BackfillState::try_from(2u8).unwrap(), - BackfillState::Complete - ); - assert!(BackfillState::try_from(3u8).is_err()); } - #[test] - fn backfill_batch_size_zero_is_rejected() { - assert!(matches!( - validate_backfill_batch_size(0), - Err(ReferenceIndexError::InvalidBackfillBatchSize) - )); + let tx = db.tx_mut()?; + let mut references_written = 0u64; + for block in blocks { + references_written += write_block( + &tx, + block.number, + block.hash, + block.timestamp, + &block.transactions, + )?; } + set_jade_first_block_number(&tx, jade_first_block)?; + update_indexed_to(&tx, last.number)?; + tx.commit()?; + Ok(references_written) } diff --git a/crates/reference-index/src/db.rs b/crates/reference-index/src/db.rs index 3c37beea..3bf8ec64 100644 --- a/crates/reference-index/src/db.rs +++ b/crates/reference-index/src/db.rs @@ -1,8 +1,11 @@ //! Reference index database opening and metadata helpers. use crate::{ - tables::{IndexedBlockKey, IndexedBlocks, MetaKey, MetaValue, ReferenceIndexTables}, - types::{BackfillState, ReferenceIndexError, SCHEMA_VERSION}, + tables::{ + BlockReferenceIndex, IndexMeta, IndexedBlockKey, IndexedBlocks, MetaKey, MetaValue, + ReferenceIndex, ReferenceIndexTables, + }, + types::{ReferenceIndexError, SCHEMA_VERSION}, }; use alloy_primitives::B256; use reth_db::{DatabaseEnv, mdbx::DatabaseArguments}; @@ -10,96 +13,73 @@ use reth_db_api::{ Database, transaction::{DbTx, DbTxMut}, }; -use std::{ - path::Path, - sync::{ - Arc, - atomic::{AtomicBool, Ordering}, - }, -}; - -// ── meta key discriminants ──────────────────────────────────────────────────── +use std::{path::Path, sync::Arc}; /// Discriminant values for the `IndexMeta` table. #[derive(Debug, Clone, Copy, PartialEq, Eq)] #[repr(u8)] -pub enum IndexMetaKey { - IndexedFrom = 1, - IndexedTo = 2, - BackfillState = 3, - BackfillCurrent = 4, - ChainId = 5, - GenesisHash = 6, - SchemaVersion = 7, - JadeFirstBlockNumber = 8, +pub(crate) enum IndexMetaKey { + IndexedTo = 1, + ChainId = 2, + GenesisHash = 3, + SchemaVersion = 4, + JadeFirstBlockNumber = 5, } impl From for MetaKey { - fn from(k: IndexMetaKey) -> Self { - Self(k as u8) + fn from(key: IndexMetaKey) -> Self { + Self(key as u8) } } -// ── codec helpers ───────────────────────────────────────────────────────────── - -pub fn encode_u64(value: u64) -> MetaValue { +pub(crate) fn encode_u64(value: u64) -> MetaValue { MetaValue(value.to_be_bytes().to_vec()) } -pub fn decode_u64(value: MetaValue) -> Result { +pub(crate) fn decode_u64(value: MetaValue) -> Result { let bytes: [u8; 8] = value .0 .as_slice() .try_into() - .map_err(|_| ReferenceIndexError::Other(eyre::eyre!("invalid u64 metadata length")))?; + .map_err(|_| ReferenceIndexError::CorruptMetadata("invalid u64 value"))?; Ok(u64::from_be_bytes(bytes)) } -pub fn encode_u32(value: u32) -> MetaValue { +pub(crate) fn encode_u32(value: u32) -> MetaValue { MetaValue(value.to_be_bytes().to_vec()) } -pub fn decode_u32(value: MetaValue) -> Result { +pub(crate) fn decode_u32(value: MetaValue) -> Result { let bytes: [u8; 4] = value .0 .as_slice() .try_into() - .map_err(|_| ReferenceIndexError::Other(eyre::eyre!("invalid u32 metadata length")))?; + .map_err(|_| ReferenceIndexError::CorruptMetadata("invalid u32 value"))?; Ok(u32::from_be_bytes(bytes)) } -pub fn encode_b256(value: B256) -> MetaValue { +pub(crate) fn encode_b256(value: B256) -> MetaValue { MetaValue(value.as_slice().to_vec()) } -pub fn decode_b256(value: MetaValue) -> Result { +pub(crate) fn decode_b256(value: MetaValue) -> Result { let bytes: [u8; 32] = value .0 .as_slice() .try_into() - .map_err(|_| ReferenceIndexError::Other(eyre::eyre!("invalid B256 metadata length")))?; + .map_err(|_| ReferenceIndexError::CorruptMetadata("invalid B256 value"))?; Ok(B256::new(bytes)) } -// ── database handle ─────────────────────────────────────────────────────────── - -/// Handle to the reference index MDBX database. -/// -/// Wraps the raw [`DatabaseEnv`] and exposes metadata read helpers and an -/// atomic readiness flag. All writes go through `tx_mut()`. +/// Cloneable handle to the reference-index MDBX environment. #[derive(Debug, Clone)] -pub struct ReferenceIndexDb { +pub(crate) struct ReferenceIndexDb { db: Arc, - ready: Arc, } impl ReferenceIndexDb { - /// Open (or create) the reference index at `path` and validate chain identity. - /// - /// On the first open (empty DB), writes `chain_id`, `genesis_hash`, and - /// `schema_version` into `IndexMeta`. On every subsequent open, re-reads - /// those values and returns an error if any mismatch is detected. - pub fn open( + /// Open (or create) the index and validate its complete chain identity. + pub(crate) fn open( path: impl AsRef, chain_id: u64, genesis_hash: B256, @@ -108,20 +88,14 @@ impl ReferenceIndexDb { path, DatabaseArguments::new(reth_db::models::ClientVersion::default()), ) - .map_err(|e| { - ReferenceIndexError::Other(eyre::eyre!("failed to open reference index DB: {e}")) + .map_err(|error| { + ReferenceIndexError::Other(eyre::eyre!("failed to open reference index DB: {error}")) })?; - - let this = Self { - db: Arc::new(db), - ready: Arc::new(AtomicBool::new(false)), - }; - + let this = Self { db: Arc::new(db) }; this.validate_or_init_chain_identity(chain_id, genesis_hash)?; Ok(this) } - /// Check (or initialise) the persisted chain identity in `IndexMeta`. fn validate_or_init_chain_identity( &self, chain_id: u64, @@ -129,125 +103,100 @@ impl ReferenceIndexDb { ) -> Result<(), ReferenceIndexError> { let tx = self.tx()?; let stored_chain_id = tx - .get::(IndexMetaKey::ChainId.into())? + .get::(IndexMetaKey::ChainId.into())? .map(decode_u64) .transpose()?; - - // First-ever open: write identity and return. - if stored_chain_id.is_none() { - drop(tx); - let tx = self.tx_mut()?; - tx.put::(IndexMetaKey::ChainId.into(), encode_u64(chain_id))?; - tx.put::( - IndexMetaKey::GenesisHash.into(), - encode_b256(genesis_hash), - )?; - tx.put::( - IndexMetaKey::SchemaVersion.into(), - encode_u32(SCHEMA_VERSION), - )?; - tx.commit()?; - return Ok(()); - } - - // Subsequent opens: validate every field. - if stored_chain_id != Some(chain_id) { - return Err(ReferenceIndexError::ChainIdentityMismatch("chain_id")); - } - let stored_genesis = tx - .get::(IndexMetaKey::GenesisHash.into())? + .get::(IndexMetaKey::GenesisHash.into())? .map(decode_b256) .transpose()?; - if stored_genesis != Some(genesis_hash) { - return Err(ReferenceIndexError::ChainIdentityMismatch("genesis_hash")); - } - let stored_schema = tx - .get::(IndexMetaKey::SchemaVersion.into())? + .get::(IndexMetaKey::SchemaVersion.into())? .map(decode_u32) .transpose()?; - match stored_schema { - Some(v) if v != SCHEMA_VERSION => { - return Err(ReferenceIndexError::SchemaMismatch { - expected: SCHEMA_VERSION, - actual: v, - }); + let has_runtime_meta = tx + .get::(IndexMetaKey::IndexedTo.into())? + .is_some() + || tx + .get::(IndexMetaKey::JadeFirstBlockNumber.into())? + .is_some(); + let has_index_data = tx.entries::()? != 0 + || tx.entries::()? != 0 + || tx.entries::()? != 0; + + match (stored_chain_id, stored_genesis, stored_schema) { + (None, None, None) if !has_runtime_meta && !has_index_data => { + drop(tx); + let tx = self.tx_mut()?; + tx.put::(IndexMetaKey::ChainId.into(), encode_u64(chain_id))?; + tx.put::(IndexMetaKey::GenesisHash.into(), encode_b256(genesis_hash))?; + tx.put::( + IndexMetaKey::SchemaVersion.into(), + encode_u32(SCHEMA_VERSION), + )?; + tx.commit()?; + return Ok(()); + } + (Some(stored_chain_id), Some(stored_genesis), Some(stored_schema)) => { + if stored_chain_id != chain_id { + return Err(ReferenceIndexError::ChainIdentityMismatch("chain_id")); + } + if stored_genesis != genesis_hash { + return Err(ReferenceIndexError::ChainIdentityMismatch("genesis_hash")); + } + if stored_schema != SCHEMA_VERSION { + return Err(ReferenceIndexError::SchemaMismatch { + expected: SCHEMA_VERSION, + actual: stored_schema, + }); + } + } + _ => { + return Err(ReferenceIndexError::CorruptMetadata( + "chain identity fields must be all present or all absent", + )); } - _ => {} } Ok(()) } - // ── readiness ───────────────────────────────────────────────────────────── - - pub fn is_ready(&self) -> bool { - self.ready.load(Ordering::Acquire) - } - - pub fn set_ready(&self, ready: bool) { - self.ready.store(ready, Ordering::Release); - } - - // ── transaction factory ─────────────────────────────────────────────────── - - pub fn tx(&self) -> Result<::TX, ReferenceIndexError> { + pub(crate) fn tx(&self) -> Result<::TX, ReferenceIndexError> { Ok(self.db.tx()?) } - pub fn tx_mut(&self) -> Result<::TXMut, ReferenceIndexError> { + pub(crate) fn tx_mut(&self) -> Result<::TXMut, ReferenceIndexError> { Ok(self.db.tx_mut()?) } - // ── metadata reads ──────────────────────────────────────────────────────── - - pub fn backfill_state(&self) -> Result { + pub(crate) fn indexed_to(&self) -> Result, ReferenceIndexError> { let tx = self.tx()?; - match tx.get::(IndexMetaKey::BackfillState.into())? { - Some(v) => BackfillState::try_from(*v.0.first().unwrap_or(&0)), - None => Ok(BackfillState::NotStarted), - } - } - - pub fn indexed_to(&self) -> Result { - let tx = self.tx()?; - tx.get::(IndexMetaKey::IndexedTo.into())? - .map(decode_u64) - .transpose() - .map(|v| v.unwrap_or(0)) - } - - pub fn indexed_from(&self) -> Result, ReferenceIndexError> { - let tx = self.tx()?; - tx.get::(IndexMetaKey::IndexedFrom.into())? + tx.get::(IndexMetaKey::IndexedTo.into())? .map(decode_u64) .transpose() } - pub fn jade_first_block_number(&self) -> Result, ReferenceIndexError> { + pub(crate) fn jade_first_block_number(&self) -> Result, ReferenceIndexError> { let tx = self.tx()?; - tx.get::(IndexMetaKey::JadeFirstBlockNumber.into())? + tx.get::(IndexMetaKey::JadeFirstBlockNumber.into())? .map(decode_u64) .transpose() } - /// Returns the canonical block hash stored in `IndexedBlocks` for `block_number`. - pub fn indexed_block_hash( + pub(crate) fn indexed_block_hash( &self, block_number: u64, ) -> Result, ReferenceIndexError> { let tx = self.tx()?; Ok(tx .get::(IndexedBlockKey { block_number })? - .map(|v| v.0)) + .map(|value| value.0)) } } #[cfg(test)] mod tests { use super::*; - use alloy_primitives::B256; use tempfile::TempDir; fn open_temp_db() -> (TempDir, ReferenceIndexDb) { @@ -257,20 +206,18 @@ mod tests { } #[test] - fn open_writes_chain_identity_on_first_open() { + fn new_database_has_no_durable_cursor() { let (_dir, db) = open_temp_db(); - assert_eq!(db.backfill_state().unwrap(), BackfillState::NotStarted); - assert_eq!(db.indexed_to().unwrap(), 0); + assert_eq!(db.indexed_to().unwrap(), None); } #[test] fn open_rejects_mismatched_chain_id() { let dir = TempDir::new().unwrap(); ReferenceIndexDb::open(dir.path(), 2818, B256::ZERO).unwrap(); - // Re-open with wrong chain_id. - let err = ReferenceIndexDb::open(dir.path(), 9999, B256::ZERO).unwrap_err(); + let error = ReferenceIndexDb::open(dir.path(), 9999, B256::ZERO).unwrap_err(); assert!(matches!( - err, + error, ReferenceIndexError::ChainIdentityMismatch("chain_id") )); } @@ -279,22 +226,56 @@ mod tests { fn open_rejects_mismatched_genesis_hash() { let dir = TempDir::new().unwrap(); ReferenceIndexDb::open(dir.path(), 2818, B256::ZERO).unwrap(); - let err = ReferenceIndexDb::open(dir.path(), 2818, B256::repeat_byte(0xff)).unwrap_err(); + let error = ReferenceIndexDb::open(dir.path(), 2818, B256::repeat_byte(0xff)).unwrap_err(); assert!(matches!( - err, + error, ReferenceIndexError::ChainIdentityMismatch("genesis_hash") )); } + #[test] + fn open_rejects_incomplete_identity_metadata() { + let dir = TempDir::new().unwrap(); + let db = ReferenceIndexDb::open(dir.path(), 2818, B256::ZERO).unwrap(); + let tx = db.tx_mut().unwrap(); + tx.delete::(IndexMetaKey::SchemaVersion.into(), None) + .unwrap(); + tx.commit().unwrap(); + drop(db); + + let error = ReferenceIndexDb::open(dir.path(), 2818, B256::ZERO).unwrap_err(); + assert!(matches!(error, ReferenceIndexError::CorruptMetadata(_))); + } + + #[test] + fn open_rejects_the_pre_release_schema() { + let dir = TempDir::new().unwrap(); + let db = ReferenceIndexDb::open(dir.path(), 2818, B256::ZERO).unwrap(); + let tx = db.tx_mut().unwrap(); + tx.put::(IndexMetaKey::SchemaVersion.into(), encode_u32(1)) + .unwrap(); + tx.commit().unwrap(); + drop(db); + + let error = ReferenceIndexDb::open(dir.path(), 2818, B256::ZERO).unwrap_err(); + assert!(matches!( + error, + ReferenceIndexError::SchemaMismatch { + expected: SCHEMA_VERSION, + actual: 1 + } + )); + } + #[test] fn encode_decode_u64_roundtrip() { - let v = 0xDEAD_BEEF_CAFE_1234u64; - assert_eq!(decode_u64(encode_u64(v)).unwrap(), v); + let value = 0xDEAD_BEEF_CAFE_1234u64; + assert_eq!(decode_u64(encode_u64(value)).unwrap(), value); } #[test] fn encode_decode_b256_roundtrip() { - let v = B256::repeat_byte(0xab); - assert_eq!(decode_b256(encode_b256(v)).unwrap(), v); + let value = B256::repeat_byte(0xab); + assert_eq!(decode_b256(encode_b256(value)).unwrap(), value); } } diff --git a/crates/reference-index/src/lib.rs b/crates/reference-index/src/lib.rs index 692cfce6..d6421c6d 100644 --- a/crates/reference-index/src/lib.rs +++ b/crates/reference-index/src/lib.rs @@ -4,7 +4,6 @@ use alloy_consensus as _; use alloy_rlp as _; -use morph_chainspec as _; use morph_primitives as _; use reth_codecs as _; use reth_primitives_traits as _; @@ -13,26 +12,26 @@ use reth_storage_api as _; use tokio as _; use tracing as _; -pub mod backfill; -pub mod db; -pub mod reader; -pub mod reconcile; -pub mod tables; -pub mod types; -pub mod writer; +mod backfill; +mod db; +mod metrics; +mod reader; +mod reconcile; +mod runtime; +mod source; +mod tables; +mod types; +mod writer; -pub use db::ReferenceIndexDb; -pub use reader::ReferenceIndexReader; +pub use reader::{ReferenceIndexHandle, ReferenceIndexPhase}; +pub use runtime::{ReferenceIndexConfig, ReferenceIndexRuntime}; +pub use source::{CanonicalBlock, CanonicalChain}; pub use types::{ - BackfillState, JADE_NOT_ACTIVE_SENTINEL, ReferenceIndexError, ReferenceQuery, - ReferenceTransactionResult, SCHEMA_VERSION, + CanonicalTip, ReferenceIndexError, ReferenceQuery, ReferenceTransactionResult, SCHEMA_VERSION, }; -/// Default number of canonical blocks the ExEx may lag before RPC returns an error. -pub const DEFAULT_LAG_THRESHOLD: u64 = 16; - -/// Default number of blocks checked during startup reconcile for offline reorgs. -pub const DEFAULT_MAX_REORG_DEPTH: u64 = 64; - /// Default backfill batch size. -pub const DEFAULT_BACKFILL_BATCH_BLOCKS: u64 = 256; +pub(crate) const DEFAULT_BACKFILL_BATCH_BLOCKS: u64 = 512; + +/// Operational live-lag service-level objective. RPC readiness remains lag zero. +pub(crate) const LIVE_LAG_SLO: u64 = 16; diff --git a/crates/reference-index/src/metrics.rs b/crates/reference-index/src/metrics.rs new file mode 100644 index 00000000..6f1d84d5 --- /dev/null +++ b/crates/reference-index/src/metrics.rs @@ -0,0 +1,33 @@ +//! Runtime observability for the derived reference index. + +use reth_metrics::{ + Metrics, + metrics::{Counter, Gauge, Histogram}, +}; + +#[derive(Metrics, Clone)] +#[metrics(scope = "morph.reference_index")] +pub(crate) struct ReferenceIndexMetrics { + /// Current runtime phase encoded by `ReferenceIndexPhase`. + pub(crate) phase: Gauge, + /// Current canonical target block. + pub(crate) target_block: Gauge, + /// Last durably indexed canonical block, or zero before activation. + pub(crate) indexed_block: Gauge, + /// Canonical block lag observed after the latest reconciliation turn. + pub(crate) lag_blocks: Gauge, + /// One only when the runtime phase is live; zero otherwise. + pub(crate) rpc_ready: Gauge, + /// Canonical blocks committed to the index. + pub(crate) indexed_blocks_total: Counter, + /// Durable index batches committed. + pub(crate) committed_batches_total: Counter, + /// Canonical suffix repairs performed. + pub(crate) reorgs_total: Counter, + /// Canonical notification receiver lag events. + pub(crate) lagged_notifications_total: Counter, + /// Failed reconciliation turns. + pub(crate) failures_total: Counter, + /// Wall-clock duration of a reconciliation turn. + pub(crate) batch_duration_seconds: Histogram, +} diff --git a/crates/reference-index/src/reader.rs b/crates/reference-index/src/reader.rs index efe6dc7a..f415e8da 100644 --- a/crates/reference-index/src/reader.rs +++ b/crates/reference-index/src/reader.rs @@ -1,48 +1,183 @@ //! Reference index read path helpers. use crate::{ - db::ReferenceIndexDb, - tables::{ReferenceIndex, ReferenceIndexKey}, - types::{ReferenceIndexError, ReferenceQuery, ReferenceTransactionResult}, + db::{IndexMetaKey, ReferenceIndexDb, decode_u64}, + metrics::ReferenceIndexMetrics, + tables::{IndexMeta, IndexedBlockKey, IndexedBlocks, ReferenceIndex, ReferenceIndexKey}, + types::{CanonicalTip, ReferenceIndexError, ReferenceQuery, ReferenceTransactionResult}, }; use alloy_primitives::{B256, U64}; +use parking_lot::RwLock; use reth_db_api::{cursor::DbCursorRO, transaction::DbTx}; +use std::sync::{ + Arc, + atomic::{AtomicU8, AtomicU64, Ordering}, +}; + +/// Runtime phase that controls whether reference queries can be served. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +#[repr(u8)] +pub enum ReferenceIndexPhase { + /// Runtime has not opened and validated its derived database yet. + Opening = 0, + /// Canonical head predates Jade, so the complete result is empty. + PreJade = 1, + /// Historical chain sync is active and index writes are intentionally paused. + Deferred = 2, + /// A bounded historical batch is being reconciled or committed. + Backfill = 3, + /// Durable cursor and hash matched the canonical head after the latest turn. + Live = 4, + /// A non-canonical indexed suffix is being removed. + Repairing = 5, + /// Queries cannot be served until retry or a manual rebuild succeeds. + Unavailable = 6, +} + +impl ReferenceIndexPhase { + fn from_u8(value: u8) -> Self { + match value { + 0 => Self::Opening, + 1 => Self::PreJade, + 2 => Self::Deferred, + 3 => Self::Backfill, + 4 => Self::Live, + 5 => Self::Repairing, + 6 => Self::Unavailable, + _ => unreachable!("reference index phase is only written by this crate"), + } + } +} + +#[derive(Debug)] +struct ReferenceIndexShared { + db: RwLock>, + phase: AtomicU8, + /// Even values are stable snapshots; odd values mean a transition is in progress. + generation: AtomicU64, + metrics: ReferenceIndexMetrics, +} -/// Read-only facade for reference index queries. +/// Cloneable query handle shared by the runtime and RPC layer. /// -/// `canonical_tip` must be the current chain head block number so the reader -/// can detect excessive lag between the index and the live chain. +/// The runtime installs the database and advances the phase. Queries only +/// succeed in [`ReferenceIndexPhase::Live`] when the durable cursor exactly +/// matches the supplied canonical tip number and hash. #[derive(Clone, Debug)] -pub struct ReferenceIndexReader { - db: ReferenceIndexDb, - lag_threshold: u64, +pub struct ReferenceIndexHandle { + shared: Arc, + jade_timestamp: u64, } -impl ReferenceIndexReader { - pub const fn new(db: ReferenceIndexDb, lag_threshold: u64) -> Self { - Self { db, lag_threshold } +impl ReferenceIndexHandle { + /// Create a handle before the background runtime has opened the database. + pub fn new(jade_timestamp: u64) -> Self { + Self { + shared: Arc::new(ReferenceIndexShared { + db: RwLock::new(None), + phase: AtomicU8::new(ReferenceIndexPhase::Opening as u8), + generation: AtomicU64::new(0), + metrics: ReferenceIndexMetrics::default(), + }), + jade_timestamp, + } + } + + /// Current in-memory runtime phase, exposed for logs and metrics. + pub fn phase(&self) -> ReferenceIndexPhase { + ReferenceIndexPhase::from_u8(self.shared.phase.load(Ordering::Acquire)) + } + + pub(crate) fn install_db(&self, db: ReferenceIndexDb) { + *self.shared.db.write() = Some(db); + } + + pub(crate) fn set_phase(&self, phase: ReferenceIndexPhase) { + if self.phase() == phase { + return; + } + self.shared.generation.fetch_add(1, Ordering::AcqRel); + self.shared.phase.store(phase as u8, Ordering::Release); + self.shared.generation.fetch_add(1, Ordering::Release); + self.shared.metrics.phase.set(phase as u8 as f64); + self.shared + .metrics + .rpc_ready + .set(f64::from(phase == ReferenceIndexPhase::Live)); + } + + pub(crate) fn db(&self) -> Option { + self.shared.db.read().clone() + } + + pub(crate) fn metrics(&self) -> &ReferenceIndexMetrics { + &self.shared.metrics + } + + #[cfg(test)] + pub(crate) fn generation(&self) -> u64 { + self.shared.generation.load(Ordering::Acquire) } - /// Execute a paginated reference query. + /// Execute a paginated query at an exact canonical chain snapshot. /// - /// `canonical_tip` is the current best block number, used to compute lag. - pub fn query( + /// Cursor validation and result iteration share one MDBX read transaction, + /// so callers never observe a cursor from one index commit and rows from + /// another. The RPC layer must additionally verify that its canonical tip + /// did not change while this method was running. + pub fn query_at( &self, query: ReferenceQuery, - canonical_tip: u64, + canonical_tip: CanonicalTip, ) -> Result, ReferenceIndexError> { - if !self.db.is_ready() { - return Err(ReferenceIndexError::Initializing); + let generation = loop { + let generation = self.shared.generation.load(Ordering::Acquire); + if generation.is_multiple_of(2) { + break generation; + } + std::hint::spin_loop(); + }; + match self.phase() { + ReferenceIndexPhase::Opening => return Err(ReferenceIndexError::Initializing), + ReferenceIndexPhase::PreJade => { + return if canonical_tip.timestamp < self.jade_timestamp { + Ok(Vec::new()) + } else { + Err(ReferenceIndexError::IndexBehind) + }; + } + ReferenceIndexPhase::Deferred | ReferenceIndexPhase::Backfill => { + return Err(ReferenceIndexError::IndexBehind); + } + ReferenceIndexPhase::Repairing | ReferenceIndexPhase::Unavailable => { + return Err(ReferenceIndexError::IndexUnavailable); + } + ReferenceIndexPhase::Live => {} } - let indexed_to = self.db.indexed_to()?; - if canonical_tip.saturating_sub(indexed_to) > self.lag_threshold { + let db = self.db().ok_or(ReferenceIndexError::IndexUnavailable)?; + let tx = db.tx()?; + let indexed_to = tx + .get::(IndexMetaKey::IndexedTo.into())? + .map(decode_u64) + .transpose()?; + let Some(indexed_to) = indexed_to else { + return Err(ReferenceIndexError::IndexUnavailable); + }; + + let indexed_hash = tx + .get::(IndexedBlockKey { + block_number: indexed_to, + })? + .map(|value| value.0); + let Some(indexed_hash) = indexed_hash else { + return Err(ReferenceIndexError::IndexUnavailable); + }; + if indexed_to != canonical_tip.number || indexed_hash != canonical_tip.hash { return Err(ReferenceIndexError::IndexBehind); } - let tx = self.db.tx()?; let mut cursor = tx.cursor_read::()?; - let seek_key = ReferenceIndexKey { reference: query.reference, block_number: 0, @@ -53,12 +188,8 @@ impl ReferenceIndexReader { let mut skipped = 0u64; let mut results = Vec::new(); let mut next = cursor.seek(seek_key)?; - while let Some((key, value)) = next { - if key.reference != query.reference { - break; - } - if results.len() as u64 >= query.limit { + if key.reference != query.reference || results.len() as u64 >= query.limit { break; } if skipped < query.offset { @@ -74,6 +205,12 @@ impl ReferenceIndexReader { next = cursor.next()?; } + if self.shared.generation.load(Ordering::Acquire) != generation + || self.phase() != ReferenceIndexPhase::Live + { + return Err(ReferenceIndexError::IndexUnavailable); + } + Ok(results) } } @@ -82,61 +219,164 @@ impl ReferenceIndexReader { mod tests { use super::*; use crate::{ - ReferenceIndexDb, + db::ReferenceIndexDb, writer::{update_indexed_to, write_block}, }; use alloy_primitives::B256; use tempfile::TempDir; - fn open_ready_db() -> (TempDir, ReferenceIndexDb) { + fn handle_with_phase( + db: ReferenceIndexDb, + jade_timestamp: u64, + phase: ReferenceIndexPhase, + ) -> ReferenceIndexHandle { + let handle = ReferenceIndexHandle::new(jade_timestamp); + handle.install_db(db); + handle.set_phase(phase); + handle + } + + #[test] + fn query_returns_initializing_while_runtime_is_opening() { + let handle = ReferenceIndexHandle::new(200); + let query = ReferenceQuery::new(B256::ZERO, None, None).unwrap(); + let tip = CanonicalTip { + number: 0, + hash: B256::ZERO, + timestamp: 0, + }; + + assert!(matches!( + handle.query_at(query, tip), + Err(ReferenceIndexError::Initializing) + )); + } + + #[test] + fn setting_the_same_phase_does_not_invalidate_readers() { + let handle = ReferenceIndexHandle::new(200); + let generation = handle.shared.generation.load(Ordering::Acquire); + + handle.set_phase(ReferenceIndexPhase::Opening); + + assert_eq!(handle.shared.generation.load(Ordering::Acquire), generation); + } + + #[test] + fn query_returns_index_behind_when_tip_is_one_block_ahead() { let dir = TempDir::new().unwrap(); let db = ReferenceIndexDb::open(dir.path(), 2818, B256::ZERO).unwrap(); - db.set_ready(true); - (dir, db) + let tx = db.tx_mut().unwrap(); + write_block(&tx, 10, B256::repeat_byte(0xaa), 100, &[]).unwrap(); + update_indexed_to(&tx, 10).unwrap(); + tx.commit().unwrap(); + let handle = handle_with_phase(db, 0, ReferenceIndexPhase::Live); + let query = ReferenceQuery::new(B256::with_last_byte(1), None, None).unwrap(); + let tip = CanonicalTip { + number: 11, + hash: B256::repeat_byte(0xbb), + timestamp: 101, + }; + + assert!(matches!( + handle.query_at(query, tip), + Err(ReferenceIndexError::IndexBehind) + )); } #[test] - fn query_returns_initializing_when_not_ready() { + fn query_at_rejects_a_non_canonical_cursor_hash() { let dir = TempDir::new().unwrap(); let db = ReferenceIndexDb::open(dir.path(), 2818, B256::ZERO).unwrap(); - // is_ready stays false - let reader = ReferenceIndexReader::new(db, 16); - let q = ReferenceQuery::new(B256::ZERO, None, None).unwrap(); + let tx = db.tx_mut().unwrap(); + write_block(&tx, 10, B256::repeat_byte(0xaa), 100, &[]).unwrap(); + update_indexed_to(&tx, 10).unwrap(); + tx.commit().unwrap(); + let handle = handle_with_phase(db, 0, ReferenceIndexPhase::Live); + let query = ReferenceQuery::new(B256::with_last_byte(1), None, None).unwrap(); + let tip = CanonicalTip { + number: 10, + hash: B256::repeat_byte(0xbb), + timestamp: 100, + }; + assert!(matches!( - reader.query(q, 100), - Err(ReferenceIndexError::Initializing) + handle.query_at(query, tip), + Err(ReferenceIndexError::IndexBehind) )); } #[test] - fn query_returns_index_behind_when_lag_exceeds_threshold() { - let (_dir, db) = open_ready_db(); + fn live_query_treats_a_missing_cursor_as_unavailable() { + let dir = TempDir::new().unwrap(); + let db = ReferenceIndexDb::open(dir.path(), 2818, B256::ZERO).unwrap(); + let handle = handle_with_phase(db, 0, ReferenceIndexPhase::Live); + let query = ReferenceQuery::new(B256::ZERO, None, None).unwrap(); + let tip = CanonicalTip { + number: 1, + hash: B256::repeat_byte(0x11), + timestamp: 100, + }; + + assert!(matches!( + handle.query_at(query, tip), + Err(ReferenceIndexError::IndexUnavailable) + )); + } + #[test] + fn live_query_treats_a_missing_cursor_hash_as_unavailable() { + let dir = TempDir::new().unwrap(); + let db = ReferenceIndexDb::open(dir.path(), 2818, B256::ZERO).unwrap(); let tx = db.tx_mut().unwrap(); - update_indexed_to(&tx, 10).unwrap(); + update_indexed_to(&tx, 1).unwrap(); tx.commit().unwrap(); + let handle = handle_with_phase(db, 0, ReferenceIndexPhase::Live); + let query = ReferenceQuery::new(B256::ZERO, None, None).unwrap(); + let tip = CanonicalTip { + number: 1, + hash: B256::repeat_byte(0x11), + timestamp: 100, + }; - let reader = ReferenceIndexReader::new(db, 16); - let q = ReferenceQuery::new(B256::with_last_byte(1), None, None).unwrap(); - // canonical_tip=27 → lag=17 > threshold=16 assert!(matches!( - reader.query(q, 27), - Err(ReferenceIndexError::IndexBehind) + handle.query_at(query, tip), + Err(ReferenceIndexError::IndexUnavailable) )); } #[test] - fn query_returns_empty_when_no_reference_txs() { - let (_dir, db) = open_ready_db(); + fn pre_jade_query_is_complete_without_a_cursor() { + let dir = TempDir::new().unwrap(); + let db = ReferenceIndexDb::open(dir.path(), 2818, B256::ZERO).unwrap(); + let handle = handle_with_phase(db, 200, ReferenceIndexPhase::PreJade); + let query = ReferenceQuery::new(B256::with_last_byte(1), None, None).unwrap(); + let tip = CanonicalTip { + number: 100, + hash: B256::repeat_byte(0xaa), + timestamp: 199, + }; + assert!(handle.query_at(query, tip).unwrap().is_empty()); + } + + #[test] + fn live_query_returns_empty_when_no_reference_txs_exist() { + let dir = TempDir::new().unwrap(); + let db = ReferenceIndexDb::open(dir.path(), 2818, B256::ZERO).unwrap(); + let block_hash = B256::repeat_byte(0x01); let tx = db.tx_mut().unwrap(); - write_block(&tx, 1, B256::repeat_byte(0x01), 100, &[]).unwrap(); + write_block(&tx, 1, block_hash, 100, &[]).unwrap(); update_indexed_to(&tx, 1).unwrap(); tx.commit().unwrap(); + let handle = handle_with_phase(db, 0, ReferenceIndexPhase::Live); + let query = ReferenceQuery::new(B256::with_last_byte(0x42), None, None).unwrap(); + let tip = CanonicalTip { + number: 1, + hash: block_hash, + timestamp: 100, + }; - let reader = ReferenceIndexReader::new(db, 16); - let q = ReferenceQuery::new(B256::with_last_byte(0x42), None, None).unwrap(); - let results = reader.query(q, 1).unwrap(); - assert!(results.is_empty()); + assert!(handle.query_at(query, tip).unwrap().is_empty()); } } diff --git a/crates/reference-index/src/reconcile.rs b/crates/reference-index/src/reconcile.rs index 204cce62..55591de5 100644 --- a/crates/reference-index/src/reconcile.rs +++ b/crates/reference-index/src/reconcile.rs @@ -1,132 +1,48 @@ -//! Startup canonical chain reconciliation. -//! -//! After backfill completes, reconcile checks the last `max_reorg_depth` -//! indexed blocks against the current canonical chain to detect any reorgs -//! that occurred while the node was offline, then fills any suffix gap between -//! `indexed_to` and `head_at_startup`. +//! Bounded rollback helpers for canonical-chain reconciliation. use crate::{ - ReferenceIndexDb, + db::ReferenceIndexDb, types::ReferenceIndexError, - writer::{delete_block, update_indexed_to, write_block}, + writer::{clear_indexed_to, delete_block, update_indexed_to}, }; -use alloy_consensus::BlockHeader; -use morph_primitives::MorphHeader; use reth_db_api::transaction::DbTx; -use reth_provider::{BlockHashReader, BlockReader, HeaderProvider}; -use reth_storage_api::TransactionVariant; -use tracing::{debug, info}; -const TARGET: &str = "morph::reference_index"; - -/// Run the startup reconciliation pass. -/// -/// Steps: -/// A. Canonical hash check over the last `max_reorg_depth` indexed blocks. -/// On mismatch, find the lowest diverging height, delete forward entries, -/// reset `indexed_to`, and continue as if filling a gap. -/// B. Suffix gap fill: write every block from `indexed_to + 1` to -/// `current_canonical_head` into the three index tables. -pub fn run_startup_reconcile

( +/// Delete the indexed suffix after `ancestor` using bounded write transactions. +pub(crate) fn rollback_suffix( db: &ReferenceIndexDb, - provider: &P, - current_head: u64, - max_reorg_depth: u64, -) -> Result<(), ReferenceIndexError> -where - P: BlockReader - + HeaderProvider

- + BlockHashReader, -{ - let indexed_to = db.indexed_to()?; - - // ── Step A: canonical hash check ──────────────────────────────────────── - // Use indexed_from as lower bound to avoid scanning blocks that were never - // indexed (e.g. pre-Jade blocks on the sentinel path where backfill sets - // indexed_to but writes no IndexedBlocks entries). - let indexed_from = db.indexed_from()?.unwrap_or(indexed_to); - let depth_start = indexed_to.saturating_sub(max_reorg_depth.saturating_sub(1)); - let check_start = indexed_from.max(depth_start); - let mut fork_height: Option = None; - - for number in check_start..=indexed_to { - let indexed_hash = db.indexed_block_hash(number)?; + ancestor: Option, + batch_size: u64, +) -> Result<(), ReferenceIndexError> { + if batch_size == 0 { + return Err(ReferenceIndexError::InvalidBackfillBatchSize); + } - // `None` means the block was never written to IndexedBlocks (e.g. the - // sentinel path where backfill marks indexed_to = head but writes no - // IndexedBlocks entries for pre-Jade blocks). Treat as "not indexed, - // skip" rather than a hash mismatch to avoid spurious reorg detection. - let Some(indexed) = indexed_hash else { - continue; + loop { + let Some(cursor) = db.indexed_to()? else { + return Ok(()); }; - - let canonical_hash = provider.block_hash(number)?; - if Some(indexed) != canonical_hash { - debug!( - target: TARGET, - number, - ?indexed, - ?canonical_hash, - "canonical hash mismatch during reconcile" - ); - fork_height = Some(number); - break; + if ancestor.is_some_and(|ancestor| cursor <= ancestor) { + return Ok(()); } - } - - // ── Step B: apply reorg if detected ────────────────────────────────────── - let rebuild_start = if let Some(fh) = fork_height { - info!( - target: TARGET, - fork_height = fh, - old_indexed_to = indexed_to, - "offline reorg detected; rolling back index" - ); + let lower_bound = ancestor.map_or(0, |ancestor| ancestor.saturating_add(1)); + let batch_start = cursor + .saturating_sub(batch_size.saturating_sub(1)) + .max(lower_bound); let tx = db.tx_mut()?; - for number in fh..=indexed_to { + for number in batch_start..=cursor { delete_block(&tx, number)?; } - let new_indexed_to = fh.saturating_sub(1); - update_indexed_to(&tx, new_indexed_to)?; - tx.commit()?; - - fh - } else { - indexed_to.saturating_add(1) - }; - - // ── Step C: suffix gap fill ─────────────────────────────────────────────── - if rebuild_start <= current_head { - info!( - target: TARGET, - rebuild_start, - current_head, - "filling reference index suffix gap" - ); - let tx = db.tx_mut()?; - for number in rebuild_start..=current_head { - // `WithHash` is required: we persist tx hashes as part of the index keys. - let block = provider - .sealed_block_with_senders(number.into(), TransactionVariant::WithHash)? - .ok_or_else(|| { - ReferenceIndexError::Other(eyre::eyre!( - "missing block {number} during reconcile" - )) - })?; - - write_block( - &tx, - block.number(), - block.hash(), - block.timestamp(), - &block.body().transactions, - )?; + if batch_start == lower_bound { + if let Some(ancestor) = ancestor { + update_indexed_to(&tx, ancestor)?; + } else { + clear_indexed_to(&tx)?; + } + } else { + update_indexed_to(&tx, batch_start - 1)?; } - update_indexed_to(&tx, current_head)?; tx.commit()?; } - - Ok(()) } diff --git a/crates/reference-index/src/runtime.rs b/crates/reference-index/src/runtime.rs new file mode 100644 index 00000000..112cc71a --- /dev/null +++ b/crates/reference-index/src/runtime.rs @@ -0,0 +1,787 @@ +//! Background canonical-cursor runtime for the Morph transaction reference index. + +use crate::{ + CanonicalChain, DEFAULT_BACKFILL_BATCH_BLOCKS, LIVE_LAG_SLO, ReferenceIndexError, + ReferenceIndexHandle, ReferenceIndexPhase, + backfill::{commit_canonical_batch, resolve_jade_first_block}, + db::ReferenceIndexDb, + reconcile::rollback_suffix, +}; +use alloy_primitives::B256; +use morph_primitives::MorphPrimitives; +use reth_chain_state::{CanonStateNotification, CanonStateNotifications}; +use reth_tasks::TaskExecutor; +use std::{ + collections::VecDeque, + path::{Path, PathBuf}, + time::{Duration, Instant}, +}; +use tokio::sync::broadcast::{ + Receiver, + error::{RecvError, TryRecvError}, +}; +use tokio::time::MissedTickBehavior; +use tracing::{debug, info, warn}; + +const RAPID_SYNC_BLOCK_THRESHOLD: u64 = 64; +const RAPID_SYNC_WINDOW: Duration = Duration::from_secs(2); +const RAPID_SYNC_QUIET_PERIOD: Duration = Duration::from_secs(5); +const RECONCILE_POLL_INTERVAL: Duration = Duration::from_secs(1); +const MAX_RETRY_BACKOFF: Duration = Duration::from_secs(30); +const TARGET: &str = "morph::reference_index"; + +#[derive(Debug)] +struct RapidSyncDetector { + observations: VecDeque<(Instant, u64)>, + blocks_in_window: u64, + defer_until: Option, +} + +impl RapidSyncDetector { + fn new() -> Self { + Self { + observations: VecDeque::new(), + blocks_in_window: 0, + defer_until: None, + } + } + + fn record_blocks(&mut self, now: Instant, blocks: usize) { + if blocks == 0 { + return; + } + while self.observations.front().is_some_and(|(observed_at, _)| { + now.saturating_duration_since(*observed_at) > RAPID_SYNC_WINDOW + }) { + let (_, expired_blocks) = self.observations.pop_front().expect("front was present"); + self.blocks_in_window = self.blocks_in_window.saturating_sub(expired_blocks); + } + let already_deferred = self.should_defer(now); + self.observations.push_back((now, blocks as u64)); + self.blocks_in_window = self.blocks_in_window.saturating_add(blocks as u64); + if already_deferred || self.blocks_in_window > RAPID_SYNC_BLOCK_THRESHOLD { + self.defer_until = Some(now + RAPID_SYNC_QUIET_PERIOD); + } + } + + fn record_lagged(&mut self, now: Instant) { + self.defer_until = Some(now + RAPID_SYNC_QUIET_PERIOD); + } + + fn should_defer(&self, now: Instant) -> bool { + self.defer_until.is_some_and(|until| now < until) + } +} + +#[derive(Debug)] +enum PendingBroadcastEvent { + Item(T), + Lagged(u64), +} + +/// Drain notifications already queued before starting another blocking batch. +fn drain_ready_broadcast( + receiver: &mut Receiver, + mut on_event: impl FnMut(PendingBroadcastEvent), +) -> bool { + loop { + match receiver.try_recv() { + Ok(item) => on_event(PendingBroadcastEvent::Item(item)), + Err(TryRecvError::Lagged(skipped)) => { + on_event(PendingBroadcastEvent::Lagged(skipped)); + } + Err(TryRecvError::Empty) => return true, + Err(TryRecvError::Closed) => return false, + } + } +} + +fn observe_canonical_notification( + notification: CanonStateNotification, + detector: &mut RapidSyncDetector, +) { + let changed_blocks = + notification.committed().len() + notification.reverted().map_or(0, |chain| chain.len()); + detector.record_blocks(Instant::now(), changed_blocks); + debug!(target: TARGET, changed_blocks, "canonical notification woke reference index"); +} + +fn observe_lagged_notification( + skipped: u64, + detector: &mut RapidSyncDetector, + handle: &ReferenceIndexHandle, +) { + handle.metrics().lagged_notifications_total.increment(1); + detector.record_lagged(Instant::now()); + warn!(target: TARGET, skipped, "reference index lagged canonical notifications; reconciling from provider"); +} + +/// Immutable runtime configuration derived from the node configuration. +#[derive(Debug, Clone)] +pub struct ReferenceIndexConfig { + path: PathBuf, + chain_id: u64, + genesis_hash: B256, + jade_timestamp: u64, +} + +impl ReferenceIndexConfig { + /// Create the immutable identity and activation configuration for one index DB. + pub fn new( + path: impl AsRef, + chain_id: u64, + genesis_hash: B256, + jade_timestamp: u64, + ) -> Self { + Self { + path: path.as_ref().to_path_buf(), + chain_id, + genesis_hash, + jade_timestamp, + } + } +} + +/// Single-writer runtime that reconciles a durable cursor with the canonical chain. +pub struct ReferenceIndexRuntime { + config: ReferenceIndexConfig, + chain: C, + handle: ReferenceIndexHandle, +} + +impl ReferenceIndexRuntime { + /// Create a single-writer runtime and its cloneable RPC query handle. + pub fn new(config: ReferenceIndexConfig, chain: C) -> (Self, ReferenceIndexHandle) { + let handle = ReferenceIndexHandle::new(config.jade_timestamp); + ( + Self { + config, + chain, + handle: handle.clone(), + }, + handle, + ) + } + + fn open_db(&self) -> Result { + if let Some(db) = self.handle.db() { + return Ok(db); + } + let db = ReferenceIndexDb::open( + &self.config.path, + self.config.chain_id, + self.config.genesis_hash, + )?; + self.handle.install_db(db.clone()); + Ok(db) + } + + fn reconcile_cursor( + &self, + db: &ReferenceIndexDb, + jade_first_block: u64, + head: crate::CanonicalTip, + ) -> Result<(), ReferenceIndexError> { + let Some(cursor) = db.indexed_to()? else { + return Ok(()); + }; + + let candidate = cursor.min(head.number); + let indexed_hash = db.indexed_block_hash(candidate)?.ok_or_else(|| { + ReferenceIndexError::CorruptMetadata( + "durable cursor range is missing an IndexedBlocks hash", + ) + })?; + if cursor == head.number && indexed_hash == head.hash { + return Ok(()); + } + if cursor < head.number && self.chain.canonical_hash(cursor)? == Some(indexed_hash) { + return Ok(()); + } + + self.handle.set_phase(ReferenceIndexPhase::Repairing); + self.handle.metrics().reorgs_total.increment(1); + let mut ancestor = None; + let mut number = candidate; + loop { + let indexed_hash = db.indexed_block_hash(number)?.ok_or_else(|| { + ReferenceIndexError::CorruptMetadata( + "indexed canonical range contains a block-hash gap", + ) + })?; + if self.chain.canonical_hash(number)? == Some(indexed_hash) { + ancestor = Some(number); + break; + } + if number == jade_first_block { + break; + } + number -= 1; + } + + let ancestor = + ancestor.ok_or(ReferenceIndexError::ManualRebuildRequired { jade_first_block })?; + rollback_suffix(db, Some(ancestor), DEFAULT_BACKFILL_BATCH_BLOCKS) + } + + /// Perform one synchronous reconciliation/backfill turn. + pub fn synchronize_once(&mut self, defer: bool) -> Result<(), ReferenceIndexError> { + let started_at = Instant::now(); + let result = self.synchronize_once_inner(defer); + self.handle + .metrics() + .batch_duration_seconds + .record(started_at.elapsed()); + if result.is_err() { + self.handle.metrics().failures_total.increment(1); + self.handle.set_phase(ReferenceIndexPhase::Unavailable); + } + result + } + + fn synchronize_once_inner(&mut self, defer: bool) -> Result<(), ReferenceIndexError> { + let db = self.open_db()?; + let head = self.chain.head()?; + self.handle.metrics().target_block.set(head.number as f64); + if head.timestamp < self.config.jade_timestamp { + self.handle.metrics().indexed_block.set(0.0); + self.handle.metrics().lag_blocks.set(0.0); + self.handle.set_phase(ReferenceIndexPhase::PreJade); + return Ok(()); + } + if defer { + let indexed_to = db.indexed_to()?.unwrap_or_default(); + self.handle.metrics().indexed_block.set(indexed_to as f64); + self.handle + .metrics() + .lag_blocks + .set(head.number.saturating_sub(indexed_to) as f64); + self.handle.set_phase(ReferenceIndexPhase::Deferred); + return Ok(()); + } + + let jade_first_block = match db.jade_first_block_number()? { + Some(block) => block, + None => resolve_jade_first_block(&self.chain, head, self.config.jade_timestamp)? + .ok_or_else(|| { + ReferenceIndexError::Other(eyre::eyre!( + "Jade is active at the canonical head but its first block was not found" + )) + })?, + }; + self.reconcile_cursor(&db, jade_first_block, head)?; + let start = db + .indexed_to()? + .map_or(jade_first_block, |cursor| cursor.saturating_add(1)); + if start > head.number { + self.handle.metrics().indexed_block.set(head.number as f64); + self.handle.metrics().lag_blocks.set(0.0); + self.handle.set_phase(ReferenceIndexPhase::Live); + return Ok(()); + } + + self.handle.set_phase(ReferenceIndexPhase::Backfill); + let end = start + .saturating_add(DEFAULT_BACKFILL_BATCH_BLOCKS - 1) + .min(head.number); + let blocks = self.chain.canonical_blocks(start..=end)?; + let expected_len = end.saturating_sub(start).saturating_add(1) as usize; + if blocks.len() != expected_len { + return Err(ReferenceIndexError::Other(eyre::eyre!( + "canonical block range {start}..={end} returned {} of {expected_len} blocks", + blocks.len() + ))); + } + for (offset, block) in blocks.iter().enumerate() { + let expected_number = start + offset as u64; + let canonical_hash = self.chain.canonical_hash(expected_number)?; + if block.number != expected_number || canonical_hash != Some(block.hash) { + return Err(ReferenceIndexError::Other(eyre::eyre!( + "canonical block changed while reading backfill batch at {expected_number}" + ))); + } + } + commit_canonical_batch(&db, jade_first_block, start, &blocks)?; + self.handle + .metrics() + .indexed_blocks_total + .increment(blocks.len() as u64); + self.handle.metrics().committed_batches_total.increment(1); + + let current_head = self.chain.head()?; + self.handle + .metrics() + .target_block + .set(current_head.number as f64); + self.handle.metrics().indexed_block.set(end as f64); + let lag = current_head.number.saturating_sub(end); + self.handle.metrics().lag_blocks.set(lag as f64); + if lag > LIVE_LAG_SLO { + debug!(target: TARGET, lag, slo = LIVE_LAG_SLO, "reference index remains behind its live-lag SLO"); + } + if end == current_head.number + && blocks + .last() + .is_some_and(|block| block.hash == current_head.hash) + { + self.handle.set_phase(ReferenceIndexPhase::Live); + } + Ok(()) + } + + /// Run the non-critical background reconciler until node shutdown. + /// + /// Notifications are wake-ups and rate signals only. Canonical data is + /// always re-read through [`CanonicalChain`] before a durable commit. + pub async fn run( + mut self, + task_executor: TaskExecutor, + mut notifications: CanonStateNotifications, + ) { + let handle = self.handle.clone(); + let mut poll = tokio::time::interval(RECONCILE_POLL_INTERVAL); + poll.set_missed_tick_behavior(MissedTickBehavior::Skip); + // Consume tokio's immediate first tick so a queued canonical + // notification can classify rapid sync before the first DB batch. + poll.tick().await; + let mut detector = RapidSyncDetector::new(); + let mut notifications_open = true; + let mut sync_requested = false; + let mut retry_backoff = Duration::from_secs(1); + let mut logged_phase = handle.phase(); + + loop { + if notifications_open { + notifications_open = drain_ready_broadcast(&mut notifications, |event| { + sync_requested = true; + match event { + PendingBroadcastEvent::Item(notification) => { + observe_canonical_notification(notification, &mut detector); + } + PendingBroadcastEvent::Lagged(skipped) => { + observe_lagged_notification(skipped, &mut detector, &handle); + } + } + }); + if !notifications_open { + warn!(target: TARGET, "canonical notification channel closed; reference index falling back to polling"); + } + } + if sync_requested + && detector.should_defer(Instant::now()) + && handle.phase() == ReferenceIndexPhase::Deferred + { + // Stay completely off the index DB while rapid historical + // sync is still active. Notifications continue to extend the + // quiet-period deadline; polling resumes catch-up afterwards. + sync_requested = false; + } + if sync_requested { + let defer = detector.should_defer(Instant::now()); + let job = task_executor.spawn_blocking(move || { + let result = self.synchronize_once(defer); + (self, result) + }); + let (returned_runtime, result) = match job.await { + Ok(output) => output, + Err(error) if error.is_cancelled() => { + debug!(target: TARGET, "reference index blocking task cancelled during shutdown"); + return; + } + Err(error) => { + handle.set_phase(ReferenceIndexPhase::Unavailable); + warn!(target: TARGET, %error, "reference index blocking task stopped unexpectedly"); + return; + } + }; + self = returned_runtime; + + match result { + Ok(()) => { + retry_backoff = Duration::from_secs(1); + let phase = handle.phase(); + if phase != logged_phase { + info!(target: TARGET, ?phase, "reference index phase changed"); + logged_phase = phase; + } + sync_requested = phase == ReferenceIndexPhase::Backfill; + if sync_requested { + tokio::task::yield_now().await; + // Loop back through the non-blocking receiver drain before + // starting another bounded backfill batch. + continue; + } + } + Err(error) => { + if error.requires_manual_rebuild() { + warn!(target: TARGET, %error, "reference index requires manual rebuild; execution continues"); + return; + } + warn!(target: TARGET, %error, ?retry_backoff, "reference index reconciliation failed; execution continues"); + tokio::time::sleep(retry_backoff).await; + retry_backoff = retry_backoff.saturating_mul(2).min(MAX_RETRY_BACKOFF); + sync_requested = true; + continue; + } + } + } + + tokio::select! { + biased; + notification = notifications.recv(), if notifications_open => { + sync_requested = true; + match notification { + Ok(notification) => { + observe_canonical_notification(notification, &mut detector); + } + Err(RecvError::Lagged(skipped)) => { + observe_lagged_notification(skipped, &mut detector, &handle); + } + Err(RecvError::Closed) => { + notifications_open = false; + warn!(target: TARGET, "canonical notification channel closed; reference index falling back to polling"); + } + } + } + _ = poll.tick() => { + sync_requested = true; + } + } + } + } +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::{CanonicalBlock, CanonicalChain, CanonicalTip, ReferenceQuery}; + use alloy_primitives::B256; + use parking_lot::RwLock; + use std::{ + ops::RangeInclusive, + sync::Arc, + time::{Duration, Instant}, + }; + + #[test] + fn rapid_sync_detector_defers_after_more_than_64_blocks_and_recovers_after_quiet_period() { + let start = Instant::now(); + let mut detector = RapidSyncDetector::new(); + + detector.record_blocks(start + Duration::from_secs(1), 64); + assert!(!detector.should_defer(start + Duration::from_secs(1))); + + detector.record_blocks(start + Duration::from_millis(1500), 1); + assert!(detector.should_defer(start + Duration::from_secs(4))); + assert!(!detector.should_defer(start + Duration::from_millis(6501))); + } + + #[test] + fn rapid_sync_detector_uses_a_sliding_two_second_window() { + let start = Instant::now(); + let mut detector = RapidSyncDetector::new(); + + detector.record_blocks(start + Duration::from_millis(1900), 32); + detector.record_blocks(start + Duration::from_millis(2100), 33); + + assert!(detector.should_defer(start + Duration::from_secs(3))); + } + + #[test] + fn deferred_sync_waits_five_seconds_after_low_rate_tail_activity() { + let start = Instant::now(); + let mut detector = RapidSyncDetector::new(); + + detector.record_blocks(start, 65); + detector.record_blocks(start + Duration::from_secs(3), 1); + + assert!(detector.should_defer(start + Duration::from_millis(7999))); + assert!(!detector.should_defer(start + Duration::from_millis(8001))); + } + + #[test] + fn lagged_notifications_enter_the_same_quiet_period() { + let start = Instant::now(); + let mut detector = RapidSyncDetector::new(); + + detector.record_lagged(start); + + assert!(detector.should_defer(start + Duration::from_secs(4))); + assert!(!detector.should_defer(start + Duration::from_millis(5001))); + } + + #[test] + fn queued_notifications_are_drained_before_the_next_batch() { + let (sender, mut receiver) = tokio::sync::broadcast::channel(128); + for _ in 0..65 { + sender.send(1usize).unwrap(); + } + let now = Instant::now(); + let mut detector = RapidSyncDetector::new(); + let mut received = 0usize; + + let remains_open = drain_ready_broadcast(&mut receiver, |event| match event { + PendingBroadcastEvent::Item(blocks) => { + received += 1; + detector.record_blocks(now, blocks); + } + PendingBroadcastEvent::Lagged(_) => panic!("receiver should not lag"), + }); + + assert!(remains_open); + assert_eq!(received, 65); + assert!(detector.should_defer(now)); + } + + #[derive(Clone, Debug)] + struct TestChain { + blocks: Arc>>, + } + + impl TestChain { + fn linear(count: u64, first_timestamp: u64) -> Self { + let blocks = (0..count) + .map(|number| CanonicalBlock { + number, + hash: B256::with_last_byte(number as u8), + timestamp: first_timestamp + number, + transactions: Vec::new(), + }) + .collect(); + Self { + blocks: Arc::new(RwLock::new(blocks)), + } + } + + fn replace_suffix(&self, from: u64) { + for block in self.blocks.write().iter_mut().skip(from as usize) { + block.hash = B256::repeat_byte(0x80 | block.number as u8); + } + } + + fn truncate(&self, len: u64) { + self.blocks.write().truncate(len as usize); + } + } + + impl CanonicalChain for TestChain { + fn head(&self) -> Result { + let blocks = self.blocks.read(); + let block = blocks.last().unwrap(); + Ok(CanonicalTip { + number: block.number, + hash: block.hash, + timestamp: block.timestamp, + }) + } + + fn canonical_hash(&self, number: u64) -> Result, crate::ReferenceIndexError> { + Ok(self + .blocks + .read() + .get(number as usize) + .map(|block| block.hash)) + } + + fn block_timestamp(&self, number: u64) -> Result, crate::ReferenceIndexError> { + Ok(self + .blocks + .read() + .get(number as usize) + .map(|block| block.timestamp)) + } + + fn canonical_blocks( + &self, + range: RangeInclusive, + ) -> Result, crate::ReferenceIndexError> { + let blocks = self.blocks.read(); + Ok(range + .filter_map(|number| blocks.get(number as usize).cloned()) + .collect()) + } + } + + #[test] + fn pre_jade_sync_serves_empty_results_without_a_cursor() { + let dir = tempfile::tempdir().unwrap(); + let chain = TestChain::linear(10, 100); + let config = ReferenceIndexConfig::new(dir.path(), 2818, B256::ZERO, 200); + let (mut runtime, handle) = ReferenceIndexRuntime::new(config, chain.clone()); + + runtime.synchronize_once(false).unwrap(); + + assert_eq!(handle.phase(), crate::ReferenceIndexPhase::PreJade); + let query = ReferenceQuery::new(B256::with_last_byte(0xaa), None, None).unwrap(); + assert!( + handle + .query_at(query, chain.head().unwrap()) + .unwrap() + .is_empty() + ); + assert_eq!(handle.db().unwrap().indexed_to().unwrap(), None); + } + + #[test] + fn post_jade_backfill_commits_at_most_512_blocks_per_turn() { + let dir = tempfile::tempdir().unwrap(); + let chain = TestChain::linear(600, 200); + let config = ReferenceIndexConfig::new(dir.path(), 2818, B256::ZERO, 200); + let (mut runtime, handle) = ReferenceIndexRuntime::new(config, chain.clone()); + + runtime.synchronize_once(false).unwrap(); + assert_eq!(handle.phase(), crate::ReferenceIndexPhase::Backfill); + assert_eq!(handle.db().unwrap().indexed_to().unwrap(), Some(511)); + + runtime.synchronize_once(false).unwrap(); + assert_eq!(handle.phase(), crate::ReferenceIndexPhase::Live); + assert_eq!(handle.db().unwrap().indexed_to().unwrap(), Some(599)); + + let query = ReferenceQuery::new(B256::with_last_byte(0xaa), None, None).unwrap(); + assert!( + handle + .query_at(query, chain.head().unwrap()) + .unwrap() + .is_empty() + ); + } + + #[test] + fn an_idle_reconciliation_keeps_the_live_snapshot_stable() { + let dir = tempfile::tempdir().unwrap(); + let chain = TestChain::linear(10, 200); + let config = ReferenceIndexConfig::new(dir.path(), 2818, B256::ZERO, 200); + let (mut runtime, handle) = ReferenceIndexRuntime::new(config, chain); + runtime.synchronize_once(false).unwrap(); + let generation = handle.generation(); + + runtime.synchronize_once(false).unwrap(); + + assert_eq!(handle.phase(), ReferenceIndexPhase::Live); + assert_eq!(handle.generation(), generation); + } + + #[test] + fn reorged_suffix_is_rolled_back_and_replayed() { + let dir = tempfile::tempdir().unwrap(); + let chain = TestChain::linear(10, 200); + let config = ReferenceIndexConfig::new(dir.path(), 2818, B256::ZERO, 200); + let (mut runtime, handle) = ReferenceIndexRuntime::new(config, chain.clone()); + runtime.synchronize_once(false).unwrap(); + assert_eq!(handle.phase(), crate::ReferenceIndexPhase::Live); + + chain.replace_suffix(7); + runtime.synchronize_once(false).unwrap(); + + assert_eq!(handle.phase(), crate::ReferenceIndexPhase::Live); + let query = ReferenceQuery::new(B256::with_last_byte(0xaa), None, None).unwrap(); + assert!( + handle + .query_at(query, chain.head().unwrap()) + .unwrap() + .is_empty() + ); + assert_eq!( + handle.db().unwrap().indexed_block_hash(7).unwrap(), + chain.canonical_hash(7).unwrap() + ); + } + + #[test] + fn pure_revert_rolls_the_cursor_back_to_the_new_head() { + let dir = tempfile::tempdir().unwrap(); + let chain = TestChain::linear(10, 200); + let config = ReferenceIndexConfig::new(dir.path(), 2818, B256::ZERO, 200); + let (mut runtime, handle) = ReferenceIndexRuntime::new(config, chain.clone()); + runtime.synchronize_once(false).unwrap(); + + chain.truncate(7); + runtime.synchronize_once(false).unwrap(); + + assert_eq!(handle.phase(), ReferenceIndexPhase::Live); + assert_eq!(handle.db().unwrap().indexed_to().unwrap(), Some(6)); + assert_eq!( + handle.db().unwrap().indexed_block_hash(6).unwrap(), + chain.canonical_hash(6).unwrap() + ); + } + + #[test] + fn reorg_without_a_post_jade_ancestor_requires_a_manual_rebuild() { + let dir = tempfile::tempdir().unwrap(); + let chain = TestChain::linear(10, 200); + let config = ReferenceIndexConfig::new(dir.path(), 2818, B256::ZERO, 200); + let (mut runtime, handle) = ReferenceIndexRuntime::new(config, chain.clone()); + runtime.synchronize_once(false).unwrap(); + + chain.replace_suffix(0); + let error = runtime.synchronize_once(false).unwrap_err(); + + assert!(matches!( + error, + ReferenceIndexError::ManualRebuildRequired { + jade_first_block: 0 + } + )); + assert_eq!(handle.phase(), ReferenceIndexPhase::Unavailable); + assert!( + dir.path().exists(), + "runtime must never delete the index DB" + ); + assert_eq!(handle.db().unwrap().indexed_to().unwrap(), Some(9)); + } + + #[derive(Clone, Debug)] + struct ShortRangeChain(TestChain); + + impl CanonicalChain for ShortRangeChain { + fn head(&self) -> Result { + self.0.head() + } + + fn canonical_hash(&self, number: u64) -> Result, ReferenceIndexError> { + self.0.canonical_hash(number) + } + + fn block_timestamp(&self, number: u64) -> Result, ReferenceIndexError> { + self.0.block_timestamp(number) + } + + fn canonical_blocks( + &self, + range: RangeInclusive, + ) -> Result, ReferenceIndexError> { + let mut blocks = self.0.canonical_blocks(range)?; + blocks.pop(); + Ok(blocks) + } + } + + #[test] + fn incomplete_provider_batch_does_not_advance_the_cursor() { + let dir = tempfile::tempdir().unwrap(); + let chain = ShortRangeChain(TestChain::linear(10, 200)); + let config = ReferenceIndexConfig::new(dir.path(), 2818, B256::ZERO, 200); + let (mut runtime, handle) = ReferenceIndexRuntime::new(config, chain); + + let error = runtime.synchronize_once(false).unwrap_err(); + + assert!(error.to_string().contains("returned 9 of 10 blocks")); + assert_eq!(handle.phase(), ReferenceIndexPhase::Unavailable); + assert_eq!(handle.db().unwrap().indexed_to().unwrap(), None); + } + + #[test] + fn deferred_sync_performs_no_per_block_index_writes() { + let dir = tempfile::tempdir().unwrap(); + let chain = TestChain::linear(10, 200); + let config = ReferenceIndexConfig::new(dir.path(), 2818, B256::ZERO, 200); + let (mut runtime, handle) = ReferenceIndexRuntime::new(config, chain); + + runtime.synchronize_once(true).unwrap(); + + assert_eq!(handle.phase(), ReferenceIndexPhase::Deferred); + assert_eq!(handle.db().unwrap().indexed_to().unwrap(), None); + assert_eq!(handle.db().unwrap().indexed_block_hash(0).unwrap(), None); + } +} diff --git a/crates/reference-index/src/source.rs b/crates/reference-index/src/source.rs new file mode 100644 index 00000000..1040b756 --- /dev/null +++ b/crates/reference-index/src/source.rs @@ -0,0 +1,86 @@ +//! Canonical-chain read seam used by the reference-index runtime. + +use crate::{CanonicalTip, ReferenceIndexError}; +use alloy_consensus::{BlockHeader, Sealable}; +use alloy_primitives::B256; +use morph_primitives::MorphTxEnvelope; +use reth_provider::{BlockHashReader, BlockReader, BlockReaderIdExt, HeaderProvider}; +use std::ops::RangeInclusive; + +/// Canonical block fields required to build the reference index. +#[derive(Debug, Clone)] +pub struct CanonicalBlock { + /// Canonical block number. + pub number: u64, + /// Header hash observed for this canonical block. + pub hash: B256, + /// Header timestamp. + pub timestamp: u64, + /// Transactions in their canonical block order. + pub transactions: Vec, +} + +/// Minimal canonical-chain API consumed by the runtime. +pub trait CanonicalChain: Clone + Send + Sync + 'static { + /// Return one internally consistent canonical head snapshot. + fn head(&self) -> Result; + + /// Return the canonical hash at `number`, if the block is retained. + fn canonical_hash(&self, number: u64) -> Result, ReferenceIndexError>; + + /// Return the canonical timestamp at `number`, if the header is retained. + fn block_timestamp(&self, number: u64) -> Result, ReferenceIndexError>; + + /// Load an inclusive range of canonical blocks with their transaction bodies. + fn canonical_blocks( + &self, + range: RangeInclusive, + ) -> Result, ReferenceIndexError>; +} + +impl

CanonicalChain for P +where + P: BlockReaderIdExt + + Clone + + Send + + Sync + + 'static, +{ + fn head(&self) -> Result { + let header = BlockReaderIdExt::latest_header(self)? + .ok_or_else(|| ReferenceIndexError::Other(eyre::eyre!("missing canonical head")))?; + Ok(CanonicalTip { + number: header.number(), + hash: header.hash(), + timestamp: header.timestamp(), + }) + } + + fn canonical_hash(&self, number: u64) -> Result, ReferenceIndexError> { + Ok(BlockHashReader::block_hash(self, number)?) + } + + fn block_timestamp(&self, number: u64) -> Result, ReferenceIndexError> { + Ok(HeaderProvider::header_by_number(self, number)?.map(|header| header.timestamp())) + } + + fn canonical_blocks( + &self, + range: RangeInclusive, + ) -> Result, ReferenceIndexError> { + BlockReader::block_range(self, range)? + .into_iter() + .map(|block| { + let number = block.header.number(); + let timestamp = block.header.timestamp(); + let hash = block.header.hash_slow(); + Ok(CanonicalBlock { + number, + hash, + timestamp, + transactions: block.body.transactions, + }) + }) + .collect() + } +} diff --git a/crates/reference-index/src/tables.rs b/crates/reference-index/src/tables.rs index 43b6e586..4cd8bf88 100644 --- a/crates/reference-index/src/tables.rs +++ b/crates/reference-index/src/tables.rs @@ -1,5 +1,7 @@ //! Reference index table declarations. +#![allow(unreachable_pub)] // `tables!` generates public marker types inside this private module. + use alloy_primitives::B256; use reth_codecs::DecompressError; use reth_db_api::{ diff --git a/crates/reference-index/src/types.rs b/crates/reference-index/src/types.rs index e8a060f7..3bfa3a50 100644 --- a/crates/reference-index/src/types.rs +++ b/crates/reference-index/src/types.rs @@ -1,33 +1,22 @@ use alloy_primitives::{B256, U64}; use serde::{Deserialize, Serialize}; -/// Current reference index database schema version. -pub const SCHEMA_VERSION: u32 = 1; - -/// Stored Jade activation sentinel for chains where Jade has not activated. -pub const JADE_NOT_ACTIVE_SENTINEL: u64 = u64::MAX; - -/// Persistent backfill progress state. -#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)] -#[repr(u8)] -pub enum BackfillState { - NotStarted = 0, - InProgress = 1, - Complete = 2, +/// Canonical chain snapshot used to linearize reference-index queries. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub struct CanonicalTip { + /// Canonical head block number. + pub number: u64, + /// Canonical head block hash. + pub hash: B256, + /// Canonical head timestamp. + pub timestamp: u64, } -impl TryFrom for BackfillState { - type Error = ReferenceIndexError; - - fn try_from(value: u8) -> Result { - match value { - 0 => Ok(Self::NotStarted), - 1 => Ok(Self::InProgress), - 2 => Ok(Self::Complete), - other => Err(ReferenceIndexError::InvalidBackfillState(other)), - } - } -} +/// Current reference index database schema version. +/// +/// Version 1 was used only by the pre-release ExEx prototype. There is no +/// migration path: operators remove that derived database manually. +pub const SCHEMA_VERSION: u32 = 2; /// Validated query parameters for reference lookups. #[derive(Debug, Clone, Copy, PartialEq, Eq)] @@ -38,10 +27,14 @@ pub struct ReferenceQuery { } impl ReferenceQuery { + /// Default page size. pub const DEFAULT_LIMIT: u64 = 100; + /// Maximum accepted page size. pub const MAX_LIMIT: u64 = 100; + /// Maximum accepted result offset. pub const MAX_OFFSET: u64 = 10_000; + /// Validate and normalize public RPC query arguments. pub fn new( reference: B256, offset: Option, @@ -69,9 +62,13 @@ impl ReferenceQuery { #[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] #[serde(rename_all = "camelCase")] pub struct ReferenceTransactionResult { + /// Hash of the matching Morph transaction. pub transaction_hash: B256, + /// Canonical block number containing the transaction. pub block_number: U64, + /// Timestamp of the containing canonical block. pub block_timestamp: U64, + /// Zero-based transaction position within the block. pub transaction_index: U64, } @@ -82,18 +79,24 @@ pub enum ReferenceIndexError { Initializing, #[error("reference index is behind")] IndexBehind, + #[error("reference index is unavailable")] + IndexUnavailable, #[error("reference query limit too large: {limit}")] LimitTooLarge { limit: u64 }, #[error("reference query offset too large: {offset}")] OffsetTooLarge { offset: u64 }, - #[error("invalid backfill state: {0}")] - InvalidBackfillState(u8), #[error("reference index backfill batch size must be greater than zero")] InvalidBackfillBatchSize, + #[error("reference index metadata is incomplete: {0}")] + CorruptMetadata(&'static str), #[error("reference index chain identity mismatch: {0}")] ChainIdentityMismatch(&'static str), #[error("reference index schema mismatch: expected {expected}, got {actual}")] SchemaMismatch { expected: u32, actual: u32 }, + #[error( + "reference index has no canonical ancestor at or after Jade block {jade_first_block}; manual rebuild required" + )] + ManualRebuildRequired { jade_first_block: u64 }, #[error(transparent)] Database(#[from] reth_db_api::DatabaseError), #[error(transparent)] @@ -101,3 +104,16 @@ pub enum ReferenceIndexError { #[error(transparent)] Other(#[from] eyre::Report), } + +impl ReferenceIndexError { + /// Errors that cannot be repaired while the existing derived DB remains. + pub(crate) fn requires_manual_rebuild(&self) -> bool { + matches!( + self, + Self::CorruptMetadata(_) + | Self::ChainIdentityMismatch(_) + | Self::SchemaMismatch { .. } + | Self::ManualRebuildRequired { .. } + ) + } +} diff --git a/crates/reference-index/src/writer.rs b/crates/reference-index/src/writer.rs index 9ce37108..936a59fc 100644 --- a/crates/reference-index/src/writer.rs +++ b/crates/reference-index/src/writer.rs @@ -8,10 +8,9 @@ use crate::{ db::{IndexMetaKey, encode_u64}, tables::{ BlockHashValue, BlockReferenceIndex, BlockReferenceKey, BlockTimestampValue, IndexMeta, - IndexedBlockKey, IndexedBlocks, MetaValue, ReferenceIndex, ReferenceIndexKey, - ReferenceValue, + IndexedBlockKey, IndexedBlocks, ReferenceIndex, ReferenceIndexKey, ReferenceValue, }, - types::{BackfillState, ReferenceIndexError}, + types::ReferenceIndexError, }; use alloy_consensus::transaction::TxHashRef; use alloy_primitives::B256; @@ -26,7 +25,7 @@ use reth_db_api::{cursor::DbCursorRO, transaction::DbTxMut}; /// the same block number before re-writing to avoid leaving stale entries /// (keys contain `tx_hash`, so a re-write with a different tx set would leave /// old-tx ghost rows). Reconcile and reorg paths follow this contract. -pub fn write_block( +pub(crate) fn write_block( tx: &Tx, block_number: u64, block_hash: B256, @@ -69,7 +68,10 @@ pub fn write_block( /// Implements the reverse of [`write_block`]: reads every /// `BlockReferenceIndex` row for `block_number`, reconstructs each /// `ReferenceIndex` key, and removes entries from all three tables. -pub fn delete_block(tx: &Tx, block_number: u64) -> Result<(), ReferenceIndexError> { +pub(crate) fn delete_block( + tx: &Tx, + block_number: u64, +) -> Result<(), ReferenceIndexError> { // 1. Collect all BlockReferenceIndex rows for this block. let mut entries = Vec::new(); { @@ -110,7 +112,7 @@ pub fn delete_block(tx: &Tx, block_number: u64) -> Result<(), Refer // ── metadata writes ────────────────────────────────────────────────────────── -pub fn update_indexed_to( +pub(crate) fn update_indexed_to( tx: &Tx, block_number: u64, ) -> Result<(), ReferenceIndexError> { @@ -118,37 +120,12 @@ pub fn update_indexed_to( Ok(()) } -pub fn update_indexed_from( - tx: &Tx, - block_number: u64, -) -> Result<(), ReferenceIndexError> { - tx.put::(IndexMetaKey::IndexedFrom.into(), encode_u64(block_number))?; +pub(crate) fn clear_indexed_to(tx: &Tx) -> Result<(), ReferenceIndexError> { + tx.delete::(IndexMetaKey::IndexedTo.into(), None)?; Ok(()) } -pub fn update_backfill_current( - tx: &Tx, - block_number: u64, -) -> Result<(), ReferenceIndexError> { - tx.put::( - IndexMetaKey::BackfillCurrent.into(), - encode_u64(block_number), - )?; - Ok(()) -} - -pub fn set_backfill_state( - tx: &Tx, - state: BackfillState, -) -> Result<(), ReferenceIndexError> { - tx.put::( - IndexMetaKey::BackfillState.into(), - MetaValue(vec![state as u8]), - )?; - Ok(()) -} - -pub fn set_jade_first_block_number( +pub(crate) fn set_jade_first_block_number( tx: &Tx, block_number: u64, ) -> Result<(), ReferenceIndexError> { @@ -162,7 +139,7 @@ pub fn set_jade_first_block_number( #[cfg(test)] mod tests { use super::*; - use crate::ReferenceIndexDb; + use crate::db::ReferenceIndexDb; use alloy_consensus::transaction::TxEip1559; use alloy_primitives::{Address, Signature, TxKind, U256}; use morph_primitives::MorphTxEnvelope; @@ -209,7 +186,7 @@ mod tests { tx.commit().unwrap(); assert_eq!(written, 0); - assert_eq!(db.indexed_to().unwrap(), 42); + assert_eq!(db.indexed_to().unwrap(), Some(42)); assert_eq!( db.indexed_block_hash(42).unwrap(), Some(B256::repeat_byte(0xaa)) @@ -237,32 +214,32 @@ mod tests { } #[test] - fn metadata_updates_are_visible_after_commit() { + fn cursor_and_jade_activation_are_visible_after_commit() { let dir = TempDir::new().unwrap(); let db = ReferenceIndexDb::open(dir.path(), 2818, B256::ZERO).unwrap(); let tx = db.tx_mut().unwrap(); - update_indexed_from(&tx, 100).unwrap(); update_indexed_to(&tx, 200).unwrap(); - update_backfill_current(&tx, 150).unwrap(); - set_backfill_state(&tx, BackfillState::Complete).unwrap(); set_jade_first_block_number(&tx, 100).unwrap(); tx.commit().unwrap(); - assert_eq!(db.indexed_from().unwrap(), Some(100)); - assert_eq!(db.indexed_to().unwrap(), 200); - assert_eq!(db.backfill_state().unwrap(), BackfillState::Complete); + assert_eq!(db.indexed_to().unwrap(), Some(200)); assert_eq!(db.jade_first_block_number().unwrap(), Some(100)); + } - // Sanity: a fresh read transaction also sees the backfill_current we set. - let tx = db.tx().unwrap(); - let raw = tx - .get::(IndexMetaKey::BackfillCurrent.into()) - .unwrap() - .unwrap(); - assert_eq!( - u64::from_be_bytes(raw.0.as_slice().try_into().unwrap()), - 150 - ); + #[test] + fn clearing_cursor_restores_no_progress_state() { + let dir = TempDir::new().unwrap(); + let db = ReferenceIndexDb::open(dir.path(), 2818, B256::ZERO).unwrap(); + + let tx = db.tx_mut().unwrap(); + update_indexed_to(&tx, 7).unwrap(); + tx.commit().unwrap(); + assert_eq!(db.indexed_to().unwrap(), Some(7)); + + let tx = db.tx_mut().unwrap(); + clear_indexed_to(&tx).unwrap(); + tx.commit().unwrap(); + assert_eq!(db.indexed_to().unwrap(), None); } } diff --git a/crates/rpc/Cargo.toml b/crates/rpc/Cargo.toml index 07885dba..356a7cb7 100644 --- a/crates/rpc/Cargo.toml +++ b/crates/rpc/Cargo.toml @@ -61,6 +61,7 @@ tracing.workspace = true [dev-dependencies] serde_json.workspace = true +tempfile.workspace = true [features] default = [] diff --git a/crates/rpc/src/morph/handler.rs b/crates/rpc/src/morph/handler.rs index 6616ca0f..928ee90a 100644 --- a/crates/rpc/src/morph/handler.rs +++ b/crates/rpc/src/morph/handler.rs @@ -1,14 +1,16 @@ //! `MorphRpc` handler implementation. use crate::morph::rpc::{MorphRpcServer, ReferenceQueryArgs}; +use alloy_consensus::BlockHeader as _; use jsonrpsee::{ core::RpcResult, types::{ErrorCode, ErrorObjectOwned}, }; use morph_reference_index::{ - ReferenceIndexError, ReferenceIndexReader, ReferenceQuery, ReferenceTransactionResult, + CanonicalTip, ReferenceIndexError, ReferenceIndexHandle, ReferenceQuery, + ReferenceTransactionResult, }; -use reth_storage_api::BlockNumReader; +use reth_storage_api::{BlockNumReader, HeaderProvider}; use tracing; const TARGET: &str = "morph::reference_index_rpc"; @@ -17,16 +19,16 @@ const TARGET: &str = "morph::reference_index_rpc"; /// `morph_` namespace context. All dependencies are required; no `Option<>`. /// -/// `Provider` must implement [`BlockNumReader`] so the handler can compare the -/// current canonical tip against the index's `indexed_to` for lag detection. +/// `Provider` supplies an exact canonical tip snapshot (number, hash, and +/// timestamp) for strict index-read consistency. #[derive(Debug, Clone)] pub struct MorphRpc { - pub reference_index: ReferenceIndexReader, + pub reference_index: ReferenceIndexHandle, pub provider: Provider, } -impl MorphRpc { - pub const fn new(reference_index: ReferenceIndexReader, provider: Provider) -> Self { +impl MorphRpc { + pub const fn new(reference_index: ReferenceIndexHandle, provider: Provider) -> Self { Self { reference_index, provider, @@ -42,14 +44,15 @@ pub struct MorphRpcHandler { ctx: MorphRpc, } -impl MorphRpcHandler { +impl MorphRpcHandler { pub const fn new(ctx: MorphRpc) -> Self { Self { ctx } } } -impl MorphRpcServer - for MorphRpcHandler +impl MorphRpcServer for MorphRpcHandler +where + Provider: BlockNumReader + HeaderProvider + Clone + Send + Sync + 'static, { fn get_transaction_hashes_by_reference( &self, @@ -58,17 +61,46 @@ impl MorphRpcServer let query = ReferenceQuery::new(args.reference, args.offset, args.limit).map_err(to_rpc_error)?; - let canonical_tip = self + let before = self .ctx .provider - .best_block_number() + .chain_info() .map_err(ReferenceIndexError::from) .map_err(to_rpc_error)?; + let header = self + .ctx + .provider + .header(before.best_hash) + .map_err(ReferenceIndexError::from) + .map_err(to_rpc_error)? + .ok_or(ReferenceIndexError::IndexBehind) + .map_err(to_rpc_error)?; + if header.number() != before.best_number { + return Err(to_rpc_error(ReferenceIndexError::IndexBehind)); + } + let canonical_tip = CanonicalTip { + number: before.best_number, + hash: before.best_hash, + timestamp: header.timestamp(), + }; - self.ctx + let result = self + .ctx .reference_index - .query(query, canonical_tip) - .map_err(to_rpc_error) + .query_at(query, canonical_tip) + .map_err(to_rpc_error)?; + + let after = self + .ctx + .provider + .chain_info() + .map_err(ReferenceIndexError::from) + .map_err(to_rpc_error)?; + if after != before { + return Err(to_rpc_error(ReferenceIndexError::IndexBehind)); + } + + Ok(result) } } @@ -82,6 +114,9 @@ fn to_rpc_error(error: ReferenceIndexError) -> ErrorObjectOwned { ReferenceIndexError::IndexBehind => { ErrorObjectOwned::owned(-32000, "reference index is behind", None::<()>) } + ReferenceIndexError::IndexUnavailable => { + ErrorObjectOwned::owned(-32000, "reference index is unavailable", None::<()>) + } ReferenceIndexError::LimitTooLarge { .. } | ReferenceIndexError::OffsetTooLarge { .. } => { ErrorObjectOwned::owned( ErrorCode::InvalidParams.code(), @@ -105,3 +140,255 @@ fn to_rpc_error(error: ReferenceIndexError) -> ErrorObjectOwned { } } } + +#[cfg(test)] +mod tests { + use super::*; + use alloy_primitives::B256; + use morph_primitives::MorphHeader; + use morph_reference_index::{ + CanonicalBlock, CanonicalChain, CanonicalTip, ReferenceIndexConfig, ReferenceIndexHandle, + ReferenceIndexRuntime, + }; + use reth_chainspec::ChainInfo; + use reth_errors::ProviderResult; + use reth_primitives_traits::SealedHeader; + use reth_storage_api::{BlockHashReader, HeaderProvider}; + use std::{ + collections::{BTreeMap, VecDeque}, + ops::{RangeBounds, RangeInclusive}, + sync::{Arc, Mutex}, + }; + + #[derive(Clone, Debug)] + struct TestChain { + block: CanonicalBlock, + } + + impl CanonicalChain for TestChain { + fn head(&self) -> Result { + Ok(CanonicalTip { + number: self.block.number, + hash: self.block.hash, + timestamp: self.block.timestamp, + }) + } + + fn canonical_hash(&self, number: u64) -> Result, ReferenceIndexError> { + Ok((number == self.block.number).then_some(self.block.hash)) + } + + fn block_timestamp(&self, number: u64) -> Result, ReferenceIndexError> { + Ok((number == self.block.number).then_some(self.block.timestamp)) + } + + fn canonical_blocks( + &self, + range: RangeInclusive, + ) -> Result, ReferenceIndexError> { + Ok(range + .contains(&self.block.number) + .then(|| self.block.clone()) + .into_iter() + .collect()) + } + } + + #[derive(Clone, Debug)] + struct TestProvider { + chain_info: Arc>>, + headers: Arc>, + } + + impl TestProvider { + fn new( + chain_info: impl IntoIterator, + headers: BTreeMap, + ) -> Self { + Self { + chain_info: Arc::new(Mutex::new(chain_info.into_iter().collect())), + headers: Arc::new(headers), + } + } + + fn info(&self) -> ChainInfo { + let mut infos = self.chain_info.lock().unwrap(); + if infos.len() > 1 { + infos.pop_front().unwrap() + } else { + *infos.front().unwrap() + } + } + } + + impl BlockHashReader for TestProvider { + fn block_hash(&self, number: u64) -> ProviderResult> { + Ok(self + .headers + .iter() + .find_map(|(hash, header)| (header.number() == number).then_some(*hash))) + } + + fn canonical_hashes_range(&self, start: u64, end: u64) -> ProviderResult> { + Ok(self + .headers + .iter() + .filter_map(|(hash, header)| { + (start..end).contains(&header.number()).then_some(*hash) + }) + .collect()) + } + } + + impl BlockNumReader for TestProvider { + fn chain_info(&self) -> ProviderResult { + Ok(self.info()) + } + + fn best_block_number(&self) -> ProviderResult { + Ok(self.info().best_number) + } + + fn last_block_number(&self) -> ProviderResult { + self.best_block_number() + } + + fn block_number(&self, hash: B256) -> ProviderResult> { + Ok(self.headers.get(&hash).map(|header| header.number())) + } + } + + impl HeaderProvider for TestProvider { + type Header = MorphHeader; + + fn header(&self, block_hash: B256) -> ProviderResult> { + Ok(self.headers.get(&block_hash).cloned()) + } + + fn header_by_number(&self, number: u64) -> ProviderResult> { + Ok(self + .headers + .values() + .find(|header| header.number() == number) + .cloned()) + } + + fn headers_range( + &self, + _range: impl RangeBounds, + ) -> ProviderResult> { + Ok(self.headers.values().cloned().collect()) + } + + fn sealed_header(&self, number: u64) -> ProviderResult>> { + Ok(self.headers.iter().find_map(|(hash, header)| { + (header.number() == number).then(|| SealedHeader::new(header.clone(), *hash)) + })) + } + + fn sealed_headers_while( + &self, + _range: impl RangeBounds, + mut predicate: impl FnMut(&SealedHeader) -> bool, + ) -> ProviderResult>> { + Ok(self + .headers + .iter() + .map(|(hash, header)| SealedHeader::new(header.clone(), *hash)) + .take_while(|header| predicate(header)) + .collect()) + } + } + + fn header(number: u64, timestamp: u64) -> MorphHeader { + let mut header = MorphHeader::default(); + header.inner.number = number; + header.inner.timestamp = timestamp; + header + } + + fn synced_handle( + jade_timestamp: u64, + block_timestamp: u64, + hash: B256, + ) -> (tempfile::TempDir, ReferenceIndexHandle) { + let dir = tempfile::tempdir().unwrap(); + let chain = TestChain { + block: CanonicalBlock { + number: 0, + hash, + timestamp: block_timestamp, + transactions: Vec::new(), + }, + }; + let config = ReferenceIndexConfig::new(dir.path(), 2818, B256::ZERO, jade_timestamp); + let (mut runtime, handle) = ReferenceIndexRuntime::new(config, chain); + runtime.synchronize_once(false).unwrap(); + (dir, handle) + } + + fn args() -> ReferenceQueryArgs { + ReferenceQueryArgs { + reference: B256::repeat_byte(0x42), + offset: None, + limit: None, + } + } + + #[test] + fn rejects_results_if_canonical_head_changes_during_query() { + let indexed_hash = B256::repeat_byte(0x11); + let changed_hash = B256::repeat_byte(0x22); + let (_dir, handle) = synced_handle(0, 100, indexed_hash); + let provider = TestProvider::new( + [ + ChainInfo { + best_hash: indexed_hash, + best_number: 0, + }, + ChainInfo { + best_hash: changed_hash, + best_number: 0, + }, + ], + BTreeMap::from([(indexed_hash, header(0, 100))]), + ); + let handler = MorphRpcHandler::new(MorphRpc::new(handle, provider)); + + let error = handler + .get_transaction_hashes_by_reference(args()) + .unwrap_err(); + + assert_eq!(error.code(), -32000); + assert_eq!(error.message(), "reference index is behind"); + } + + #[test] + fn passes_the_exact_head_timestamp_to_the_index() { + let hash = B256::repeat_byte(0x11); + let (_dir, handle) = synced_handle(200, 199, hash); + let provider = TestProvider::new( + [ChainInfo { + best_hash: hash, + best_number: 0, + }], + BTreeMap::from([(hash, header(0, 200))]), + ); + let handler = MorphRpcHandler::new(MorphRpc::new(handle, provider)); + + let error = handler + .get_transaction_hashes_by_reference(args()) + .unwrap_err(); + + assert_eq!(error.code(), -32000); + assert_eq!(error.message(), "reference index is behind"); + } + + #[test] + fn maps_unavailable_to_an_explicit_server_error() { + let error = to_rpc_error(ReferenceIndexError::IndexUnavailable); + + assert_eq!(error.code(), -32000); + assert_eq!(error.message(), "reference index is unavailable"); + } +} diff --git a/crates/rpc/src/morph/rpc.rs b/crates/rpc/src/morph/rpc.rs index 932fe227..2dd4109d 100644 --- a/crates/rpc/src/morph/rpc.rs +++ b/crates/rpc/src/morph/rpc.rs @@ -29,11 +29,14 @@ pub trait MorphRpc { /// ordered by (block_number, tx_index) ascending, with offset-based /// pagination. /// - /// Returns `-32000 "reference index initializing"` while the startup - /// backfill/reconcile has not yet completed. + /// Returns `-32000 "reference index initializing"` while the runtime is + /// opening the index. /// - /// Returns `-32000 "reference index is behind"` when the index lags the - /// current chain tip by more than the configured threshold (default 16). + /// Returns `-32000 "reference index is behind"` unless the durable index + /// cursor exactly matches the canonical head number and hash. + /// + /// Returns `-32000 "reference index is unavailable"` while the runtime is + /// repairing the index or after a non-recoverable index error. /// /// Marked `blocking` because the handler runs synchronous MDBX reads. #[method(name = "getTransactionHashesByReference", blocking)] From 372fd4686e99840d9caad5571469fb7a70e7649e Mon Sep 17 00:00:00 2001 From: panos-xyz Date: Tue, 14 Jul 2026 19:56:28 +0800 Subject: [PATCH 5/6] feat(proofs): add bounded historical proof history --- Cargo.lock | 215 +- Cargo.toml | 13 + README.md | 38 +- bin/morph-reth/Cargo.toml | 9 + bin/morph-reth/src/main.rs | 110 +- bin/morph-reth/src/proofs.rs | 254 ++ crates/node/Cargo.toml | 4 + crates/node/src/add_ons.rs | 44 +- crates/node/src/args.rs | 63 + crates/node/src/node.rs | 1 + crates/node/tests/it/main.rs | 1 + crates/node/tests/it/proof_history.rs | 178 + crates/proofs-exex/Cargo.toml | 37 + crates/proofs-exex/NOTICE.md | 31 + crates/proofs-exex/src/lib.rs | 1192 ++++++ crates/proofs-exex/src/sync_target.rs | 637 +++ crates/proofs/Cargo.toml | 50 + crates/proofs/NOTICE.md | 32 + crates/proofs/src/api.rs | 435 ++ crates/proofs/src/batch_provider.rs | 313 ++ crates/proofs/src/cursor.rs | 129 + crates/proofs/src/cursor_factory.rs | 260 ++ crates/proofs/src/db/batch.rs | 143 + crates/proofs/src/db/cursor.rs | 1545 +++++++ crates/proofs/src/db/mod.rs | 15 + crates/proofs/src/db/models/block.rs | 82 + crates/proofs/src/db/models/change_set.rs | 131 + crates/proofs/src/db/models/kv.rs | 85 + crates/proofs/src/db/models/metadata.rs | 56 + crates/proofs/src/db/models/mod.rs | 93 + crates/proofs/src/db/models/storage.rs | 272 ++ crates/proofs/src/db/models/version.rs | 205 + crates/proofs/src/db/store.rs | 4707 +++++++++++++++++++++ crates/proofs/src/error.rs | 233 + crates/proofs/src/in_memory.rs | 1138 +++++ crates/proofs/src/initialize.rs | 1357 ++++++ crates/proofs/src/lib.rs | 61 + crates/proofs/src/live.rs | 432 ++ crates/proofs/src/metrics.rs | 97 + crates/proofs/src/proof.rs | 667 +++ crates/proofs/src/provider.rs | 340 ++ crates/proofs/src/prune/error.rs | 102 + crates/proofs/src/prune/metrics.rs | 54 + crates/proofs/src/prune/mod.rs | 12 + crates/proofs/src/prune/pruner.rs | 735 ++++ crates/proofs/src/prune/task.rs | 80 + crates/proofs/tests/identity.rs | 24 + crates/rpc/Cargo.toml | 6 + crates/rpc/src/eth/mod.rs | 1 + crates/rpc/src/eth/proofs.rs | 146 + crates/rpc/src/lib.rs | 3 + crates/rpc/src/proof_status.rs | 130 + crates/rpc/src/state.rs | 185 + local-test/README.md | 2 +- local-test/reset.sh | 2 + local-test/reth-start.sh | 1 - 56 files changed, 17177 insertions(+), 11 deletions(-) create mode 100644 bin/morph-reth/src/proofs.rs create mode 100644 crates/node/tests/it/proof_history.rs create mode 100644 crates/proofs-exex/Cargo.toml create mode 100644 crates/proofs-exex/NOTICE.md create mode 100644 crates/proofs-exex/src/lib.rs create mode 100644 crates/proofs-exex/src/sync_target.rs create mode 100644 crates/proofs/Cargo.toml create mode 100644 crates/proofs/NOTICE.md create mode 100644 crates/proofs/src/api.rs create mode 100644 crates/proofs/src/batch_provider.rs create mode 100644 crates/proofs/src/cursor.rs create mode 100644 crates/proofs/src/cursor_factory.rs create mode 100644 crates/proofs/src/db/batch.rs create mode 100644 crates/proofs/src/db/cursor.rs create mode 100644 crates/proofs/src/db/mod.rs create mode 100644 crates/proofs/src/db/models/block.rs create mode 100644 crates/proofs/src/db/models/change_set.rs create mode 100644 crates/proofs/src/db/models/kv.rs create mode 100644 crates/proofs/src/db/models/metadata.rs create mode 100644 crates/proofs/src/db/models/mod.rs create mode 100644 crates/proofs/src/db/models/storage.rs create mode 100644 crates/proofs/src/db/models/version.rs create mode 100644 crates/proofs/src/db/store.rs create mode 100644 crates/proofs/src/error.rs create mode 100644 crates/proofs/src/in_memory.rs create mode 100644 crates/proofs/src/initialize.rs create mode 100644 crates/proofs/src/lib.rs create mode 100644 crates/proofs/src/live.rs create mode 100644 crates/proofs/src/metrics.rs create mode 100644 crates/proofs/src/proof.rs create mode 100644 crates/proofs/src/provider.rs create mode 100644 crates/proofs/src/prune/error.rs create mode 100644 crates/proofs/src/prune/metrics.rs create mode 100644 crates/proofs/src/prune/mod.rs create mode 100644 crates/proofs/src/prune/pruner.rs create mode 100644 crates/proofs/src/prune/task.rs create mode 100644 crates/proofs/tests/identity.rs create mode 100644 crates/rpc/src/eth/proofs.rs create mode 100644 crates/rpc/src/proof_status.rs create mode 100644 crates/rpc/src/state.rs diff --git a/Cargo.lock b/Cargo.lock index 16307f55..0b9cd49c 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1488,6 +1488,26 @@ dependencies = [ "serde", ] +[[package]] +name = "bincode" +version = "2.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "36eaf5d7b090263e8150820482d5d93cd964a81e4019913c972f4edcc6edb740" +dependencies = [ + "bincode_derive", + "serde", + "unty", +] + +[[package]] +name = "bincode_derive" +version = "2.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bf95709a440f45e986983918d0e8a1f30a9b1df04918fc828670606804ac3c09" +dependencies = [ + "virtue", +] + [[package]] name = "bindgen" version = "0.72.1" @@ -2583,6 +2603,12 @@ dependencies = [ "litrs", ] +[[package]] +name = "downcast" +version = "0.11.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1435fa1053d8b2fbbe9be7e97eca7f33d37b28409959813daefc1446a14247f1" + [[package]] name = "dunce" version = "1.0.5" @@ -2956,6 +2982,15 @@ dependencies = [ "percent-encoding", ] +[[package]] +name = "fragile" +version = "2.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8878864ba14bb86e818a412bfd6f18f9eabd4ec0f008a28e8f7eb61db532fcf9" +dependencies = [ + "futures-core", +] + [[package]] name = "fs_extra" version = "1.3.0" @@ -4793,6 +4828,32 @@ dependencies = [ "windows-sys 0.61.2", ] +[[package]] +name = "mockall" +version = "0.14.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f58d964098a5f9c6b63d0798e5372fd04708193510a7af313c22e9f29b7b620b" +dependencies = [ + "cfg-if", + "downcast", + "fragile", + "mockall_derive", + "predicates", + "predicates-tree", +] + +[[package]] +name = "mockall_derive" +version = "0.14.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ca41ce716dda6a9be188b385aa78ee5260fc25cd3802cb2a8afdc6afbe6b6dbf" +dependencies = [ + "cfg-if", + "proc-macro2", + "quote", + "syn 2.0.117", +] + [[package]] name = "modular-bitfield" version = "0.13.1" @@ -4959,6 +5020,8 @@ dependencies = [ "morph-payload-builder", "morph-payload-types", "morph-primitives", + "morph-proofs", + "morph-proofs-exex", "morph-reference-index", "morph-rpc", "morph-txpool", @@ -4982,6 +5045,7 @@ dependencies = [ "reth-provider", "reth-rpc-builder", "reth-rpc-eth-api", + "reth-rpc-server-types", "reth-tasks", "reth-tracing", "reth-transaction-pool", @@ -4989,6 +5053,7 @@ dependencies = [ "reth-trie-db", "serde", "serde_json", + "tempfile", "tokio", "tokio-stream", "tracing", @@ -5068,6 +5133,63 @@ dependencies = [ "serde_json", ] +[[package]] +name = "morph-proofs" +version = "1.0.0" +dependencies = [ + "alloy-eips", + "alloy-primitives", + "auto_impl", + "bincode 2.0.1", + "bytes", + "derive_more", + "metrics", + "mockall", + "parking_lot", + "reth-codecs", + "reth-db", + "reth-db-api", + "reth-evm", + "reth-execution-errors", + "reth-metrics", + "reth-primitives-traits", + "reth-provider", + "reth-revm", + "reth-storage-errors", + "reth-tasks", + "reth-trie", + "reth-trie-common", + "serde", + "strum", + "tempfile", + "thiserror 2.0.18", + "tokio", + "tracing", +] + +[[package]] +name = "morph-proofs-exex" +version = "1.0.0" +dependencies = [ + "alloy-consensus", + "alloy-eips", + "alloy-primitives", + "eyre", + "futures", + "morph-proofs", + "reth-db", + "reth-ethereum-primitives", + "reth-execution-types", + "reth-exex", + "reth-exex-test-utils", + "reth-node-api", + "reth-primitives-traits", + "reth-provider", + "reth-trie", + "tokio", + "tracing", +] + [[package]] name = "morph-reference-index" version = "1.0.0" @@ -5100,17 +5222,26 @@ dependencies = [ name = "morph-reth" version = "1.0.0" dependencies = [ + "alloy-consensus", "alloy-primitives", "clap", + "eyre", "morph-chainspec", "morph-consensus", "morph-evm", "morph-node", + "morph-proofs", + "morph-proofs-exex", + "reth-chainspec", "reth-cli", + "reth-cli-commands", + "reth-cli-runner", "reth-cli-util", "reth-ethereum-cli", "reth-node-builder", + "reth-provider", "reth-rpc-server-types", + "reth-tasks", "tracing", ] @@ -5148,26 +5279,32 @@ dependencies = [ "alloy-primitives", "alloy-rpc-types-eth", "alloy-serde", + "async-trait", "derive_more", "eyre", "jsonrpsee", + "metrics", "morph-chainspec", "morph-evm", "morph-primitives", + "morph-proofs", "morph-reference-index", "morph-revm", "reth-chainspec", "reth-errors", "reth-evm", + "reth-metrics", "reth-node-api", "reth-node-builder", "reth-primitives-traits", "reth-provider", "reth-revm", "reth-rpc", + "reth-rpc-api", "reth-rpc-convert", "reth-rpc-eth-api", "reth-rpc-eth-types", + "reth-rpc-server-types", "reth-storage-api", "reth-tasks", "reth-transaction-pool", @@ -5887,6 +6024,32 @@ dependencies = [ "zerocopy", ] +[[package]] +name = "predicates" +version = "3.1.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ada8f2932f28a27ee7b70dd6c1c39ea0675c55a36879ab92f3a715eaa1e63cfe" +dependencies = [ + "anstyle", + "predicates-core", +] + +[[package]] +name = "predicates-core" +version = "1.0.10" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cad38746f3166b4031b1a0d39ad9f954dd291e7854fcc0eed52ee41a0b50d144" + +[[package]] +name = "predicates-tree" +version = "1.0.13" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d0de1b847b39c8131db0467e9df1ff60e6d0562ab8e9a16e568ad0fdb372e2f2" +dependencies = [ + "predicates-core", + "termtree", +] + [[package]] name = "prefix-trie" version = "0.8.4" @@ -7633,6 +7796,38 @@ dependencies = [ "tracing", ] +[[package]] +name = "reth-exex-test-utils" +version = "2.3.0" +source = "git+https://github.com/paradigmxyz/reth?rev=f0a41795d72ae5bb0b6c3661ac42ffb7b78ef105#f0a41795d72ae5bb0b6c3661ac42ffb7b78ef105" +dependencies = [ + "alloy-eips", + "eyre", + "futures-util", + "reth-chainspec", + "reth-config", + "reth-consensus", + "reth-db", + "reth-db-common", + "reth-ethereum-primitives", + "reth-evm-ethereum", + "reth-execution-types", + "reth-exex", + "reth-network", + "reth-node-api", + "reth-node-builder", + "reth-node-core", + "reth-node-ethereum", + "reth-payload-builder", + "reth-primitives-traits", + "reth-provider", + "reth-tasks", + "reth-transaction-pool", + "tempfile", + "thiserror 2.0.18", + "tokio", +] + [[package]] name = "reth-exex-types" version = "2.3.0" @@ -7907,7 +8102,7 @@ version = "2.3.0" source = "git+https://github.com/paradigmxyz/reth?rev=f0a41795d72ae5bb0b6c3661ac42ffb7b78ef105#f0a41795d72ae5bb0b6c3661ac42ffb7b78ef105" dependencies = [ "anyhow", - "bincode", + "bincode 1.3.3", "derive_more", "lz4_flex", "memmap2", @@ -10425,6 +10620,12 @@ dependencies = [ "windows-sys 0.61.2", ] +[[package]] +name = "termtree" +version = "0.5.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8f50febec83f5ee1df3015341d8bd429f2d1cc62bcba7ea2076759d315084683" + [[package]] name = "thiserror" version = "1.0.69" @@ -11188,6 +11389,12 @@ version = "0.9.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "8ecb6da28b8a351d773b68d5825ac39017e680750f980f3a1a85cd8dd28a47c1" +[[package]] +name = "unty" +version = "0.0.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6d49784317cd0d1ee7ec5c716dd598ec5b4483ea832a2dced265471cc0f690ae" + [[package]] name = "url" version = "2.5.8" @@ -11287,6 +11494,12 @@ version = "0.9.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "0b928f33d975fc6ad9f86c8f283853ad26bdd5b10b7f1542aa2fa15e2289105a" +[[package]] +name = "virtue" +version = "0.0.18" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "051eb1abcf10076295e815102942cc58f9d5e3b4560e46e53c21e8ff6f3af7b1" + [[package]] name = "visibility" version = "0.1.1" diff --git a/Cargo.toml b/Cargo.toml index cd966912..c5674299 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -19,6 +19,8 @@ members = [ "crates/payload/builder", "crates/payload/types", "crates/primitives", + "crates/proofs", + "crates/proofs-exex", "crates/reference-index", "crates/revm", "crates/txpool", @@ -94,6 +96,8 @@ morph-node = { path = "crates/node"} morph-payload-builder = { path = "crates/payload/builder", default-features = false } morph-payload-types = { path = "crates/payload/types", default-features = false } morph-primitives = { path = "crates/primitives", default-features = false } +morph-proofs = { path = "crates/proofs", default-features = false } +morph-proofs-exex = { path = "crates/proofs-exex", default-features = false } morph-reference-index = { path = "crates/reference-index", default-features = false } morph-rpc = { path = "crates/rpc" } morph-revm = { path = "crates/revm", default-features = false } @@ -104,6 +108,7 @@ reth-chain-state = { git = "https://github.com/paradigmxyz/reth", rev = "f0a4179 reth-chainspec = { git = "https://github.com/paradigmxyz/reth", rev = "f0a41795d72ae5bb0b6c3661ac42ffb7b78ef105" } reth-cli = { git = "https://github.com/paradigmxyz/reth", rev = "f0a41795d72ae5bb0b6c3661ac42ffb7b78ef105" } reth-cli-commands = { git = "https://github.com/paradigmxyz/reth", rev = "f0a41795d72ae5bb0b6c3661ac42ffb7b78ef105" } +reth-cli-runner = { git = "https://github.com/paradigmxyz/reth", rev = "f0a41795d72ae5bb0b6c3661ac42ffb7b78ef105" } reth-cli-util = { git = "https://github.com/paradigmxyz/reth", rev = "f0a41795d72ae5bb0b6c3661ac42ffb7b78ef105" } reth-codecs = { version = "0.5.0", default-features = false } reth-codecs-derive = "0.5.0" @@ -116,6 +121,9 @@ reth-engine-local = { git = "https://github.com/paradigmxyz/reth", rev = "f0a417 reth-engine-primitives = { git = "https://github.com/paradigmxyz/reth", rev = "f0a41795d72ae5bb0b6c3661ac42ffb7b78ef105" } reth-engine-tree = { git = "https://github.com/paradigmxyz/reth", rev = "f0a41795d72ae5bb0b6c3661ac42ffb7b78ef105" } reth-errors = { git = "https://github.com/paradigmxyz/reth", rev = "f0a41795d72ae5bb0b6c3661ac42ffb7b78ef105" } +reth-exex = { git = "https://github.com/paradigmxyz/reth", rev = "f0a41795d72ae5bb0b6c3661ac42ffb7b78ef105" } +reth-exex-test-utils = { git = "https://github.com/paradigmxyz/reth", rev = "f0a41795d72ae5bb0b6c3661ac42ffb7b78ef105" } +reth-execution-errors = { git = "https://github.com/paradigmxyz/reth", rev = "f0a41795d72ae5bb0b6c3661ac42ffb7b78ef105" } reth-eth-wire-types = { git = "https://github.com/paradigmxyz/reth", rev = "f0a41795d72ae5bb0b6c3661ac42ffb7b78ef105" } reth-ethereum = { git = "https://github.com/paradigmxyz/reth", rev = "f0a41795d72ae5bb0b6c3661ac42ffb7b78ef105" } reth-ethereum-cli = { git = "https://github.com/paradigmxyz/reth", rev = "f0a41795d72ae5bb0b6c3661ac42ffb7b78ef105" } @@ -145,9 +153,11 @@ reth-rpc-eth-api = { git = "https://github.com/paradigmxyz/reth", rev = "f0a4179 reth-rpc-eth-types = { git = "https://github.com/paradigmxyz/reth", rev = "f0a41795d72ae5bb0b6c3661ac42ffb7b78ef105" } reth-rpc-server-types = { git = "https://github.com/paradigmxyz/reth", rev = "f0a41795d72ae5bb0b6c3661ac42ffb7b78ef105" } reth-storage-api = { git = "https://github.com/paradigmxyz/reth", rev = "f0a41795d72ae5bb0b6c3661ac42ffb7b78ef105" } +reth-storage-errors = { git = "https://github.com/paradigmxyz/reth", rev = "f0a41795d72ae5bb0b6c3661ac42ffb7b78ef105", default-features = false } reth-tasks = { git = "https://github.com/paradigmxyz/reth", rev = "f0a41795d72ae5bb0b6c3661ac42ffb7b78ef105" } reth-tracing = { git = "https://github.com/paradigmxyz/reth", rev = "f0a41795d72ae5bb0b6c3661ac42ffb7b78ef105" } reth-trie = { git = "https://github.com/paradigmxyz/reth", rev = "f0a41795d72ae5bb0b6c3661ac42ffb7b78ef105" } +reth-trie-common = { git = "https://github.com/paradigmxyz/reth", rev = "f0a41795d72ae5bb0b6c3661ac42ffb7b78ef105" } reth-transaction-pool = { git = "https://github.com/paradigmxyz/reth", rev = "f0a41795d72ae5bb0b6c3661ac42ffb7b78ef105" } reth-zstd-compressors = { version = "0.5.0", default-features = false } reth-execution-cache = { git = "https://github.com/paradigmxyz/reth", rev = "f0a41795d72ae5bb0b6c3661ac42ffb7b78ef105" } @@ -196,6 +206,7 @@ axum = "0.8.4" base64 = "0.22" blake3 = "1.8" bytes = "1.8" +bincode = "2" clap = { version = "4.5.45", features = ["derive"] } const-hex = { version = "1.15.0" } dashmap = "6.0" @@ -209,6 +220,7 @@ itertools = "0.14.0" jiff = { version = "0.2.15", default-features = false } jsonrpsee = { version = "0.26.0", features = ["server", "client", "macros"] } metrics = "0.24.0" +mockall = "0.14" p256 = "0.13" parking_lot = "0.12.4" prometheus-client = "0.24.0" @@ -219,6 +231,7 @@ rand_core = "0.6.4" rayon = "1.10" serde = { version = "1.0.219", features = ["derive"] } serde_json = "1.0.142" +strum = { version = "0.27", default-features = false } sha2 = "0.10" tempfile = "3.20.0" test-fuzz = "7" diff --git a/README.md b/README.md index a2239452..697f15ff 100644 --- a/README.md +++ b/README.md @@ -16,6 +16,7 @@ Morph Reth is the next-generation execution client for [Morph](https://www.morph - **Morph Hardforks**: Implements Morph hardfork logic through Jade, with bundled Mainnet and Hoodi chainspecs scheduled through Jade - **Custom Engine API**: L2-specific Engine API for sequencer block building and validation - **L1 Fee Validation**: Transaction pool with L1 data fee affordability checks +- **Bounded Historical Proofs**: Optional forward-only EIP-1186 proof history in an independent MDBX database ### Supported Networks @@ -42,6 +43,9 @@ morph-reth/ │ ├── builder/ # Block building logic │ └── types/ # Engine API types (MorphExecutionData, etc.) ├── primitives/ # Core types (transactions, receipts) + ├── proofs/ # Versioned MPT history and proof state provider + ├── proofs-exex/ # Forward accumulation, reorg, unwind, and pruning + ├── reference-index/ # Independent transaction reference index ├── revm/ # L1 fee calculation, token fee logic ├── rpc/ # RPC implementation and type conversions └── txpool/ # Transaction pool with L1 fee validation @@ -60,6 +64,9 @@ morph-reth/ | `morph-payload-types` | Engine API payload types | | `morph-payload-builder` | Block building implementation | | `morph-primitives` | Transaction and receipt types | +| `morph-proofs` | Bounded, versioned MPT history in MDBX | +| `morph-proofs-exex` | Forward-only proof-history execution extension | +| `morph-reference-index` | Independent Morph transaction reference index | | `morph-revm` | L1 fee and token fee calculations | | `morph-rpc` | RPC implementation and type conversions | | `morph-txpool` | Transaction pool with L1 fee and MorphTx validation | @@ -114,7 +121,36 @@ openssl rand -hex 32 > jwt.hex |------|---------|-------------| | `--morph.max-tx-payload-bytes` | 122880 (120KB) | Maximum transaction payload bytes per block | | `--morph.max-tx-per-block` | None (unlimited) | Maximum number of transactions per block | -| `--rpc.eth-proof-window` | 0 (disabled) | Max historical blocks for `eth_getProof` (up to 1209600) | +| `--proofs-history` | false | Enable historical `eth_getProof` and proof-history accumulation | +| `--proofs-history.storage-path` | `/historical-proofs` | Override the proof MDBX directory | +| `--proofs-history.window` | 1296000 | Number of canonical blocks retained | +| `--proofs-history.verification-interval` | 0 | Re-execute every Nth indexed block; 0 disables verification | +| `--rpc.eth-proof-window` | 0 | Reth overlay limit used only while proof history is disabled | + +#### Initializing Historical Proofs + +Proof history starts at the current canonical tip and accumulates forward. Stop the node before +initialization so the source state remains fixed: + +```bash +./target/release/morph-reth proofs init --chain hoodi --datadir /path/to/reth-data + +./target/release/morph-reth node \ + --chain hoodi \ + --datadir /path/to/reth-data \ + --proofs-history \ + --http \ + --authrpc.jwtsecret jwt.hex +``` + +The standard `eth_getProof` method is then served only for the inclusive range reported by +`debug_proofsSyncStatus`; requests outside that range never fall back to Reth's historical overlay. +The same `eth_getProof` override is installed on the authenticated RPC server. For manual +maintenance use `morph-reth proofs prune` and `morph-reth proofs unwind --target `. + +For cold copies, stop the source node and copy the complete chain data directory, including +`historical-proofs`, as one consistent unit. Startup validates the proof database schema, chain ID, +genesis hash, and latest canonical block hash before resuming. ### Running Tests diff --git a/bin/morph-reth/Cargo.toml b/bin/morph-reth/Cargo.toml index 485617c2..ecd361c0 100644 --- a/bin/morph-reth/Cargo.toml +++ b/bin/morph-reth/Cargo.toml @@ -20,17 +20,26 @@ morph-chainspec = { workspace = true, features = ["cli"] } morph-consensus.workspace = true morph-evm.workspace = true morph-node.workspace = true +morph-proofs.workspace = true +morph-proofs-exex.workspace = true # Reth CLI +reth-chainspec.workspace = true reth-cli.workspace = true +reth-cli-commands.workspace = true +reth-cli-runner.workspace = true reth-cli-util.workspace = true reth-ethereum-cli.workspace = true reth-node-builder.workspace = true +reth-provider.workspace = true reth-rpc-server-types.workspace = true +reth-tasks.workspace = true # Other alloy-primitives.workspace = true +alloy-consensus.workspace = true clap.workspace = true +eyre.workspace = true tracing.workspace = true [features] diff --git a/bin/morph-reth/src/main.rs b/bin/morph-reth/src/main.rs index a39d8adf..207597fd 100644 --- a/bin/morph-reth/src/main.rs +++ b/bin/morph-reth/src/main.rs @@ -3,6 +3,8 @@ //! This is the main entry point for the Morph L2 execution layer client. //! It extends reth with Morph-specific functionality. +mod proofs; + #[global_allocator] static ALLOC: reth_cli_util::allocator::Allocator = reth_cli_util::allocator::new_allocator(); @@ -21,6 +23,10 @@ use morph_chainspec::{MORPH_DEFAULT_PRIORITY_FEE, MorphChainSpec, MorphChainSpec use morph_consensus::MorphConsensus; use morph_evm::{MorphEvmConfig, evm::MorphEvmFactory}; use morph_node::{MorphAddOns, MorphArgs, MorphNode}; +use morph_proofs::{MdbxProofsStorage, MorphProofsStorage, MorphProofsStore, ProofDbIdentity}; +use morph_proofs_exex::MorphProofsExEx; +use proofs::MorphSubcommand; +use reth_chainspec::EthChainSpec; use reth_cli_util::sigsegv_handler; use reth_ethereum_cli::{Cli, Commands}; use reth_node_builder::Node; @@ -33,7 +39,7 @@ fn morph_default_suggested_fee() -> U256 { } fn apply_morph_cli_defaults( - cli: &mut Cli, + cli: &mut Cli, ) { if let Commands::Node(command) = &mut cli.command { command @@ -67,7 +73,8 @@ fn main() { ) }; - let mut cli = Cli::::parse(); + let mut cli = + Cli::::parse(); apply_morph_cli_defaults(&mut cli); // Run the node @@ -75,14 +82,76 @@ fn main() { cli.run_with_components::(components, async move |builder, morph_args| { info!(target: "morph::cli", "Starting Morph-Reth node"); + let chain_spec = builder.config().chain.clone(); + let datadir = builder.config().datadir(); + let chain_id = chain_spec.chain().id(); + let genesis_hash = chain_spec.genesis_hash(); + + let proof_history = if morph_args.proofs_history { + let path = morph_args + .proofs_history_storage_path + .clone() + .unwrap_or_else(|| datadir.data_dir().join("historical-proofs")); + info!( + target: "morph::proofs", + path = %path.display(), + chain_id, + "opening Morph historical proof database" + ); + let storage: MorphProofsStorage> = Arc::new( + MdbxProofsStorage::open( + &path, + ProofDbIdentity::new(chain_id, genesis_hash), + ) + .map_err(|error| { + eyre::eyre!( + "failed to open historical proof database at {}: {error}", + path.display() + ) + })?, + ); + if storage.get_earliest_block_number()?.is_none() + || storage.get_latest_block_number()?.is_none() + { + return Err(eyre::eyre!( + "proof history is enabled but {} is not initialized; run `morph-reth proofs init` first", + path.display() + )); + } + Some(( + storage, + morph_args.proofs_history_window, + morph_args.proofs_history_verification_interval, + )) + } else { + None + }; let node = MorphNode::new(morph_args); - let handle = builder + let mut add_ons = MorphAddOns::new(); + if let Some((storage, _, _)) = &proof_history { + add_ons = add_ons.with_proof_history(storage.clone()); + } + + let mut node_builder = builder .with_types::() .with_components(node.components_builder()) - .with_add_ons(MorphAddOns::new()) - .launch_with_debug_capabilities() - .await?; + .with_add_ons(add_ons); + + if let Some((storage, window, verification_interval)) = proof_history { + node_builder = node_builder.install_exex( + "morph-proof-history", + async move |ctx| { + let exex = MorphProofsExEx::builder(ctx, storage) + .with_proofs_history_window(window) + .with_verification_interval(verification_interval) + .build(); + Ok(async move { exex.run().await }) + }, + ); + } + + let handle = node_builder.launch_with_debug_capabilities().await?; info!(target: "morph::cli", "Node started successfully"); @@ -94,3 +163,32 @@ fn main() { std::process::exit(1); } } + +#[cfg(test)] +mod tests { + use super::*; + + type MorphCli = + Cli; + + #[test] + fn parses_top_level_proofs_command() { + let cli = MorphCli::try_parse_from(["morph-reth", "proofs", "init", "--chain", "hoodi"]) + .expect("top-level proofs command must parse"); + assert!(matches!( + cli.command, + Commands::Ext(MorphSubcommand::Proofs(_)) + )); + } + + #[test] + fn disabled_mode_keeps_reth_historical_overlay_off() { + let cli = MorphCli::try_parse_from(["morph-reth", "node", "--chain", "hoodi"]) + .expect("node command must parse"); + let Commands::Node(command) = cli.command else { + panic!("expected node command") + }; + assert!(!command.ext.proofs_history); + assert_eq!(command.rpc.rpc_eth_proof_window, 0); + } +} diff --git a/bin/morph-reth/src/proofs.rs b/bin/morph-reth/src/proofs.rs new file mode 100644 index 00000000..e7865c07 --- /dev/null +++ b/bin/morph-reth/src/proofs.rs @@ -0,0 +1,254 @@ +//! Management commands for the independent historical-proof database. + +use std::{path::PathBuf, sync::Arc}; + +use alloy_consensus::BlockHeader; +use clap::{Parser, Subcommand}; +use morph_chainspec::{MorphChainSpec, MorphChainSpecParser}; +use morph_node::{MorphNode, args::DEFAULT_PROOFS_HISTORY_WINDOW}; +use morph_proofs::{ + InitializationJob, MdbxProofsStorage, MorphProofStoragePruner, MorphProofsStorage, + MorphProofsStore, ProofDbIdentity, +}; +use reth_chainspec::EthChainSpec; +use reth_cli_commands::common::{AccessRights, Environment, EnvironmentArgs}; +use reth_cli_runner::CliRunner; +use reth_ethereum_cli::ExtendedCommand; +use reth_provider::{ + BlockNumReader, BlockReader, DBProvider, DatabaseProviderFactory, TransactionVariant, +}; +use tracing::{info, warn}; + +/// Morph-specific top-level CLI commands. +#[derive(Debug, Subcommand)] +pub(crate) enum MorphSubcommand { + /// Manage the historical proof database. + #[command(name = "proofs")] + Proofs(ProofsCommand), +} + +impl ExtendedCommand for MorphSubcommand { + fn execute(self, runner: CliRunner) -> eyre::Result<()> { + match self { + Self::Proofs(command) => { + let runtime = runner.runtime(); + runner.run_blocking_until_ctrl_c(command.execute(runtime)) + } + } + } +} + +/// Manage Morph historical proofs. +#[derive(Debug, Parser)] +pub(crate) struct ProofsCommand { + #[command(subcommand)] + command: ProofsSubcommand, +} + +impl ProofsCommand { + async fn execute(self, runtime: reth_tasks::Runtime) -> eyre::Result<()> { + match self.command { + ProofsSubcommand::Init(command) => command.execute(runtime), + ProofsSubcommand::Prune(command) => command.execute(runtime), + ProofsSubcommand::Unwind(command) => command.execute(runtime), + } + } +} + +#[derive(Debug, Subcommand)] +enum ProofsSubcommand { + /// Initialize proof history from the current canonical tip. + Init(InitCommand), + /// Remove proof history outside the configured window. + Prune(PruneCommand), + /// Remove proof history at and after a canonical block. + Unwind(UnwindCommand), +} + +#[derive(Debug, Parser)] +struct InitCommand { + #[command(flatten)] + env: EnvironmentArgs, + + /// Proof-history MDBX directory (defaults to `/historical-proofs`). + #[arg(long = "proofs-history.storage-path", value_name = "PATH")] + storage_path: Option, +} + +impl InitCommand { + fn execute(self, runtime: reth_tasks::Runtime) -> eyre::Result<()> { + let Environment { + provider_factory, + data_dir, + .. + } = self.env.init::(AccessRights::RO, runtime)?; + let path = self + .storage_path + .unwrap_or_else(|| data_dir.data_dir().join("historical-proofs")); + let storage = open_storage(&path, &self.env.chain)?; + + if let Some((number, hash)) = storage.get_earliest_block_number()? { + info!(target: "morph::proofs", block_number = number, ?hash, "Proof history is already initialized"); + return Ok(()); + } + + let chain_info = provider_factory.chain_info()?; + info!( + target: "morph::proofs", + path = %path.display(), + block_number = chain_info.best_number, + block_hash = ?chain_info.best_hash, + "Initializing proof history from canonical state" + ); + + let provider = provider_factory + .database_provider_ro()? + .disable_long_read_transaction_safety(); + InitializationJob::new(storage, provider.into_tx()) + .run(chain_info.best_number, chain_info.best_hash)?; + + info!(target: "morph::proofs", "Proof history initialized"); + Ok(()) + } +} + +#[derive(Debug, Parser)] +struct PruneCommand { + #[command(flatten)] + env: EnvironmentArgs, + + /// Proof-history MDBX directory (defaults to `/historical-proofs`). + #[arg(long = "proofs-history.storage-path", value_name = "PATH")] + storage_path: Option, + + /// Number of canonical blocks to retain. + #[arg( + long = "proofs-history.window", + value_name = "BLOCKS", + default_value_t = DEFAULT_PROOFS_HISTORY_WINDOW, + value_parser = clap::value_parser!(u64).range(1..) + )] + window: u64, + + /// Maximum number of blocks pruned per transaction. + #[arg( + long = "proofs-history.prune-batch-size", + value_name = "BLOCKS", + default_value_t = 1_000, + value_parser = clap::value_parser!(u64).range(1..) + )] + batch_size: u64, +} + +impl PruneCommand { + fn execute(self, runtime: reth_tasks::Runtime) -> eyre::Result<()> { + let Environment { + provider_factory, + data_dir, + .. + } = self.env.init::(AccessRights::RO, runtime)?; + let path = self + .storage_path + .unwrap_or_else(|| data_dir.data_dir().join("historical-proofs")); + let storage = open_storage(&path, &self.env.chain)?; + + let output = + MorphProofStoragePruner::new(storage, provider_factory, self.window, self.batch_size) + .try_run()?; + info!(target: "morph::proofs", ?output, "Proof-history pruning completed"); + Ok(()) + } +} + +#[derive(Debug, Parser)] +struct UnwindCommand { + #[command(flatten)] + env: EnvironmentArgs, + + /// Proof-history MDBX directory (defaults to `/historical-proofs`). + #[arg(long = "proofs-history.storage-path", value_name = "PATH")] + storage_path: Option, + + /// First block to remove, inclusive. + #[arg(long, value_name = "BLOCK")] + target: u64, +} + +impl UnwindCommand { + fn execute(self, runtime: reth_tasks::Runtime) -> eyre::Result<()> { + let Environment { + provider_factory, + data_dir, + .. + } = self.env.init::(AccessRights::RO, runtime)?; + let path = self + .storage_path + .unwrap_or_else(|| data_dir.data_dir().join("historical-proofs")); + let storage = open_storage(&path, &self.env.chain)?; + + let (Some((earliest, _)), Some((latest, _))) = ( + storage.get_earliest_block_number()?, + storage.get_latest_block_number()?, + ) else { + warn!(target: "morph::proofs", "Proof history is empty; nothing to unwind"); + return Ok(()); + }; + + if self.target <= earliest || self.target > latest { + return Err(eyre::eyre!( + "unwind target {} must be in ({earliest}, {latest}]", + self.target + )); + } + + let block = provider_factory + .recovered_block(self.target.into(), TransactionVariant::NoHash)? + .ok_or_else(|| eyre::eyre!("canonical block {} not found", self.target))?; + info!( + target: "morph::proofs", + block_number = block.number(), + block_hash = ?block.hash(), + "Unwinding proof history" + ); + storage.unwind_history(block.block_with_parent())?; + Ok(()) + } +} + +fn open_storage( + path: &std::path::Path, + chain: &MorphChainSpec, +) -> eyre::Result>> { + let identity = ProofDbIdentity::new(chain.chain().id(), chain.genesis_hash()); + let backend = MdbxProofsStorage::open(path, identity).map_err(|error| { + eyre::eyre!( + "failed to open proof history at {}: {error}", + path.display() + ) + })?; + Ok(Arc::new(backend)) +} + +#[cfg(test)] +mod tests { + use clap::Parser; + + use super::ProofsCommand; + + #[test] + fn parses_all_proof_commands() { + ProofsCommand::try_parse_from(["proofs", "init", "--chain", "hoodi"]) + .expect("init command must parse"); + ProofsCommand::try_parse_from([ + "proofs", + "prune", + "--chain", + "hoodi", + "--proofs-history.window", + "64", + ]) + .expect("prune command must parse"); + ProofsCommand::try_parse_from(["proofs", "unwind", "--chain", "hoodi", "--target", "42"]) + .expect("unwind command must parse"); + } +} diff --git a/crates/node/Cargo.toml b/crates/node/Cargo.toml index bb6427b6..df9b5617 100644 --- a/crates/node/Cargo.toml +++ b/crates/node/Cargo.toml @@ -18,6 +18,7 @@ morph-evm.workspace = true morph-payload-builder.workspace = true morph-payload-types.workspace = true morph-primitives = { workspace = true, features = ["reth-codec"] } +morph-proofs.workspace = true morph-reference-index.workspace = true morph-rpc.workspace = true morph-txpool.workspace = true @@ -84,6 +85,7 @@ reth-db = { workspace = true, features = ["test-utils"] } reth-e2e-test-utils.workspace = true reth-node-core.workspace = true reth-payload-primitives.workspace = true +reth-rpc-server-types.workspace = true reth-tasks.workspace = true reth-tracing.workspace = true alloy-consensus.workspace = true @@ -94,7 +96,9 @@ jsonrpsee.workspace = true morph-chainspec.workspace = true morph-payload-types.workspace = true morph-primitives.workspace = true +morph-proofs-exex.workspace = true serde_json.workspace = true +tempfile.workspace = true [[test]] name = "it" diff --git a/crates/node/src/add_ons.rs b/crates/node/src/add_ons.rs index 65249d57..b3bdd127 100644 --- a/crates/node/src/add_ons.rs +++ b/crates/node/src/add_ons.rs @@ -8,9 +8,12 @@ use alloy_hardforks::ForkCondition; use morph_chainspec::{MorphHardfork, MorphHardforks}; use morph_evm::MorphEvmConfig; use morph_primitives::{Block, MorphHeader, MorphReceipt}; +use morph_proofs::{MdbxProofsStorage, MorphProofsStorage}; use morph_reference_index::{ReferenceIndexConfig, ReferenceIndexRuntime}; use morph_rpc::{ - MorphEthApiBuilder, MorphEthConfigApiServer, MorphEthConfigHandler, + MorphEthApiBuilder, MorphEthConfigApiServer, MorphEthConfigHandler, ProofStatusApiExt, + ProofStatusApiOverrideServer, + eth::proofs::{EthProofApiExt, EthProofApiOverrideServer}, morph::{MorphRpc, MorphRpcHandler, MorphRpcServer}, }; use reth_chain_state::CanonStateSubscriptions; @@ -29,6 +32,7 @@ use reth_provider::{ use reth_rpc_builder::Identity; use reth_rpc_eth_api::RpcNodeCore; use reth_tracing::tracing; +use std::sync::Arc; /// Morph node add-ons for RPC and Engine API. /// @@ -47,6 +51,8 @@ pub struct MorphAddOns< > { /// Inner RPC add-ons from reth. inner: RpcAddOns, + /// Optional proof-history storage used to replace `eth_getProof` on normal and auth RPC. + proof_history: Option>>, } impl MorphAddOns, MorphEthApiBuilder> @@ -68,8 +74,18 @@ where Identity::default(), Identity::default(), ), + proof_history: None, } } + + /// Attach initialized proof-history storage to the RPC add-ons. + pub fn with_proof_history( + mut self, + storage: MorphProofsStorage>, + ) -> Self { + self.proof_history = Some(storage); + self + } } fn ensure_reference_index_pruning_compatible( @@ -171,6 +187,7 @@ where let morph_rpc_ctx = MorphRpc::new(reference_index_handle, provider.clone()); let reference_rpc_handler = MorphRpcHandler::new(morph_rpc_ctx); + let proof_history = self.proof_history; // Use launch_add_ons_with to register custom Engine API and eth_config self.inner @@ -178,9 +195,34 @@ where let reth_node_builder::rpc::RpcModuleContainer { modules, auth_module, + registry, .. } = container; + if let Some(storage) = proof_history { + let eth_api = registry.eth_api().clone(); + modules + .replace_configured( + EthProofApiExt::new(eth_api.clone(), storage.clone()).into_rpc(), + ) + .map_err(|error| { + eyre::eyre!("Failed to replace normal RPC eth_getProof: {error}") + })?; + auth_module + .replace_auth_methods( + EthProofApiExt::new(eth_api, storage.clone()).into_rpc(), + ) + .map_err(|error| { + eyre::eyre!("Failed to replace auth RPC eth_getProof: {error}") + })?; + modules + .replace_configured(ProofStatusApiExt::new(storage).into_rpc()) + .map_err(|error| { + eyre::eyre!("Failed to register debug_proofsSyncStatus: {error}") + })?; + tracing::info!(target: "morph::node", "Historical proof RPCs registered"); + } + // Register Morph eth_config handler (EIP-7910 + morph extension) // This provides eth_config on HTTP/WS/IPC for morphnode compatibility. tracing::debug!(target: "morph::node", "Registering Morph eth_config handler"); diff --git a/crates/node/src/args.rs b/crates/node/src/args.rs index 7e5e9233..16846daa 100644 --- a/crates/node/src/args.rs +++ b/crates/node/src/args.rs @@ -1,5 +1,7 @@ //! Morph node CLI arguments. +use std::path::PathBuf; + use clap::Args; /// Default maximum transaction payload bytes per block (120KB). @@ -7,6 +9,9 @@ use clap::Args; /// This matches Morph's go-ethereum configuration. pub const MORPH_DEFAULT_MAX_TX_PAYLOAD_BYTES: u64 = 122_880; +/// Default proof-history retention window: 30 days at a two-second block time. +pub const DEFAULT_PROOFS_HISTORY_WINDOW: u64 = 1_296_000; + /// Morph-specific CLI arguments. /// /// These arguments extend the standard reth CLI with Morph-specific options @@ -33,6 +38,33 @@ pub struct MorphArgs { /// Morph Holesky testnet uses 1000 as the default limit. #[arg(long = "morph.max-tx-per-block", value_name = "COUNT")] pub max_tx_per_block: Option, + + /// Enable the forward-only historical proof index and its RPC overrides. + #[arg(long = "proofs-history", default_value_t = false)] + pub proofs_history: bool, + + /// Proof-history MDBX directory. + /// + /// Defaults to `/historical-proofs` when proof history is enabled. + #[arg(long = "proofs-history.storage-path", value_name = "PATH")] + pub proofs_history_storage_path: Option, + + /// Number of canonical blocks retained in proof history. + #[arg( + long = "proofs-history.window", + value_name = "BLOCKS", + default_value_t = DEFAULT_PROOFS_HISTORY_WINDOW, + value_parser = clap::value_parser!(u64).range(1..) + )] + pub proofs_history_window: u64, + + /// Re-execute every Nth block to verify notification trie data; zero disables verification. + #[arg( + long = "proofs-history.verification-interval", + value_name = "BLOCKS", + default_value_t = 0 + )] + pub proofs_history_verification_interval: u64, } impl Default for MorphArgs { @@ -40,6 +72,10 @@ impl Default for MorphArgs { Self { max_tx_payload_bytes: MORPH_DEFAULT_MAX_TX_PAYLOAD_BYTES, max_tx_per_block: None, + proofs_history: false, + proofs_history_storage_path: None, + proofs_history_window: DEFAULT_PROOFS_HISTORY_WINDOW, + proofs_history_verification_interval: 0, } } } @@ -63,6 +99,10 @@ mod tests { MORPH_DEFAULT_MAX_TX_PAYLOAD_BYTES ); assert_eq!(args.max_tx_per_block, None); + assert!(!args.proofs_history); + assert_eq!(args.proofs_history_storage_path, None); + assert_eq!(args.proofs_history_window, 1_296_000); + assert_eq!(args.proofs_history_verification_interval, 0); } #[test] @@ -105,6 +145,29 @@ mod tests { ); } + #[test] + fn test_proofs_history_args() { + let args = CommandParser::::parse_from([ + "test", + "--proofs-history", + "--proofs-history.storage-path", + "/tmp/morph-proofs", + "--proofs-history.window", + "2592000", + "--proofs-history.verification-interval", + "64", + ]) + .args; + + assert!(args.proofs_history); + assert_eq!( + args.proofs_history_storage_path.as_deref(), + Some(std::path::Path::new("/tmp/morph-proofs")) + ); + assert_eq!(args.proofs_history_window, 2_592_000); + assert_eq!(args.proofs_history_verification_interval, 64); + } + #[test] fn test_default_trait_impl() { let args = MorphArgs::default(); diff --git a/crates/node/src/node.rs b/crates/node/src/node.rs index 347cef93..1f6d7887 100644 --- a/crates/node/src/node.rs +++ b/crates/node/src/node.rs @@ -252,6 +252,7 @@ mod tests { let args = super::super::args::MorphArgs { max_tx_payload_bytes: 200_000, max_tx_per_block: Some(500), + ..Default::default() }; let node = MorphNode::new(args); assert_eq!(node.args.max_tx_payload_bytes, 200_000); diff --git a/crates/node/tests/it/main.rs b/crates/node/tests/it/main.rs index fd66dea6..66388bc5 100644 --- a/crates/node/tests/it/main.rs +++ b/crates/node/tests/it/main.rs @@ -13,6 +13,7 @@ mod evm; mod hardfork; mod l1_messages; mod morph_tx; +mod proof_history; mod reference_index; mod rpc; mod sync; diff --git a/crates/node/tests/it/proof_history.rs b/crates/node/tests/it/proof_history.rs new file mode 100644 index 00000000..2e8a95f3 --- /dev/null +++ b/crates/node/tests/it/proof_history.rs @@ -0,0 +1,178 @@ +//! End-to-end coexistence coverage for proof history and the reference index. + +use std::{sync::Arc, time::Duration}; + +use alloy_genesis::Genesis; +use alloy_primitives::{Address, B256}; +use alloy_rpc_types_eth::EIP1186AccountProofResponse; +use jsonrpsee::{core::client::ClientT, http_client::HttpClient, rpc_params}; +use morph_chainspec::MorphChainSpec; +use morph_node::{ + MorphAddOns, MorphNode, + test_utils::{MorphTestNode, advance_empty_block, morph_payload_attributes}, +}; +use morph_proofs::{InitializationJob, MdbxProofsStorage, MorphProofsStorage, ProofDbIdentity}; +use morph_proofs_exex::MorphProofsExEx; +use morph_reference_index::ReferenceTransactionResult; +use morph_rpc::{ProofsSyncStatus, ReferenceQueryArgs}; +use reth_chainspec::EthChainSpec; +use reth_e2e_test_utils::node::NodeTestContext; +use reth_node_builder::{EngineNodeLauncher, Node, NodeBuilder, NodeConfig, NodeHandle}; +use reth_node_core::args::{DiscoveryArgs, NetworkArgs, RpcServerArgs}; +use reth_provider::{DBProvider, DatabaseProviderFactory, providers::BlockchainProvider}; +use reth_rpc_server_types::RpcModuleSelection; +use reth_tasks::Runtime; + +async fn launch_node_with_proof_history( + chain_spec: Arc, + storage: MorphProofsStorage>, +) -> eyre::Result { + let runtime = Runtime::test(); + let network = NetworkArgs { + discovery: DiscoveryArgs { + disable_discovery: true, + ..DiscoveryArgs::default() + }, + ..NetworkArgs::default() + }; + let node_config = NodeConfig::new(chain_spec.clone()) + .with_network(network) + .with_unused_ports() + .with_rpc( + RpcServerArgs::default() + .with_unused_ports() + .with_http() + .with_http_api(RpcModuleSelection::All), + ); + let node = MorphNode::default(); + let exex_storage = storage.clone(); + + let NodeHandle { + node, + node_exit_future: _, + } = NodeBuilder::new(node_config) + .testing_node(runtime.clone()) + .with_types_and_provider::>() + .with_components(node.components_builder()) + .with_add_ons(MorphAddOns::new().with_proof_history(storage)) + .install_exex("morph-proof-history", async move |ctx| { + let head = ctx.head; + let provider = ctx + .provider() + .database_provider_ro()? + .disable_long_read_transaction_safety(); + InitializationJob::new(exex_storage.clone(), provider.into_tx()) + .run(head.number, head.hash)?; + + let exex = MorphProofsExEx::builder(ctx, exex_storage).build(); + Ok(async move { exex.run().await }) + }) + .launch_with_fn(|builder| { + let launcher = EngineNodeLauncher::new( + builder.task_executor().clone(), + builder.config().datadir(), + reth_node_api::TreeConfig::default().with_cross_block_cache_size(1024 * 1024), + ); + builder.launch_with(launcher) + }) + .await?; + + let node = NodeTestContext::new(node, morph_payload_attributes).await?; + let genesis_hash = chain_spec.genesis_hash(); + node.update_forkchoice(genesis_hash, genesis_hash).await?; + Ok(node) +} + +async fn wait_for_proof_tip(client: &HttpClient, expected: u64) -> eyre::Result { + let deadline = tokio::time::Instant::now() + Duration::from_secs(10); + let mut last_status = None; + while tokio::time::Instant::now() < deadline { + if let Ok(status) = client + .request::("debug_proofsSyncStatus", rpc_params![]) + .await + { + if status.latest == Some(expected) { + return Ok(status); + } + last_status = Some(status); + } + tokio::time::sleep(Duration::from_millis(25)).await; + } + Err(eyre::eyre!( + "proof history did not reach block {expected}: {last_status:?}" + )) +} + +async fn wait_for_reference_index(client: &HttpClient) -> eyre::Result<()> { + let args = ReferenceQueryArgs { + reference: B256::repeat_byte(0x99), + offset: None, + limit: None, + }; + let deadline = tokio::time::Instant::now() + Duration::from_secs(10); + let mut last_error = None; + while tokio::time::Instant::now() < deadline { + match client + .request::, _>( + "morph_getTransactionHashesByReference", + (args.clone(),), + ) + .await + { + Ok(results) if results.is_empty() => return Ok(()), + Ok(results) => { + return Err(eyre::eyre!( + "unexpected reference-index results: {results:?}" + )); + } + Err(error) => last_error = Some(error.to_string()), + } + tokio::time::sleep(Duration::from_millis(25)).await; + } + Err(eyre::eyre!( + "reference index did not become ready: {}", + last_error.as_deref().unwrap_or("no RPC response") + )) +} + +#[tokio::test(flavor = "multi_thread")] +async fn proof_history_and_reference_index_advance_together() -> eyre::Result<()> { + reth_tracing::init_test_tracing(); + + let genesis: Genesis = serde_json::from_str(include_str!("../assets/test-genesis.json"))?; + let chain_spec = Arc::new(MorphChainSpec::from_genesis(genesis)); + let proof_directory = tempfile::tempdir()?; + let proof_storage = Arc::new(MdbxProofsStorage::open( + proof_directory.path(), + ProofDbIdentity::new(chain_spec.chain().id(), chain_spec.genesis_hash()), + )?); + let mut node = launch_node_with_proof_history(chain_spec, proof_storage).await?; + + advance_empty_block(&mut node).await?; + let client = node + .rpc_client() + .ok_or_else(|| eyre::eyre!("HTTP RPC client not available"))?; + + let status = wait_for_proof_tip(&client, 1).await?; + assert_eq!(status.earliest, Some(0)); + wait_for_reference_index(&client).await?; + + let normal_proof: EIP1186AccountProofResponse = client + .request( + "eth_getProof", + rpc_params![Address::ZERO, Vec::::new(), "0x0"], + ) + .await?; + let auth_proof: EIP1186AccountProofResponse = node + .auth_server_handle() + .http_client() + .request( + "eth_getProof", + rpc_params![Address::ZERO, Vec::::new(), "0x0"], + ) + .await?; + assert_eq!(normal_proof.address, Address::ZERO); + assert_eq!(normal_proof, auth_proof); + + Ok(()) +} diff --git a/crates/proofs-exex/Cargo.toml b/crates/proofs-exex/Cargo.toml new file mode 100644 index 00000000..6c1a6728 --- /dev/null +++ b/crates/proofs-exex/Cargo.toml @@ -0,0 +1,37 @@ +[package] +name = "morph-proofs-exex" +description = "Forward-only proof-history execution extension for Morph" +version.workspace = true +edition.workspace = true +rust-version.workspace = true +license.workspace = true +publish.workspace = true + +[lints] +workspace = true + +[dependencies] +morph-proofs.workspace = true + +alloy-consensus.workspace = true +alloy-eips.workspace = true +eyre.workspace = true +futures.workspace = true +reth-execution-types.workspace = true +reth-exex.workspace = true +reth-node-api.workspace = true +reth-provider.workspace = true +reth-trie.workspace = true +tokio.workspace = true +tracing.workspace = true + +[dev-dependencies] +alloy-primitives.workspace = true +morph-proofs = { workspace = true, features = ["test-utils"] } +reth-db = { workspace = true, features = ["test-utils"] } +reth-ethereum-primitives = { workspace = true, features = ["test-utils"] } +reth-exex-test-utils.workspace = true +reth-primitives-traits = { workspace = true, features = ["test-utils"] } + +[features] +default = [] diff --git a/crates/proofs-exex/NOTICE.md b/crates/proofs-exex/NOTICE.md new file mode 100644 index 00000000..db03f0a4 --- /dev/null +++ b/crates/proofs-exex/NOTICE.md @@ -0,0 +1,31 @@ +# Attribution + +This execution extension is adapted from Base at commit +`b2673bbd927cb34d7cfad4d448bfbd5bd30eae88` under the MIT license: + + + +Morph-specific changes include MDBX-only storage, canonical-tip validation, Morph CLI +messages, and Morph metrics/logging names. + +## Base MIT License + +Copyright (c) 2025 Base + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. diff --git a/crates/proofs-exex/src/lib.rs b/crates/proofs-exex/src/lib.rs new file mode 100644 index 00000000..e1ad5d98 --- /dev/null +++ b/crates/proofs-exex/src/lib.rs @@ -0,0 +1,1192 @@ +//! Forward-only proof-history execution extension for Morph. +//! +//! Adapted from Base commit `b2673bbd927cb34d7cfad4d448bfbd5bd30eae88` +//! under the MIT license. + +#![cfg_attr(not(test), warn(unused_crate_dependencies))] + +mod sync_target; +use std::{sync::Arc, time::Duration}; + +use alloy_consensus::BlockHeader; +use alloy_eips::eip1898::BlockWithParent; +use futures::TryStreamExt; +use morph_proofs::{ + MorphProofStoragePrunerTask, MorphProofsBatchStore, MorphProofsStorage, + live::{BatchBlock, LiveTrieCollector}, + metrics::ProofHistoryMetrics, +}; +use reth_execution_types::Chain; +use reth_exex::{ExExContext, ExExEvent, ExExNotification, ExExNotificationsStream}; +use reth_node_api::{FullNodeComponents, NodePrimitives, NodeTypes}; +use reth_provider::{BlockHashReader, BlockNumReader, BlockReader, TransactionVariant}; +pub use sync_target::{CachedBlockTrieData, SyncTarget, SyncTargetState}; +use tokio::task; +use tracing::{debug, error, info}; + +/// Default safety threshold for the gap between stored earliest block and the configured +/// window target. When exceeded on startup, the node refuses to auto-prune and asks the +/// operator to run `proofs prune` manually. The threshold is a backstop against accidental +/// misconfiguration (e.g. shrinking the proofs window by orders of magnitude) — it is not a +/// measure of pruner throughput. Override via +/// [`MorphProofsExExBuilder::with_max_prune_blocks_startup`]. +const DEFAULT_MAX_PRUNE_BLOCKS_STARTUP: u64 = 1_000; + +/// How many blocks to process in a single sync turn before yielding. +const SYNC_BLOCKS_PER_TURN: usize = 50; + +/// Default proofs history window: 1 month of blocks at 2s block time +const DEFAULT_PROOFS_HISTORY_WINDOW: u64 = 1_296_000; + +/// Default interval between proof-storage prune runs. Default is 15 seconds. +const DEFAULT_PRUNE_INTERVAL: Duration = Duration::from_secs(15); + +/// Default verification interval: disabled +const DEFAULT_VERIFICATION_INTERVAL: u64 = 0; // disabled + +/// Builder for [`MorphProofsExEx`]. +#[derive(Debug)] +pub struct MorphProofsExExBuilder +where + Node: FullNodeComponents, +{ + ctx: ExExContext, + storage: MorphProofsStorage, + proofs_history_window: u64, + proofs_history_prune_interval: Duration, + verification_interval: u64, + max_prune_blocks_startup: u64, +} + +impl MorphProofsExExBuilder +where + Node: FullNodeComponents, +{ + /// Create a new builder with required parameters and defaults. + pub const fn new(ctx: ExExContext, storage: MorphProofsStorage) -> Self { + Self { + ctx, + storage, + proofs_history_window: DEFAULT_PROOFS_HISTORY_WINDOW, + proofs_history_prune_interval: DEFAULT_PRUNE_INTERVAL, + verification_interval: DEFAULT_VERIFICATION_INTERVAL, + max_prune_blocks_startup: DEFAULT_MAX_PRUNE_BLOCKS_STARTUP, + } + } + + /// Sets the window to span blocks for proofs history. + pub const fn with_proofs_history_window(mut self, window: u64) -> Self { + self.proofs_history_window = window; + self + } + + /// Sets the interval between proof-storage prune runs. + pub const fn with_proofs_history_prune_interval(mut self, interval: Duration) -> Self { + self.proofs_history_prune_interval = interval; + self + } + + /// Sets the verification interval. + pub const fn with_verification_interval(mut self, interval: u64) -> Self { + self.verification_interval = interval; + self + } + + /// Sets the safety threshold for blocks that may be auto-pruned at startup. + /// See `DEFAULT_MAX_PRUNE_BLOCKS_STARTUP`. + pub const fn with_max_prune_blocks_startup(mut self, max_blocks: u64) -> Self { + self.max_prune_blocks_startup = max_blocks; + self + } + + /// Builds the [`MorphProofsExEx`]. + pub fn build(self) -> MorphProofsExEx { + MorphProofsExEx { + ctx: self.ctx, + storage: self.storage, + proofs_history_window: self.proofs_history_window, + proofs_history_prune_interval: self.proofs_history_prune_interval, + verification_interval: self.verification_interval, + max_prune_blocks_startup: self.max_prune_blocks_startup, + } + } +} + +/// Proofs `ExEx` - processes blocks and tracks state changes within fault proof window. +/// +/// Saves and serves trie nodes to make proofs faster. This handles the process of +/// saving the current state, new blocks as they're added, and serving proof RPCs +/// based on the saved data. +/// +#[derive(Debug)] +pub struct MorphProofsExEx +where + Node: FullNodeComponents, +{ + /// The `ExEx` context containing the node related utilities e.g. provider, notifications, + /// events. + ctx: ExExContext, + /// The type of storage DB. + storage: MorphProofsStorage, + /// The window to span blocks for proofs history. Value is the number of blocks, received as + /// cli arg. + proofs_history_window: u64, + /// Interval between proof-storage prune runs + proofs_history_prune_interval: Duration, + /// Verification interval: perform full block execution every N blocks for data integrity. + /// If 0, verification is disabled (always use fast path when available). + /// If 1, verification is always enabled (always execute blocks). + verification_interval: u64, + /// Maximum blocks the startup check is willing to schedule for auto-prune. See + /// [`DEFAULT_MAX_PRUNE_BLOCKS_STARTUP`]. + max_prune_blocks_startup: u64, +} + +impl MorphProofsExEx +where + Node: FullNodeComponents, +{ + /// Create a new `MorphProofsExEx` instance. + pub fn new(ctx: ExExContext, storage: MorphProofsStorage) -> Self { + MorphProofsExExBuilder::new(ctx, storage).build() + } + + /// Create a new builder for `MorphProofsExEx`. + pub const fn builder( + ctx: ExExContext, + storage: MorphProofsStorage, + ) -> MorphProofsExExBuilder { + MorphProofsExExBuilder::new(ctx, storage) + } +} + +impl MorphProofsExEx +where + Node: FullNodeComponents>, + Primitives: NodePrimitives, + Storage: MorphProofsBatchStore + Clone + 'static, +{ + /// Main execution loop for the `ExEx` + pub async fn run(mut self) -> eyre::Result<()> { + self.ensure_initialized()?; + let sync_target = self.spawn_sync_task(); + + // If storage is behind tip, start syncing immediately rather than waiting + // for the first notification. + let best_block = self.ctx.provider().best_block_number()?; + let latest_stored = self + .storage + .get_latest_block_number()? + .map(|(n, _)| n) + .unwrap_or(0); + if latest_stored < best_block { + info!( + target: "morph::proofs_exex", + latest_stored, + best_block, + "Storage behind tip, starting sync immediately" + ); + sync_target.update_state(SyncTargetState::SyncUpTo { to: best_block }); + } + + let prune_task = MorphProofStoragePrunerTask::new( + self.storage.clone(), + self.ctx.provider().clone(), + self.proofs_history_window, + self.proofs_history_prune_interval, + ); + self.ctx + .task_executor() + .spawn_with_graceful_shutdown_signal(|signal| Box::pin(prune_task.run(signal))); + + self.ctx.notifications.set_without_head(); + + while let Some(notification) = self.ctx.notifications.try_next().await? { + self.handle_notification(notification, &sync_target)?; + } + + Ok(()) + } + + /// Ensure proofs storage is initialized + fn ensure_initialized(&self) -> eyre::Result<()> { + // Check if proofs storage is initialized + let earliest_block_number = match self.storage.get_earliest_block_number()? { + Some((n, _)) => n, + None => { + return Err(eyre::eyre!( + "Proofs storage not initialized. Please run 'morph-reth proofs init --proofs-history.storage-path ' first." + )); + } + }; + + let (latest_block_number, latest_block_hash) = match self + .storage + .get_latest_block_number()? + { + Some(latest) => latest, + None => { + return Err(eyre::eyre!( + "Proofs storage not initialized. Please run 'morph-reth proofs init --proofs-history.storage-path ' first." + )); + } + }; + + // Check if we have accumulated too much history for the configured window. + // If the gap between what we have and what we want to keep is too large, the auto-pruner + // will stall the node. + let target_earliest = latest_block_number.saturating_sub(self.proofs_history_window); + if target_earliest > earliest_block_number { + let blocks_to_prune = target_earliest - earliest_block_number; + if blocks_to_prune > self.max_prune_blocks_startup { + return Err(eyre::eyre!( + "Configuration requires pruning {} blocks, which exceeds the safety threshold of {}. \ + Huge prune operations can stall the node. \ + Please run 'morph-reth proofs prune' manually before starting the node.", + blocks_to_prune, + self.max_prune_blocks_startup + )); + } + } + + let canonical_hash = self + .ctx + .provider() + .block_hash(latest_block_number)? + .ok_or_else(|| eyre::eyre!( + "Proofs storage latest block {latest_block_number} is missing from the canonical chain" + ))?; + if canonical_hash != latest_block_hash { + return Err(eyre::eyre!( + "Proofs storage latest block hash mismatch at {latest_block_number}: stored {latest_block_hash}, canonical {canonical_hash}" + )); + } + + ProofHistoryMetrics::set_window(earliest_block_number, latest_block_number); + + Ok(()) + } + + /// Spawn the background sync task and return the shared [`SyncTarget`]. + /// + /// The sync target buffers trie data from notifications so the sync loop + /// can use pre-computed data for blocks even when it is many blocks behind + /// the chain tip. Blocks whose trie data was evicted from the cache fall + /// back to full execution. + fn spawn_sync_task(&self) -> Arc { + let sync_target = Arc::new(SyncTarget::new()); + let task_sync_target = Arc::clone(&sync_target); + + let task_storage = self.storage.clone(); + let task_provider = self.ctx.provider().clone(); + let task_evm_config = self.ctx.evm_config().clone(); + let verification_interval = self.verification_interval; + + self.ctx.task_executor().spawn_critical_task( + "morph::proofs_exex::proofs_storage_sync_loop", + async move { + let storage = task_storage.clone(); + let task_collector = + LiveTrieCollector::new(task_evm_config, task_provider.clone(), &storage); + Self::sync_loop( + task_sync_target, + task_storage, + task_provider, + &task_collector, + verification_interval, + ) + .await + .unwrap_or_else(|error| panic!("proof storage sync loop failed: {error:?}")); + }, + ); + + sync_target + } + + async fn sync_loop( + sync_target: Arc, + storage: MorphProofsStorage, + provider: Node::Provider, + collector: &LiveTrieCollector<'_, Node::Evm, Node::Provider, Storage>, + verification_interval: u64, + ) -> eyre::Result<()> { + info!(target: "morph::proofs_exex", "Starting proofs storage sync loop"); + + loop { + let Some(state) = sync_target.take_state() else { + sync_target.notified().await; + continue; + }; + + match state { + SyncTargetState::Revert { revert_to } => { + Self::handle_revert(&storage, collector, revert_to)?; + sync_target.mark_revert_complete(&revert_to); + } + SyncTargetState::RevertThenSync { revert_to, sync_to } => { + Self::handle_revert(&storage, collector, revert_to)?; + sync_target.mark_revert_complete(&revert_to); + Self::sync_forward( + &sync_target, + &storage, + &provider, + collector, + verification_interval, + sync_to, + ) + .await; + } + SyncTargetState::SyncUpTo { to } => { + Self::sync_forward( + &sync_target, + &storage, + &provider, + collector, + verification_interval, + to, + ) + .await; + } + } + } + } + + fn handle_revert( + storage: &MorphProofsStorage, + collector: &LiveTrieCollector<'_, Node::Evm, Node::Provider, Storage>, + revert_to: BlockWithParent, + ) -> eyre::Result<()> { + let latest = match storage.get_latest_block_number()? { + Some((n, _)) => n, + None => return Ok(()), + }; + + if latest >= revert_to.block.number { + info!( + target: "morph::proofs_exex", + revert_to = revert_to.block.number, + latest, + "Reverting proofs storage" + ); + collector.unwind_history(revert_to).map_err(|error| { + error!(target: "morph::proofs_exex", ?error, "Failed to revert proofs storage"); + error + })?; + } else { + debug!( + target: "morph::proofs_exex", + revert_to = revert_to.block.number, + latest, + "Revert target beyond stored blocks, skipping" + ); + } + Ok(()) + } + + #[cfg(test)] + fn handle_revert_for_test(&self, revert_to: BlockWithParent) -> eyre::Result<()> { + let collector = LiveTrieCollector::new( + self.ctx.evm_config().clone(), + self.ctx.provider().clone(), + &self.storage, + ); + Self::handle_revert(&self.storage, &collector, revert_to) + } + + async fn sync_forward( + sync_target: &SyncTarget, + storage: &MorphProofsStorage, + provider: &Node::Provider, + collector: &LiveTrieCollector<'_, Node::Evm, Node::Provider, Storage>, + verification_interval: u64, + target: u64, + ) { + loop { + // Check for higher-priority state (e.g. revert) before processing. + if sync_target.has_pending_state() { + return; + } + + let latest = match storage.get_latest_block_number() { + Ok(Some((n, _))) => n, + Ok(None) => { + error!(target: "morph::proofs_exex", "No blocks stored in proofs storage during sync"); + return; + } + Err(e) => { + error!(target: "morph::proofs_exex", error = ?e, "Failed to get latest block"); + return; + } + }; + + if latest >= target { + return; + } + + let end = latest + .saturating_add(SYNC_BLOCKS_PER_TURN as u64) + .min(target); + info!( + target: "morph::proofs_exex", + start = latest + 1, + end, + target, + blocks = end - latest, + "Processing proofs storage sync turn" + ); + + let mut batch: Vec> = + Vec::with_capacity((end - latest) as usize); + for block_num in (latest + 1)..=end { + let cached = sync_target.take(block_num); + match Self::build_batch_entry(block_num, cached, provider, verification_interval) { + Ok(entry) => batch.push(entry), + Err(e) => { + error!(target: "morph::proofs_exex", block_number = block_num, error = ?e, "Preparing block for batch failed"); + return; + } + } + } + + if let Err(e) = collector.execute_and_store_batch(batch) { + error!(target: "morph::proofs_exex", start = latest + 1, end, error = ?e, "Batch processing failed"); + return; + } + + info!(target: "morph::proofs_exex", latest_stored = latest, target, "Batch processed, yielding"); + task::yield_now().await; + } + } + + fn build_batch_entry( + block_number: u64, + cached: Option, + provider: &Node::Provider, + verification_interval: u64, + ) -> eyre::Result> { + let should_verify = + verification_interval > 0 && block_number.is_multiple_of(verification_interval); + let has_cached = cached.is_some(); + + if let Some(cached) = cached + && !should_verify + { + debug!( + target: "morph::proofs_exex", + block_number, + "Using pre-computed state from notification" + ); + return Ok(BatchBlock::Cached { + block_with_parent: cached.block_with_parent, + sorted_trie_updates: cached.trie_data.trie_updates(), + sorted_post_state: cached.trie_data.hashed_state(), + }); + } + + if has_cached { + info!( + target: "morph::proofs_exex", + block_number, + verification_interval, + "Periodic verification: performing full block execution despite cached data" + ); + } else { + debug!( + target: "morph::proofs_exex", + block_number, + "No cached trie data, falling back to full execution" + ); + } + + debug!( + target: "morph::proofs_exex", + block_number, + "Fetching block from provider for execution", + ); + + let block = provider + .recovered_block(block_number.into(), TransactionVariant::NoHash)? + .ok_or_else(|| eyre::eyre!("Missing block {} in provider", block_number))?; + + Ok(BatchBlock::Execute(Box::new(block))) + } + + fn handle_notification( + &self, + notification: ExExNotification, + sync_target: &SyncTarget, + ) -> eyre::Result<()> { + match ¬ification { + ExExNotification::ChainCommitted { new } => { + self.handle_chain_committed(Arc::clone(new), sync_target)? + } + ExExNotification::ChainReorged { old, new } => { + self.handle_chain_reorged(Arc::clone(old), Arc::clone(new), sync_target)? + } + ExExNotification::ChainReverted { old } => { + self.handle_chain_reverted(Arc::clone(old), sync_target)? + } + } + + if let Some(committed_chain) = notification.committed_chain() { + let tip = committed_chain.tip().num_hash(); + debug!( + target: "morph::proofs_exex", + block_number = tip.number, + block_hash = ?tip.hash, + "Sending FinishedHeight event" + ); + self.ctx.events.send(ExExEvent::FinishedHeight(tip))?; + } + + Ok(()) + } + + fn handle_chain_committed( + &self, + new: Arc>, + sync_target: &SyncTarget, + ) -> eyre::Result<()> { + debug!( + target: "morph::proofs_exex", + block_number = new.tip().number(), + block_hash = ?new.tip().hash(), + "ChainCommitted notification received", + ); + + // Cache trie data for all blocks in the chain so the sync loop can + // use pre-computed data even when it is many blocks behind. + let total_blocks = new.blocks().len(); + let mut cached_count = 0usize; + for (&block_number, block) in new.blocks() { + if let Some(trie_data) = new.trie_data_at(block_number) { + sync_target.insert( + block_number, + CachedBlockTrieData { + block_with_parent: block.block_with_parent(), + trie_data: trie_data.clone(), + }, + ); + cached_count += 1; + } else { + debug!( + target: "morph::proofs_exex", + block_number, + "Notification block missing trie data" + ); + } + } + + debug!( + target: "morph::proofs_exex", + tip = new.tip().number(), + total_blocks, + cached_count, + missing = total_blocks - cached_count, + "Cached notification trie data" + ); + + sync_target.update_state(SyncTargetState::SyncUpTo { + to: new.tip().number(), + }); + Ok(()) + } + + fn handle_chain_reorged( + &self, + old: Arc>, + new: Arc>, + sync_target: &SyncTarget, + ) -> eyre::Result<()> { + info!( + target: "morph::proofs_exex", + old_block_number = old.tip().number(), + old_block_hash = ?old.tip().hash(), + new_block_number = new.tip().number(), + new_block_hash = ?new.tip().hash(), + "ChainReorged notification received", + ); + + if old.fork_block() != new.fork_block() { + return Err(eyre::eyre!( + "Fork blocks do not match: old fork block {:?}, new fork block {:?}", + old.fork_block(), + new.fork_block() + )); + } + + let first_old = old.first().block_with_parent(); + + // Invalidate any cached blocks from the old chain. + sync_target.clear_from(first_old.block.number); + + // Cache trie data for all blocks in the new chain. + for (&block_number, block) in new.blocks() { + if let Some(trie_data) = new.trie_data_at(block_number) { + sync_target.insert( + block_number, + CachedBlockTrieData { + block_with_parent: block.block_with_parent(), + trie_data: trie_data.clone(), + }, + ); + } else { + debug!( + target: "morph::proofs_exex", + block_number, + "Reorged block missing trie data" + ); + } + } + + sync_target.update_state(SyncTargetState::RevertThenSync { + revert_to: first_old, + sync_to: new.tip().number(), + }); + + Ok(()) + } + + fn handle_chain_reverted( + &self, + old: Arc>, + sync_target: &SyncTarget, + ) -> eyre::Result<()> { + info!( + target: "morph::proofs_exex", + old_block_number = old.tip().number(), + old_block_hash = ?old.tip().hash(), + "ChainReverted notification received", + ); + + let first_old = old.first().block_with_parent(); + + // Invalidate any cached blocks that are being reverted. + sync_target.clear_from(first_old.block.number); + + sync_target.update_state(SyncTargetState::Revert { + revert_to: first_old, + }); + Ok(()) + } +} + +#[cfg(test)] +mod tests { + use std::{collections::BTreeMap, default::Default, sync::Arc, time::Duration}; + + use alloy_consensus::private::alloy_primitives::B256; + use alloy_eips::{BlockNumHash, NumHash, eip1898::BlockWithParent}; + use morph_proofs::{BlockStateDiff, MdbxProofsStorage, MorphProofsStorage, MorphProofsStore}; + use reth_db::test_utils::tempdir_path; + use reth_ethereum_primitives::{Block, Receipt}; + use reth_execution_types::{Chain, ExecutionOutcome}; + use reth_primitives_traits::RecoveredBlock; + use reth_trie::{ + ComputedTrieData, HashedPostStateSorted, LazyTrieData, updates::TrieUpdatesSorted, + }; + + use super::*; + + // ------------------------------------------------------------------------- + // Helpers: deterministic blocks and deterministic Chain with precomputed updates + // ------------------------------------------------------------------------- + fn b256(byte: u8) -> B256 { + B256::new([byte; 32]) + } + + // deterministic hash from block number: 0 -> 0x00.., 1 -> 0x01.., etc. + fn hash_for_num(num: u64) -> B256 { + // if you only care about small test numbers, this is enough: + b256(num as u8) + + // If you want to avoid wrapping when num > 255, use something like: + // let mut out = [0u8; 32]; + // out[0..8].copy_from_slice(&num.to_be_bytes()); + // B256::new(out) + } + + fn mk_block(num: u64) -> RecoveredBlock { + let mut b: RecoveredBlock = Default::default(); + b.set_block_number(num); + b.set_hash(hash_for_num(num)); + b.set_parent_hash(hash_for_num(num - 1)); + b + } + + fn mk_chain_with_updates( + from: u64, + to: u64, + hash_override: Option, + ) -> Chain { + let mut blocks: Vec> = Vec::new(); + let mut trie_data = BTreeMap::new(); + + for n in from..=to { + let mut b = mk_block(n); + if let Some(hash) = hash_override { + b.set_hash(hash); + } + blocks.push(b); + + let data = LazyTrieData::ready(ComputedTrieData::new( + Arc::new(HashedPostStateSorted::default()), + Arc::new(TrieUpdatesSorted::default()), + )); + trie_data.insert(n, data); + } + + let execution_outcome: ExecutionOutcome = ExecutionOutcome { + bundle: Default::default(), + receipts: Vec::new(), + requests: Vec::new(), + first_block: from, + }; + + Chain::new(blocks, execution_outcome, trie_data) + } + + /// Store blocks directly into proofs storage (bypasses the sync loop). + fn store_blocks(from: u64, to: u64, storage: &MorphProofsStorage) { + for n in from..=to { + let chain = mk_chain_with_updates(n, n, None); + let block = chain.blocks().get(&n).unwrap(); + storage + .store_trie_updates(block.block_with_parent(), BlockStateDiff::default()) + .expect("store trie update"); + } + } + + fn init_storage_at( + storage: MorphProofsStorage, + genesis_block: NumHash, + ) { + storage + .set_earliest_block_number(genesis_block.number, genesis_block.hash) + .expect("set earliest"); + storage + .store_trie_updates( + BlockWithParent::new(genesis_block.hash, genesis_block), + BlockStateDiff::default(), + ) + .expect("store trie update"); + } + + fn init_storage(storage: MorphProofsStorage) { + init_storage_at(storage, NumHash::new(0, b256(0x00))); + } + + // Initialize exex with config + fn build_test_exex( + ctx: ExExContext, + storage: MorphProofsStorage, + ) -> MorphProofsExEx + where + NodeT: FullNodeComponents, + Store: MorphProofsStore + Clone + 'static, + { + MorphProofsExEx::builder(ctx, storage) + .with_proofs_history_window(20) + .with_proofs_history_prune_interval(Duration::from_secs(3600)) + .with_verification_interval(1000) + .with_max_prune_blocks_startup(1000) + .build() + } + + #[tokio::test] + async fn handle_notification_chain_committed() { + // MDBX proofs storage + let dir = tempdir_path(); + let store = Arc::new(MdbxProofsStorage::new(dir.as_path()).expect("env")); + let proofs: MorphProofsStorage> = Arc::clone(&store); + + init_storage(proofs.clone()); + + let (ctx, _handle) = reth_exex_test_utils::test_exex_context() + .await + .expect("exex test context"); + + let exex = build_test_exex(ctx, proofs.clone()); + + let new_chain = Arc::new(mk_chain_with_updates(1, 1, None)); + let notif = ExExNotification::ChainCommitted { new: new_chain }; + + let sync_target = SyncTarget::new(); + + exex.handle_notification(notif, &sync_target) + .expect("handle chain commit"); + + // Committed blocks are cached in sync target, not stored directly + assert!(sync_target.take(1).is_some(), "block 1 should be cached"); + let state = sync_target.take_state().expect("should have pending state"); + assert!(matches!(state, SyncTargetState::SyncUpTo { to: 1 })); + } + + #[tokio::test] + async fn handle_notification_chain_committed_caches_already_stored_blocks() { + // MDBX proofs storage + let dir = tempdir_path(); + let store = Arc::new(MdbxProofsStorage::new(dir.as_path()).expect("env")); + let proofs: MorphProofsStorage> = Arc::clone(&store); + + init_storage(proofs.clone()); + + // Pre-store blocks 1..5 so storage is at block 5 + store_blocks(1, 5, &proofs); + + let (ctx, _handle) = reth_exex_test_utils::test_exex_context() + .await + .expect("exex test context"); + + let exex = build_test_exex(ctx, proofs.clone()); + + let sync_target = SyncTarget::new(); + + // Handle notification for block 5 which is already stored - still caches + let new_chain = Arc::new(mk_chain_with_updates(5, 5, Some(hash_for_num(10)))); + let notif = ExExNotification::ChainCommitted { new: new_chain }; + exex.handle_notification(notif, &sync_target) + .expect("handle chain commit"); + + // State is set (sync loop will see latest >= target and skip) + let state = sync_target.take_state().expect("should have pending state"); + assert!(matches!(state, SyncTargetState::SyncUpTo { to: 5 })); + + // Storage is unchanged (notification handler doesn't write to storage) + let latest = proofs + .get_latest_block_number() + .expect("get latest block") + .expect("ok"); + assert_eq!(latest.0, 5); + assert_eq!(latest.1, hash_for_num(5)); + } + + #[tokio::test] + async fn handle_notification_chain_reorged() { + // MDBX proofs storage + let dir = tempdir_path(); + let store = Arc::new(MdbxProofsStorage::new(dir.as_path()).expect("env")); + let proofs: MorphProofsStorage> = Arc::clone(&store); + + init_storage(proofs.clone()); + store_blocks(1, 10, &proofs); + + let (ctx, _handle) = reth_exex_test_utils::test_exex_context() + .await + .expect("exex test context"); + + let exex = build_test_exex(ctx, proofs.clone()); + + let sync_target = SyncTarget::new(); + + // Now the tip is 10, and we want to reorg from block 6..12 + let old_chain = Arc::new(mk_chain_with_updates(6, 10, None)); + let new_chain = Arc::new(mk_chain_with_updates(6, 12, None)); + + // Notification: chain reorged 6..12 + let notif = ExExNotification::ChainReorged { + new: new_chain, + old: old_chain, + }; + + exex.handle_notification(notif, &sync_target) + .expect("handle chain re-orged"); + + // Should have RevertThenSync state + let state = sync_target.take_state().expect("should have pending state"); + assert!(matches!( + state, + SyncTargetState::RevertThenSync { revert_to, sync_to: 12 } + if revert_to.block.number == 6 + )); + + // New chain blocks should be cached + for n in 6..=12 { + assert!(sync_target.take(n).is_some(), "block {n} should be cached"); + } + + // Storage unchanged (sync loop handles the actual revert) + let latest = proofs + .get_latest_block_number() + .expect("get latest block") + .expect("ok") + .0; + assert_eq!(latest, 10); + } + + #[tokio::test] + async fn handle_notification_chain_reorged_beyond_stored_blocks() { + // MDBX proofs storage + let dir = tempdir_path(); + let store = Arc::new(MdbxProofsStorage::new(dir.as_path()).expect("env")); + let proofs: MorphProofsStorage> = Arc::clone(&store); + + init_storage(proofs.clone()); + store_blocks(1, 10, &proofs); + + let (ctx, _handle) = reth_exex_test_utils::test_exex_context() + .await + .expect("exex test context"); + + let exex = build_test_exex(ctx, proofs.clone()); + + let sync_target = SyncTarget::new(); + + // Now the tip is 10, and we want to reorg from block 12..15 + // Both chains share the same fork block (block 11) + let old_chain = Arc::new(mk_chain_with_updates(12, 15, None)); + let new_chain = Arc::new(mk_chain_with_updates(12, 20, None)); + + // Notification: chain reorged 12..20 + let notif = ExExNotification::ChainReorged { + new: new_chain, + old: old_chain, + }; + + exex.handle_notification(notif, &sync_target) + .expect("handle chain re-orged"); + + // State is set; sync loop will detect revert is beyond stored blocks + let state = sync_target.take_state().expect("should have pending state"); + assert!(matches!( + state, + SyncTargetState::RevertThenSync { revert_to, sync_to: 20 } + if revert_to.block.number == 12 + )); + + // Storage unchanged + let latest = proofs + .get_latest_block_number() + .expect("get latest block") + .expect("ok") + .0; + assert_eq!(latest, 10); + } + + #[tokio::test] + async fn handle_notification_chain_reverted() { + // MDBX proofs storage + let dir = tempdir_path(); + let store = Arc::new(MdbxProofsStorage::new(dir.as_path()).expect("env")); + let proofs: MorphProofsStorage> = Arc::clone(&store); + + init_storage(proofs.clone()); + store_blocks(1, 10, &proofs); + + let (ctx, _handle) = reth_exex_test_utils::test_exex_context() + .await + .expect("exex test context"); + + let exex = build_test_exex(ctx, proofs.clone()); + + let sync_target = SyncTarget::new(); + + // Now the tip is 10, and we want to revert from block 9..10 + let old_chain = Arc::new(mk_chain_with_updates(9, 10, None)); + + // Notification: chain reverted 9..10 + let notif = ExExNotification::ChainReverted { old: old_chain }; + + exex.handle_notification(notif, &sync_target) + .expect("handle chain reverted"); + + // Should have Revert state + let state = sync_target.take_state().expect("should have pending state"); + assert!(matches!( + state, + SyncTargetState::Revert { revert_to } + if revert_to.block.number == 9 + )); + + // Storage unchanged (sync loop handles the actual revert) + let latest = proofs + .get_latest_block_number() + .expect("get latest block") + .expect("ok") + .0; + assert_eq!(latest, 10); + } + + #[tokio::test] + async fn handle_notification_chain_reverted_beyond_stored_blocks() { + // MDBX proofs storage + let dir = tempdir_path(); + let store = Arc::new(MdbxProofsStorage::new(dir.as_path()).expect("env")); + let proofs: MorphProofsStorage> = Arc::clone(&store); + + init_storage(proofs.clone()); + store_blocks(1, 5, &proofs); + + let (ctx, _handle) = reth_exex_test_utils::test_exex_context() + .await + .expect("exex test context"); + + let exex = build_test_exex(ctx, proofs.clone()); + + let sync_target = SyncTarget::new(); + + // Now the tip is 5, and we want to revert from block 9..10 + let old_chain = Arc::new(mk_chain_with_updates(9, 10, None)); + + // Notification: chain reverted 9..10 + let notif = ExExNotification::ChainReverted { old: old_chain }; + + exex.handle_notification(notif, &sync_target) + .expect("handle chain reverted"); + + // State is set; sync loop will detect revert is beyond stored blocks + let state = sync_target.take_state().expect("should have pending state"); + assert!(matches!( + state, + SyncTargetState::Revert { revert_to } + if revert_to.block.number == 9 + )); + + // Storage unchanged + let latest = proofs + .get_latest_block_number() + .expect("get latest block") + .expect("ok") + .0; + assert_eq!(latest, 5); + } + + #[tokio::test] + async fn handle_revert_propagates_unwind_failure() { + let dir = tempdir_path(); + let store = Arc::new(MdbxProofsStorage::new(dir.as_path()).expect("env")); + let proofs: MorphProofsStorage> = Arc::clone(&store); + init_storage(proofs.clone()); + + let (ctx, _handle) = reth_exex_test_utils::test_exex_context() + .await + .expect("exex test context"); + let exex = build_test_exex(ctx, proofs.clone()); + + let initial_anchor = BlockWithParent::new(b256(0x00), NumHash::new(0, b256(0x00))); + let error = exex + .handle_revert_for_test(initial_anchor) + .expect_err("unwinding the initial anchor must fail"); + + assert!( + error.to_string().contains("earliest"), + "unexpected unwind error: {error:?}" + ); + assert_eq!( + proofs + .get_latest_block_number() + .expect("latest") + .expect("initialized") + .0, + 0, + "failed unwind must leave proof history unchanged" + ); + } + + #[tokio::test] + async fn ensure_initialized_errors_on_storage_not_initialized() { + // MDBX proofs storage + let dir = tempdir_path(); + let store = Arc::new(MdbxProofsStorage::new(dir.as_path()).expect("env")); + let proofs: MorphProofsStorage> = Arc::clone(&store); + + let (ctx, _handle) = reth_exex_test_utils::test_exex_context() + .await + .expect("exex test context"); + + let exex = build_test_exex(ctx, proofs.clone()); + let _ = exex.ensure_initialized().expect_err("should return error"); + } + + #[tokio::test] + async fn ensure_initialized_errors_when_prune_exceeds_threshold() { + // MDBX proofs storage + let dir = tempdir_path(); + let store = Arc::new(MdbxProofsStorage::new(dir.as_path()).expect("env")); + let proofs: MorphProofsStorage> = Arc::clone(&store); + + init_storage(proofs.clone()); + + for i in 1..1100 { + proofs + .store_trie_updates( + BlockWithParent::new( + hash_for_num(i - 1), + BlockNumHash::new(i, hash_for_num(i)), + ), + BlockStateDiff::default(), + ) + .expect("store trie update"); + } + + let (ctx, _handle) = reth_exex_test_utils::test_exex_context() + .await + .expect("exex test context"); + + let exex = build_test_exex(ctx, proofs.clone()); + let _ = exex.ensure_initialized().expect_err("should return error"); + } + + #[tokio::test] + async fn ensure_initialized_succeeds() { + // MDBX proofs storage + let dir = tempdir_path(); + let store = Arc::new(MdbxProofsStorage::new(dir.as_path()).expect("env")); + let proofs: MorphProofsStorage> = Arc::clone(&store); + + let (ctx, handle) = reth_exex_test_utils::test_exex_context() + .await + .expect("exex test context"); + init_storage_at(proofs.clone(), handle.genesis.num_hash()); + + let exex = build_test_exex(ctx, proofs.clone()); + exex.ensure_initialized().expect("should not return error"); + } + + #[tokio::test] + async fn handle_notification_schedules_async_on_gap() { + // MDBX proofs storage + let dir = tempdir_path(); + let store = Arc::new(MdbxProofsStorage::new(dir.as_path()).expect("env")); + let proofs: MorphProofsStorage> = Arc::clone(&store); + + init_storage(proofs.clone()); + + let (ctx, _handle) = reth_exex_test_utils::test_exex_context() + .await + .expect("exex test context"); + + let exex = build_test_exex(ctx, proofs.clone()); + + // Notification: chain committed 5..10 (Blocks 1,2,3,4 are missing from storage) + let new_chain = Arc::new(mk_chain_with_updates(5, 10, None)); + let notif = ExExNotification::ChainCommitted { new: new_chain }; + + let sync_target = SyncTarget::new(); + + // Process notification + exex.handle_notification(notif, &sync_target) + .expect("handle chain commit should return ok immediately"); + + // Verify the sync target state was set + let state = sync_target.take_state().expect("should have pending state"); + assert!( + matches!(state, SyncTargetState::SyncUpTo { to: 10 }), + "Should have scheduled sync to block 10" + ); + + // Verify Main Thread did NOT process it + // Because we didn't spawn the actual worker thread in this test, storage should still be at + // 0. This proves the 'handle_notification' returned instantly without doing the + // heavy lifting. + let latest = proofs + .get_latest_block_number() + .expect("get") + .expect("ok") + .0; + assert_eq!( + latest, 0, + "Main thread should not have processed the blocks synchronously" + ); + } +} diff --git a/crates/proofs-exex/src/sync_target.rs b/crates/proofs-exex/src/sync_target.rs new file mode 100644 index 00000000..5a26b6fa --- /dev/null +++ b/crates/proofs-exex/src/sync_target.rs @@ -0,0 +1,637 @@ +//! Sync target with trie data cache for the background sync loop. +//! +//! Buffers trie data from exex notifications so the sync loop can use +//! pre-computed data even when it is many blocks behind the chain tip. +//! Routes committed, reverted, and reorged notifications through a +//! [`SyncTargetState`] state machine so the sync loop is the single +//! writer to proofs storage. + +use std::{collections::BTreeMap, sync::Mutex}; + +use alloy_eips::eip1898::BlockWithParent; +use reth_trie::LazyTrieData; +use tokio::sync::Notify; +use tracing::debug; + +/// Maximum number of blocks to cache trie data for. +const CACHE_CAPACITY: usize = 1024; + +/// Cached trie data for a single block. +#[derive(Debug)] +pub struct CachedBlockTrieData { + /// The block identifier with its parent hash. + pub block_with_parent: BlockWithParent, + /// The lazy trie data (hashed state + trie updates). + pub trie_data: LazyTrieData, +} + +/// The state of the sync target, describing what the sync loop should do next. +#[derive(Debug)] +pub enum SyncTargetState { + /// Sync forward to a specific block number. + SyncUpTo { + /// The target block number to sync up to. + to: u64, + }, + /// Revert to a specific block, then sync forward to a target. + RevertThenSync { + /// The first block to remove (inclusive) during the revert. + revert_to: BlockWithParent, + /// The target block number to sync up to after reverting. + sync_to: u64, + }, + /// Revert to a specific block without syncing forward. + Revert { + /// The first block to remove (inclusive) during the revert. + revert_to: BlockWithParent, + }, +} + +impl SyncTargetState { + const fn apply_next(&mut self, new: Self) { + *self = match (&*self, new) { + // If we are just syncing to tip already, replace with the new state. + (Self::SyncUpTo { .. }, new) => new, + + // If the new state is a revert, replace with the new state. + (_, Self::Revert { revert_to }) => Self::Revert { revert_to }, + (_, Self::RevertThenSync { revert_to, sync_to }) => { + Self::RevertThenSync { revert_to, sync_to } + } + + // If we're currently reverting, replace the sync to value with the new + // state. + ( + Self::RevertThenSync { revert_to, .. } | Self::Revert { revert_to }, + Self::SyncUpTo { to }, + ) => Self::RevertThenSync { + revert_to: *revert_to, + sync_to: to, + }, + }; + } +} + +/// Sync target that buffers trie data from recent exex notifications. +/// +/// Routes all notification types (committed, reverted, reorged) through a +/// [`SyncTargetState`] state machine so the sync loop is the single writer +/// to proofs storage. Uses a [`Notify`] to wake the sync loop when new +/// state is available. +/// +/// Trie data is accumulated in a bounded [`BTreeMap`] so the sync loop +/// can still use pre-computed trie data for blocks from earlier notifications. +pub struct SyncTarget { + cache: Mutex>, + state: Mutex>, + notify: Notify, +} + +impl Default for SyncTarget { + fn default() -> Self { + Self::new() + } +} + +impl SyncTarget { + /// Create a new `SyncTarget` with no cached data and no pending state. + pub fn new() -> Self { + Self { + cache: Mutex::new(BTreeMap::new()), + state: Mutex::new(None), + notify: Notify::new(), + } + } + + /// Update the sync target state and wake the sync loop. + /// + /// If there is already a pending state, the new state is merged using + /// `SyncTargetState::apply_next`. + pub fn update_state(&self, new: SyncTargetState) { + let mut state = self.state.lock().expect("SyncTarget lock poisoned"); + match state.as_mut() { + Some(current) => current.apply_next(new), + None => *state = Some(new), + } + drop(state); + self.notify.notify_one(); + } + + /// Take the current pending state, leaving `None` in its place. + /// + /// Used by the sync loop to consume the next action to perform. + pub fn take_state(&self) -> Option { + self.state.lock().expect("SyncTarget lock poisoned").take() + } + + /// Notify the sync target that a revert has been processed up to the given block. + /// + /// Only strips the revert portion if the pending revert target is at or above + /// `reverted_to` (i.e. already covered by the completed revert). If a deeper + /// revert arrived while processing, the pending state is left unchanged. + /// + /// - Covered `RevertThenSync` → `SyncUpTo` (keeps the sync target) + /// - Covered `Revert` → `None` + /// - Deeper revert or other states → unchanged + pub fn mark_revert_complete(&self, reverted_to: &BlockWithParent) { + let mut state = self.state.lock().expect("SyncTarget lock poisoned"); + match &*state { + Some(SyncTargetState::RevertThenSync { revert_to, sync_to }) + if revert_to.block.number >= reverted_to.block.number => + { + *state = Some(SyncTargetState::SyncUpTo { to: *sync_to }); + } + Some(SyncTargetState::Revert { revert_to }) + if revert_to.block.number >= reverted_to.block.number => + { + *state = None; + } + _ => {} + } + } + + /// Check if there is a pending state without consuming it. + /// + /// Used by the sync loop to interrupt forward sync when a higher-priority + /// state (e.g. revert) arrives. + pub fn has_pending_state(&self) -> bool { + self.state + .lock() + .expect("SyncTarget lock poisoned") + .is_some() + } + + /// Wait for a state change notification. + /// + /// Returns immediately if a notification arrived since the last call. + pub async fn notified(&self) { + self.notify.notified().await; + } + + /// Insert cached trie data for a block. + /// + /// Evicts the oldest entries when the cache exceeds capacity. + pub fn insert(&self, block_number: u64, data: CachedBlockTrieData) { + let mut cache = self.cache.lock().expect("SyncTarget lock poisoned"); + cache.insert(block_number, data); + let mut evicted = 0u64; + while cache.len() > CACHE_CAPACITY { + cache.pop_first(); + evicted += 1; + } + if evicted > 0 { + debug!( + target: "morph::proofs_exex::sync_target", + block_number, + evicted, + "Cache full, evicted oldest entries" + ); + } + debug!( + target: "morph::proofs_exex::sync_target", + block_number, + cached_blocks = cache.len(), + "Cached trie data for block" + ); + } + + /// Take cached trie data for a specific block, removing it from the cache. + pub fn take(&self, block_number: u64) -> Option { + let result = self + .cache + .lock() + .expect("SyncTarget lock poisoned") + .remove(&block_number); + if result.is_some() { + debug!( + target: "morph::proofs_exex::sync_target", + block_number, + "Cache hit: trie data found for block" + ); + } else { + debug!( + target: "morph::proofs_exex::sync_target", + block_number, + "Cache miss: no trie data for block, will re-execute" + ); + } + result + } + + /// Remove all cached entries at or above the given block number. + /// + /// Used when a revert or reorg invalidates cached blocks. + pub fn clear_from(&self, block_number: u64) { + let mut cache = self.cache.lock().expect("SyncTarget lock poisoned"); + let removed = cache.split_off(&block_number); + if !removed.is_empty() { + debug!( + target: "morph::proofs_exex::sync_target", + block_number, + cleared = removed.len(), + "Cleared cached entries from block onward" + ); + } + } +} + +impl std::fmt::Debug for SyncTarget { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + f.debug_struct("SyncTarget") + .field( + "has_pending_state", + &self + .state + .lock() + .expect("SyncTarget lock poisoned") + .is_some(), + ) + .field( + "cached_blocks", + &self.cache.lock().expect("SyncTarget lock poisoned").len(), + ) + .finish() + } +} + +#[cfg(test)] +mod tests { + use std::sync::Arc; + + use alloy_consensus::private::alloy_primitives::B256; + use alloy_eips::{NumHash, eip1898::BlockWithParent}; + use reth_trie::{ + ComputedTrieData, HashedPostStateSorted, LazyTrieData, updates::TrieUpdatesSorted, + }; + + use super::*; + + fn b256(byte: u8) -> B256 { + B256::new([byte; 32]) + } + + fn block_with_parent(num: u64) -> BlockWithParent { + BlockWithParent::new( + b256(num.wrapping_sub(1) as u8), + NumHash::new(num, b256(num as u8)), + ) + } + + fn dummy_cached_data(num: u64) -> CachedBlockTrieData { + CachedBlockTrieData { + block_with_parent: block_with_parent(num), + trie_data: LazyTrieData::ready(ComputedTrieData::new( + Arc::new(HashedPostStateSorted::default()), + Arc::new(TrieUpdatesSorted::default()), + )), + } + } + + // ---- SyncTargetState::apply_next tests ---- + + #[test] + fn apply_next_sync_replaced_by_sync() { + let mut state = SyncTargetState::SyncUpTo { to: 10 }; + state.apply_next(SyncTargetState::SyncUpTo { to: 20 }); + assert!(matches!(state, SyncTargetState::SyncUpTo { to: 20 })); + } + + #[test] + fn apply_next_sync_replaced_by_revert() { + let mut state = SyncTargetState::SyncUpTo { to: 10 }; + let revert_to = block_with_parent(5); + state.apply_next(SyncTargetState::Revert { revert_to }); + assert!( + matches!(state, SyncTargetState::Revert { revert_to } if revert_to.block.number == 5) + ); + } + + #[test] + fn apply_next_sync_replaced_by_revert_then_sync() { + let mut state = SyncTargetState::SyncUpTo { to: 10 }; + let revert_to = block_with_parent(5); + state.apply_next(SyncTargetState::RevertThenSync { + revert_to, + sync_to: 20, + }); + assert!(matches!( + state, + SyncTargetState::RevertThenSync { revert_to, sync_to: 20 } + if revert_to.block.number == 5 + )); + } + + #[test] + fn apply_next_revert_then_sync_keeps_revert_on_new_sync() { + let revert_to = block_with_parent(5); + let mut state = SyncTargetState::RevertThenSync { + revert_to, + sync_to: 10, + }; + state.apply_next(SyncTargetState::SyncUpTo { to: 30 }); + assert!(matches!( + state, + SyncTargetState::RevertThenSync { revert_to, sync_to: 30 } + if revert_to.block.number == 5 + )); + } + + #[test] + fn apply_next_revert_then_sync_replaced_by_new_revert() { + let revert_to = block_with_parent(5); + let mut state = SyncTargetState::RevertThenSync { + revert_to, + sync_to: 10, + }; + let new_revert = block_with_parent(3); + state.apply_next(SyncTargetState::Revert { + revert_to: new_revert, + }); + assert!(matches!( + state, + SyncTargetState::Revert { revert_to } if revert_to.block.number == 3 + )); + } + + #[test] + fn apply_next_revert_gains_sync_target() { + let revert_to = block_with_parent(5); + let mut state = SyncTargetState::Revert { revert_to }; + state.apply_next(SyncTargetState::SyncUpTo { to: 15 }); + assert!(matches!( + state, + SyncTargetState::RevertThenSync { revert_to, sync_to: 15 } + if revert_to.block.number == 5 + )); + } + + #[test] + fn apply_next_revert_replaced_by_revert_then_sync() { + let revert_to = block_with_parent(5); + let mut state = SyncTargetState::Revert { revert_to }; + let new_revert = block_with_parent(3); + state.apply_next(SyncTargetState::RevertThenSync { + revert_to: new_revert, + sync_to: 20, + }); + assert!(matches!( + state, + SyncTargetState::RevertThenSync { revert_to, sync_to: 20 } + if revert_to.block.number == 3 + )); + } + + // ---- SyncTarget state management tests ---- + + #[test] + fn new_sync_target_has_no_state() { + let target = SyncTarget::new(); + assert!(!target.has_pending_state()); + assert!(target.take_state().is_none()); + } + + #[test] + fn update_state_sets_pending() { + let target = SyncTarget::new(); + target.update_state(SyncTargetState::SyncUpTo { to: 10 }); + assert!(target.has_pending_state()); + } + + #[test] + fn take_state_clears_pending() { + let target = SyncTarget::new(); + target.update_state(SyncTargetState::SyncUpTo { to: 10 }); + let state = target.take_state(); + assert!(matches!(state, Some(SyncTargetState::SyncUpTo { to: 10 }))); + assert!(!target.has_pending_state()); + } + + #[test] + fn update_state_merges_with_existing() { + let target = SyncTarget::new(); + let revert_to = block_with_parent(5); + target.update_state(SyncTargetState::Revert { revert_to }); + target.update_state(SyncTargetState::SyncUpTo { to: 20 }); + + let state = target.take_state().expect("should have state"); + assert!(matches!( + state, + SyncTargetState::RevertThenSync { revert_to, sync_to: 20 } + if revert_to.block.number == 5 + )); + } + + // ---- mark_revert_complete tests ---- + + #[test] + fn mark_revert_complete_clears_covered_revert() { + let target = SyncTarget::new(); + let revert_to = block_with_parent(5); + target.update_state(SyncTargetState::Revert { revert_to }); + + target.mark_revert_complete(&block_with_parent(5)); + assert!(!target.has_pending_state()); + } + + #[test] + fn mark_revert_complete_strips_covered_revert_then_sync() { + let target = SyncTarget::new(); + let revert_to = block_with_parent(5); + target.update_state(SyncTargetState::RevertThenSync { + revert_to, + sync_to: 20, + }); + + target.mark_revert_complete(&block_with_parent(5)); + let state = target.take_state().expect("should have state"); + assert!(matches!(state, SyncTargetState::SyncUpTo { to: 20 })); + } + + #[test] + fn mark_revert_complete_keeps_deeper_revert() { + let target = SyncTarget::new(); + // A deeper revert (block 3) arrived while we were reverting to block 5 + let revert_to = block_with_parent(3); + target.update_state(SyncTargetState::Revert { revert_to }); + + target.mark_revert_complete(&block_with_parent(5)); + let state = target.take_state().expect("should still have state"); + assert!(matches!( + state, + SyncTargetState::Revert { revert_to } if revert_to.block.number == 3 + )); + } + + #[test] + fn mark_revert_complete_keeps_deeper_revert_then_sync() { + let target = SyncTarget::new(); + let revert_to = block_with_parent(3); + target.update_state(SyncTargetState::RevertThenSync { + revert_to, + sync_to: 20, + }); + + target.mark_revert_complete(&block_with_parent(5)); + let state = target.take_state().expect("should still have state"); + assert!(matches!( + state, + SyncTargetState::RevertThenSync { revert_to, sync_to: 20 } + if revert_to.block.number == 3 + )); + } + + #[test] + fn mark_revert_complete_clears_shallower_revert() { + let target = SyncTarget::new(); + // Pending revert to block 8 is shallower than what we reverted to (block 5) + let revert_to = block_with_parent(8); + target.update_state(SyncTargetState::Revert { revert_to }); + + target.mark_revert_complete(&block_with_parent(5)); + assert!(!target.has_pending_state()); + } + + #[test] + fn mark_revert_complete_noop_on_sync_up_to() { + let target = SyncTarget::new(); + target.update_state(SyncTargetState::SyncUpTo { to: 20 }); + + target.mark_revert_complete(&block_with_parent(5)); + let state = target.take_state().expect("should still have state"); + assert!(matches!(state, SyncTargetState::SyncUpTo { to: 20 })); + } + + #[test] + fn mark_revert_complete_noop_on_empty() { + let target = SyncTarget::new(); + target.mark_revert_complete(&block_with_parent(5)); + assert!(!target.has_pending_state()); + } + + // ---- Cache tests ---- + + #[test] + fn insert_and_take_cached_data() { + let target = SyncTarget::new(); + target.insert(42, dummy_cached_data(42)); + + let taken = target.take(42); + assert!(taken.is_some()); + assert_eq!(taken.unwrap().block_with_parent.block.number, 42); + + // Second take returns None (was removed) + assert!(target.take(42).is_none()); + } + + #[test] + fn take_nonexistent_block_returns_none() { + let target = SyncTarget::new(); + assert!(target.take(999).is_none()); + } + + #[test] + fn clear_from_removes_blocks_at_and_above() { + let target = SyncTarget::new(); + for n in 1..=10 { + target.insert(n, dummy_cached_data(n)); + } + + target.clear_from(6); + + // Blocks 1..=5 remain + for n in 1..=5 { + assert!(target.take(n).is_some(), "block {n} should still be cached"); + } + // Blocks 6..=10 removed + for n in 6..=10 { + assert!( + target.take(n).is_none(), + "block {n} should have been cleared" + ); + } + } + + #[test] + fn clear_from_at_start_clears_everything() { + let target = SyncTarget::new(); + for n in 5..=10 { + target.insert(n, dummy_cached_data(n)); + } + target.clear_from(5); + for n in 5..=10 { + assert!(target.take(n).is_none()); + } + } + + #[test] + fn clear_from_beyond_cache_is_noop() { + let target = SyncTarget::new(); + for n in 1..=5 { + target.insert(n, dummy_cached_data(n)); + } + target.clear_from(100); + for n in 1..=5 { + assert!(target.take(n).is_some()); + } + } + + #[test] + fn cache_evicts_oldest_when_full() { + let target = SyncTarget::new(); + + // Fill to capacity + 10 + for n in 1..=(CACHE_CAPACITY as u64 + 10) { + target.insert(n, dummy_cached_data(n)); + } + + // Oldest 10 blocks should have been evicted + for n in 1..=10 { + assert!( + target.take(n).is_none(), + "block {n} should have been evicted" + ); + } + + // Block 11 onward should still be present + assert!(target.take(11).is_some()); + assert!(target.take(CACHE_CAPACITY as u64 + 10).is_some()); + } + + #[test] + fn default_creates_empty_sync_target() { + let target = SyncTarget::default(); + assert!(!target.has_pending_state()); + assert!(target.take(0).is_none()); + } + + #[test] + fn debug_impl_does_not_panic() { + let target = SyncTarget::new(); + target.insert(1, dummy_cached_data(1)); + target.update_state(SyncTargetState::SyncUpTo { to: 5 }); + let debug_str = format!("{target:?}"); + assert!(debug_str.contains("SyncTarget")); + assert!(debug_str.contains("has_pending_state")); + assert!(debug_str.contains("cached_blocks")); + } + + #[tokio::test] + async fn notified_wakes_on_update_state() { + let target = Arc::new(SyncTarget::new()); + let target_clone = Arc::clone(&target); + + // Spawn a task that waits for notification + let handle = tokio::spawn(async move { + target_clone.notified().await; + target_clone.take_state() + }); + + // Give the task time to start waiting + tokio::task::yield_now().await; + + target.update_state(SyncTargetState::SyncUpTo { to: 42 }); + + let result = handle.await.expect("task should complete"); + assert!(matches!(result, Some(SyncTargetState::SyncUpTo { to: 42 }))); + } +} diff --git a/crates/proofs/Cargo.toml b/crates/proofs/Cargo.toml new file mode 100644 index 00000000..db4b1603 --- /dev/null +++ b/crates/proofs/Cargo.toml @@ -0,0 +1,50 @@ +[package] +name = "morph-proofs" +description = "Bounded historical MPT storage for Morph eth_getProof" +version.workspace = true +edition.workspace = true +rust-version.workspace = true +license.workspace = true +publish.workspace = true + +[lints] +workspace = true + +[dependencies] +alloy-eips.workspace = true +alloy-primitives.workspace = true +auto_impl.workspace = true +bincode = { workspace = true, features = ["serde"] } +bytes.workspace = true +derive_more.workspace = true +metrics.workspace = true +parking_lot.workspace = true +serde.workspace = true +strum = { workspace = true, features = ["derive"] } +thiserror.workspace = true +tokio = { workspace = true, features = ["sync"] } +tracing.workspace = true + +reth-codecs.workspace = true +reth-db = { workspace = true, features = ["mdbx"] } +reth-evm.workspace = true +reth-execution-errors.workspace = true +reth-metrics.workspace = true +reth-primitives-traits.workspace = true +reth-provider.workspace = true +reth-revm = { workspace = true, features = ["witness"] } +reth-tasks.workspace = true +reth-trie = { workspace = true, features = ["serde"] } +reth-trie-common = { workspace = true, features = ["serde"] } + +[dev-dependencies] +alloy-primitives = { workspace = true, features = ["rand"] } +mockall.workspace = true +reth-db = { workspace = true, features = ["test-utils"] } +reth-db-api = { workspace = true, features = ["test-utils"] } +reth-storage-errors.workspace = true +tempfile.workspace = true + +[features] +default = [] +test-utils = [] diff --git a/crates/proofs/NOTICE.md b/crates/proofs/NOTICE.md new file mode 100644 index 00000000..f60916ef --- /dev/null +++ b/crates/proofs/NOTICE.md @@ -0,0 +1,32 @@ +# Attribution + +The versioned trie storage and proof-history algorithms in this crate are adapted from +the Base repository at commit `b2673bbd927cb34d7cfad4d448bfbd5bd30eae88`: + + + +The adapted code is used under Base's MIT license. Morph-specific changes include an +MDBX-only surface, durable chain-identity metadata, and integration with Morph's node, +RPC, and lifecycle rules. + +## Base MIT License + +Copyright (c) 2025 Base + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. diff --git a/crates/proofs/src/api.rs b/crates/proofs/src/api.rs new file mode 100644 index 00000000..0d014093 --- /dev/null +++ b/crates/proofs/src/api.rs @@ -0,0 +1,435 @@ +//! Storage API for external storage of intermediary trie nodes. + +use std::{fmt::Debug, time::Duration}; + +use alloy_eips::{BlockNumHash, eip1898::BlockWithParent}; +use alloy_primitives::{B256, U256}; +use auto_impl::auto_impl; +use derive_more::{AddAssign, Constructor}; +use reth_primitives_traits::Account; +use reth_trie::{ + hashed_cursor::{HashedCursor, HashedStorageCursor}, + trie_cursor::{TrieCursor, TrieStorageCursor}, +}; +use reth_trie_common::{ + BranchNodeCompact, HashedPostStateSorted, Nibbles, StoredNibbles, updates::TrieUpdatesSorted, +}; + +use crate::{ + MorphProofsStorageResult, + db::{HashedStorageKey, StorageTrieKey}, +}; + +/// Type alias for bulk storage branch entries: a list of (hashed address, storage nodes) pairs. +pub type StorageBranchEntries = Vec<(B256, Vec<(Nibbles, Option)>)>; + +/// Diff of trie updates and post state for a block. +#[derive(Debug, Clone, Default)] +pub struct BlockStateDiff { + /// Trie updates for branch nodes + pub sorted_trie_updates: TrieUpdatesSorted, + /// Post state for leaf nodes (accounts and storage) + pub sorted_post_state: HashedPostStateSorted, +} + +impl BlockStateDiff { + /// Extend the [` BlockStateDiff`] from other latest [`BlockStateDiff`] + pub fn extend_ref(&mut self, other: &Self) { + self.sorted_trie_updates + .extend_ref_and_sort(&other.sorted_trie_updates); + self.sorted_post_state + .extend_ref_and_sort(&other.sorted_post_state); + } +} + +/// Counts of trie updates written to storage. +#[derive(Debug, Clone, Default, AddAssign, Constructor, Eq, PartialEq)] +pub struct WriteCounts { + /// Number of account trie updates written + pub account_trie_updates_written_total: u64, + /// Number of storage trie updates written + pub storage_trie_updates_written_total: u64, + /// Number of hashed accounts written + pub hashed_accounts_written_total: u64, + /// Number of hashed storages written + pub hashed_storages_written_total: u64, +} + +/// Duration metrics for block processing. +#[derive(Debug, Default, Clone)] +pub struct OperationDurations { + /// Total time to process a block (end-to-end) in seconds + pub total_duration_seconds: Duration, + /// Time spent executing the block (EVM) in seconds + pub execution_duration_seconds: Duration, + /// Time spent calculating state root in seconds + pub state_root_duration_seconds: Duration, + /// Time spent writing trie updates to storage in seconds + pub write_duration_seconds: Duration, +} + +/// Trait for reading trie nodes from the database. +/// +/// Only leaf nodes and some branch nodes are stored. The bottom layer of branch nodes +/// are not stored to reduce write amplification. This matches Reth's non-historical trie storage. +#[auto_impl(Arc)] +pub trait MorphProofsStore: Send + Sync + Debug { + /// Cursor for iterating over trie branches. + type StorageTrieCursor<'tx>: TrieStorageCursor + 'tx + where + Self: 'tx; + + /// Cursor for iterating over account trie branches. + type AccountTrieCursor<'tx>: TrieCursor + 'tx + where + Self: 'tx; + + /// Cursor for iterating over storage leaves. + type StorageCursor<'tx>: HashedStorageCursor + Send + Sync + 'tx + where + Self: 'tx; + + /// Cursor for iterating over account leaves. + type AccountHashedCursor<'tx>: HashedCursor + Send + Sync + 'tx + where + Self: 'tx; + + /// Transaction type used by request-scoped cursor factories. + type Tx<'tx>: Debug + Send + Sync + where + Self: 'tx; + + /// Get the earliest block number and hash that has been stored + /// + /// This is used to determine the block number of trie nodes with block number 0. + /// All earliest block numbers are stored in 0 to reduce updates required to prune trie nodes. + fn get_earliest_block_number(&self) -> MorphProofsStorageResult>; + + /// Get the latest block number and hash that has been stored + fn get_latest_block_number(&self) -> MorphProofsStorageResult>; + + /// Get the earliest block number and hash visible to an existing read transaction. + fn get_earliest_block_number_with_tx<'tx, 'db>( + &self, + tx: &'tx Self::Tx<'db>, + ) -> MorphProofsStorageResult> + where + Self: 'db, + 'db: 'tx; + + /// Get the latest block number and hash visible to an existing read transaction. + fn get_latest_block_number_with_tx<'tx, 'db>( + &self, + tx: &'tx Self::Tx<'db>, + ) -> MorphProofsStorageResult> + where + Self: 'db, + 'db: 'tx; + + /// Get a trie cursor for the storage backend + fn storage_trie_cursor<'tx>( + &'tx self, + hashed_address: B256, + max_block_number: u64, + ) -> MorphProofsStorageResult>; + + /// Get a trie cursor for the account backend + fn account_trie_cursor<'tx>( + &'tx self, + max_block_number: u64, + ) -> MorphProofsStorageResult>; + + /// Get a storage cursor for the storage backend + fn storage_hashed_cursor<'tx>( + &'tx self, + hashed_address: B256, + max_block_number: u64, + ) -> MorphProofsStorageResult>; + + /// Get an account hashed cursor for the storage backend + fn account_hashed_cursor<'tx>( + &'tx self, + max_block_number: u64, + ) -> MorphProofsStorageResult>; + + /// Open a transaction for a request-scoped cursor factory. + fn ro_tx<'tx>(&'tx self) -> MorphProofsStorageResult>; + + /// Storage trie cursor reusing `tx`. + fn storage_trie_cursor_with_tx<'tx, 'db>( + &self, + tx: &'tx Self::Tx<'db>, + hashed_address: B256, + max_block_number: u64, + ) -> MorphProofsStorageResult> + where + Self: 'db, + 'db: 'tx; + + /// Account trie cursor reusing `tx`. + fn account_trie_cursor_with_tx<'tx, 'db>( + &self, + tx: &'tx Self::Tx<'db>, + max_block_number: u64, + ) -> MorphProofsStorageResult> + where + Self: 'db, + 'db: 'tx; + + /// Storage hashed cursor reusing `tx`. + fn storage_hashed_cursor_with_tx<'tx, 'db>( + &self, + tx: &'tx Self::Tx<'db>, + hashed_address: B256, + max_block_number: u64, + ) -> MorphProofsStorageResult> + where + Self: 'db, + 'db: 'tx; + + /// Account hashed cursor reusing `tx`. + fn account_hashed_cursor_with_tx<'tx, 'db>( + &self, + tx: &'tx Self::Tx<'db>, + max_block_number: u64, + ) -> MorphProofsStorageResult> + where + Self: 'db, + 'db: 'tx; + + /// Store a batch of trie updates. + /// + /// If wiped is true, the entire storage trie is wiped, but this is unsupported going forward, + /// so should only happen for legacy reasons. + fn store_trie_updates( + &self, + block_ref: BlockWithParent, + block_state_diff: BlockStateDiff, + ) -> MorphProofsStorageResult; + + /// Fetch all updates for a given block number. + fn fetch_trie_updates(&self, block_number: u64) -> MorphProofsStorageResult; + + /// Applies [`BlockStateDiff`] to the earliest state (updating/deleting nodes) and updates the + /// earliest block number. + fn prune_earliest_state( + &self, + new_earliest_block_ref: BlockWithParent, + ) -> MorphProofsStorageResult; + + /// Remove account, storage and trie updates from historical storage for all blocks till + /// the specified block (inclusive). + fn unwind_history(&self, to: BlockWithParent) -> MorphProofsStorageResult<()>; + + /// Deletes all updates > `latest_common_block` and replaces them with the new updates. + fn replace_updates( + &self, + latest_common_block: BlockNumHash, + blocks_to_add: Vec<(BlockWithParent, BlockStateDiff)>, + ) -> MorphProofsStorageResult<()>; + + /// Set the earliest block number and hash that has been stored + fn set_earliest_block_number( + &self, + block_number: u64, + hash: B256, + ) -> MorphProofsStorageResult<()>; +} + +/// Session-scoped store of trie updates that amortizes write overhead across multiple +/// block writes within a single batch. +/// +/// Reads through the session observe writes from earlier [`MorphProofsBatchSession::store_trie_updates`] +/// calls in the same session, enabling cold catch-up where `block N+1` is computed against +/// `block N` written earlier in the same session. +/// +/// All cursor methods mirror [`MorphProofsStore`] but read through the session's active view. +#[auto_impl(&mut)] +pub trait MorphProofsBatchSession: Send + Sync + Debug { + /// Cursor for iterating over storage trie branches in the active session. + type StorageTrieCursor<'a>: TrieStorageCursor + 'a + where + Self: 'a; + + /// Cursor for iterating over account trie branches in the active session. + type AccountTrieCursor<'a>: TrieCursor + 'a + where + Self: 'a; + + /// Cursor for iterating over storage leaves in the active session. + type StorageCursor<'a>: HashedStorageCursor + Send + Sync + 'a + where + Self: 'a; + + /// Cursor for iterating over account leaves in the active session. + type AccountHashedCursor<'a>: HashedCursor + Send + Sync + 'a + where + Self: 'a; + + /// Earliest stored block number/hash visible to the active transaction. + fn get_earliest_block_number(&self) -> MorphProofsStorageResult>; + + /// Latest stored block number/hash visible to this session (including writes from + /// earlier [`Self::store_trie_updates`] calls in this session). + fn get_latest_block_number(&self) -> MorphProofsStorageResult>; + + /// Storage trie cursor reading through the active transaction. + fn storage_trie_cursor( + &self, + hashed_address: B256, + max_block_number: u64, + ) -> MorphProofsStorageResult>; + + /// Account trie cursor reading through the active transaction. + fn account_trie_cursor( + &self, + max_block_number: u64, + ) -> MorphProofsStorageResult>; + + /// Storage hashed cursor reading through the active transaction. + fn storage_hashed_cursor( + &self, + hashed_address: B256, + max_block_number: u64, + ) -> MorphProofsStorageResult>; + + /// Account hashed cursor reading through the active transaction. + fn account_hashed_cursor( + &self, + max_block_number: u64, + ) -> MorphProofsStorageResult>; + + /// Append-only write of `block_state_diff` for `block_ref` to the active session. + /// Subsequent reads through this session observe the new state immediately. + /// Durability semantics are backend-dependent; see [`MorphProofsBatchStore`]. + fn store_trie_updates( + &mut self, + block_ref: BlockWithParent, + block_state_diff: BlockStateDiff, + ) -> MorphProofsStorageResult; +} + +/// Storage that can open a [`MorphProofsBatchSession`] for amortized multi-block writes. +/// +/// MDBX commits the session atomically on `Ok` and rolls it back on `Err`. +/// The in-memory implementation is a test double and does not provide rollback. +pub trait MorphProofsBatchStore: MorphProofsStore { + /// Session type for this backend. + type BatchSession<'a>: MorphProofsBatchSession + 'a + where + Self: 'a; + + /// Run `f` inside one batch session. Atomicity and error-recovery semantics are + /// backend-dependent; see the trait-level documentation. + fn with_batch_session(&self, f: F) -> MorphProofsStorageResult + where + F: FnOnce(&mut Self::BatchSession<'_>) -> MorphProofsStorageResult; +} + +impl MorphProofsBatchStore for std::sync::Arc { + type BatchSession<'a> + = T::BatchSession<'a> + where + Self: 'a; + + fn with_batch_session(&self, f: F) -> MorphProofsStorageResult + where + F: FnOnce(&mut Self::BatchSession<'_>) -> MorphProofsStorageResult, + { + (**self).with_batch_session(f) + } +} + +/// Status of the initial state anchor. +#[derive(Debug, Clone, Copy, Default)] +pub enum InitialStateStatus { + /// Init isn't yet started + #[default] + NotStarted, + /// Init is in progress (some tables may already be populated) + InProgress, + /// Init completed successfully (all tables done + earliest block set) + Completed, +} + +/// Anchor for the initial state. +#[derive(Debug, Clone, Default)] +pub struct InitialStateAnchor { + /// The block for which the initial state is being initialized. None if initialization is not + /// yet started. + pub block: Option, + /// Whether initialization is still running or completed. + pub status: InitialStateStatus, + /// The latest key stored for `AccountTrieHistory`. + pub latest_account_trie_key: Option, + /// The latest key stored for `StorageTrieHistory`. + pub latest_storage_trie_key: Option, + /// The latest key stored for `HashedAccountHistory`. + pub latest_hashed_account_key: Option, + /// The latest key stored for `HashedStorageHistory`. + pub latest_hashed_storage_key: Option, +} + +/// Trait for storing and retrieving the initial state anchor. +#[auto_impl(Arc)] +pub trait MorphProofsInitialStateStore: Send + Sync + Debug { + /// Read the current anchor. + fn initial_state_anchor(&self) -> MorphProofsStorageResult; + + /// Create the anchor if it doesn't exist. + /// Returns `Err` if an anchor already exists (prevents accidental overwrite). + fn set_initial_state_anchor(&self, anchor: BlockNumHash) -> MorphProofsStorageResult<()>; + + /// Store a batch of account trie branches. Used for saving existing state. For live state + /// capture, use [store_trie_updates](MorphProofsStore::store_trie_updates). + fn store_account_branches( + &self, + account_nodes: Vec<(Nibbles, Option)>, + ) -> MorphProofsStorageResult<()>; + + /// Store a batch of storage trie branches. Used for saving existing state. + fn store_storage_branches( + &self, + hashed_address: B256, + storage_nodes: Vec<(Nibbles, Option)>, + ) -> MorphProofsStorageResult<()>; + + /// Store storage trie branches for multiple addresses in one initialization batch. + fn store_storage_branches_bulk( + &self, + entries: StorageBranchEntries, + ) -> MorphProofsStorageResult<()> { + for (hashed_address, nodes) in entries { + self.store_storage_branches(hashed_address, nodes)?; + } + Ok(()) + } + + /// Store a batch of account trie leaf nodes. Used for saving existing state. + fn store_hashed_accounts( + &self, + accounts: Vec<(B256, Option)>, + ) -> MorphProofsStorageResult<()>; + + /// Store a batch of storage trie leaf nodes. Used for saving existing state. + fn store_hashed_storages( + &self, + hashed_address: B256, + storages: Vec<(B256, U256)>, + ) -> MorphProofsStorageResult<()>; + + /// Store hashed storage slots for multiple addresses in one initialization batch. + fn store_hashed_storages_bulk( + &self, + entries: Vec<(B256, Vec<(B256, U256)>)>, + ) -> MorphProofsStorageResult<()> { + for (hashed_address, storages) in entries { + self.store_hashed_storages(hashed_address, storages)?; + } + Ok(()) + } + + /// Commit the initial state - mark the anchor as completed and also set the earliest block + /// number to anchor. + fn commit_initial_state(&self) -> MorphProofsStorageResult; +} diff --git a/crates/proofs/src/batch_provider.rs b/crates/proofs/src/batch_provider.rs new file mode 100644 index 00000000..11b67e09 --- /dev/null +++ b/crates/proofs/src/batch_provider.rs @@ -0,0 +1,313 @@ +//! State provider for an active [`MorphProofsBatchSession`] enabling reads to observe +//! uncommitted writes performed earlier in the same session. + +use std::fmt::Debug; + +use alloy_primitives::{ + keccak256, + map::{B256Map, HashMap}, +}; +use derive_more::Constructor; +use reth_primitives_traits::{Account, Bytecode}; +use reth_provider::{ + AccountReader, BlockHashReader, BytecodeReader, HashedPostStateProvider, ProviderError, + ProviderResult, StateProofProvider, StateProvider, StateRootProvider, StorageRootProvider, +}; +use reth_revm::{ + db::BundleState, + primitives::{Address, B256, Bytes, StorageValue, alloy_primitives::BlockNumber}, +}; +use reth_trie::{ + StateRoot, StorageRoot, TrieType, + hashed_cursor::{HashedCursor, HashedPostStateCursorFactory}, + metrics::TrieRootMetrics, + proof, + trie_cursor::InMemoryTrieCursorFactory, + witness::TrieWitness, +}; +use reth_trie_common::{ + AccountProof, ExecutionWitnessMode, HashedPostState, HashedPostStateSorted, HashedStorage, + KeccakKeyHasher, MultiProof, MultiProofTargets, StorageMultiProof, StorageProof, TrieInput, + updates::TrieUpdates, +}; + +use crate::{ + MorphProofsBatchHashedAccountCursorFactory, MorphProofsBatchTrieCursorFactory, + api::MorphProofsBatchSession, +}; + +/// State provider that reads through an active [`MorphProofsBatchSession`]'s transaction. +#[derive(Constructor)] +pub struct MorphProofsBatchStateProviderRef<'a, S: MorphProofsBatchSession> { + latest: Box, + session: &'a S, + block_number: BlockNumber, +} + +impl Debug for MorphProofsBatchStateProviderRef<'_, S> +where + S: MorphProofsBatchSession, +{ + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + f.debug_struct("MorphProofsBatchStateProviderRef") + .field("session", &self.session) + .field("block_number", &self.block_number) + .finish() + } +} + +impl<'a, S: MorphProofsBatchSession> MorphProofsBatchStateProviderRef<'a, S> { + const fn factories( + &self, + ) -> ( + MorphProofsBatchTrieCursorFactory<'a, S>, + MorphProofsBatchHashedAccountCursorFactory<'a, S>, + ) { + ( + MorphProofsBatchTrieCursorFactory::new(self.session, self.block_number), + MorphProofsBatchHashedAccountCursorFactory::new(self.session, self.block_number), + ) + } +} + +impl BlockHashReader for MorphProofsBatchStateProviderRef<'_, S> { + fn block_hash(&self, number: BlockNumber) -> ProviderResult> { + self.latest.block_hash(number) + } + + fn canonical_hashes_range( + &self, + start: BlockNumber, + end: BlockNumber, + ) -> ProviderResult> { + self.latest.canonical_hashes_range(start, end) + } +} + +impl StateRootProvider for MorphProofsBatchStateProviderRef<'_, S> { + fn state_root(&self, state: HashedPostState) -> ProviderResult { + let prefix_sets = state.construct_prefix_sets().freeze(); + let state_sorted = state.into_sorted(); + let (trie_factory, hashed_factory) = self.factories(); + StateRoot::new( + trie_factory, + HashedPostStateCursorFactory::new(hashed_factory, &state_sorted), + ) + .with_prefix_sets(prefix_sets) + .root() + .map_err(ProviderError::from) + } + + fn state_root_from_nodes(&self, input: TrieInput) -> ProviderResult { + let state_sorted = input.state.into_sorted(); + let nodes_sorted = input.nodes.into_sorted(); + let (trie_factory, hashed_factory) = self.factories(); + StateRoot::new( + InMemoryTrieCursorFactory::new(trie_factory, &nodes_sorted), + HashedPostStateCursorFactory::new(hashed_factory, &state_sorted), + ) + .with_prefix_sets(input.prefix_sets.freeze()) + .root() + .map_err(ProviderError::from) + } + + fn state_root_with_updates( + &self, + state: HashedPostState, + ) -> ProviderResult<(B256, TrieUpdates)> { + let prefix_sets = state.construct_prefix_sets().freeze(); + let state_sorted = state.into_sorted(); + let (trie_factory, hashed_factory) = self.factories(); + StateRoot::new( + trie_factory, + HashedPostStateCursorFactory::new(hashed_factory, &state_sorted), + ) + .with_prefix_sets(prefix_sets) + .root_with_updates() + .map_err(ProviderError::from) + } + + fn state_root_from_nodes_with_updates( + &self, + input: TrieInput, + ) -> ProviderResult<(B256, TrieUpdates)> { + let state_sorted = input.state.into_sorted(); + let nodes_sorted = input.nodes.into_sorted(); + let (trie_factory, hashed_factory) = self.factories(); + StateRoot::new( + InMemoryTrieCursorFactory::new(trie_factory, &nodes_sorted), + HashedPostStateCursorFactory::new(hashed_factory, &state_sorted), + ) + .with_prefix_sets(input.prefix_sets.freeze()) + .root_with_updates() + .map_err(ProviderError::from) + } +} + +impl StorageRootProvider for MorphProofsBatchStateProviderRef<'_, S> { + fn storage_root(&self, address: Address, storage: HashedStorage) -> ProviderResult { + let prefix_set = storage.construct_prefix_set().freeze(); + let state_sorted = + HashedPostState::from_hashed_storage(keccak256(address), storage).into_sorted(); + let (trie_factory, hashed_factory) = self.factories(); + StorageRoot::new( + trie_factory, + HashedPostStateCursorFactory::new(hashed_factory, &state_sorted), + address, + prefix_set, + TrieRootMetrics::new(TrieType::Custom("morph_historical_proofs_storage_batch")), + ) + .root() + .map_err(|err| ProviderError::Database(err.into())) + } + + fn storage_proof( + &self, + address: Address, + slot: B256, + hashed_storage: HashedStorage, + ) -> ProviderResult { + let hashed_address = keccak256(address); + let prefix_set = hashed_storage.construct_prefix_set(); + let state_sorted = HashedPostStateSorted::new( + Default::default(), + HashMap::from_iter([(hashed_address, hashed_storage.into_sorted())]), + ); + let (trie_factory, hashed_factory) = self.factories(); + proof::StorageProof::new(trie_factory, hashed_factory.clone(), address) + .with_hashed_cursor_factory(HashedPostStateCursorFactory::new( + hashed_factory, + &state_sorted, + )) + .with_prefix_set_mut(prefix_set) + .storage_proof(slot) + .map_err(ProviderError::from) + } + + fn storage_multiproof( + &self, + address: Address, + slots: &[B256], + hashed_storage: HashedStorage, + ) -> ProviderResult { + let hashed_address = keccak256(address); + let targets = slots.iter().map(keccak256).collect(); + let prefix_set = hashed_storage.construct_prefix_set(); + let state_sorted = HashedPostStateSorted::new( + Default::default(), + HashMap::from_iter([(hashed_address, hashed_storage.into_sorted())]), + ); + let (trie_factory, hashed_factory) = self.factories(); + proof::StorageProof::new(trie_factory, hashed_factory.clone(), address) + .with_hashed_cursor_factory(HashedPostStateCursorFactory::new( + hashed_factory, + &state_sorted, + )) + .with_prefix_set_mut(prefix_set) + .storage_multiproof(targets) + .map_err(ProviderError::from) + } +} + +impl StateProofProvider for MorphProofsBatchStateProviderRef<'_, S> { + fn proof( + &self, + input: TrieInput, + address: Address, + slots: &[B256], + ) -> ProviderResult { + let nodes_sorted = input.nodes.into_sorted(); + let state_sorted = input.state.into_sorted(); + let (trie_factory, hashed_factory) = self.factories(); + proof::Proof::new(trie_factory.clone(), hashed_factory.clone()) + .with_trie_cursor_factory(InMemoryTrieCursorFactory::new(trie_factory, &nodes_sorted)) + .with_hashed_cursor_factory(HashedPostStateCursorFactory::new( + hashed_factory, + &state_sorted, + )) + .with_prefix_sets_mut(input.prefix_sets) + .account_proof(address, slots) + .map_err(ProviderError::from) + } + + fn multiproof( + &self, + input: TrieInput, + targets: MultiProofTargets, + ) -> ProviderResult { + let nodes_sorted = input.nodes.into_sorted(); + let state_sorted = input.state.into_sorted(); + let (trie_factory, hashed_factory) = self.factories(); + proof::Proof::new(trie_factory.clone(), hashed_factory.clone()) + .with_trie_cursor_factory(InMemoryTrieCursorFactory::new(trie_factory, &nodes_sorted)) + .with_hashed_cursor_factory(HashedPostStateCursorFactory::new( + hashed_factory, + &state_sorted, + )) + .with_prefix_sets_mut(input.prefix_sets) + .multiproof(targets) + .map_err(ProviderError::from) + } + + fn witness( + &self, + input: TrieInput, + target: HashedPostState, + _mode: ExecutionWitnessMode, + ) -> ProviderResult> { + let nodes_sorted = input.nodes.into_sorted(); + let state_sorted = input.state.into_sorted(); + let (trie_factory, hashed_factory) = self.factories(); + let result: B256Map = TrieWitness::new(trie_factory.clone(), hashed_factory.clone()) + .with_trie_cursor_factory(InMemoryTrieCursorFactory::new(trie_factory, &nodes_sorted)) + .with_hashed_cursor_factory(HashedPostStateCursorFactory::new( + hashed_factory, + &state_sorted, + )) + .with_prefix_sets_mut(input.prefix_sets) + .always_include_root_node() + .compute(target) + .map_err(ProviderError::from)?; + Ok(result.into_values().collect()) + } +} + +impl HashedPostStateProvider + for MorphProofsBatchStateProviderRef<'_, S> +{ + fn hashed_post_state(&self, bundle_state: &BundleState) -> HashedPostState { + HashedPostState::from_bundle_state::(bundle_state.state()) + } +} + +impl AccountReader for MorphProofsBatchStateProviderRef<'_, S> { + fn basic_account(&self, address: &Address) -> ProviderResult> { + let hashed_key = keccak256(address.0); + Ok(self + .session + .account_hashed_cursor(self.block_number) + .map_err(Into::::into)? + .seek(hashed_key) + .map_err(Into::::into)? + .and_then(|(key, account)| (key == hashed_key).then_some(account))) + } +} + +impl StateProvider for MorphProofsBatchStateProviderRef<'_, S> { + fn storage(&self, address: Address, storage_key: B256) -> ProviderResult> { + let hashed_key = keccak256(storage_key); + Ok(self + .session + .storage_hashed_cursor(keccak256(address.0), self.block_number) + .map_err(Into::::into)? + .seek(hashed_key) + .map_err(Into::::into)? + .and_then(|(key, storage)| (key == hashed_key).then_some(storage))) + } +} + +impl BytecodeReader for MorphProofsBatchStateProviderRef<'_, S> { + fn bytecode_by_hash(&self, code_hash: &B256) -> ProviderResult> { + self.latest.bytecode_by_hash(code_hash) + } +} diff --git a/crates/proofs/src/cursor.rs b/crates/proofs/src/cursor.rs new file mode 100644 index 00000000..ec04141e --- /dev/null +++ b/crates/proofs/src/cursor.rs @@ -0,0 +1,129 @@ +//! Implementation of [`HashedCursor`] and [`TrieCursor`] for +//! [`MorphProofsStorage`](crate::MorphProofsStorage). + +use alloy_primitives::{B256, U256}; +use derive_more::Constructor; +use reth_db::DatabaseError; +use reth_primitives_traits::Account; +use reth_trie::{ + hashed_cursor::{HashedCursor, HashedStorageCursor}, + trie_cursor::{TrieCursor, TrieStorageCursor}, +}; +use reth_trie_common::{BranchNodeCompact, Nibbles}; + +/// Manages reading storage or account trie nodes from [`TrieCursor`]. +#[derive(Debug, Clone, Constructor)] +pub struct MorphProofsTrieCursor(pub C); + +impl TrieCursor for MorphProofsTrieCursor +where + C: TrieCursor, +{ + #[inline] + fn seek_exact( + &mut self, + key: Nibbles, + ) -> Result, DatabaseError> { + self.0.seek_exact(key) + } + + #[inline] + fn seek( + &mut self, + key: Nibbles, + ) -> Result, DatabaseError> { + self.0.seek(key) + } + + #[inline] + fn next(&mut self) -> Result, DatabaseError> { + self.0.next() + } + + #[inline] + fn current(&mut self) -> Result, DatabaseError> { + self.0.current() + } + + #[inline] + fn reset(&mut self) { + self.0.reset() + } +} + +impl TrieStorageCursor for MorphProofsTrieCursor +where + C: TrieStorageCursor, +{ + #[inline] + fn set_hashed_address(&mut self, hashed_address: B256) { + self.0.set_hashed_address(hashed_address) + } +} + +/// Manages reading hashed account nodes from external storage. +#[derive(Debug, Clone, Constructor)] +pub struct MorphProofsHashedAccountCursor(pub C); + +impl HashedCursor for MorphProofsHashedAccountCursor +where + C: HashedCursor + Send + Sync, +{ + type Value = Account; + + #[inline] + fn seek(&mut self, key: B256) -> Result, DatabaseError> { + self.0.seek(key) + } + + #[inline] + fn next(&mut self) -> Result, DatabaseError> { + self.0.next() + } + + #[inline] + fn reset(&mut self) { + self.0.reset() + } +} + +/// Manages reading hashed storage nodes from external storage. +#[derive(Debug, Clone, Constructor)] +pub struct MorphProofsHashedStorageCursor(pub C); + +impl HashedCursor for MorphProofsHashedStorageCursor +where + C: HashedCursor + Send + Sync, +{ + type Value = U256; + + #[inline] + fn seek(&mut self, key: B256) -> Result, DatabaseError> { + self.0.seek(key) + } + + #[inline] + fn next(&mut self) -> Result, DatabaseError> { + self.0.next() + } + + #[inline] + fn reset(&mut self) { + self.0.reset() + } +} + +impl HashedStorageCursor for MorphProofsHashedStorageCursor +where + C: HashedStorageCursor + Send + Sync, +{ + #[inline] + fn is_storage_empty(&mut self) -> Result { + self.0.is_storage_empty() + } + + #[inline] + fn set_hashed_address(&mut self, hashed_address: B256) { + self.0.set_hashed_address(hashed_address) + } +} diff --git a/crates/proofs/src/cursor_factory.rs b/crates/proofs/src/cursor_factory.rs new file mode 100644 index 00000000..fa2dfcbb --- /dev/null +++ b/crates/proofs/src/cursor_factory.rs @@ -0,0 +1,260 @@ +//! Implements [`TrieCursorFactory`] and [`HashedCursorFactory`] for [`MorphProofsStore`] types. +//! +//! Both factories borrow one read-only [`MorphProofsStore::Tx`] for their entire lifetime +//! and route every cursor allocation through the `*_with_tx` fast path. This mirrors +//! reth's own `DatabaseTrieCursorFactory` / `DatabaseHashedCursorFactory` pattern and is +//! what lets proof, state-root, and witness requests acquire exactly one MDBX +//! transaction. + +use alloy_primitives::B256; +use reth_db::DatabaseError; +use reth_trie::{hashed_cursor::HashedCursorFactory, trie_cursor::TrieCursorFactory}; + +use crate::{ + MorphProofsHashedAccountCursor, MorphProofsHashedStorageCursor, MorphProofsStorage, + MorphProofsStore, MorphProofsTrieCursor, + api::MorphProofsBatchSession, + cursor::{ + MorphProofsHashedAccountCursor as RawHashedAccountCursor, + MorphProofsHashedStorageCursor as RawHashedStorageCursor, + MorphProofsTrieCursor as RawTrieCursor, + }, +}; + +/// Request-scoped factory that opens trie cursors against a shared read-only transaction. +/// +/// Holds a borrow of the transaction so every cursor allocation reuses the same MDBX +/// reader slot. See [`MorphProofsStore::Tx`] for the underlying contention story. +#[derive(Debug, Clone)] +pub struct MorphProofsTrieCursorFactory<'tx, 'db, S: MorphProofsStore> { + storage: &'db MorphProofsStorage, + tx: &'tx as MorphProofsStore>::Tx<'db>, + block_number: u64, +} + +impl<'tx, 'db, S: MorphProofsStore> MorphProofsTrieCursorFactory<'tx, 'db, S> { + /// Initializes a request-scoped trie cursor factory bound to `tx`. + pub const fn new( + storage: &'db MorphProofsStorage, + tx: &'tx as MorphProofsStore>::Tx<'db>, + block_number: u64, + ) -> Self { + Self { + storage, + tx, + block_number, + } + } +} + +impl<'tx, 'db, S> TrieCursorFactory for MorphProofsTrieCursorFactory<'tx, 'db, S> +where + for<'a> S: MorphProofsStore + 'db, + 'db: 'tx, +{ + type AccountTrieCursor<'a> + = MorphProofsTrieCursor> + where + Self: 'a; + type StorageTrieCursor<'a> + = MorphProofsTrieCursor> + where + Self: 'a; + + fn account_trie_cursor(&self) -> Result, DatabaseError> { + Ok(MorphProofsTrieCursor::new( + self.storage + .account_trie_cursor_with_tx(self.tx, self.block_number) + .map_err(Into::::into)?, + )) + } + + fn storage_trie_cursor( + &self, + hashed_address: B256, + ) -> Result, DatabaseError> { + Ok(MorphProofsTrieCursor::new( + self.storage + .storage_trie_cursor_with_tx(self.tx, hashed_address, self.block_number) + .map_err(Into::::into)?, + )) + } +} + +/// Request-scoped factory that opens hashed cursors against a shared read-only transaction. +/// +/// Mirror of [`MorphProofsTrieCursorFactory`] for the hashed account/storage tries. +#[derive(Debug, Clone)] +pub struct MorphProofsHashedAccountCursorFactory<'tx, 'db, S: MorphProofsStore> { + storage: &'db MorphProofsStorage, + tx: &'tx as MorphProofsStore>::Tx<'db>, + block_number: u64, +} + +impl<'tx, 'db, S: MorphProofsStore> MorphProofsHashedAccountCursorFactory<'tx, 'db, S> { + /// Initializes a request-scoped hashed cursor factory bound to `tx`. + pub const fn new( + storage: &'db MorphProofsStorage, + tx: &'tx as MorphProofsStore>::Tx<'db>, + block_number: u64, + ) -> Self { + Self { + storage, + tx, + block_number, + } + } +} + +impl<'tx, 'db, S> HashedCursorFactory for MorphProofsHashedAccountCursorFactory<'tx, 'db, S> +where + for<'a> S: MorphProofsStore + 'db, + 'db: 'tx, +{ + type AccountCursor<'a> + = MorphProofsHashedAccountCursor> + where + Self: 'a; + type StorageCursor<'a> + = MorphProofsHashedStorageCursor> + where + Self: 'a; + + fn hashed_account_cursor(&self) -> Result, DatabaseError> { + Ok(MorphProofsHashedAccountCursor::new( + self.storage + .account_hashed_cursor_with_tx(self.tx, self.block_number)?, + )) + } + + fn hashed_storage_cursor( + &self, + hashed_address: B256, + ) -> Result, DatabaseError> { + Ok(MorphProofsHashedStorageCursor::new( + self.storage.storage_hashed_cursor_with_tx( + self.tx, + hashed_address, + self.block_number, + )?, + )) + } +} + +/// Session-scoped trie cursor factory backed by a [`MorphProofsBatchSession`]. +/// +/// Cursors read from the session's active transaction and therefore observe writes +/// from earlier `store_trie_updates` calls in the same session. +#[derive(Debug)] +pub struct MorphProofsBatchTrieCursorFactory<'a, S: MorphProofsBatchSession> { + session: &'a S, + block_number: u64, +} + +impl Clone for MorphProofsBatchTrieCursorFactory<'_, S> { + fn clone(&self) -> Self { + Self { + session: self.session, + block_number: self.block_number, + } + } +} + +impl<'a, S: MorphProofsBatchSession> MorphProofsBatchTrieCursorFactory<'a, S> { + /// Initializes a session-scoped trie cursor factory. + pub const fn new(session: &'a S, block_number: u64) -> Self { + Self { + session, + block_number, + } + } +} + +impl TrieCursorFactory for MorphProofsBatchTrieCursorFactory<'_, S> +where + S: MorphProofsBatchSession, +{ + type AccountTrieCursor<'a> + = RawTrieCursor> + where + Self: 'a; + type StorageTrieCursor<'a> + = RawTrieCursor> + where + Self: 'a; + + fn account_trie_cursor(&self) -> Result, DatabaseError> { + Ok(RawTrieCursor::new( + self.session + .account_trie_cursor(self.block_number) + .map_err(Into::::into)?, + )) + } + + fn storage_trie_cursor( + &self, + hashed_address: B256, + ) -> Result, DatabaseError> { + Ok(RawTrieCursor::new( + self.session + .storage_trie_cursor(hashed_address, self.block_number) + .map_err(Into::::into)?, + )) + } +} + +/// Session-scoped hashed cursor factory backed by a [`MorphProofsBatchSession`]. +#[derive(Debug)] +pub struct MorphProofsBatchHashedAccountCursorFactory<'a, S: MorphProofsBatchSession> { + session: &'a S, + block_number: u64, +} + +impl Clone for MorphProofsBatchHashedAccountCursorFactory<'_, S> { + fn clone(&self) -> Self { + Self { + session: self.session, + block_number: self.block_number, + } + } +} + +impl<'a, S: MorphProofsBatchSession> MorphProofsBatchHashedAccountCursorFactory<'a, S> { + /// Initializes a session-scoped hashed cursor factory. + pub const fn new(session: &'a S, block_number: u64) -> Self { + Self { + session, + block_number, + } + } +} + +impl HashedCursorFactory for MorphProofsBatchHashedAccountCursorFactory<'_, S> +where + S: MorphProofsBatchSession, +{ + type AccountCursor<'a> + = RawHashedAccountCursor> + where + Self: 'a; + type StorageCursor<'a> + = RawHashedStorageCursor> + where + Self: 'a; + + fn hashed_account_cursor(&self) -> Result, DatabaseError> { + Ok(RawHashedAccountCursor::new( + self.session.account_hashed_cursor(self.block_number)?, + )) + } + + fn hashed_storage_cursor( + &self, + hashed_address: B256, + ) -> Result, DatabaseError> { + Ok(RawHashedStorageCursor::new( + self.session + .storage_hashed_cursor(hashed_address, self.block_number)?, + )) + } +} diff --git a/crates/proofs/src/db/batch.rs b/crates/proofs/src/db/batch.rs new file mode 100644 index 00000000..1b8ba868 --- /dev/null +++ b/crates/proofs/src/db/batch.rs @@ -0,0 +1,143 @@ +//! Batch write session for [`MdbxProofsStorage`] enabling multiple block writes inside one MDBX +//! RW transaction. Reads through the session observe uncommitted writes from earlier blocks in +//! the same session, which is required for cold catch-up where block `N+1` must execute against +//! block `N` written but not yet committed. + +use alloy_eips::eip1898::BlockWithParent; +use alloy_primitives::B256; +use reth_db::{ + Database, DatabaseEnv, + table::{DupSort, Table}, + transaction::DbTx, +}; + +use crate::{ + BlockStateDiff, MorphProofsStorageError, MorphProofsStorageResult, + api::{MorphProofsBatchSession, WriteCounts}, + db::{ + AccountTrieHistory, HashedAccountHistory, HashedStorageHistory, MdbxAccountCursor, + MdbxProofsStorage, MdbxStorageCursor, MdbxTrieCursor, StorageTrieHistory, + }, +}; + +/// Alias for the dup-sorted cursor type produced by an MDBX RW transaction. +pub type DupRw<'tx, T> = <::TXMut as DbTx>::DupCursor; + +/// Active write batch holding one MDBX RW transaction across multiple block writes. +#[derive(Debug)] +pub struct MdbxBatchSession<'tx> { + storage: &'tx MdbxProofsStorage, + tx: Option<::TXMut>, +} + +impl<'tx> MdbxBatchSession<'tx> { + pub(crate) const fn new( + storage: &'tx MdbxProofsStorage, + tx: ::TXMut, + ) -> Self { + Self { + storage, + tx: Some(tx), + } + } + + pub(crate) fn commit(mut self) -> MorphProofsStorageResult<()> { + if let Some(tx) = self.tx.take() { + tx.commit()?; + } + Ok(()) + } + + fn tx_ref(&self) -> MorphProofsStorageResult<&::TXMut> { + self.tx + .as_ref() + .ok_or(MorphProofsStorageError::BatchSessionClosed) + } + + fn dup_cursor(&self) -> MorphProofsStorageResult> { + Ok(self.tx_ref()?.cursor_dup_read::()?) + } +} + +impl MorphProofsBatchSession for MdbxBatchSession<'_> { + type StorageTrieCursor<'a> + = MdbxTrieCursor> + where + Self: 'a; + type AccountTrieCursor<'a> + = MdbxTrieCursor> + where + Self: 'a; + type StorageCursor<'a> + = MdbxStorageCursor> + where + Self: 'a; + type AccountHashedCursor<'a> + = MdbxAccountCursor> + where + Self: 'a; + + fn get_earliest_block_number(&self) -> MorphProofsStorageResult> { + self.storage + .inner_get_earliest_block_number_hash(self.tx_ref()?) + } + + fn get_latest_block_number(&self) -> MorphProofsStorageResult> { + self.storage + .inner_get_latest_block_number_hash(self.tx_ref()?) + } + + fn storage_trie_cursor( + &self, + hashed_address: B256, + max_block_number: u64, + ) -> MorphProofsStorageResult> { + Ok(MdbxTrieCursor::new( + self.dup_cursor::()?, + max_block_number, + Some(hashed_address), + )) + } + + fn account_trie_cursor( + &self, + max_block_number: u64, + ) -> MorphProofsStorageResult> { + Ok(MdbxTrieCursor::new( + self.dup_cursor::()?, + max_block_number, + None, + )) + } + + fn storage_hashed_cursor( + &self, + hashed_address: B256, + max_block_number: u64, + ) -> MorphProofsStorageResult> { + Ok(MdbxStorageCursor::new( + self.dup_cursor::()?, + max_block_number, + hashed_address, + )) + } + + fn account_hashed_cursor( + &self, + max_block_number: u64, + ) -> MorphProofsStorageResult> { + Ok(MdbxAccountCursor::new( + self.dup_cursor::()?, + max_block_number, + )) + } + + fn store_trie_updates( + &mut self, + block_ref: BlockWithParent, + block_state_diff: BlockStateDiff, + ) -> MorphProofsStorageResult { + self.storage + .store_trie_updates_append_only(self.tx_ref()?, block_ref, block_state_diff) + } +} diff --git a/crates/proofs/src/db/cursor.rs b/crates/proofs/src/db/cursor.rs new file mode 100644 index 00000000..057a1a23 --- /dev/null +++ b/crates/proofs/src/db/cursor.rs @@ -0,0 +1,1545 @@ +use std::marker::PhantomData; + +use alloy_primitives::{B256, U256}; +use reth_db::{ + Database, DatabaseEnv, DatabaseError, + cursor::{DbCursorRO, DbDupCursorRO}, + table::{DupSort, Table}, + transaction::DbTx, +}; +use reth_primitives_traits::Account; +use reth_trie::{ + hashed_cursor::{HashedCursor, HashedStorageCursor}, + trie_cursor::{TrieCursor, TrieStorageCursor}, +}; +use reth_trie_common::{BranchNodeCompact, Nibbles, StoredNibbles}; + +use crate::{ + MorphProofsStorageResult, + db::{ + AccountTrieHistory, HashedAccountHistory, HashedStorageHistory, HashedStorageKey, + MaybeDeleted, StorageTrieHistory, StorageTrieKey, VersionedValue, + }, +}; + +/// Generic alias for dup cursor for T +pub type Dup<'tx, T> = <::TX as DbTx>::DupCursor; + +/// Iterates versioned dup-sorted rows and returns the latest value (<= `max_block_number`), +/// skipping tombstones. +#[derive(Debug, Clone)] +pub struct BlockNumberVersionedCursor { + _table: PhantomData, + cursor: Cursor, + max_block_number: u64, +} + +impl BlockNumberVersionedCursor +where + T: Table> + DupSort, + Cursor: DbCursorRO + DbDupCursorRO, +{ + /// Initializes new [`BlockNumberVersionedCursor`]. + pub const fn new(cursor: Cursor, max_block_number: u64) -> Self { + Self { + _table: PhantomData, + cursor, + max_block_number, + } + } + + /// Check if the cursor is currently positioned at a valid row. + fn is_positioned(&mut self) -> MorphProofsStorageResult { + Ok(self.cursor.current()?.is_some()) + } + + /// Resolve the latest version for `key` with `block_number` <= `max_block_number`. + /// Strategy: + /// - `seek_by_key_subkey(key, max)` gives first dup >= max. + /// - if exactly == max → it's our latest + /// - if > max → `prev_dup()` is latest < max (or None) + /// - if no dup >= max: + /// - if key exists → `last_dup()` is latest < max + /// - else → None + fn latest_version_for_key( + &mut self, + key: T::Key, + ) -> MorphProofsStorageResult> { + // First dup with subkey >= max_block_number + let seek_res = self + .cursor + .seek_by_key_subkey(key.clone(), self.max_block_number)?; + + if let Some(vv) = seek_res { + if vv.block_number > self.max_block_number { + // step back to the last dup < max + return Ok(self.cursor.prev_dup()?); + } + // already at the dup = max + return Ok(Some((key, vv))); + } + + // No dup >= max ⇒ either key absent or all dups < max. Check if key exists: + if self.cursor.seek_exact(key.clone())?.is_none() { + return Ok(None); + } + + // Key exists ⇒ take last dup (< max). + if let Some(vv) = self.cursor.last_dup()? { + return Ok(Some((key, vv))); + } + Ok(None) + } + + /// Returns a non-deleted latest version for exactly `key`, if any. + fn seek_exact(&mut self, key: T::Key) -> MorphProofsStorageResult> { + if let Some((latest_key, latest_value)) = self.latest_version_for_key(key)? + && let MaybeDeleted(Some(v)) = latest_value.value + { + return Ok(Some((latest_key, v))); + } + Ok(None) + } + + /// Walk forward from `first_key` (inclusive) until we find a *live* latest-≤-max value. + /// `first_key` must already be a *real key* in the table. + fn next_live_from( + &mut self, + mut first_key: T::Key, + ) -> MorphProofsStorageResult> { + loop { + // Compute latest version ≤ max for this key + if let Some((k, v)) = self.seek_exact(first_key.clone())? { + return Ok(Some((k, v))); + } + + // Move to next distinct key, or EOF + let Some((next_key, _)) = self.cursor.next_no_dup()? else { + return Ok(None); + }; + + first_key = next_key; + } + } + + /// Seek to the first non-deleted latest version at or after `start_key`. + /// Logic: + /// - Try exact key first (above). If alive, return it. + /// - Otherwise hop to next distinct key and repeat until we find a live version or hit EOF. + fn seek(&mut self, start_key: T::Key) -> MorphProofsStorageResult> { + // Position MDBX at first key >= start_key + if let Some((first_key, _)) = self.cursor.seek(start_key)? { + return self.next_live_from(first_key); + } + Ok(None) + } + + /// Advance to the next distinct key from the current MDBX position + /// and return its non-deleted latest version, if any. + /// Next distinct key; if not positioned, start from `T::Key::default()`. + fn next(&mut self) -> MorphProofsStorageResult> + where + T::Key: Default, + { + // If not positioned, start from the beginning (default key). + if self.cursor.current()?.is_none() { + let Some((first_key, _)) = self.cursor.seek(T::Key::default())? else { + return Ok(None); + }; + return self.next_live_from(first_key); + } + + // Otherwise advance to next distinct key and resume the walk. + let Some((next_key, _)) = self.cursor.next_no_dup()? else { + return Ok(None); + }; + self.next_live_from(next_key) + } +} + +/// MDBX implementation of [`TrieCursor`]. +#[derive(Debug)] +pub struct MdbxTrieCursor { + inner: BlockNumberVersionedCursor, + hashed_address: Option, +} + +impl< + V, + T: Table> + DupSort, + Cursor: DbCursorRO + DbDupCursorRO, +> MdbxTrieCursor +{ + /// Initializes new [`MdbxTrieCursor`]. + pub const fn new(cursor: Cursor, max_block_number: u64, hashed_address: Option) -> Self { + Self { + inner: BlockNumberVersionedCursor::new(cursor, max_block_number), + hashed_address, + } + } +} + +impl TrieCursor for MdbxTrieCursor +where + Cursor: DbCursorRO + DbDupCursorRO + Send + Sync, +{ + fn seek_exact( + &mut self, + path: Nibbles, + ) -> Result, DatabaseError> { + Ok(self + .inner + .seek_exact(StoredNibbles(path)) + .map(|opt| opt.map(|(StoredNibbles(n), node)| (n, node)))?) + } + + fn seek( + &mut self, + path: Nibbles, + ) -> Result, DatabaseError> { + Ok(self + .inner + .seek(StoredNibbles(path)) + .map(|opt| opt.map(|(StoredNibbles(n), node)| (n, node)))?) + } + + fn next(&mut self) -> Result, DatabaseError> { + Ok(self + .inner + .next() + .map(|opt| opt.map(|(StoredNibbles(n), node)| (n, node)))?) + } + + fn current(&mut self) -> Result, DatabaseError> { + self.inner + .cursor + .current() + .map(|opt| opt.map(|(StoredNibbles(n), _)| n)) + } + + fn reset(&mut self) { + // Database cursors are stateless, no reset needed + } +} + +impl TrieCursor for MdbxTrieCursor +where + Cursor: DbCursorRO + DbDupCursorRO + Send + Sync, +{ + fn seek_exact( + &mut self, + path: Nibbles, + ) -> Result, DatabaseError> { + if let Some(address) = self.hashed_address { + let key = StorageTrieKey::new(address, StoredNibbles(path)); + return Ok(self.inner.seek_exact(key).map(|opt| { + opt.and_then(|(k, node)| (k.hashed_address == address).then_some((k.path.0, node))) + })?); + } + Ok(None) + } + + fn seek( + &mut self, + path: Nibbles, + ) -> Result, DatabaseError> { + if let Some(address) = self.hashed_address { + let key = StorageTrieKey::new(address, StoredNibbles(path)); + return Ok(self.inner.seek(key).map(|opt| { + opt.and_then(|(k, node)| (k.hashed_address == address).then_some((k.path.0, node))) + })?); + } + Ok(None) + } + + fn next(&mut self) -> Result, DatabaseError> { + if let Some(address) = self.hashed_address { + // If the cursor is not positioned, we need to seek to the first key for our bound + // address to ensure we start iterating from the correct position in the + // table. This is necessary because BlockNumberVersionedCursor::next() would + // otherwise start from T::Key::default() (the beginning of the entire + // table), which would cause us to miss entries for non-first addresses. + if !self.inner.is_positioned()? { + return self.seek(Nibbles::default()); + } + + return Ok(self.inner.next().map(|opt| { + opt.and_then(|(k, node)| (k.hashed_address == address).then_some((k.path.0, node))) + })?); + } + Ok(None) + } + + fn current(&mut self) -> Result, DatabaseError> { + if let Some(address) = self.hashed_address { + return self.inner.cursor.current().map(|opt| { + opt.and_then(|(k, _)| (k.hashed_address == address).then_some(k.path.0)) + }); + } + Ok(None) + } + + fn reset(&mut self) { + // Database cursors are stateless, no reset needed + } +} + +impl TrieStorageCursor for MdbxTrieCursor +where + Cursor: DbCursorRO + DbDupCursorRO + Send + Sync, +{ + fn set_hashed_address(&mut self, hashed_address: B256) { + self.hashed_address = Some(hashed_address); + } +} + +/// MDBX implementation of [`HashedCursor`] for storage state. +#[derive(Debug)] +pub struct MdbxStorageCursor { + inner: BlockNumberVersionedCursor, + hashed_address: B256, +} + +impl MdbxStorageCursor +where + Cursor: DbCursorRO + DbDupCursorRO + Send + Sync, +{ + /// Initializes new [`MdbxStorageCursor`] + pub const fn new(cursor: Cursor, block_number: u64, hashed_address: B256) -> Self { + Self { + inner: BlockNumberVersionedCursor::new(cursor, block_number), + hashed_address, + } + } +} + +impl HashedCursor for MdbxStorageCursor +where + Cursor: DbCursorRO + DbDupCursorRO + Send + Sync, +{ + type Value = U256; + + fn seek(&mut self, key: B256) -> Result, DatabaseError> { + let storage_key = HashedStorageKey::new(self.hashed_address, key); + + // hashed storage values can be zero, which means the storage slot is deleted, so we should + // skip those + let result = self.inner.seek(storage_key).map(|opt| { + opt.and_then(|(k, v)| { + // Only return entries that belong to the bound address + (k.hashed_address == self.hashed_address).then_some((k.hashed_storage_key, v.0)) + }) + })?; + + if let Some((_, v)) = result + && v.is_zero() + { + return self.next(); + } + + Ok(result) + } + + fn next(&mut self) -> Result, DatabaseError> { + // If the cursor is not positioned, we need to seek to the first key for our bound address + // to ensure we start iterating from the correct position in the table. + // This is necessary because BlockNumberVersionedCursor::next() would otherwise start + // from T::Key::default() (the beginning of the entire table), which would cause us + // to miss entries for non-first addresses. + if !self.inner.is_positioned()? { + return self.seek(B256::ZERO); + } + + loop { + let result = self.inner.next().map(|opt| { + opt.and_then(|(k, v)| { + // Only return entries that belong to the bound address + (k.hashed_address == self.hashed_address).then_some((k.hashed_storage_key, v.0)) + }) + })?; + + // hashed storage values can be zero, which means the storage slot is deleted, so we + // should skip those + if let Some((_, v)) = result + && v.is_zero() + { + continue; + } + + return Ok(result); + } + } + + fn reset(&mut self) { + // Database cursors are stateless, no reset needed + } +} + +impl HashedStorageCursor for MdbxStorageCursor +where + Cursor: DbCursorRO + DbDupCursorRO + Send + Sync, +{ + fn is_storage_empty(&mut self) -> Result { + Ok(self.seek(B256::ZERO)?.is_none()) + } + + fn set_hashed_address(&mut self, hashed_address: B256) { + self.hashed_address = hashed_address + } +} + +/// MDBX implementation of [`HashedCursor`] for account state. +#[derive(Debug)] +pub struct MdbxAccountCursor { + inner: BlockNumberVersionedCursor, +} + +impl MdbxAccountCursor +where + Cursor: DbCursorRO + DbDupCursorRO + Send + Sync, +{ + /// Initializes new `MdbxAccountCursor` + pub const fn new(cursor: Cursor, block_number: u64) -> Self { + Self { + inner: BlockNumberVersionedCursor::new(cursor, block_number), + } + } +} + +impl HashedCursor for MdbxAccountCursor +where + Cursor: DbCursorRO + DbDupCursorRO + Send + Sync, +{ + type Value = Account; + + fn seek(&mut self, key: B256) -> Result, DatabaseError> { + Ok(self.inner.seek(key)?) + } + + fn next(&mut self) -> Result, DatabaseError> { + Ok(self.inner.next()?) + } + + fn reset(&mut self) { + // Database cursors are stateless, no reset needed + } +} + +#[cfg(test)] +mod tests { + use reth_db::{ + DatabaseEnv, + mdbx::{DatabaseArguments, init_db_for}, + }; + use reth_db_api::{ + Database, + cursor::DbDupCursorRW, + transaction::{DbTx, DbTxMut}, + }; + use reth_trie::{BranchNodeCompact, Nibbles, StoredNibbles}; + use tempfile::TempDir; + + use super::*; + use crate::db::{StorageValue, models}; + + fn setup_db() -> DatabaseEnv { + let tmp = TempDir::new().expect("create tmpdir"); + init_db_for::<_, models::Tables>(tmp, DatabaseArguments::default()).expect("init db") + } + + fn stored(path: Nibbles) -> StoredNibbles { + StoredNibbles(path) + } + + fn node() -> BranchNodeCompact { + BranchNodeCompact::default() + } + + fn append_account_trie( + wtx: &::TXMut, + key: StoredNibbles, + block: u64, + val: Option, + ) { + let mut c = wtx + .cursor_dup_write::() + .expect("dup write cursor"); + let vv = VersionedValue { + block_number: block, + value: MaybeDeleted(val), + }; + c.append_dup(key, vv).expect("append dup"); + } + + fn append_storage_trie( + wtx: &::TXMut, + address: B256, + path: Nibbles, + block: u64, + val: Option, + ) { + let mut c = wtx + .cursor_dup_write::() + .expect("dup write cursor"); + let key = StorageTrieKey::new(address, StoredNibbles(path)); + let vv = VersionedValue { + block_number: block, + value: MaybeDeleted(val), + }; + c.append_dup(key, vv).expect("append dup"); + } + + fn append_hashed_storage( + wtx: &::TXMut, + addr: B256, + slot: B256, + block: u64, + val: Option, + ) { + let mut c = wtx + .cursor_dup_write::() + .expect("dup write"); + let key = HashedStorageKey::new(addr, slot); + let vv = VersionedValue { + block_number: block, + value: MaybeDeleted(val.map(StorageValue)), + }; + c.append_dup(key, vv).expect("append dup"); + } + + fn append_hashed_account( + wtx: &::TXMut, + key: B256, + block: u64, + val: Option, + ) { + let mut c = wtx + .cursor_dup_write::() + .expect("dup write"); + let vv = VersionedValue { + block_number: block, + value: MaybeDeleted(val), + }; + c.append_dup(key, vv).expect("append dup"); + } + + // Open a dup-RO cursor and wrap it in a BlockNumberVersionedCursor with a given bound. + fn version_cursor( + tx: &::TX, + max_block: u64, + ) -> BlockNumberVersionedCursor> { + let cur = tx + .cursor_dup_read::() + .expect("dup ro cursor"); + BlockNumberVersionedCursor::new(cur, max_block) + } + + fn account_trie_cursor( + tx: &'_ ::TX, + max_block: u64, + ) -> MdbxTrieCursor> { + let c = tx + .cursor_dup_read::() + .expect("dup ro cursor"); + // For account trie the address is not used; pass None. + MdbxTrieCursor::new(c, max_block, None) + } + + // Helper: build a Storage trie cursor bound to an address + fn storage_trie_cursor( + tx: &'_ ::TX, + max_block: u64, + address: B256, + ) -> MdbxTrieCursor> { + let c = tx + .cursor_dup_read::() + .expect("dup ro cursor"); + MdbxTrieCursor::new(c, max_block, Some(address)) + } + + fn storage_cursor( + tx: &'_ ::TX, + max_block: u64, + address: B256, + ) -> MdbxStorageCursor> { + let c = tx + .cursor_dup_read::() + .expect("dup ro cursor"); + MdbxStorageCursor::new(c, max_block, address) + } + + fn account_cursor( + tx: &'_ ::TX, + max_block: u64, + ) -> MdbxAccountCursor> { + let c = tx + .cursor_dup_read::() + .expect("dup ro cursor"); + MdbxAccountCursor::new(c, max_block) + } + + // Assert helper: ensure the chosen VersionedValue has the expected block and deletion flag. + fn assert_block( + got: Option<(StoredNibbles, VersionedValue)>, + expected_block: u64, + expect_deleted: bool, + ) { + let (_, vv) = got.expect("expected Some(..)"); + assert_eq!(vv.block_number, expected_block, "wrong block chosen"); + let is_deleted = matches!(vv.value, MaybeDeleted(None)); + assert_eq!(is_deleted, expect_deleted, "tombstone mismatch"); + } + + /// No entry for key → None. + #[test] + fn latest_version_for_key_none_when_key_absent() { + let db = setup_db(); + let tx = db.tx().expect("ro tx"); + let mut cursor = version_cursor(&tx, 100); + + let out = cursor + .latest_version_for_key(stored(Nibbles::default())) + .expect("should not return error"); + assert!(out.is_none(), "absent key must return None"); + } + + /// Exact match at max (live) → pick it. + #[test] + fn latest_version_for_key_picks_value_at_max_if_present() { + let db = setup_db(); + let k = stored(Nibbles::from_nibbles([0x0A])); + { + let wtx = db.tx_mut().expect("rw tx"); + append_account_trie(&wtx, k.clone(), 10, Some(node())); + append_account_trie(&wtx, k.clone(), 50, Some(node())); // == max + wtx.commit().expect("commit"); + } + + let tx = db.tx().expect("ro tx"); + let mut core = version_cursor(&tx, 50); + + let out = core.latest_version_for_key(k).expect("ok"); + assert_block(out, 50, false); + } + + /// When `seek_by_key_subkey` points to the subkey > max - fallback to the prev. + #[test] + fn latest_version_for_key_picks_latest_below_max_when_next_is_above() { + let db = setup_db(); + let k = stored(Nibbles::from_nibbles([0x0A])); + { + let wtx = db.tx_mut().expect("rw tx"); + append_account_trie(&wtx, k.clone(), 10, Some(node())); + append_account_trie(&wtx, k.clone(), 30, Some(node())); // expected + append_account_trie(&wtx, k.clone(), 70, Some(node())); // > max + wtx.commit().expect("commit"); + } + + let tx = db.tx().expect("ro tx"); + let mut core = version_cursor(&tx, 50); + + let out = core.latest_version_for_key(k).expect("ok"); + assert_block(out, 30, false); + } + + /// No ≥ max but key exists → use last < max. + #[test] + fn latest_version_for_key_picks_last_below_max_when_none_at_or_above() { + let db = setup_db(); + let k = stored(Nibbles::from_nibbles([0x0A])); + { + let wtx = db.tx_mut().expect("rw tx"); + append_account_trie(&wtx, k.clone(), 10, Some(node())); + append_account_trie(&wtx, k.clone(), 40, Some(node())); // expected (max=100) + wtx.commit().expect("commit"); + } + + let tx = db.tx().expect("ro tx"); + let mut core = version_cursor(&tx, 100); + + let out = core.latest_version_for_key(k).expect("ok"); + assert_block(out, 40, false); + } + + /// All entries are > max → None. + #[test] + fn latest_version_for_key_none_when_everything_is_above_max() { + let db = setup_db(); + let k1 = stored(Nibbles::from_nibbles([0x0A])); + let k2 = stored(Nibbles::from_nibbles([0x0B])); + + { + let wtx = db.tx_mut().expect("rw tx"); + append_account_trie(&wtx, k1.clone(), 60, Some(node())); + append_account_trie(&wtx, k1.clone(), 70, Some(node())); + append_account_trie(&wtx, k2, 40, Some(node())); + wtx.commit().expect("commit"); + } + + let tx = db.tx().expect("ro tx"); + let mut core = version_cursor(&tx, 50); + + let out = core.latest_version_for_key(k1).expect("ok"); + assert!(out.is_none(), "no dup ≤ max ⇒ None"); + } + + /// Single dup < max → pick it. + #[test] + fn latest_version_for_key_picks_single_below_max() { + let db = setup_db(); + let k = stored(Nibbles::from_nibbles([0x0A])); + { + let wtx = db.tx_mut().expect("rw tx"); + append_account_trie(&wtx, k.clone(), 25, Some(node())); // < max + wtx.commit().expect("commit"); + } + + let tx = db.tx().expect("ro tx"); + let mut core = version_cursor(&tx, 50); + + let out = core.latest_version_for_key(k).expect("ok"); + assert_block(out, 25, false); + } + + /// Single dup == max → pick it. + #[test] + fn latest_version_for_key_picks_single_at_max() { + let db = setup_db(); + let k = stored(Nibbles::from_nibbles([0x0A])); + { + let wtx = db.tx_mut().expect("rw tx"); + append_account_trie(&wtx, k.clone(), 50, Some(node())); // == max + wtx.commit().expect("commit"); + } + + let tx = db.tx().expect("ro tx"); + let mut core = version_cursor(&tx, 50); + + let out = core.latest_version_for_key(k).expect("ok"); + assert_block(out, 50, false); + } + + /// Latest ≤ max is a tombstone → return it (this API doesn't filter). + #[test] + fn latest_version_for_key_returns_tombstone_if_latest_is_deleted() { + let db = setup_db(); + let k = stored(Nibbles::from_nibbles([0x0A])); + { + let wtx = db.tx_mut().expect("rw tx"); + append_account_trie(&wtx, k.clone(), 10, Some(node())); + append_account_trie(&wtx, k.clone(), 90, None); // latest ≤ max, but deleted + wtx.commit().expect("commit"); + } + + let tx = db.tx().expect("ro tx"); + let mut core = version_cursor(&tx, 100); + + let out = core.latest_version_for_key(k).expect("ok"); + assert_block(out, 90, true); + } + + /// Should skip tombstones and return None when the latest ≤ max is deleted. + #[test] + fn seek_exact_skips_tombstone_returns_none() { + let db = setup_db(); + let k = stored(Nibbles::from_nibbles([0x0A])); + { + let wtx = db.tx_mut().expect("rw tx"); + append_account_trie(&wtx, k.clone(), 10, Some(node())); + append_account_trie(&wtx, k.clone(), 90, None); // latest ≤ max is tombstoned + wtx.commit().expect("commit"); + } + + let tx = db.tx().expect("ro tx"); + let mut core = version_cursor(&tx, 100); + + let out = core.seek_exact(k).expect("ok"); + assert!( + out.is_none(), + "seek_exact must filter out deleted latest value" + ); + } + + /// Empty table → None. + #[test] + fn seek_empty_returns_none() { + let db = setup_db(); + let tx = db.tx().expect("ro tx"); + let mut cur = version_cursor(&tx, 100); + + let out = cur.seek(stored(Nibbles::from_nibbles([0x0A]))).expect("ok"); + assert!(out.is_none()); + } + + /// Start at an existing key whose latest ≤ max is live → returns that key. + #[test] + fn seek_at_live_key_returns_it() { + let db = setup_db(); + let k = stored(Nibbles::from_nibbles([0x0A])); + { + let wtx = db.tx_mut().expect("rw tx"); + append_account_trie(&wtx, k.clone(), 10, Some(node())); + append_account_trie(&wtx, k.clone(), 20, Some(node())); // latest ≤ max + wtx.commit().expect("commit"); + } + let tx = db.tx().expect("ro tx"); + let mut cur = version_cursor(&tx, 50); + + let out = cur.seek(k.clone()).expect("ok").expect("some"); + assert_eq!(out.0, k); + } + + /// Start at an existing key whose latest ≤ max is tombstoned → skip to next key with live + /// value. + #[test] + fn seek_skips_tombstoned_key_to_next_live_key() { + let db = setup_db(); + let k1 = stored(Nibbles::from_nibbles([0x0A])); + let k2 = stored(Nibbles::from_nibbles([0x0B])); + + { + let wtx = db.tx_mut().expect("rw tx"); + // Key 0x10 latest ≤ max is deleted + append_account_trie(&wtx, k1.clone(), 10, Some(node())); + append_account_trie(&wtx, k1.clone(), 20, None); // tombstone at latest ≤ max + // Next key has live + append_account_trie(&wtx, k2.clone(), 5, Some(node())); + wtx.commit().expect("commit"); + } + let tx = db.tx().expect("ro tx"); + let mut cur = version_cursor(&tx, 50); + + let out = cur.seek(k1).expect("ok").expect("some"); + assert_eq!(out.0, k2); + } + + /// Start between keys → returns the next key’s live latest ≤ max. + #[test] + fn seek_between_keys_returns_next_key() { + let db = setup_db(); + let k1 = stored(Nibbles::from_nibbles([0x0A])); + let k2 = stored(Nibbles::from_nibbles([0x0C])); + let k3 = stored(Nibbles::from_nibbles([0x0B])); + + { + let wtx = db.tx_mut().expect("rw tx"); + append_account_trie(&wtx, k1, 10, Some(node())); + append_account_trie(&wtx, k2.clone(), 10, Some(node())); + wtx.commit().expect("commit"); + } + let tx = db.tx().expect("ro tx"); + let mut cur = version_cursor(&tx, 100); + + // Start at 0x15 (between 0x10 and 0x20) + + let out = cur.seek(k3).expect("ok").expect("some"); + assert_eq!(out.0, k2); + } + + /// Start after the last key → None. + #[test] + fn seek_after_last_returns_none() { + let db = setup_db(); + let k1 = stored(Nibbles::from_nibbles([0x0A])); + let k2 = stored(Nibbles::from_nibbles([0x0B])); + let k3 = stored(Nibbles::from_nibbles([0x0C])); + + { + let wtx = db.tx_mut().expect("rw tx"); + append_account_trie(&wtx, k1, 10, Some(node())); + append_account_trie(&wtx, k2, 10, Some(node())); + wtx.commit().expect("commit"); + } + let tx = db.tx().expect("ro tx"); + let mut cur = version_cursor(&tx, 100); + + let out = cur.seek(k3).expect("ok"); + assert!(out.is_none()); + } + + /// If the first key at-or-after has only versions > max, it is effectively not visible → skip + /// to next. + #[test] + fn seek_skips_keys_with_only_versions_above_max() { + let db = setup_db(); + let k1 = stored(Nibbles::from_nibbles([0x0A])); + let k2 = stored(Nibbles::from_nibbles([0x0B])); + + { + let wtx = db.tx_mut().expect("rw tx"); + append_account_trie(&wtx, k1.clone(), 60, Some(node())); + append_account_trie(&wtx, k2.clone(), 40, Some(node())); + wtx.commit().expect("commit"); + } + let tx = db.tx().expect("ro tx"); + let mut cur = version_cursor(&tx, 50); + + let out = cur.seek(k1).expect("ok").expect("some"); + assert_eq!(out.0, k2); + } + + /// Start at a key with mixed versions; latest ≤ max is tombstone → skip to next key with live. + #[test] + fn seek_mixed_versions_tombstone_latest_skips_to_next_key() { + let db = setup_db(); + let k1 = stored(Nibbles::from_nibbles([0x0A])); + let k2 = stored(Nibbles::from_nibbles([0x0B])); + + { + let wtx = db.tx_mut().expect("rw tx"); + append_account_trie(&wtx, k1.clone(), 10, Some(node())); + append_account_trie(&wtx, k1.clone(), 30, None); + append_account_trie(&wtx, k2.clone(), 5, Some(node())); + wtx.commit().expect("commit"); + } + let tx = db.tx().expect("ro tx"); + let mut cur = version_cursor(&tx, 30); + + let out = cur.seek(k1).expect("ok").expect("some"); + assert_eq!(out.0, k2); + } + + /// When not positioned should start from default key and return the first live key. + #[test] + fn next_unpositioned_starts_from_default_returns_first_live() { + let db = setup_db(); + let k1 = stored(Nibbles::from_nibbles([0x0A])); + let k2 = stored(Nibbles::from_nibbles([0x0B])); + + { + let wtx = db.tx_mut().expect("rw tx"); + append_account_trie(&wtx, k1.clone(), 10, Some(node())); // first live + append_account_trie(&wtx, k2, 10, Some(node())); + wtx.commit().expect("commit"); + } + + let tx = db.tx().expect("ro tx"); + // Unpositioned cursor + let mut cur = version_cursor(&tx, 100); + + let out = cur.next().expect("ok").expect("some"); + assert_eq!(out.0, k1); + } + + /// After positioning on a live key via `seek()`, `next()` should advance to the next live key. + #[test] + fn next_advances_from_current_live_to_next_live() { + let db = setup_db(); + let k1 = stored(Nibbles::from_nibbles([0x0A])); + let k2 = stored(Nibbles::from_nibbles([0x0B])); + + { + let wtx = db.tx_mut().expect("rw tx"); + append_account_trie(&wtx, k1.clone(), 10, Some(node())); // live + append_account_trie(&wtx, k2.clone(), 10, Some(node())); // next live + wtx.commit().expect("commit"); + } + + let tx = db.tx().expect("ro tx"); + let mut cur = version_cursor(&tx, 100); + + // Position at k1 + let _ = cur.seek(k1).expect("ok").expect("some"); + // Next should yield k2 + let out = cur.next().expect("ok").expect("some"); + assert_eq!(out.0, k2); + } + + /// If the next key's latest ≤ max is tombstone, `next()` should skip to the next live key. + #[test] + fn next_skips_tombstoned_key_to_next_live() { + let db = setup_db(); + let k1 = stored(Nibbles::from_nibbles([0x0A])); + let k2 = stored(Nibbles::from_nibbles([0x0B])); // will be tombstoned at latest ≤ max + let k3 = stored(Nibbles::from_nibbles([0x0C])); // next live + + { + let wtx = db.tx_mut().expect("rw tx"); + // k1 live + append_account_trie(&wtx, k1.clone(), 10, Some(node())); + // k2: latest ≤ max is tombstone + append_account_trie(&wtx, k2.clone(), 10, Some(node())); + append_account_trie(&wtx, k2, 20, None); + // k3 live + append_account_trie(&wtx, k3.clone(), 10, Some(node())); + wtx.commit().expect("commit"); + } + + let tx = db.tx().expect("ro tx"); + let mut cur = version_cursor(&tx, 50); + + // Position at k1 + let _ = cur.seek(k1).expect("ok").expect("some"); + // next should skip k2 (tombstoned latest) and return k3 + let out = cur.next().expect("ok").expect("some"); + assert_eq!(out.0, k3); + } + + /// If positioned on the last live key, `next()` should return None (EOF). + #[test] + fn next_returns_none_at_eof() { + let db = setup_db(); + let k1 = stored(Nibbles::from_nibbles([0x0A])); + let k2 = stored(Nibbles::from_nibbles([0x0B])); // last key + + { + let wtx = db.tx_mut().expect("rw tx"); + append_account_trie(&wtx, k1, 10, Some(node())); + append_account_trie(&wtx, k2.clone(), 10, Some(node())); // last live + wtx.commit().expect("commit"); + } + + let tx = db.tx().expect("ro tx"); + let mut cur = version_cursor(&tx, 100); + + // Position at the last key k2 + let _ = cur.seek(k2).expect("ok").expect("some"); + // `next()` should hit EOF + let out = cur.next().expect("ok"); + assert!(out.is_none()); + } + + /// If the first key has only versions > max, `next()` should skip it and return the next live + /// key. + #[test] + fn next_skips_keys_with_only_versions_above_max() { + let db = setup_db(); + let k1 = stored(Nibbles::from_nibbles([0x0A])); // only > max + let k2 = stored(Nibbles::from_nibbles([0x0B])); // ≤ max live + + { + let wtx = db.tx_mut().expect("rw tx"); + // k1 only above max (max=50) + append_account_trie(&wtx, k1, 60, Some(node())); + // k2 within max + append_account_trie(&wtx, k2.clone(), 40, Some(node())); + wtx.commit().expect("commit"); + } + + let tx = db.tx().expect("ro tx"); + // Unpositioned; `next()` will start from default and walk + let mut cur = version_cursor(&tx, 50); + + let out = cur.next().expect("ok").expect("some"); + assert_eq!(out.0, k2); + } + + /// Empty table: `next()` should return None. + #[test] + fn next_on_empty_returns_none() { + let db = setup_db(); + let tx = db.tx().expect("ro tx"); + let mut cur = version_cursor(&tx, 100); + + let out = cur.next().expect("ok"); + assert!(out.is_none()); + } + + // ----------------- Account trie cursor thin-wrapper checks ----------------- + + #[test] + fn account_seek_exact_live_maps_key_and_value() { + let db = setup_db(); + let k = Nibbles::from_nibbles([0x0A]); + + { + let wtx = db.tx_mut().expect("rw tx"); + append_account_trie(&wtx, StoredNibbles(k), 10, Some(node())); + wtx.commit().expect("commit"); + } + + let tx = db.tx().expect("ro tx"); + + // Build wrapper + let mut cur = account_trie_cursor(&tx, 100); + + // Wrapper should return (Nibbles, BranchNodeCompact) + let out = TrieCursor::seek_exact(&mut cur, k) + .expect("ok") + .expect("some"); + assert_eq!(out.0, k); + } + + #[test] + fn account_seek_exact_filters_tombstone() { + let db = setup_db(); + let k = Nibbles::from_nibbles([0x0B]); + + { + let wtx = db.tx_mut().expect("rw tx"); + append_account_trie(&wtx, StoredNibbles(k), 5, Some(node())); + append_account_trie(&wtx, StoredNibbles(k), 9, None); // latest ≤ max tombstone + wtx.commit().expect("commit"); + } + + let tx = db.tx().expect("ro tx"); + let mut cur = account_trie_cursor(&tx, 10); + + let out = TrieCursor::seek_exact(&mut cur, k).expect("ok"); + assert!(out.is_none(), "account seek_exact must filter tombstone"); + } + + #[test] + fn account_seek_and_next_and_current_roundtrip() { + let db = setup_db(); + let k1 = Nibbles::from_nibbles([0x01]); + let k2 = Nibbles::from_nibbles([0x02]); + + { + let wtx = db.tx_mut().expect("rw tx"); + append_account_trie(&wtx, StoredNibbles(k1), 10, Some(node())); + append_account_trie(&wtx, StoredNibbles(k2), 10, Some(node())); + wtx.commit().expect("commit"); + } + + let tx = db.tx().expect("ro tx"); + let mut cur = account_trie_cursor(&tx, 100); + + // seek at k1 + let out1 = TrieCursor::seek(&mut cur, k1).expect("ok").expect("some"); + assert_eq!(out1.0, k1); + + // current should be k1 + let cur_k = TrieCursor::current(&mut cur).expect("ok").expect("some"); + assert_eq!(cur_k, k1); + + // next should move to k2 + let out2 = TrieCursor::next(&mut cur).expect("ok").expect("some"); + assert_eq!(out2.0, k2); + } + + // ----------------- Storage trie cursor thin-wrapper checks ----------------- + + #[test] + fn storage_seek_exact_respects_address_filter() { + let db = setup_db(); + + let addr_a = B256::from([0xAA; 32]); + let addr_b = B256::from([0xBB; 32]); + + let path = Nibbles::from_nibbles([0x0D]); + + { + let wtx = db.tx_mut().expect("rw tx"); + // insert only under B + append_storage_trie(&wtx, addr_b, path, 10, Some(node())); + wtx.commit().expect("commit"); + } + + let tx = db.tx().expect("ro tx"); + + // Cursor bound to A must not see B’s data + let mut cur_a = storage_trie_cursor(&tx, 100, addr_a); + let out_a = TrieCursor::seek_exact(&mut cur_a, path).expect("ok"); + assert!(out_a.is_none(), "no data for addr A"); + + // Cursor bound to B should see it + let mut cur_b = storage_trie_cursor(&tx, 100, addr_b); + let out_b = TrieCursor::seek_exact(&mut cur_b, path) + .expect("ok") + .expect("some"); + assert_eq!(out_b.0, path); + } + + #[test] + fn storage_seek_returns_first_key_for_bound_address() { + let db = setup_db(); + + let addr_a = B256::from([0x11; 32]); + let addr_b = B256::from([0x22; 32]); + + let p1 = Nibbles::from_nibbles([0x01]); + let p2 = Nibbles::from_nibbles([0x02]); + let p3 = Nibbles::from_nibbles([0x03]); + + { + let wtx = db.tx_mut().expect("rw tx"); + // For A: only p2 + append_storage_trie(&wtx, addr_a, p2, 10, Some(node())); + // For B: p1 + append_storage_trie(&wtx, addr_b, p1, 10, Some(node())); + wtx.commit().expect("commit"); + } + + // test seek behaviour + { + let tx = db.tx().expect("ro tx"); + let mut cur_a = storage_trie_cursor(&tx, 100, addr_a); + + // seek at p1: for A there is no p1; the next key >= p1 under A is p2 + let out = TrieCursor::seek(&mut cur_a, p1).expect("ok").expect("some"); + assert_eq!(out.0, p2); + + // seek at p2: exact match + let out = TrieCursor::seek(&mut cur_a, p2).expect("ok").expect("some"); + assert_eq!(out.0, p2); + + // seek at p3: no p3 under A; no next key ≥ p3 under A → None + let out = TrieCursor::seek(&mut cur_a, p3).expect("ok"); + assert!(out.is_none(), "no key ≥ p3 under A"); + } + + // test next behaviour + { + let tx = db.tx().expect("ro tx"); + let mut cur_a = storage_trie_cursor(&tx, 100, addr_a); + + let out = TrieCursor::next(&mut cur_a).expect("ok").expect("some"); + assert_eq!(out.0, p2); + + // next should yield None as there is no further key under A + let out = TrieCursor::next(&mut cur_a).expect("ok"); + assert!(out.is_none(), "no more keys under A"); + + // current should return None + let out = TrieCursor::current(&mut cur_a).expect("ok"); + assert!(out.is_none(), "no current key after EOF"); + } + + // test seek_exact behaviour + { + let tx = db.tx().expect("ro tx"); + let mut cur_a = storage_trie_cursor(&tx, 100, addr_a); + + // seek_exact at p1: no exact match + let out = TrieCursor::seek_exact(&mut cur_a, p1).expect("ok"); + assert!(out.is_none(), "no exact p1 under A"); + + // seek_exact at p2: exact match + let out = TrieCursor::seek_exact(&mut cur_a, p2) + .expect("ok") + .expect("some"); + assert_eq!(out.0, p2); + + // seek_exact at p3: no exact match + let out = TrieCursor::seek_exact(&mut cur_a, p3).expect("ok"); + assert!(out.is_none(), "no exact p3 under A"); + } + } + + #[test] + fn storage_next_stops_at_address_boundary() { + let db = setup_db(); + + let addr_a = B256::from([0x33; 32]); + let addr_b = B256::from([0x44; 32]); + + let p1 = Nibbles::from_nibbles([0x05]); // under A + let p2 = Nibbles::from_nibbles([0x06]); // under B (next key overall) + + { + let wtx = db.tx_mut().expect("rw tx"); + append_storage_trie(&wtx, addr_a, p1, 10, Some(node())); + append_storage_trie(&wtx, addr_b, p2, 10, Some(node())); + wtx.commit().expect("commit"); + } + + let tx = db.tx().expect("ro tx"); + let mut cur_a = storage_trie_cursor(&tx, 100, addr_a); + + // position at p1 (A) + let _ = TrieCursor::seek_exact(&mut cur_a, p1) + .expect("ok") + .expect("some"); + + // next should reach boundary; impl filters different address and returns None + let out = TrieCursor::next(&mut cur_a).expect("ok"); + assert!( + out.is_none(), + "next() should stop when next key is a different address" + ); + } + + #[test] + fn storage_current_maps_key() { + let db = setup_db(); + + let addr = B256::from([0x55; 32]); + let p = Nibbles::from_nibbles([0x09]); + + { + let wtx = db.tx_mut().expect("rw tx"); + append_storage_trie(&wtx, addr, p, 10, Some(node())); + wtx.commit().expect("commit"); + } + + let tx = db.tx().expect("ro tx"); + let mut cur = storage_trie_cursor(&tx, 100, addr); + + let _ = TrieCursor::seek_exact(&mut cur, p) + .expect("ok") + .expect("some"); + + let now = TrieCursor::current(&mut cur).expect("ok").expect("some"); + assert_eq!(now, p); + } + + #[test] + fn hashed_storage_seek_maps_slot_and_value() { + let db = setup_db(); + let addr = B256::from([0xAA; 32]); + let slot = B256::from([0x10; 32]); + + { + let wtx = db.tx_mut().expect("rw"); + append_hashed_storage(&wtx, addr, slot, 10, Some(U256::from(7))); + wtx.commit().expect("commit"); + } + + let tx = db.tx().expect("ro"); + let mut cur = storage_cursor(&tx, 100, addr); + + let (got_slot, got_val) = cur.seek(slot).expect("ok").expect("some"); + assert_eq!(got_slot, slot); + assert_eq!(got_val, U256::from(7)); + } + + #[test] + fn hashed_storage_seek_filters_tombstone() { + let db = setup_db(); + let addr = B256::from([0xAB; 32]); + let slot = B256::from([0x11; 32]); + + { + let wtx = db.tx_mut().expect("rw"); + append_hashed_storage(&wtx, addr, slot, 5, Some(U256::from(1))); + append_hashed_storage(&wtx, addr, slot, 9, None); // latest ≤ max is tombstone + wtx.commit().expect("commit"); + } + + let tx = db.tx().expect("ro"); + let mut cur = storage_cursor(&tx, 10, addr); + + let out = cur.seek(slot).expect("ok"); + assert!(out.is_none(), "wrapper must filter tombstoned latest"); + } + + #[test] + fn hashed_storage_seek_and_next_roundtrip() { + let db = setup_db(); + let addr = B256::from([0xAC; 32]); + let s1 = B256::from([0x01; 32]); + let s2 = B256::from([0x02; 32]); + + { + let wtx = db.tx_mut().expect("rw"); + append_hashed_storage(&wtx, addr, s1, 10, Some(U256::from(11))); + append_hashed_storage(&wtx, addr, s2, 10, Some(U256::from(22))); + wtx.commit().expect("commit"); + } + + let tx = db.tx().expect("ro"); + let mut cur = storage_cursor(&tx, 100, addr); + + let (k1, v1) = cur.seek(s1).expect("ok").expect("some"); + assert_eq!((k1, v1), (s1, U256::from(11))); + + let (k2, v2) = cur.next().expect("ok").expect("some"); + assert_eq!((k2, v2), (s2, U256::from(22))); + } + + #[test] + fn hashed_storage_address_boundary() { + let db = setup_db(); + let addr1 = B256::from([0xAC; 32]); + let addr2 = B256::from([0xAD; 32]); + let s1 = B256::from([0x01; 32]); + let s2 = B256::from([0x02; 32]); + let s3 = B256::from([0x03; 32]); + + { + let wtx = db.tx_mut().expect("rw"); + append_hashed_storage(&wtx, addr1, s1, 10, Some(U256::from(11))); + append_hashed_storage(&wtx, addr1, s2, 10, Some(U256::from(22))); + wtx.commit().expect("commit"); + } + + { + let wtx = db.tx_mut().expect("rw"); + append_hashed_storage(&wtx, addr2, s1, 10, Some(U256::from(33))); + append_hashed_storage(&wtx, addr2, s2, 10, Some(U256::from(44))); + wtx.commit().expect("commit"); + } + + let tx = db.tx().expect("ro"); + let mut cur = storage_cursor(&tx, 100, addr1); + + let (k1, v1) = cur.next().expect("ok").expect("some"); + assert_eq!((k1, v1), (s1, U256::from(11))); + + let (k2, v2) = cur.next().expect("ok").expect("some"); + assert_eq!((k2, v2), (s2, U256::from(22))); + + let out = cur.next().expect("ok"); + assert!(out.is_none(), "should stop at address boundary"); + + let (k1, v1) = cur.seek(s1).expect("ok").expect("some"); + assert_eq!((k1, v1), (s1, U256::from(11))); + + let (k2, v2) = cur.seek(s2).expect("ok").expect("some"); + assert_eq!((k2, v2), (s2, U256::from(22))); + + let out = cur.seek(s3).expect("ok"); + assert!(out.is_none(), "should not see keys from other address"); + } + + #[test] + fn hashed_account_seek_maps_key_and_value() { + let db = setup_db(); + let key = B256::from([0x20; 32]); + + { + let wtx = db.tx_mut().expect("rw"); + append_hashed_account(&wtx, key, 10, Some(Account::default())); + wtx.commit().expect("commit"); + } + + let tx = db.tx().expect("ro"); + let mut cur = account_cursor(&tx, 100); + + let (got_key, _acc) = cur.seek(key).expect("ok").expect("some"); + assert_eq!(got_key, key); + } + + #[test] + fn hashed_account_seek_filters_tombstone() { + let db = setup_db(); + let key = B256::from([0x21; 32]); + + { + let wtx = db.tx_mut().expect("rw"); + append_hashed_account(&wtx, key, 5, Some(Account::default())); + append_hashed_account(&wtx, key, 9, None); // latest ≤ max is tombstone + wtx.commit().expect("commit"); + } + + let tx = db.tx().expect("ro"); + let mut cur = account_cursor(&tx, 10); + + let out = cur.seek(key).expect("ok"); + assert!(out.is_none(), "wrapper must filter tombstoned latest"); + } + + #[test] + fn hashed_account_seek_and_next_roundtrip() { + let db = setup_db(); + let k1 = B256::from([0x01; 32]); + let k2 = B256::from([0x02; 32]); + + { + let wtx = db.tx_mut().expect("rw"); + append_hashed_account(&wtx, k1, 10, Some(Account::default())); + append_hashed_account(&wtx, k2, 10, Some(Account::default())); + wtx.commit().expect("commit"); + } + + let tx = db.tx().expect("ro"); + let mut cur = account_cursor(&tx, 100); + + let (got1, _) = cur.seek(k1).expect("ok").expect("some"); + assert_eq!(got1, k1); + + let (got2, _) = cur.next().expect("ok").expect("some"); + assert_eq!(got2, k2); + } + + /// Regression test: `MdbxStorageCursor` `next()` should work without explicit `seek()` + /// when cursor is constructed for a non-first key. + /// + /// Bug: When a storage cursor is created for a specific address (e.g., 0x02), + /// calling `next()` without first calling `seek()` returns None instead of the first + /// slot for that address. This only manifests when the address is not the first + /// in the table. + #[test] + fn storage_cursor_next_without_seek_for_non_first_address() { + let db = setup_db(); + let addr1 = B256::from([0x01; 32]); // First address + let addr2 = B256::from([0x02; 32]); // Second address (non-first) + let slot1 = B256::from([0x11; 32]); + let slot2 = B256::from([0x12; 32]); + + { + let wtx = db.tx_mut().expect("rw"); + // Add storage for first address + append_hashed_storage(&wtx, addr1, slot1, 10, Some(U256::from(100))); + + // Add storage for second address + append_hashed_storage(&wtx, addr2, slot2, 10, Some(U256::from(200))); + wtx.commit().expect("commit"); + } + + let tx = db.tx().expect("ro"); + + // Test with addr1 (first address) - this typically works + let mut cur1 = storage_cursor(&tx, 100, addr1); + let result1 = cur1.next().expect("ok"); + assert!( + result1.is_some(), + "next() should return data for first address without seek()" + ); + if let Some((key, val)) = result1 { + assert_eq!(key, slot1); + assert_eq!(val, U256::from(100)); + } + + // Test with addr2 (non-first address) - this demonstrates the bug fix + let mut cur2 = storage_cursor(&tx, 100, addr2); + let result2_without_seek = cur2.next().expect("ok"); + + assert!( + result2_without_seek.is_some(), + "next() should return data for non-first address without seek()" + ); + if let Some((key, val)) = result2_without_seek { + assert_eq!(key, slot2); + assert_eq!(val, U256::from(200)); + } + + // Verify that seek() works correctly + let mut cur3 = storage_cursor(&tx, 100, addr2); + let result3_with_seek = cur3.seek(slot2).expect("ok"); + assert!( + result3_with_seek.is_some(), + "seek() should find the slot for addr2" + ); + if let Some((key, val)) = result3_with_seek { + assert_eq!(key, slot2); + assert_eq!(val, U256::from(200)); + } + } + + /// Regression test: `MdbxTrieCursor` `next()` should work without `seek()` + /// for non-first addresses. + #[test] + fn storage_trie_cursor_next_without_seek_for_non_first_address() { + let db = setup_db(); + let addr1 = B256::from([0x01; 32]); + let addr2 = B256::from([0x02; 32]); + let path1 = Nibbles::from_nibbles([0x0A]); + let path2 = Nibbles::from_nibbles([0x0B]); + + { + let wtx = db.tx_mut().expect("rw"); + append_storage_trie(&wtx, addr1, path1, 10, Some(node())); + append_storage_trie(&wtx, addr2, path2, 10, Some(node())); + wtx.commit().expect("commit"); + } + + let tx = db.tx().expect("ro"); + + // Test addr1 (first) - works + let mut cur1 = storage_trie_cursor(&tx, 100, addr1); + let result1 = TrieCursor::next(&mut cur1).expect("ok"); + assert!(result1.is_some()); + assert_eq!(result1.unwrap().0, path1); + + // Test addr2 (non-first) - should also work now + let mut cur2 = storage_trie_cursor(&tx, 100, addr2); + let result2 = TrieCursor::next(&mut cur2).expect("ok"); + assert!( + result2.is_some(), + "next() should work for non-first address without seek()" + ); + assert_eq!(result2.unwrap().0, path2); + } +} diff --git a/crates/proofs/src/db/mod.rs b/crates/proofs/src/db/mod.rs new file mode 100644 index 00000000..7ebcfc34 --- /dev/null +++ b/crates/proofs/src/db/mod.rs @@ -0,0 +1,15 @@ +//! MDBX-backed implementation of [`MorphProofsStore`](crate::MorphProofsStore). + +mod models; +pub use models::*; + +mod store; +pub use store::{MdbxProofsStorage, MdbxProofsStorageOptions, ProofDbIdentity, ProofWindowBounds}; + +mod cursor; +pub use cursor::{ + BlockNumberVersionedCursor, Dup, MdbxAccountCursor, MdbxStorageCursor, MdbxTrieCursor, +}; + +mod batch; +pub use batch::{DupRw, MdbxBatchSession}; diff --git a/crates/proofs/src/db/models/block.rs b/crates/proofs/src/db/models/block.rs new file mode 100644 index 00000000..7475c364 --- /dev/null +++ b/crates/proofs/src/db/models/block.rs @@ -0,0 +1,82 @@ +use alloy_eips::BlockNumHash; +use alloy_primitives::B256; +use bytes::BufMut; +use derive_more::{From, Into}; +use reth_codecs::DecompressError; +use reth_db::{ + DatabaseError, + table::{Compress, Decompress}, +}; +use serde::{Deserialize, Serialize}; + +/// Wrapper for block number and block hash tuple to implement [`Compress`]/[`Decompress`]. +/// +/// Used for storing block metadata (number + hash). +#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize, From, Into)] +pub struct BlockNumberHash(BlockNumHash); + +impl Compress for BlockNumberHash { + type Compressed = Vec; + + fn compress_to_buf>(&self, buf: &mut B) { + // Encode block number (8 bytes, big-endian) + hash (32 bytes) = 40 bytes total + buf.put_u64(self.0.number); + buf.put_slice(self.0.hash.as_slice()); + } +} + +impl Decompress for BlockNumberHash { + fn decompress(value: &[u8]) -> Result { + if value.len() != 40 { + return Err(DecompressError::new(DatabaseError::Decode)); + } + + let number = u64::from_be_bytes( + value[..8] + .try_into() + .map_err(|_| DecompressError::new(DatabaseError::Decode))?, + ); + let hash = B256::from_slice(&value[8..40]); + + Ok(Self(BlockNumHash { number, hash })) + } +} + +impl BlockNumberHash { + /// Create new instance. + pub const fn new(number: u64, hash: B256) -> Self { + Self(BlockNumHash { number, hash }) + } + + /// Get the block number. + pub const fn number(&self) -> u64 { + self.0.number + } + + /// Get the block hash. + pub const fn hash(&self) -> &B256 { + &self.0.hash + } +} + +#[cfg(test)] +mod tests { + use alloy_primitives::B256; + + use super::*; + + #[test] + fn test_block_number_hash_roundtrip() { + let test_cases = vec![ + BlockNumberHash::new(0, B256::ZERO), + BlockNumberHash::new(42, B256::repeat_byte(0xaa)), + BlockNumberHash::new(u64::MAX, B256::repeat_byte(0xff)), + ]; + + for original in test_cases { + let compressed = original.compress(); + let decompressed = BlockNumberHash::decompress(&compressed).unwrap(); + assert_eq!(original, decompressed); + } + } +} diff --git a/crates/proofs/src/db/models/change_set.rs b/crates/proofs/src/db/models/change_set.rs new file mode 100644 index 00000000..d9f7d806 --- /dev/null +++ b/crates/proofs/src/db/models/change_set.rs @@ -0,0 +1,131 @@ +use alloy_primitives::B256; +use reth_codecs::DecompressError; +use reth_db::{ + DatabaseError, + table::{self, Decode, Encode}, +}; +use reth_trie_common::StoredNibbles; +use serde::{Deserialize, Serialize}; + +use crate::db::{HashedStorageKey, StorageTrieKey}; + +/// The keys of the entries in the history tables. +#[derive(Debug, Default, Clone, PartialEq, Eq, PartialOrd, Ord, Hash, Serialize, Deserialize)] +pub struct ChangeSet { + /// Keys changed in [`AccountTrieHistory`](super::AccountTrieHistory) table. + pub account_trie_keys: Vec, + /// Keys changed in [`StorageTrieHistory`](super::StorageTrieHistory) table. + pub storage_trie_keys: Vec, + /// Keys changed in [`HashedAccountHistory`](super::HashedAccountHistory) table. + pub hashed_account_keys: Vec, + /// Keys changed in [`HashedStorageHistory`](super::HashedStorageHistory) table. + pub hashed_storage_keys: Vec, +} + +impl table::Encode for ChangeSet { + type Encoded = Vec; + + fn encode(self) -> Self::Encoded { + bincode::serde::encode_to_vec(&self, bincode::config::standard()) + .expect("ChangeSet serialization should not fail") + } +} + +impl table::Decode for ChangeSet { + fn decode(value: &[u8]) -> Result { + bincode::serde::decode_from_slice(value, bincode::config::standard()) + .map(|(v, _)| v) + .map_err(|_| DatabaseError::Decode) + } +} + +impl table::Compress for ChangeSet { + type Compressed = Vec; + + fn compress_to_buf>(&self, buf: &mut B) { + let encoded = self.clone().encode(); + buf.put_slice(&encoded); + } +} + +impl table::Decompress for ChangeSet { + fn decompress(value: &[u8]) -> Result { + Self::decode(value).map_err(DecompressError::new) + } +} + +#[cfg(test)] +mod tests { + use alloy_primitives::B256; + use reth_db::table::{Compress, Decompress}; + + use super::*; + + #[test] + fn test_encode_decode_empty_change_set() { + let change_set = ChangeSet { + account_trie_keys: vec![], + storage_trie_keys: vec![], + hashed_account_keys: vec![], + hashed_storage_keys: vec![], + }; + + let encoded = change_set.clone().encode(); + let decoded = ChangeSet::decode(&encoded).expect("Failed to decode"); + assert_eq!(change_set, decoded); + } + + #[test] + fn test_encode_decode_populated_change_set() { + let account_key = StoredNibbles::from(vec![1, 2, 3, 4]); + let storage_key = StorageTrieKey { + hashed_address: B256::repeat_byte(0x11), + path: StoredNibbles::from(vec![5, 6, 7, 8]), + }; + let hashed_storage_key = HashedStorageKey { + hashed_address: B256::repeat_byte(0x22), + hashed_storage_key: B256::repeat_byte(0x33), + }; + + let change_set = ChangeSet { + account_trie_keys: vec![account_key], + storage_trie_keys: vec![storage_key], + hashed_account_keys: vec![B256::repeat_byte(0x44)], + hashed_storage_keys: vec![hashed_storage_key], + }; + + let encoded = change_set.clone().encode(); + let decoded = ChangeSet::decode(&encoded).expect("Failed to decode"); + assert_eq!(change_set, decoded); + } + + #[test] + fn test_decode_invalid_data() { + let invalid_data = vec![0xFF; 32]; + let result = ChangeSet::decode(&invalid_data); + assert!(result.is_err()); + assert!(matches!(result.unwrap_err(), DatabaseError::Decode)); + } + + #[test] + fn test_compress_decompress() { + let change_set = ChangeSet { + account_trie_keys: vec![StoredNibbles::from(vec![1, 2, 3])], + storage_trie_keys: vec![StorageTrieKey { + hashed_address: B256::ZERO, + path: StoredNibbles::from(vec![4, 5, 6]), + }], + hashed_account_keys: vec![B256::ZERO], + hashed_storage_keys: vec![HashedStorageKey { + hashed_address: B256::ZERO, + hashed_storage_key: B256::repeat_byte(0x42), + }], + }; + + let mut buf = Vec::new(); + change_set.compress_to_buf(&mut buf); + + let decompressed = ChangeSet::decompress(&buf).expect("Failed to decompress"); + assert_eq!(change_set, decompressed); + } +} diff --git a/crates/proofs/src/db/models/kv.rs b/crates/proofs/src/db/models/kv.rs new file mode 100644 index 00000000..05f85807 --- /dev/null +++ b/crates/proofs/src/db/models/kv.rs @@ -0,0 +1,85 @@ +use alloy_primitives::B256; +use reth_db::table::{DupSort, Table}; +use reth_primitives_traits::Account; +use reth_trie_common::{BranchNodeCompact, Nibbles, StoredNibbles}; + +use crate::db::{ + AccountTrieHistory, HashedAccountHistory, HashedStorageHistory, HashedStorageKey, MaybeDeleted, + StorageTrieHistory, StorageTrieKey, StorageValue, VersionedValue, +}; + +/// Helper to convert inputs into a table key or kv pair. +pub trait IntoKV { + /// Convert `self` into the table key. + fn into_key(self) -> Tab::Key; + + /// Convert `self` into kv for the given `block_number`. + fn into_kv(self, block_number: u64) -> (Tab::Key, Tab::Value); +} + +impl IntoKV for (Nibbles, Option) { + fn into_key(self) -> StoredNibbles { + StoredNibbles::from(self.0) + } + + fn into_kv(self, block_number: u64) -> (StoredNibbles, VersionedValue) { + let (path, node) = self; + ( + StoredNibbles::from(path), + VersionedValue { + block_number, + value: MaybeDeleted(node), + }, + ) + } +} + +impl IntoKV for (B256, Nibbles, Option) { + fn into_key(self) -> StorageTrieKey { + let (hashed_address, path, _) = self; + StorageTrieKey::new(hashed_address, StoredNibbles::from(path)) + } + fn into_kv(self, block_number: u64) -> (StorageTrieKey, VersionedValue) { + let (hashed_address, path, node) = self; + ( + StorageTrieKey::new(hashed_address, StoredNibbles::from(path)), + VersionedValue { + block_number, + value: MaybeDeleted(node), + }, + ) + } +} + +impl IntoKV for (B256, Option) { + fn into_key(self) -> B256 { + self.0 + } + fn into_kv(self, block_number: u64) -> (B256, VersionedValue) { + let (hashed_address, account) = self; + ( + hashed_address, + VersionedValue { + block_number, + value: MaybeDeleted(account), + }, + ) + } +} + +impl IntoKV for (B256, B256, Option) { + fn into_key(self) -> HashedStorageKey { + let (hashed_address, hashed_storage_key, _) = self; + HashedStorageKey::new(hashed_address, hashed_storage_key) + } + fn into_kv(self, block_number: u64) -> (HashedStorageKey, VersionedValue) { + let (hashed_address, hashed_storage_key, value) = self; + ( + HashedStorageKey::new(hashed_address, hashed_storage_key), + VersionedValue { + block_number, + value: MaybeDeleted(value), + }, + ) + } +} diff --git a/crates/proofs/src/db/models/metadata.rs b/crates/proofs/src/db/models/metadata.rs new file mode 100644 index 00000000..999795e7 --- /dev/null +++ b/crates/proofs/src/db/models/metadata.rs @@ -0,0 +1,56 @@ +use bytes::BufMut; +use reth_codecs::DecompressError; +use reth_db::{ + DatabaseError, + table::{Compress, Decode, Decompress, Encode}, +}; +use serde::{Deserialize, Serialize}; + +/// Durable proof-database metadata keys. +#[derive(Debug, Clone, Copy, PartialEq, Eq, PartialOrd, Ord, Hash, Serialize, Deserialize)] +#[repr(u8)] +pub enum ProofMetadataKey { + /// EIP-155 chain identifier. + ChainId = 0, + /// Canonical genesis block hash. + GenesisHash = 1, + /// Proof-database schema version. + SchemaVersion = 2, +} + +impl Encode for ProofMetadataKey { + type Encoded = [u8; 1]; + + fn encode(self) -> Self::Encoded { + [self as u8] + } +} + +impl Decode for ProofMetadataKey { + fn decode(value: &[u8]) -> Result { + match value.first() { + Some(0) => Ok(Self::ChainId), + Some(1) => Ok(Self::GenesisHash), + Some(2) => Ok(Self::SchemaVersion), + _ => Err(DatabaseError::Decode), + } + } +} + +/// Opaque metadata bytes whose shape is validated by the store. +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] +pub struct ProofMetadataValue(pub Vec); + +impl Compress for ProofMetadataValue { + type Compressed = Vec; + + fn compress_to_buf>(&self, buf: &mut B) { + buf.put_slice(&self.0); + } +} + +impl Decompress for ProofMetadataValue { + fn decompress(value: &[u8]) -> Result { + Ok(Self(value.to_vec())) + } +} diff --git a/crates/proofs/src/db/models/mod.rs b/crates/proofs/src/db/models/mod.rs new file mode 100644 index 00000000..de5b9786 --- /dev/null +++ b/crates/proofs/src/db/models/mod.rs @@ -0,0 +1,93 @@ +//! MDBX implementation of [`MorphProofsStore`](crate::MorphProofsStore). +//! +//! This module provides a complete MDBX implementation of the +//! [`MorphProofsStore`](crate::MorphProofsStore) trait. It uses the [`reth_db`] crate for +//! database interactions and defines the necessary tables and models for storing trie branches, +//! accounts, and storage leaves. + +mod block; +pub use block::*; +mod version; +pub use version::*; +mod storage; +pub use storage::*; +mod change_set; +mod kv; +mod metadata; +use std::fmt; + +use alloy_primitives::B256; +pub use change_set::*; +pub use kv::*; +pub use metadata::*; +use reth_db::{ + TableSet, TableType, TableViewer, + table::{DupSort, TableInfo}, + tables, +}; +use reth_primitives_traits::Account; +use reth_trie_common::{BranchNodeCompact, StoredNibbles}; + +tables! { + /// Stores historical branch nodes for the account state trie. + /// + /// Each entry maps a compact-encoded trie path (`StoredNibbles`) to its versioned branch node. + /// Multiple versions of the same node are stored using the block number as a subkey. + table AccountTrieHistory { + type Key = StoredNibbles; + type Value = VersionedValue; + type SubKey = u64; // block number + } + + /// Stores historical branch nodes for the storage trie of each account. + /// + /// Each entry is identified by a composite key combining the account’s hashed address and the + /// compact-encoded trie path. Versions are tracked using block numbers as subkeys. + table StorageTrieHistory { + type Key = StorageTrieKey; + type Value = VersionedValue; + type SubKey = u64; // block number + } + + /// Stores versioned account state across block history. + /// + /// Each entry maps a hashed account address to its serialized account data (balance, nonce, + /// code hash, storage root). + table HashedAccountHistory { + type Key = B256; + type Value = VersionedValue; + type SubKey = u64; // block number + } + + /// Stores versioned storage state across block history. + /// + /// Each entry maps a composite key of (hashed address, storage key) to its stored value. + /// Used for reconstructing contract storage at any historical block height. + table HashedStorageHistory { + type Key = HashedStorageKey; + type Value = VersionedValue; + type SubKey = u64; // block number + } + + /// Tracks the active proof window in the external historical storage. + /// + /// Stores the earliest and latest block numbers (and corresponding hashes) + /// for which historical trie data is retained. + table ProofWindow { + type Key = ProofWindowKey; + type Value = BlockNumberHash; + } + + /// Stores schema and canonical-chain identity for fail-fast cold-copy validation. + table ProofMetadata { + type Key = ProofMetadataKey; + type Value = ProofMetadataValue; + } + + /// A reverse mapping of block numbers to a keys of the tables. + /// This is used for efficiently locating data by block number. + table BlockChangeSet { + type Key = u64; // Block number + type Value = ChangeSet; + } +} diff --git a/crates/proofs/src/db/models/storage.rs b/crates/proofs/src/db/models/storage.rs new file mode 100644 index 00000000..bcf30dfb --- /dev/null +++ b/crates/proofs/src/db/models/storage.rs @@ -0,0 +1,272 @@ +use alloy_primitives::{B256, U256}; +use derive_more::{Constructor, From, Into}; +use reth_codecs::DecompressError; +use reth_db::{ + DatabaseError, + table::{Compress, Decode, Decompress, Encode}, +}; +use reth_trie_common::StoredNibbles; +use serde::{Deserialize, Serialize}; + +/// Composite key: `(hashed-address, path)` for storage trie branches +/// +/// Used to efficiently index storage branches by both account address and trie path. +/// The encoding ensures lexicographic ordering: first by address, then by path. +#[derive(Default, Debug, Clone, PartialEq, Eq, PartialOrd, Ord, Hash, Serialize, Deserialize)] +pub struct StorageTrieKey { + /// Hashed account address + pub hashed_address: B256, + /// Trie path as nibbles + pub path: StoredNibbles, +} + +impl StorageTrieKey { + /// Create a new storage branch key + pub const fn new(hashed_address: B256, path: StoredNibbles) -> Self { + Self { + hashed_address, + path, + } + } +} + +impl Encode for StorageTrieKey { + type Encoded = Vec; + + fn encode(self) -> Self::Encoded { + let mut buf = Vec::with_capacity(32 + self.path.0.len()); + // First encode the address (32 bytes) + buf.extend_from_slice(self.hashed_address.as_slice()); + // Then encode the path + buf.extend_from_slice(&self.path.encode()); + buf + } +} + +impl Decode for StorageTrieKey { + fn decode(value: &[u8]) -> Result { + if value.len() < 32 { + return Err(DatabaseError::Decode); + } + + // First 32 bytes are the address + let hashed_address = B256::from_slice(&value[..32]); + + // Remaining bytes are the path + let path = StoredNibbles::decode(&value[32..])?; + + Ok(Self { + hashed_address, + path, + }) + } +} + +/// Composite key: (`hashed_address`, `hashed_storage_key`) for hashed storage values +/// +/// Used to efficiently index storage values by both account address and storage key. +/// The encoding ensures lexicographic ordering: first by address, then by storage key. +#[derive(Debug, Default, Clone, PartialEq, Eq, PartialOrd, Ord, Hash, Serialize, Deserialize)] +pub struct HashedStorageKey { + /// Hashed account address + pub hashed_address: B256, + /// Hashed storage key + pub hashed_storage_key: B256, +} + +impl HashedStorageKey { + /// Create a new hashed storage key + pub const fn new(hashed_address: B256, hashed_storage_key: B256) -> Self { + Self { + hashed_address, + hashed_storage_key, + } + } +} + +impl Encode for HashedStorageKey { + type Encoded = [u8; 64]; + + fn encode(self) -> Self::Encoded { + let mut buf = [0u8; 64]; + // First 32 bytes: address + buf[..32].copy_from_slice(self.hashed_address.as_slice()); + // Next 32 bytes: storage key + buf[32..].copy_from_slice(self.hashed_storage_key.as_slice()); + buf + } +} + +impl Decode for HashedStorageKey { + fn decode(value: &[u8]) -> Result { + if value.len() != 64 { + return Err(DatabaseError::Decode); + } + + let hashed_address = B256::from_slice(&value[..32]); + let hashed_storage_key = B256::from_slice(&value[32..64]); + + Ok(Self { + hashed_address, + hashed_storage_key, + }) + } +} + +/// Storage value wrapper for U256 values +#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize, From, Into, Constructor)] +pub struct StorageValue(pub U256); + +impl Compress for StorageValue { + type Compressed = Vec; + + fn compress_to_buf>(&self, buf: &mut B) { + let be: [u8; 32] = self.0.to_be_bytes::<32>(); + buf.put_slice(&be); + } +} + +impl Decompress for StorageValue { + fn decompress(value: &[u8]) -> Result { + if value.len() != 32 { + return Err(DecompressError::new(DatabaseError::Decode)); + } + let bytes: [u8; 32] = value + .try_into() + .map_err(|_| DecompressError::new(DatabaseError::Decode))?; + Ok(Self(U256::from_be_bytes(bytes))) + } +} + +/// Proof Window key for tracking active proof window bounds +/// +/// Used to store earliest and latest block numbers in the external storage. +#[derive(Debug, Clone, Copy, PartialEq, Eq, PartialOrd, Ord, Hash, Serialize, Deserialize)] +#[repr(u8)] +pub enum ProofWindowKey { + /// Earliest block number stored in external storage + EarliestBlock = 0, + /// Latest block number stored in external storage + LatestBlock = 1, + /// Anchor block from where the initial state initialization started + InitialStateAnchor = 2, +} + +impl Encode for ProofWindowKey { + type Encoded = [u8; 1]; + + fn encode(self) -> Self::Encoded { + [self as u8] + } +} + +impl Decode for ProofWindowKey { + fn decode(value: &[u8]) -> Result { + match value.first() { + Some(&0) => Ok(Self::EarliestBlock), + Some(&1) => Ok(Self::LatestBlock), + Some(&2) => Ok(Self::InitialStateAnchor), + _ => Err(DatabaseError::Decode), + } + } +} + +#[cfg(test)] +mod tests { + use reth_trie::Nibbles; + + use super::*; + + #[test] + fn test_storage_branch_subkey_encode_decode() { + let addr = B256::from([1u8; 32]); + let path = StoredNibbles(Nibbles::from_nibbles_unchecked([1, 2, 3, 4])); + let key = StorageTrieKey::new(addr, path.clone()); + + let encoded = key.clone().encode(); + let decoded = StorageTrieKey::decode(&encoded).unwrap(); + + assert_eq!(key, decoded); + assert_eq!(decoded.hashed_address, addr); + assert_eq!(decoded.path, path); + } + + #[test] + fn test_storage_branch_subkey_ordering() { + let addr1 = B256::from([1u8; 32]); + let addr2 = B256::from([2u8; 32]); + let path1 = StoredNibbles(Nibbles::from_nibbles_unchecked([1, 2])); + let path2 = StoredNibbles(Nibbles::from_nibbles_unchecked([1, 3])); + + let key1 = StorageTrieKey::new(addr1, path1.clone()); + let key2 = StorageTrieKey::new(addr1, path2); + let key3 = StorageTrieKey::new(addr2, path1); + + // Encoded bytes should be sortable: first by address, then by path + let enc1 = key1.encode(); + let enc2 = key2.encode(); + let enc3 = key3.encode(); + + assert!(enc1 < enc2, "Same address, path1 < path2"); + assert!(enc1 < enc3, "addr1 < addr2"); + assert!(enc2 < enc3, "addr1 < addr2 (even with larger path)"); + } + + #[test] + fn test_hashed_storage_subkey_encode_decode() { + let addr = B256::from([1u8; 32]); + let storage_key = B256::from([2u8; 32]); + let key = HashedStorageKey::new(addr, storage_key); + + let encoded = key.clone().encode(); + let decoded = HashedStorageKey::decode(&encoded).unwrap(); + + assert_eq!(key, decoded); + assert_eq!(decoded.hashed_address, addr); + assert_eq!(decoded.hashed_storage_key, storage_key); + } + + #[test] + fn test_hashed_storage_subkey_ordering() { + let addr1 = B256::from([1u8; 32]); + let addr2 = B256::from([2u8; 32]); + let storage1 = B256::from([10u8; 32]); + let storage2 = B256::from([20u8; 32]); + + let key1 = HashedStorageKey::new(addr1, storage1); + let key2 = HashedStorageKey::new(addr1, storage2); + let key3 = HashedStorageKey::new(addr2, storage1); + + // Encoded bytes should be sortable: first by address, then by storage key + let enc1 = key1.encode(); + let enc2 = key2.encode(); + let enc3 = key3.encode(); + + assert!(enc1 < enc2, "Same address, storage1 < storage2"); + assert!(enc1 < enc3, "addr1 < addr2"); + assert!(enc2 < enc3, "addr1 < addr2 (even with larger storage key)"); + } + + #[test] + fn test_hashed_storage_subkey_size() { + let addr = B256::from([1u8; 32]); + let storage_key = B256::from([2u8; 32]); + let key = HashedStorageKey::new(addr, storage_key); + + let encoded = key.encode(); + assert_eq!(encoded.len(), 64, "Encoded size should be exactly 64 bytes"); + } + + #[test] + fn test_metadata_key_encode_decode() { + let key = ProofWindowKey::EarliestBlock; + let encoded = key.encode(); + let decoded = ProofWindowKey::decode(&encoded).unwrap(); + assert_eq!(key, decoded); + + let key = ProofWindowKey::LatestBlock; + let encoded = key.encode(); + let decoded = ProofWindowKey::decode(&encoded).unwrap(); + assert_eq!(key, decoded); + } +} diff --git a/crates/proofs/src/db/models/version.rs b/crates/proofs/src/db/models/version.rs new file mode 100644 index 00000000..5e1ea7d2 --- /dev/null +++ b/crates/proofs/src/db/models/version.rs @@ -0,0 +1,205 @@ +use bytes::{Buf, BufMut}; +use reth_codecs::DecompressError; +use reth_db::{ + DatabaseError, + table::{Compress, Decompress}, +}; +use reth_primitives_traits::ValueWithSubKey; +use serde::{Deserialize, Serialize}; + +/// Wrapper type for `Option` that implements [`Compress`] and [`Decompress`] +/// +/// Encoding: +/// - `None` => empty byte array (length 0) +/// - `Some(value)` => compressed bytes of value (length > 0) +/// +/// This assumes the inner type `T` always compresses to non-empty bytes when it exists. +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] +pub struct MaybeDeleted(pub Option); + +impl From> for MaybeDeleted { + fn from(opt: Option) -> Self { + Self(opt) + } +} + +impl From> for Option { + fn from(maybe: MaybeDeleted) -> Self { + maybe.0 + } +} + +impl Compress for MaybeDeleted { + type Compressed = Vec; + + fn compress_to_buf>(&self, buf: &mut B) { + match &self.0 { + None => { + // Empty = deleted, write nothing + } + Some(value) => { + // Compress the inner value to the buffer + value.compress_to_buf(buf); + } + } + } +} + +impl Decompress for MaybeDeleted { + fn decompress(value: &[u8]) -> Result { + if value.is_empty() { + // Empty = deleted + Ok(Self(None)) + } else { + // Non-empty = present + let inner = T::decompress(value)?; + Ok(Self(Some(inner))) + } + } +} + +/// Versioned value wrapper for [`DupSort`] tables +/// +/// For [`DupSort`] tables in MDBX, the Value type must contain the [`DupSort::SubKey`] as a field. +/// This wrapper combines a [`block_number`] (the [`DupSort::SubKey`]) with +/// the actual value. +/// +/// [`DupSort`]: reth_db::table::DupSort +/// [`DupSort::SubKey`]: reth_db::table::DupSort::SubKey +/// [`block_number`]: Self::block_number +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] +pub struct VersionedValue { + /// Block number ([`DupSort::SubKey`] for [`DupSort`]) + /// + /// [`DupSort`]: reth_db::table::DupSort + /// [`DupSort::SubKey`]: reth_db::table::DupSort::SubKey + pub block_number: u64, + /// The actual value (may be deleted) + pub value: MaybeDeleted, +} + +impl VersionedValue { + /// Create a new versioned value + pub const fn new(block_number: u64, value: MaybeDeleted) -> Self { + Self { + block_number, + value, + } + } +} + +impl Compress for VersionedValue { + type Compressed = Vec; + + fn compress_to_buf>(&self, buf: &mut B) { + // Encode block number first (8 bytes, big-endian) + buf.put_u64(self.block_number); + // Then encode the value + self.value.compress_to_buf(buf); + } +} + +impl Decompress for VersionedValue { + fn decompress(value: &[u8]) -> Result { + if value.len() < 8 { + return Err(DecompressError::new(DatabaseError::Decode)); + } + + let mut buf: &[u8] = value; + let block_number = buf.get_u64(); + let value = MaybeDeleted::::decompress(&value[8..])?; + + Ok(Self { + block_number, + value, + }) + } +} + +impl ValueWithSubKey for VersionedValue { + type SubKey = u64; + + fn get_subkey(&self) -> Self::SubKey { + self.block_number + } +} + +#[cfg(test)] +mod tests { + use reth_primitives_traits::Account; + use reth_trie::BranchNodeCompact; + + use super::*; + + #[test] + fn test_maybe_deleted_none() { + let none: MaybeDeleted = MaybeDeleted(None); + let compressed = none.compress(); + assert!(compressed.is_empty(), "None should compress to empty bytes"); + + let decompressed = MaybeDeleted::::decompress(&compressed).unwrap(); + assert_eq!(decompressed.0, None); + } + + #[test] + fn test_maybe_deleted_some_account() { + let account = Account { + nonce: 42, + balance: alloy_primitives::U256::from(1000u64), + bytecode_hash: None, + }; + let some = MaybeDeleted(Some(account)); + let compressed = some.compress(); + assert!( + !compressed.is_empty(), + "Some should compress to non-empty bytes" + ); + + let decompressed = MaybeDeleted::::decompress(&compressed).unwrap(); + assert_eq!(decompressed.0, Some(account)); + } + + #[test] + fn test_maybe_deleted_some_branch() { + // Create a simple valid BranchNodeCompact (empty is valid) + let branch = BranchNodeCompact::new( + 0, // state_mask + 0, // tree_mask + 0, // hash_mask + vec![], // hashes + None, // root_hash + ); + let some = MaybeDeleted(Some(branch.clone())); + let compressed = some.compress(); + assert!( + !compressed.is_empty(), + "Some should compress to non-empty bytes" + ); + + let decompressed = MaybeDeleted::::decompress(&compressed).unwrap(); + assert_eq!(decompressed.0, Some(branch)); + } + + #[test] + fn test_maybe_deleted_roundtrip() { + let test_cases = vec![ + MaybeDeleted(None), + MaybeDeleted(Some(Account { + nonce: 0, + balance: alloy_primitives::U256::ZERO, + bytecode_hash: None, + })), + MaybeDeleted(Some(Account { + nonce: 999, + balance: alloy_primitives::U256::MAX, + bytecode_hash: Some([0xff; 32].into()), + })), + ]; + + for original in test_cases { + let compressed = original.clone().compress(); + let decompressed = MaybeDeleted::::decompress(&compressed).unwrap(); + assert_eq!(original, decompressed); + } + } +} diff --git a/crates/proofs/src/db/store.rs b/crates/proofs/src/db/store.rs new file mode 100644 index 00000000..0f7064d5 --- /dev/null +++ b/crates/proofs/src/db/store.rs @@ -0,0 +1,4707 @@ +use std::{collections::BTreeMap, ops::RangeBounds, path::Path, time::Duration}; + +use alloy_eips::{BlockNumHash, NumHash, eip1898::BlockWithParent}; +use alloy_primitives::{B256, U256, map::HashMap}; +use reth_db::{ + Database, DatabaseEnv, DatabaseError, + cursor::{DbCursorRO, DbCursorRW, DbDupCursorRO, DbDupCursorRW}, + mdbx::{DatabaseArguments, MaxReadTransactionDuration, init_db_for}, + table::{DupSort, Table}, + transaction::{DbTx, DbTxMut}, +}; +use reth_primitives_traits::Account; +use reth_trie::{hashed_cursor::HashedCursor, trie_cursor::TrieCursor}; +use reth_trie_common::{ + BranchNodeCompact, HashedPostState, Nibbles, StoredNibbles, + updates::{StorageTrieUpdates, TrieUpdates}, +}; + +use super::{ + BlockNumberHash, ProofMetadata, ProofMetadataKey, ProofMetadataValue, ProofWindow, + ProofWindowKey, Tables, +}; +use crate::{ + BlockStateDiff, MorphProofsStorageError, + MorphProofsStorageError::NoBlocksFound, + MorphProofsStorageResult, MorphProofsStore, + api::{ + InitialStateAnchor, InitialStateStatus, MorphProofsBatchStore, + MorphProofsInitialStateStore, WriteCounts, + }, + db::{ + MdbxAccountCursor, MdbxBatchSession, MdbxStorageCursor, MdbxTrieCursor, + cursor::Dup, + models::{ + AccountTrieHistory, BlockChangeSet, ChangeSet, HashedAccountHistory, + HashedStorageHistory, HashedStorageKey, IntoKV, MaybeDeleted, StorageTrieHistory, + StorageTrieKey, StorageValue, VersionedValue, + }, + }, +}; + +/// MDBX implementation of [`MorphProofsStore`]. +#[derive(Debug)] +pub struct MdbxProofsStorage { + env: DatabaseEnv, +} + +/// Options for opening [`MdbxProofsStorage`]. +#[derive(Debug, Clone, Copy, Default, PartialEq, Eq)] +pub struct MdbxProofsStorageOptions { + /// Maximum duration of a read transaction. + pub max_read_transaction_duration: Option, +} + +/// Canonical-chain identity persisted in the proof database. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub struct ProofDbIdentity { + /// EIP-155 chain identifier. + pub chain_id: u64, + /// Canonical genesis block hash. + pub genesis_hash: B256, +} + +impl ProofDbIdentity { + /// Creates a proof-database identity. + pub const fn new(chain_id: u64, genesis_hash: B256) -> Self { + Self { + chain_id, + genesis_hash, + } + } +} + +impl Default for ProofDbIdentity { + fn default() -> Self { + Self::new(0, B256::ZERO) + } +} + +/// Inclusive range currently served by the proof database. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub struct ProofWindowBounds { + /// Oldest block whose proof data is retained. + pub earliest: (u64, B256), + /// Newest block durably indexed by proof history. + pub latest: (u64, B256), +} + +/// Current on-disk schema. There is intentionally no migration path before launch. +const SCHEMA_VERSION: u32 = 1; + +fn decode_u64_metadata( + value: ProofMetadataValue, + field: &'static str, +) -> MorphProofsStorageResult { + let bytes: [u8; 8] = value + .0 + .try_into() + .map_err(|_| MorphProofsStorageError::CorruptMetadata(field))?; + Ok(u64::from_be_bytes(bytes)) +} + +fn decode_u32_metadata( + value: ProofMetadataValue, + field: &'static str, +) -> MorphProofsStorageResult { + let bytes: [u8; 4] = value + .0 + .try_into() + .map_err(|_| MorphProofsStorageError::CorruptMetadata(field))?; + Ok(u32::from_be_bytes(bytes)) +} + +fn decode_hash_metadata( + value: ProofMetadataValue, + field: &'static str, +) -> MorphProofsStorageResult { + let bytes: [u8; 32] = value + .0 + .try_into() + .map_err(|_| MorphProofsStorageError::CorruptMetadata(field))?; + Ok(B256::new(bytes)) +} + +struct ProofWindowValue { + earliest: NumHash, + latest: NumHash, +} + +/// Preprocessed prune plan for a target block number +#[derive(Debug, Clone)] +struct PrunePlan { + earliest_block: u64, + acc_survivors: Vec<(StoredNibbles, u64)>, + storage_survivors: Vec<(StorageTrieKey, u64)>, + hashed_acc_survivors: Vec<(B256, u64)>, + hashed_storage_survivors: Vec<(HashedStorageKey, u64)>, +} + +/// Preprocessed delete work for a prune range +#[derive(Debug, Default, Clone)] +struct HistoryDeleteBatch { + account_trie: Vec<(::Key, u64)>, + storage_trie: Vec<(::Key, u64)>, + hashed_account: Vec<(::Key, u64)>, + hashed_storage: Vec<(::Key, u64)>, +} + +impl MdbxProofsStorage { + /// Creates a new [`MdbxProofsStorage`] instance with the given path. + pub fn new(path: &Path) -> Result { + Self::open(path, ProofDbIdentity::default()) + } + + /// Opens an MDBX proof database and validates its canonical-chain identity. + pub fn open(path: &Path, identity: ProofDbIdentity) -> Result { + Self::open_with_options(path, identity, MdbxProofsStorageOptions::default()) + } + + /// Creates a new [`MdbxProofsStorage`] instance with the given path and options. + pub fn new_with_options( + path: &Path, + options: MdbxProofsStorageOptions, + ) -> Result { + Self::open_with_options(path, ProofDbIdentity::default(), options) + } + + /// Opens an MDBX proof database with runtime options and validates chain identity. + pub fn open_with_options( + path: &Path, + identity: ProofDbIdentity, + options: MdbxProofsStorageOptions, + ) -> Result { + let mut args = DatabaseArguments::default(); + if let Some(duration) = options.max_read_transaction_duration { + args = args.with_max_read_transaction_duration(Some(MaxReadTransactionDuration::Set( + duration, + ))); + } + let env = init_db_for::<_, Tables>(path, args) + .map_err(|e| DatabaseError::Other(format!("Failed to open database: {e}")))?; + let storage = Self { env }; + storage.validate_or_initialize_identity(identity)?; + Ok(storage) + } + + fn validate_or_initialize_identity( + &self, + identity: ProofDbIdentity, + ) -> MorphProofsStorageResult<()> { + // Use one explicit write transaction so the empty-database check and identity + // initialization are atomic. Dropping the transaction on any error aborts it instead of + // committing a partially-written identity. + let tx = self.env.tx_mut()?; + let stored_chain_id = tx.get::(ProofMetadataKey::ChainId)?; + let stored_genesis_hash = tx.get::(ProofMetadataKey::GenesisHash)?; + let stored_schema_version = tx.get::(ProofMetadataKey::SchemaVersion)?; + let metadata_entries = tx.entries::()?; + + match (stored_chain_id, stored_genesis_hash, stored_schema_version) { + (None, None, None) if metadata_entries == 0 => { + if !Self::proof_tables_are_empty(&tx)? { + return Err(MorphProofsStorageError::ProofDataWithoutIdentityMetadata); + } + tx.put::( + ProofMetadataKey::ChainId, + ProofMetadataValue(identity.chain_id.to_be_bytes().to_vec()), + )?; + tx.put::( + ProofMetadataKey::GenesisHash, + ProofMetadataValue(identity.genesis_hash.as_slice().to_vec()), + )?; + tx.put::( + ProofMetadataKey::SchemaVersion, + ProofMetadataValue(SCHEMA_VERSION.to_be_bytes().to_vec()), + )?; + } + (Some(chain_id), Some(genesis_hash), Some(schema_version)) if metadata_entries == 3 => { + let chain_id = decode_u64_metadata(chain_id, "chain_id")?; + if chain_id != identity.chain_id { + return Err(MorphProofsStorageError::ChainIdentityMismatch("chain_id")); + } + + let genesis_hash = decode_hash_metadata(genesis_hash, "genesis_hash")?; + if genesis_hash != identity.genesis_hash { + return Err(MorphProofsStorageError::ChainIdentityMismatch( + "genesis_hash", + )); + } + + let schema_version = decode_u32_metadata(schema_version, "schema_version")?; + if schema_version != SCHEMA_VERSION { + return Err(MorphProofsStorageError::SchemaMismatch { + expected: SCHEMA_VERSION, + actual: schema_version, + }); + } + } + _ => return Err(MorphProofsStorageError::IncompleteIdentityMetadata), + } + + tx.commit()?; + Ok(()) + } + + fn proof_tables_are_empty(tx: &impl DbTx) -> Result { + Ok(tx.entries::()? == 0 + && tx.entries::()? == 0 + && tx.entries::()? == 0 + && tx.entries::()? == 0 + && tx.entries::()? == 0 + && tx.entries::()? == 0) + } + + /// Returns the inclusive range currently served by proof history. + pub fn proof_window(&self) -> MorphProofsStorageResult> { + self.env.view(|tx| { + Ok(self + .inner_get_proof_window(tx)? + .map(|window| ProofWindowBounds { + earliest: (window.earliest.number, window.earliest.hash), + latest: (window.latest.number, window.latest.hash), + })) + })? + } + + pub(crate) fn inner_get_latest_block_number_hash( + &self, + tx: &impl DbTx, + ) -> MorphProofsStorageResult> { + let block = self.inner_get_block_number_hash(tx, ProofWindowKey::LatestBlock)?; + if block.is_some() { + return Ok(block); + } + + self.inner_get_block_number_hash(tx, ProofWindowKey::EarliestBlock) + } + + pub(crate) fn inner_get_earliest_block_number_hash( + &self, + tx: &impl DbTx, + ) -> MorphProofsStorageResult> { + self.inner_get_block_number_hash(tx, ProofWindowKey::EarliestBlock) + } + + fn inner_get_block_number_hash( + &self, + tx: &impl DbTx, + key: ProofWindowKey, + ) -> MorphProofsStorageResult> { + let mut cursor = tx.cursor_read::()?; + let value = cursor.seek_exact(key)?; + Ok(value.map(|(_, val)| (val.number(), *val.hash()))) + } + + fn inner_get_proof_window( + &self, + tx: &impl DbTx, + ) -> MorphProofsStorageResult> { + let mut cursor = tx.cursor_read::()?; + + let earliest = match cursor.seek_exact(ProofWindowKey::EarliestBlock)? { + Some((_, val)) => NumHash::new(val.number(), *val.hash()), + None => return Ok(None), + }; + + let latest = match cursor.seek_exact(ProofWindowKey::LatestBlock)? { + Some((_, val)) => NumHash::new(val.number(), *val.hash()), + None => earliest, + }; + + Ok(Some(ProofWindowValue { earliest, latest })) + } + + fn set_earliest_block_number_hash( + &self, + block_number: u64, + hash: B256, + ) -> MorphProofsStorageResult<()> { + let _ = self.env.update(|tx| { + Self::inner_set_earliest_block_number(tx, block_number, hash)?; + Ok::<(), DatabaseError>(()) + })?; + Ok(()) + } + + /// Internal helper to set earliest block number hash within an existing transaction + fn inner_set_earliest_block_number( + tx: &(impl DbTxMut + DbTx), + block_number: u64, + hash: B256, + ) -> MorphProofsStorageResult<()> { + let mut cursor = tx.cursor_write::()?; + cursor.upsert( + ProofWindowKey::EarliestBlock, + &BlockNumberHash::new(block_number, hash), + )?; + Ok(()) + } + + /// Internal helper to set latest block number hash within an existing transaction + fn inner_set_latest_block_number( + tx: &(impl DbTxMut + DbTx), + block_number: u64, + hash: B256, + ) -> MorphProofsStorageResult<()> { + let mut cursor = tx.cursor_write::()?; + cursor.upsert( + ProofWindowKey::LatestBlock, + &BlockNumberHash::new(block_number, hash), + )?; + Ok(()) + } + + /// Persist a batch of versioned history entries to a dup-sorted table. + /// + /// # Parameters + /// - `block_number`: Target block number for versioning entries + /// - `items`: **Must be sorted** - iterator of entries to persist + /// - `append_mode`: Mode selector for write strategy: + /// - `true` (Append): Appends all entries including tombstones for forward progress + /// - `false` (Prune): Removes tombstones, writes non-tombstones to block 0 + /// + /// The cost of pruning is the cost of (append + deleting tombstones + deleting old block 0). + /// The tombstones deletion is expensive as it requires a seek for each (key + subkey). + /// + /// Uses [`reth_db::mdbx::cursor::Cursor::upsert`] for upsert operation. + fn persist_history_batch( + &self, + tx: &(impl DbTxMut + DbTx), + block_number: T::SubKey, + items: I, + append_mode: bool, + ) -> MorphProofsStorageResult> + where + T: Table> + DupSort, + T::Key: Clone, + I: IntoIterator, + I::Item: IntoKV, + { + let mut cur = tx.cursor_dup_write::()?; + let mut keys = Vec::::new(); + + // Materialize iterator to enable partitioning and collect keys + let mut pairs: Vec<(T::Key, T::Value)> = Vec::new(); + for it in items { + let (k, vv) = it.into_kv(block_number); + pairs.push((k.clone(), vv)); + keys.push(k) + } + + if append_mode { + // Append all entries (including tombstones) to preserve full history + for (k, vv) in pairs { + cur.append_dup(k.clone(), vv)?; + } + return Ok(keys); + } + + // Drop current cursor to start clean for Phase 1 + drop(cur); + + // Phase 1: Batch Delete (Sequential) + // Remove all existing state at Block 0 for these keys. + { + let mut del_cur = tx.cursor_dup_write::()?; + for (k, _) in &pairs { + // Seek to (Key, Block 0) + if let Some(vv) = del_cur.seek_by_key_subkey(k.clone(), 0)? + && vv.block_number == 0 + { + del_cur.delete_current()?; + } + } + } + + // Phase 2: Batch Write (Sequential) + // Write new values (skipping tombstones). + { + let mut write_cur = tx.cursor_dup_write::()?; + for (k, vv) in pairs { + if vv.value.0.is_some() { + write_cur.upsert(k, &vv)?; + } + } + } + + Ok(keys) + } + + /// Delete entries for `items` at exactly `block_number` in a dup-sorted table. + /// Seeks (key, block) and deletes current if the subkey matches. + fn delete_dup_sorted( + &self, + tx: &(impl DbTxMut + DbTx), + items: I, + ) -> MorphProofsStorageResult<()> + where + T: Table> + DupSort, + T::Key: Clone, + T::SubKey: PartialEq + Clone, + I: IntoIterator, + { + let mut cur = tx.cursor_dup_write::()?; + for (key, subkey) in items { + if let Some(vv) = cur.seek_by_key_subkey(key, subkey)? { + // ensure we didn't land on a >subkey + if vv.block_number == subkey { + cur.delete_current()?; + } + } + } + Ok(()) + } + + /// Phase 1 of pruning: Calculate survivors. + /// Scans change sets to find the LATEST update for every key in the range. + fn calculate_prune_plan( + &self, + target_block: u64, + ) -> MorphProofsStorageResult> { + self.env.view(|tx| { + let Some((earliest, _)) = + self.inner_get_block_number_hash(tx, ProofWindowKey::EarliestBlock)? + else { + return Ok(None); + }; + + if earliest >= target_block { + return Ok(None); + } + + // Walk the change set in REVERSE so the first time we see a key, its block_number + // is by definition the latest version in [earliest+1, target_block] — the only + // one that must survive. This lets us use `or_insert` instead of an + // `and_modify(max).or_insert` per key, halving the per-insert work. + // + // HashMaps are pre-sized to bound rehash overhead. The chosen capacity is a rough + // average of unique keys per block on Base mainnet; the maps grow if exceeded. + const PER_BLOCK_CAPACITY_HINT: usize = 4096; + let blocks_in_range = target_block + .saturating_sub(earliest) + .try_into() + .unwrap_or(usize::MAX); + let cap = blocks_in_range + .saturating_mul(PER_BLOCK_CAPACITY_HINT) + .min(1 << 20); + + let mut acc_candidates: HashMap = + HashMap::with_capacity_and_hasher(cap, Default::default()); + let mut storage_candidates: HashMap = + HashMap::with_capacity_and_hasher(cap, Default::default()); + let mut hashed_acc_candidates: HashMap = + HashMap::with_capacity_and_hasher(cap, Default::default()); + let mut hashed_storage_candidates: HashMap = + HashMap::with_capacity_and_hasher(cap, Default::default()); + + let mut cs_cursor = tx.cursor_read::()?; + let mut walker = cs_cursor.walk_back(Some(target_block))?; + while let Some(Ok((block_number, cs))) = walker.next() { + if block_number <= earliest { + break; + } + for k in cs.account_trie_keys { + acc_candidates.entry(k).or_insert(block_number); + } + for k in cs.storage_trie_keys { + storage_candidates.entry(k).or_insert(block_number); + } + for k in cs.hashed_account_keys { + hashed_acc_candidates.entry(k).or_insert(block_number); + } + for k in cs.hashed_storage_keys { + hashed_storage_candidates.entry(k).or_insert(block_number); + } + } + + Ok(Some(PrunePlan { + earliest_block: earliest, + acc_survivors: Self::flatten_and_sort(acc_candidates), + storage_survivors: Self::flatten_and_sort(storage_candidates), + hashed_acc_survivors: Self::flatten_and_sort(hashed_acc_candidates), + hashed_storage_survivors: Self::flatten_and_sort(hashed_storage_candidates), + })) + })? + } + + /// Helper to flatten `HashMap` into a sorted Vector of survivors. + /// Sorting is required to ensure optimal sequential seek performance in MDBX. + fn flatten_and_sort(map: HashMap) -> Vec<(K, u64)> { + let mut v: Vec<_> = map.into_iter().collect(); + v.sort_unstable_by(|a, b| a.0.cmp(&b.0)); + v + } + + /// Delete history versions for `items` that are strictly older than the provided block number. + /// `items` is a list of (Key, `SurvivorBlock`). Everything strictly older than `SurvivorBlock` + /// is deleted. Returns the number of entries deleted. + fn prune_history_preceding( + &self, + tx: &(impl DbTxMut + DbTx), + cutoff_items: Vec<(T::Key, u64)>, + ) -> MorphProofsStorageResult + where + T: Table> + DupSort, + T::Key: Clone + Ord, + { + if cutoff_items.is_empty() { + return Ok(0); + } + + let mut deleted_count = 0; + let mut cur = tx.cursor_dup_write::()?; + for (key, survivor_block) in cutoff_items { + let Some(mut entry) = cur.seek_by_key_subkey(key, 0)? else { + continue; + }; + loop { + if entry.block_number >= survivor_block { + // Reached the survivor version (or newer). Stop deleting for this key. + // If the survivor is a tombstone (None), delete it too — with all older + // history already gone, a tombstone at the new earliest is redundant + // (it implies "does not exist"). + if entry.block_number == survivor_block && entry.value.0.is_none() { + cur.delete_current()?; + deleted_count += 1; + } + break; + } + + // Entry is strictly older than survivor. Delete it. + cur.delete_current()?; + deleted_count += 1; + + // libmdbx semantics: after `delete_current()` the cursor sits directly + // before the just-deleted slot, so `next_dup_val` (MDBX_NEXT_DUP) yields the + // following dup of the same key, or `None` once this key's dup-group is + // exhausted. That bound removes the need for an explicit key comparison. + match cur.next_dup_val()? { + Some(next_entry) => entry = next_entry, + None => break, + } + } + } + Ok(deleted_count) + } + + /// Tombstone every key returned by `next`, then overlay `new_entries` so collisions + /// resolve in favor of `new_entries`, and `append_dup` the merged set in sorted-key order at + /// `block_number`. Used by the wipe branches of `store_trie_updates_for_block` to keep new + /// slots / nodes that arrive in the same block as a `wiped`/`is_deleted` entry — + /// `HashedStorage::from_plain_storage` sets `wiped = was_destroyed()` (true for + /// `DestroyedChanged`), so a SELFDESTRUCT + same-block recreate produces both at once. + fn wipe_and_overlay( + &self, + tx: &(impl DbTxMut + DbTx), + block_number: u64, + hashed_address: B256, + mut next: Next, + new_entries: I, + ) -> MorphProofsStorageResult> + where + T: Table> + DupSort, + Next: FnMut() -> MorphProofsStorageResult>, + I: IntoIterator)>, + (B256, K, Option): IntoKV, + T::Key: Clone, + K: Ord, + { + let mut merged: BTreeMap> = BTreeMap::new(); + while let Some((k, _vv)) = next()? { + merged.insert(k, None); + } + for (k, v) in new_entries { + merged.insert(k, v); + } + + let mut cur = tx.cursor_dup_write::()?; + let mut keys: Vec = Vec::with_capacity(merged.len()); + for (k, value) in merged { + let key: T::Key = (hashed_address, k, Option::::None).into_key(); + let vv: T::Value = VersionedValue { + block_number, + value: MaybeDeleted(value), + }; + cur.append_dup(key.clone(), vv)?; + keys.push(key); + } + Ok(keys) + } + + /// Collect versioned history over `block_range` using `BlockChangeSet`. + fn collect_history_ranged( + &self, + tx: &impl DbTx, + block_range: impl RangeBounds, + ) -> MorphProofsStorageResult { + let mut history = HistoryDeleteBatch::default(); + let mut change_set_cursor = tx.cursor_read::()?; + let mut walker = change_set_cursor.walk_range(block_range)?; + + while let Some(Ok((block_number, change_set))) = walker.next() { + // Push (key, subkey=block_number) pairs + history.account_trie.extend( + change_set + .account_trie_keys + .into_iter() + .map(|k| (k, block_number)), + ); + history.storage_trie.extend( + change_set + .storage_trie_keys + .into_iter() + .map(|k| (k, block_number)), + ); + history.hashed_account.extend( + change_set + .hashed_account_keys + .into_iter() + .map(|k| (k, block_number)), + ); + history.hashed_storage.extend( + change_set + .hashed_storage_keys + .into_iter() + .map(|k| (k, block_number)), + ); + } + + // Sorting by tuple sorts by key first, then by block_number. + history + .account_trie + .sort_by(|(k1, b1), (k2, b2)| k1.cmp(k2).then_with(|| b1.cmp(b2))); + history + .storage_trie + .sort_by(|(k1, b1), (k2, b2)| k1.cmp(k2).then_with(|| b1.cmp(b2))); + history + .hashed_account + .sort_by(|(k1, b1), (k2, b2)| k1.cmp(k2).then_with(|| b1.cmp(b2))); + history + .hashed_storage + .sort_by(|(k1, b1), (k2, b2)| k1.cmp(k2).then_with(|| b1.cmp(b2))); + + Ok(history) + } + + /// Delete versioned history over `block_range` using history batch. + fn delete_history_ranged( + &self, + tx: &(impl DbTxMut + DbTx), + block_range: impl RangeBounds, + history: HistoryDeleteBatch, + ) -> MorphProofsStorageResult { + let mut change_set_cursor = tx.cursor_write::()?; + let mut walker = change_set_cursor.walk_range(block_range)?; + + while let Some(Ok((_, _))) = walker.next() { + walker.delete_current()?; + } + + // Delete using the simplified API: iterator of (key, subkey) + self.delete_dup_sorted::(tx, history.clone().account_trie)?; + self.delete_dup_sorted::(tx, history.clone().storage_trie)?; + self.delete_dup_sorted::(tx, history.clone().hashed_account)?; + self.delete_dup_sorted::(tx, history.clone().hashed_storage)?; + + Ok(WriteCounts { + account_trie_updates_written_total: history.account_trie.len() as u64, + storage_trie_updates_written_total: history.storage_trie.len() as u64, + hashed_accounts_written_total: history.hashed_account.len() as u64, + hashed_storages_written_total: history.hashed_storage.len() as u64, + }) + } + + /// Write trie/state history for `block_number` from `block_state_diff`. + fn store_trie_updates_for_block( + &self, + tx: &::TXMut, + block_number: u64, + block_state_diff: BlockStateDiff, + append_mode: bool, + ) -> MorphProofsStorageResult { + let BlockStateDiff { + sorted_trie_updates, + sorted_post_state, + } = block_state_diff; + + let storage_trie_len = sorted_trie_updates.storage_tries_ref().len(); + let hashed_storage_len = sorted_post_state.storages.len(); + + let account_trie_keys = self.persist_history_batch( + tx, + block_number, + sorted_trie_updates.account_nodes_ref().iter().cloned(), + append_mode, + )?; + let hashed_account_keys = self.persist_history_batch( + tx, + block_number, + sorted_post_state.accounts.iter().copied(), + append_mode, + )?; + + let mut storage_trie_keys = Vec::::with_capacity(storage_trie_len); + for (hashed_address, nodes) in sorted_trie_updates.storage_tries_ref() { + if nodes.is_deleted && append_mode { + // Lookback must use the active tx so it sees uncommitted writes from earlier + // blocks in the same batch session. Opening a fresh RO tx here would read the + // pre-batch committed snapshot and emit wrong tombstones for wipe blocks whose + // parent is staged in the same batch. + let mut ro = MdbxTrieCursor::::new( + tx.cursor_dup_read::()?, + block_number - 1, + Some(*hashed_address), + ); + let keys = self.wipe_and_overlay( + tx, + block_number, + *hashed_address, + || Ok(ro.next()?), + nodes.storage_nodes_ref().iter().cloned(), + )?; + storage_trie_keys.extend(keys); + continue; + } + + let keys = self.persist_history_batch( + tx, + block_number, + nodes + .storage_nodes_ref() + .iter() + .cloned() + .map(|(path, node)| (*hashed_address, path, node)), + append_mode, + )?; + storage_trie_keys.extend(keys); + } + + let mut hashed_storage_keys = Vec::::with_capacity(hashed_storage_len); + for (hashed_address, storage) in sorted_post_state.storages { + if append_mode && storage.is_wiped() { + // See lookback note above: must use the active tx, not a fresh RO tx. + let mut ro = MdbxStorageCursor::new( + tx.cursor_dup_read::()?, + block_number - 1, + hashed_address, + ); + let keys = self.wipe_and_overlay( + tx, + block_number, + hashed_address, + || Ok(ro.next()?), + storage + .storage_slots_ref() + .iter() + .map(|(slot, val)| (*slot, Some(StorageValue(*val)))), + )?; + hashed_storage_keys.extend(keys); + continue; + } + let keys = self.persist_history_batch( + tx, + block_number, + storage + .storage_slots_ref() + .iter() + .map(|(key, val)| (hashed_address, *key, Some(StorageValue(*val)))), + append_mode, + )?; + hashed_storage_keys.extend(keys); + } + + Ok(ChangeSet { + account_trie_keys, + storage_trie_keys, + hashed_account_keys, + hashed_storage_keys, + }) + } + + /// Append-only writer for a block: validates parent, persists diff (soft-delete=true), + /// records a `BlockChangeSet`, and advances `ProofWindow::LatestBlock`. + pub(crate) fn store_trie_updates_append_only( + &self, + tx: &::TXMut, + block_ref: BlockWithParent, + block_state_diff: BlockStateDiff, + ) -> MorphProofsStorageResult { + let block_number = block_ref.block.number; + + // Check the latest stored block is the parent of the incoming block + let latest_block_hash = self + .inner_get_latest_block_number_hash(tx)? + .map_or(B256::ZERO, |(_num, hash)| hash); + + if latest_block_hash != block_ref.parent { + return Err(MorphProofsStorageError::OutOfOrder { + block_number, + parent_block_hash: block_ref.parent, + latest_block_hash, + }); + } + + let change_set = + &self.store_trie_updates_for_block(tx, block_number, block_state_diff, true)?; + + // Cursor for recording all changes made in this block for all history tables + let mut change_set_cursor = tx.new_cursor::()?; + change_set_cursor.append(block_number, change_set)?; + + // Update proof window's latest block + Self::inner_set_latest_block_number(tx, block_number, block_ref.block.hash)?; + + Ok(WriteCounts { + account_trie_updates_written_total: change_set.account_trie_keys.len() as u64, + storage_trie_updates_written_total: change_set.storage_trie_keys.len() as u64, + hashed_accounts_written_total: change_set.hashed_account_keys.len() as u64, + hashed_storages_written_total: change_set.hashed_storage_keys.len() as u64, + }) + } + + /// Return `BlockNumHash` for the initial state anchor. + fn get_initial_state_anchor(&self) -> MorphProofsStorageResult> { + self.env.view(|tx| { + let mut cur = tx.cursor_read::()?; + Ok(cur + .seek_exact(ProofWindowKey::InitialStateAnchor)? + .map(|(_k, v)| v.into())) + })? + } + + /// Return latest key for a table + fn get_latest_key(&self) -> MorphProofsStorageResult> + where + T: Table, + { + self.env.view(|tx| { + let mut cursor = tx.cursor_read::()?; + Ok(cursor.last()?.map(|(k, _)| k)) + })? + } +} + +impl MorphProofsStore for MdbxProofsStorage { + type StorageTrieCursor<'tx> + = MdbxTrieCursor> + where + Self: 'tx; + type AccountTrieCursor<'tx> + = MdbxTrieCursor> + where + Self: 'tx; + type StorageCursor<'tx> + = MdbxStorageCursor> + where + Self: 'tx; + type AccountHashedCursor<'tx> + = MdbxAccountCursor> + where + Self: 'tx; + type Tx<'tx> + = ::TX + where + Self: 'tx; + + fn ro_tx<'tx>(&'tx self) -> MorphProofsStorageResult> { + Ok(self.env.tx()?) + } + + fn get_earliest_block_number(&self) -> MorphProofsStorageResult> { + self.env + .view(|tx| self.inner_get_earliest_block_number_hash(tx))? + } + + fn get_latest_block_number(&self) -> MorphProofsStorageResult> { + self.env + .view(|tx| self.inner_get_latest_block_number_hash(tx))? + } + + fn get_earliest_block_number_with_tx<'tx, 'db>( + &self, + tx: &'tx Self::Tx<'db>, + ) -> MorphProofsStorageResult> + where + Self: 'db, + 'db: 'tx, + { + self.inner_get_earliest_block_number_hash(tx) + } + + fn get_latest_block_number_with_tx<'tx, 'db>( + &self, + tx: &'tx Self::Tx<'db>, + ) -> MorphProofsStorageResult> + where + Self: 'db, + 'db: 'tx, + { + self.inner_get_latest_block_number_hash(tx) + } + + fn storage_trie_cursor<'tx>( + &'tx self, + hashed_address: B256, + max_block_number: u64, + ) -> MorphProofsStorageResult> { + let tx = self.env.tx()?; + let cursor = tx.cursor_dup_read::()?; + + Ok(MdbxTrieCursor::new( + cursor, + max_block_number, + Some(hashed_address), + )) + } + + fn account_trie_cursor<'tx>( + &'tx self, + max_block_number: u64, + ) -> MorphProofsStorageResult> { + let tx = self.env.tx()?; + let cursor = tx.cursor_dup_read::()?; + + Ok(MdbxTrieCursor::new(cursor, max_block_number, None)) + } + + fn storage_hashed_cursor<'tx>( + &'tx self, + hashed_address: B256, + max_block_number: u64, + ) -> MorphProofsStorageResult> { + let tx = self.env.tx()?; + let cursor = tx.cursor_dup_read::()?; + + Ok(MdbxStorageCursor::new( + cursor, + max_block_number, + hashed_address, + )) + } + + fn account_hashed_cursor<'tx>( + &'tx self, + max_block_number: u64, + ) -> MorphProofsStorageResult> { + let tx = self.env.tx()?; + let cursor = tx.cursor_dup_read::()?; + + Ok(MdbxAccountCursor::new(cursor, max_block_number)) + } + + fn storage_trie_cursor_with_tx<'tx, 'db>( + &self, + tx: &'tx Self::Tx<'db>, + hashed_address: B256, + max_block_number: u64, + ) -> MorphProofsStorageResult> + where + Self: 'db, + 'db: 'tx, + { + let cursor = tx.cursor_dup_read::()?; + + Ok(MdbxTrieCursor::new( + cursor, + max_block_number, + Some(hashed_address), + )) + } + + fn account_trie_cursor_with_tx<'tx, 'db>( + &self, + tx: &'tx Self::Tx<'db>, + max_block_number: u64, + ) -> MorphProofsStorageResult> + where + Self: 'db, + 'db: 'tx, + { + let cursor = tx.cursor_dup_read::()?; + + Ok(MdbxTrieCursor::new(cursor, max_block_number, None)) + } + + fn storage_hashed_cursor_with_tx<'tx, 'db>( + &self, + tx: &'tx Self::Tx<'db>, + hashed_address: B256, + max_block_number: u64, + ) -> MorphProofsStorageResult> + where + Self: 'db, + 'db: 'tx, + { + let cursor = tx.cursor_dup_read::()?; + + Ok(MdbxStorageCursor::new( + cursor, + max_block_number, + hashed_address, + )) + } + + fn account_hashed_cursor_with_tx<'tx, 'db>( + &self, + tx: &'tx Self::Tx<'db>, + max_block_number: u64, + ) -> MorphProofsStorageResult> + where + Self: 'db, + 'db: 'tx, + { + let cursor = tx.cursor_dup_read::()?; + + Ok(MdbxAccountCursor::new(cursor, max_block_number)) + } + + fn store_trie_updates( + &self, + block_ref: BlockWithParent, + block_state_diff: BlockStateDiff, + ) -> MorphProofsStorageResult { + self.env + .update(|tx| self.store_trie_updates_append_only(tx, block_ref, block_state_diff))? + } + + fn fetch_trie_updates(&self, block_number: u64) -> MorphProofsStorageResult { + self.env.view(|tx| { + let mut change_set_cursor = tx.cursor_read::()?; + let (_, change_set) = change_set_cursor + .seek_exact(block_number)? + .ok_or(MorphProofsStorageError::NoChangeSetForBlock(block_number))?; + + let mut account_trie_cursor = tx.new_cursor::()?; + let mut storage_trie_cursor = tx.new_cursor::()?; + let mut hashed_account_cursor = tx.new_cursor::()?; + let mut hashed_storage_cursor = tx.new_cursor::()?; + + let mut trie_updates = TrieUpdates::default(); + for key in change_set.account_trie_keys { + let entry = + match account_trie_cursor.seek_by_key_subkey(key.clone(), block_number)? { + Some(v) if v.block_number == block_number => v.value.0, + _ => { + return Err(MorphProofsStorageError::MissingAccountTrieHistory( + key.0, + block_number, + )); + } + }; + + if let Some(value) = entry { + trie_updates.account_nodes.insert(key.0, value); + } else { + trie_updates.removed_nodes.insert(key.0); + } + } + + for key in change_set.storage_trie_keys { + let entry = + match storage_trie_cursor.seek_by_key_subkey(key.clone(), block_number)? { + Some(v) if v.block_number == block_number => v.value.0, + _ => { + return Err(MorphProofsStorageError::MissingStorageTrieHistory( + key.hashed_address, + key.path.0, + block_number, + )); + } + }; + + let stu = trie_updates + .storage_tries + .entry(key.hashed_address) + .or_insert_with(StorageTrieUpdates::default); + + // handle is_deleted scenario + // Issue: https://github.com/op-rs/op-reth/issues/323 + if let Some(value) = entry { + stu.storage_nodes.insert(key.path.0, value); + } else { + stu.removed_nodes.insert(key.path.0); + } + } + + let mut post_state = + HashedPostState::with_capacity(change_set.hashed_account_keys.len()); + for key in change_set.hashed_account_keys { + let entry = match hashed_account_cursor.seek_by_key_subkey(key, block_number)? { + Some(v) if v.block_number == block_number => v.value.0, + _ => { + return Err(MorphProofsStorageError::MissingHashedAccountHistory( + key, + block_number, + )); + } + }; + + post_state.accounts.insert(key, entry); + } + + for key in change_set.hashed_storage_keys { + let entry = + match hashed_storage_cursor.seek_by_key_subkey(key.clone(), block_number)? { + Some(v) if v.block_number == block_number => v.value.0, + _ => { + return Err(MorphProofsStorageError::MissingHashedStorageHistory { + hashed_address: key.hashed_address, + hashed_storage_key: key.hashed_storage_key, + block_number, + }); + } + }; + + let hs = post_state.storages.entry(key.hashed_address).or_default(); + + // handle wiped storage scenario + // Issue: https://github.com/op-rs/op-reth/issues/323 + if let Some(value) = entry { + hs.storage.insert(key.hashed_storage_key, value.0); + } else { + hs.storage.insert(key.hashed_storage_key, U256::ZERO); + } + } + + Ok(BlockStateDiff { + sorted_trie_updates: trie_updates.into_sorted(), + sorted_post_state: post_state.into_sorted(), + }) + })? + } + + /// Update the initial state with the provided diff. + /// Prune all historical trie data till `new_earliest_block_number` (inclusive) using + /// the [`BlockChangeSet`] index. + /// + /// Arguments: + /// - `new_earliest_block_ref`: The new earliest block reference (with parent hash). + /// - `diff`: The state diff to apply to the initial state (block 0). This diff represents all + /// the changes from the old earliest block to the new earliest block (inclusive). + fn prune_earliest_state( + &self, + new_earliest_block_ref: BlockWithParent, + ) -> MorphProofsStorageResult { + let target_block = new_earliest_block_ref.block.number; + + // --- PHASE 1: READ (Calculate Deletions) --- + let plan = self.calculate_prune_plan(target_block)?; + let Some(plan) = plan else { + return Ok(WriteCounts::default()); + }; + + // --- PHASE 2: WRITE (Execute Deletions) --- + self.env.update(|tx| { + // 1. Execute Sparse Deletions and track actual deleted rows + let acc_deleted = + self.prune_history_preceding::(tx, plan.acc_survivors)?; + + let st_deleted = + self.prune_history_preceding::(tx, plan.storage_survivors)?; + + let ha_deleted = self.prune_history_preceding::( + tx, + plan.hashed_acc_survivors, + )?; + + let hs_deleted = self.prune_history_preceding::( + tx, + plan.hashed_storage_survivors, + )?; + + let counts = WriteCounts { + account_trie_updates_written_total: acc_deleted, + storage_trie_updates_written_total: st_deleted, + hashed_accounts_written_total: ha_deleted, + hashed_storages_written_total: hs_deleted, + }; + + // 2. Delete ChangeSets + let range = (plan.earliest_block + 1)..=target_block; + let mut cs_cursor = tx.cursor_write::()?; + let mut walker = cs_cursor.walk_range(range)?; + while walker.next().is_some() { + walker.delete_current()?; + } + + // 3. Update Earliest Pointer + Self::inner_set_earliest_block_number( + tx, + target_block, + new_earliest_block_ref.block.hash, + )?; + + Ok(counts) + })? + } + + /// Unwind the historical state to `unwind_upto_block` (inclusive), deleting all history + /// starting from provided block. Also updates the `ProofWindow::LatestBlock` to parent of + /// `unwind_upto_block`. + fn unwind_history(&self, to: BlockWithParent) -> MorphProofsStorageResult<()> { + let history_to_delete = self + .env + .view(|tx| self.collect_history_ranged(tx, to.block.number..))??; + + self.env.update(|tx| { + let proof_window = match self.inner_get_proof_window(tx)? { + Some(pw) => pw, + None => return Ok(()), // Nothing to unwind + }; + + if to.block.number > proof_window.latest.number { + return Ok(()); // Nothing to unwind + } + + if to.block.number <= proof_window.earliest.number { + return Err(MorphProofsStorageError::UnwindBeyondEarliest { + unwind_block_number: to.block.number, + earliest_block_number: proof_window.earliest.number, + }); + } + + self.delete_history_ranged(tx, to.block.number.., history_to_delete)?; + + let new_latest_block = + BlockNumberHash::new(to.block.number.saturating_sub(1), to.parent); + + // Update proof window's Latest block + Self::inner_set_latest_block_number( + tx, + new_latest_block.number(), + *new_latest_block.hash(), + )?; + + Ok(()) + })? + } + + fn replace_updates( + &self, + latest_common_block: BlockNumHash, + mut blocks_to_add: Vec<(BlockWithParent, BlockStateDiff)>, + ) -> MorphProofsStorageResult<()> { + // Sort the vec list by block number + blocks_to_add.sort_unstable_by_key(|(bwp, _)| bwp.block.number); + + let history_to_delete = self + .env + .view(|tx| self.collect_history_ranged(tx, latest_common_block.number + 1..))??; + + self.env.update(|tx| { + // Remove the old history + self.delete_history_ranged(tx, latest_common_block.number + 1.., history_to_delete)?; + + // Update the ProofWindow Latest Block to latest_common_block so we can perform + // `store_trie_updates_append_only`. + Self::inner_set_latest_block_number( + tx, + latest_common_block.number, + latest_common_block.hash, + )?; + + // Apply the new history + for (block_with_parent, diff) in blocks_to_add { + self.store_trie_updates_append_only(tx, block_with_parent, diff)?; + } + Ok(()) + })? + } + + fn set_earliest_block_number( + &self, + block_number: u64, + hash: B256, + ) -> MorphProofsStorageResult<()> { + self.set_earliest_block_number_hash(block_number, hash) + } +} + +impl MorphProofsBatchStore for MdbxProofsStorage { + type BatchSession<'a> + = MdbxBatchSession<'a> + where + Self: 'a; + + fn with_batch_session(&self, f: F) -> MorphProofsStorageResult + where + F: FnOnce(&mut Self::BatchSession<'_>) -> MorphProofsStorageResult, + { + let tx = self.env.tx_mut()?; + let mut session = MdbxBatchSession::new(self, tx); + match f(&mut session) { + Ok(result) => { + session.commit()?; + Ok(result) + } + Err(err) => Err(err), + } + } +} + +impl MorphProofsInitialStateStore for MdbxProofsStorage { + fn initial_state_anchor(&self) -> MorphProofsStorageResult { + // 1) NotStarted: no anchor row + let Some(block) = self.get_initial_state_anchor()? else { + return Ok(InitialStateAnchor::default()); + }; + + // 2) Completed: anchor exists + earliest is set + let completed = self.get_earliest_block_number()?.is_some(); + + // 3) InProgress / Completed: populate details + Ok(InitialStateAnchor { + block: Some(block), + status: if completed { + InitialStateStatus::Completed + } else { + InitialStateStatus::InProgress + }, + latest_account_trie_key: self.get_latest_key::()?, + latest_storage_trie_key: self.get_latest_key::()?, + latest_hashed_account_key: self.get_latest_key::()?, + latest_hashed_storage_key: self.get_latest_key::()?, + }) + } + + fn set_initial_state_anchor(&self, anchor: BlockNumHash) -> MorphProofsStorageResult<()> { + self.env.update(|tx| { + let mut cur = tx.cursor_write::()?; + cur.insert(ProofWindowKey::InitialStateAnchor, &anchor.into())?; + Ok(()) + })? + } + + fn store_account_branches( + &self, + account_nodes: Vec<(Nibbles, Option)>, + ) -> MorphProofsStorageResult<()> { + let mut account_nodes = account_nodes; + if account_nodes.is_empty() { + return Ok(()); + } + + account_nodes.sort_by_key(|(key, _)| *key); + + self.env.update(|tx| { + self.persist_history_batch(tx, 0, account_nodes, true)?; + Ok(()) + })? + } + + fn store_storage_branches( + &self, + hashed_address: B256, + storage_nodes: Vec<(Nibbles, Option)>, + ) -> MorphProofsStorageResult<()> { + let mut storage_nodes = storage_nodes; + if storage_nodes.is_empty() { + return Ok(()); + } + + storage_nodes.sort_by_key(|(key, _)| *key); + + self.env.update(|tx| { + self.persist_history_batch( + tx, + 0, + storage_nodes + .into_iter() + .map(|(path, node)| (hashed_address, path, node)), + true, + )?; + Ok(()) + })? + } + + fn store_hashed_accounts( + &self, + accounts: Vec<(B256, Option)>, + ) -> MorphProofsStorageResult<()> { + let mut accounts = accounts; + if accounts.is_empty() { + return Ok(()); + } + + // sort the accounts by key to ensure insertion is efficient + accounts.sort_by_key(|(key, _)| *key); + + self.env.update(|tx| { + self.persist_history_batch(tx, 0, accounts, true)?; + Ok(()) + })? + } + + fn store_hashed_storages( + &self, + hashed_address: B256, + storages: Vec<(B256, U256)>, + ) -> MorphProofsStorageResult<()> { + let mut storages = storages; + if storages.is_empty() { + return Ok(()); + } + + // sort the storages by key to ensure insertion is efficient + storages.sort_by_key(|(key, _)| *key); + + self.env.update(|tx| { + self.persist_history_batch( + tx, + 0, + storages + .into_iter() + .map(|(key, val)| (hashed_address, key, Some(StorageValue(val)))), + true, + )?; + Ok(()) + })? + } + + fn commit_initial_state(&self) -> MorphProofsStorageResult { + let anchor = self.get_initial_state_anchor()?.ok_or(NoBlocksFound)?; + self.set_earliest_block_number(anchor.number, anchor.hash)?; + Ok(anchor) + } +} + +#[cfg(test)] +mod tests { + use alloy_eips::NumHash; + use alloy_primitives::B256; + use reth_db::{ + DatabaseError, + cursor::DbDupCursorRO, + transaction::{DbTx, DbTxMut}, + }; + use reth_trie::{ + BranchNodeCompact, HashedPostStateSorted, HashedStorage, Nibbles, StoredNibbles, + updates::{StorageTrieUpdates, TrieUpdatesSorted}, + }; + use tempfile::TempDir; + + use super::*; + use crate::db::{ + StorageTrieKey, + models::{AccountTrieHistory, StorageTrieHistory}, + }; + + const B0: u64 = 0; + + #[test] + fn open_rejects_partial_identity_metadata_and_requires_manual_deletion() { + let dir = TempDir::new().unwrap(); + let env = init_db_for::<_, Tables>(dir.path(), DatabaseArguments::default()) + .expect("raw proof database"); + env.update(|tx| { + tx.put::( + ProofMetadataKey::ChainId, + ProofMetadataValue(2818u64.to_be_bytes().to_vec()), + ) + }) + .expect("write transaction") + .expect("write partial metadata"); + drop(env); + + let error = MdbxProofsStorage::open( + dir.path(), + ProofDbIdentity::new(2818, B256::repeat_byte(0x11)), + ) + .expect_err("partial identity metadata must fail closed"); + + assert!( + error.to_string().contains("manually delete"), + "operator must receive an explicit manual-deletion instruction: {error}" + ); + } + + #[test] + fn open_rejects_proof_data_without_identity_metadata() { + let identity = ProofDbIdentity::new(2818, B256::repeat_byte(0x11)); + + let proof_window_dir = TempDir::new().unwrap(); + let env = init_db_for::<_, Tables>(proof_window_dir.path(), DatabaseArguments::default()) + .expect("raw proof database"); + env.update(|tx| { + tx.put::( + ProofWindowKey::EarliestBlock, + BlockNumberHash::new(1, B256::repeat_byte(0x22)), + ) + }) + .expect("write transaction") + .expect("write proof window"); + drop(env); + + let error = MdbxProofsStorage::open(proof_window_dir.path(), identity) + .expect_err("proof-window data without identity metadata must fail closed"); + assert!( + error.to_string().contains("manually delete"), + "operator must receive an explicit manual-deletion instruction: {error}" + ); + + let history_dir = TempDir::new().unwrap(); + let env = init_db_for::<_, Tables>(history_dir.path(), DatabaseArguments::default()) + .expect("raw proof database"); + env.update(|tx| tx.put::(1, ChangeSet::default())) + .expect("write transaction") + .expect("write history change set"); + drop(env); + + let error = MdbxProofsStorage::open(history_dir.path(), identity) + .expect_err("history data without identity metadata must fail closed"); + assert!( + error.to_string().contains("manually delete"), + "operator must receive an explicit manual-deletion instruction: {error}" + ); + } + + #[test] + fn new_with_options_accepts_custom_max_read_transaction_duration() { + let dir = TempDir::new().unwrap(); + let options = MdbxProofsStorageOptions { + max_read_transaction_duration: Some(Duration::from_secs(30)), + }; + let store = MdbxProofsStorage::new_with_options(dir.path(), options).expect("env"); + + let _tx = store.env.tx().expect("ro tx"); + } + + #[test] + fn store_hashed_accounts_writes_versioned_values() { + let dir = TempDir::new().unwrap(); + let store = MdbxProofsStorage::new(dir.path()).expect("env"); + + let addr = B256::from([0xAA; 32]); + let account = Account::default(); + store + .store_hashed_accounts(vec![(addr, Some(account))]) + .expect("write accounts"); + + let tx = store.env.tx().expect("ro tx"); + let mut cur = tx.new_cursor::().expect("cursor"); + let vv = cur.seek_by_key_subkey(addr, B0).expect("seek"); + let vv = vv.expect("entry exists"); + + assert_eq!(vv.block_number, B0); + assert_eq!(vv.value.0, Some(account)); + } + + #[test] + fn store_hashed_accounts_multiple_items_unsorted() { + let dir = TempDir::new().unwrap(); + let store = MdbxProofsStorage::new(dir.path()).expect("env"); + + // Unsorted input, mixed Some/None (deletion) + let a1 = B256::from([0x01; 32]); + let a2 = B256::from([0x02; 32]); + let a3 = B256::from([0x03; 32]); + let acc1 = Account { + nonce: 2, + balance: U256::from(1000u64), + ..Default::default() + }; + let acc3 = Account { + nonce: 1, + balance: U256::from(10000u64), + ..Default::default() + }; + + store + .store_hashed_accounts(vec![(a2, None), (a1, Some(acc1)), (a3, Some(acc3))]) + .expect("write"); + + let tx = store.env.tx().expect("ro tx"); + let mut cur = tx.new_cursor::().expect("cursor"); + + let v1 = cur + .seek_by_key_subkey(a1, B0) + .expect("seek a1") + .expect("exists a1"); + assert_eq!(v1.block_number, B0); + assert_eq!(v1.value.0, Some(acc1)); + + let v2 = cur + .seek_by_key_subkey(a2, B0) + .expect("seek a2") + .expect("exists a2"); + assert_eq!(v2.block_number, B0); + assert!(v2.value.0.is_none(), "a2 is none"); + + let v3 = cur + .seek_by_key_subkey(a3, B0) + .expect("seek a3") + .expect("exists a3"); + assert_eq!(v3.block_number, B0); + assert_eq!(v3.value.0, Some(acc3)); + } + + #[test] + fn store_hashed_accounts_multiple_calls() { + let dir = TempDir::new().unwrap(); + let store = MdbxProofsStorage::new(dir.path()).expect("env"); + + // Unsorted input, mixed Some/None (deletion) + let a1 = B256::from([0x01; 32]); + let a2 = B256::from([0x02; 32]); + let a3 = B256::from([0x03; 32]); + let a4 = B256::from([0x04; 32]); + let a5 = B256::from([0x05; 32]); + let acc1 = Account { + nonce: 2, + balance: U256::from(1000u64), + ..Default::default() + }; + let acc3 = Account { + nonce: 1, + balance: U256::from(10000u64), + ..Default::default() + }; + let acc4 = Account { + nonce: 5, + balance: U256::from(5000u64), + ..Default::default() + }; + let acc5 = Account { + nonce: 10, + balance: U256::from(20000u64), + ..Default::default() + }; + + { + store + .store_hashed_accounts(vec![(a2, None), (a1, Some(acc1)), (a4, Some(acc4))]) + .expect("write"); + + let tx = store.env.tx().expect("ro tx"); + let mut cur = tx.new_cursor::().expect("cursor"); + + let v1 = cur + .seek_by_key_subkey(a1, B0) + .expect("seek a1") + .expect("exists a1"); + assert_eq!(v1.block_number, B0); + assert_eq!(v1.value.0, Some(acc1)); + + let v2 = cur + .seek_by_key_subkey(a2, B0) + .expect("seek a2") + .expect("exists a2"); + assert_eq!(v2.block_number, B0); + assert!(v2.value.0.is_none(), "a2 is none"); + + let v4 = cur + .seek_by_key_subkey(a4, B0) + .expect("seek a4") + .expect("exists a4"); + assert_eq!(v4.block_number, B0); + assert_eq!(v4.value.0, Some(acc4)); + } + + { + // Second call + store + .store_hashed_accounts(vec![(a5, Some(acc5)), (a3, Some(acc3))]) + .expect("write"); + + let tx = store.env.tx().expect("ro tx"); + let mut cur = tx.new_cursor::().expect("cursor"); + + let v3 = cur + .seek_by_key_subkey(a3, B0) + .expect("seek a3") + .expect("exists a3"); + assert_eq!(v3.block_number, B0); + assert_eq!(v3.value.0, Some(acc3)); + + let v5 = cur + .seek_by_key_subkey(a5, B0) + .expect("seek a5") + .expect("exists a5"); + assert_eq!(v5.block_number, B0); + assert_eq!(v5.value.0, Some(acc5)); + } + } + + #[test] + fn store_hashed_storages_writes_versioned_values() { + let dir = TempDir::new().unwrap(); + let store = MdbxProofsStorage::new(dir.path()).expect("env"); + + let addr = B256::from([0x11; 32]); + let slot = B256::from([0x22; 32]); + let val = U256::from(0x1234u64); + + store + .store_hashed_storages(addr, vec![(slot, val)]) + .expect("write storage"); + + let tx = store.env.tx().expect("ro tx"); + let mut cur = tx.new_cursor::().expect("cursor"); + let key = HashedStorageKey::new(addr, slot); + let vv = cur.seek_by_key_subkey(key, B0).expect("seek"); + let vv = vv.expect("entry exists"); + + assert_eq!(vv.block_number, B0); + let inner = vv.value.0.as_ref().expect("Some(StorageValue)"); + assert_eq!(inner.0, val); + } + + #[test] + fn store_hashed_storages_multiple_slots_unsorted() { + let dir = TempDir::new().unwrap(); + let store = MdbxProofsStorage::new(dir.path()).expect("env"); + + let addr = B256::from([0x11; 32]); + let s1 = B256::from([0x01; 32]); + let v1 = U256::from(1u64); + let s2 = B256::from([0x02; 32]); + let v2 = U256::from(2u64); + let s3 = B256::from([0x03; 32]); + let v3 = U256::from(3u64); + + store + .store_hashed_storages(addr, vec![(s2, v2), (s1, v1), (s3, v3)]) + .expect("write"); + + let tx = store.env.tx().expect("ro tx"); + let mut cur = tx.new_cursor::().expect("cursor"); + + for (slot, expected) in [(s1, v1), (s2, v2), (s3, v3)] { + let key = HashedStorageKey::new(addr, slot); + let vv = cur + .seek_by_key_subkey(key, B0) + .expect("seek") + .expect("exists"); + assert_eq!(vv.block_number, B0); + let inner = vv.value.0.as_ref().expect("Some(StorageValue)"); + assert_eq!(inner.0, expected); + } + } + + #[test] + fn store_hashed_storages_multiple_calls() { + let dir = TempDir::new().unwrap(); + let store = MdbxProofsStorage::new(dir.path()).expect("env"); + + let addr = B256::from([0x11; 32]); + let s1 = B256::from([0x01; 32]); + let v1 = U256::from(1u64); + let s2 = B256::from([0x02; 32]); + let v2 = U256::from(2u64); + let s3 = B256::from([0x03; 32]); + let v3 = U256::from(3u64); + let s4 = B256::from([0x04; 32]); + let v4 = U256::from(4u64); + let s5 = B256::from([0x05; 32]); + let v5 = U256::from(5u64); + + { + store + .store_hashed_storages(addr, vec![(s2, v2), (s1, v1), (s5, v5)]) + .expect("write"); + + let tx = store.env.tx().expect("ro tx"); + let mut cur = tx.new_cursor::().expect("cursor"); + + for (slot, expected) in [(s1, v1), (s2, v2), (s5, v5)] { + let key = HashedStorageKey::new(addr, slot); + let vv = cur + .seek_by_key_subkey(key, B0) + .expect("seek") + .expect("exists"); + assert_eq!(vv.block_number, B0); + let inner = vv.value.0.as_ref().expect("Some(StorageValue)"); + assert_eq!(inner.0, expected); + } + } + + { + // Second call + store + .store_hashed_storages(addr, vec![(s4, v4), (s3, v3)]) + .expect("write"); + + let tx = store.env.tx().expect("ro tx"); + let mut cur = tx.new_cursor::().expect("cursor"); + + for (slot, expected) in [(s4, v4), (s3, v3)] { + let key = HashedStorageKey::new(addr, slot); + let vv = cur + .seek_by_key_subkey(key, B0) + .expect("seek") + .expect("exists"); + assert_eq!(vv.block_number, B0); + let inner = vv.value.0.as_ref().expect("Some(StorageValue)"); + assert_eq!(inner.0, expected); + } + } + } + + #[test] + fn test_store_account_branches_writes_versioned_values() { + let dir = TempDir::new().unwrap(); + let store = MdbxProofsStorage::new(dir.path()).expect("env"); + + let nibble = Nibbles::from_nibbles_unchecked([0x12, 0x34]); + let branch_node = BranchNodeCompact::new(0b1, 0, 0, vec![], Some(B256::random())); + let updates = vec![(nibble, Some(branch_node.clone()))]; + + store.store_account_branches(updates).expect("write"); + + let tx = store.env.tx().expect("ro tx"); + let mut cur = tx.cursor_dup_read::().expect("cursor"); + + let vv = cur + .seek_by_key_subkey(StoredNibbles::from(nibble), B0) + .expect("seek") + .expect("entry exists"); + + assert_eq!(vv.block_number, B0); + assert_eq!(vv.value.0, Some(branch_node)); + } + + #[test] + fn test_store_account_branches_multiple_items_unsorted() { + let dir = TempDir::new().unwrap(); + let store = MdbxProofsStorage::new(dir.path()).expect("env"); + + let n1 = Nibbles::from_nibbles_unchecked([0x01]); + let b1 = BranchNodeCompact::new(0b1, 0, 0, vec![], Some(B256::random())); + let n2 = Nibbles::from_nibbles_unchecked([0x02]); + let n3 = Nibbles::from_nibbles_unchecked([0x03]); + let b3 = BranchNodeCompact::new(0b1, 0, 0, vec![], Some(B256::random())); + + let updates = vec![(n2, None), (n1, Some(b1)), (n3, Some(b3))]; + store + .store_account_branches(updates.clone()) + .expect("write"); + + let tx = store.env.tx().expect("ro tx"); + let mut cur = tx.cursor_dup_read::().expect("cursor"); + + for (nibble, branch) in updates { + let v = cur + .seek_by_key_subkey(StoredNibbles::from(nibble), B0) + .expect("seek") + .expect("exists"); + assert_eq!(v.block_number, B0); + assert_eq!(v.value.0, branch); + } + } + + #[test] + fn store_account_branches_multiple_calls() { + let dir = TempDir::new().unwrap(); + let store = MdbxProofsStorage::new(dir.path()).expect("env"); + + let n1 = Nibbles::from_nibbles_unchecked([0x01]); + let b1 = BranchNodeCompact::new(0b1, 0, 0, vec![], Some(B256::random())); + let n2 = Nibbles::from_nibbles_unchecked([0x02]); + let n3 = Nibbles::from_nibbles_unchecked([0x03]); + let b3 = BranchNodeCompact::new(0b1, 0, 0, vec![], Some(B256::random())); + let n4 = Nibbles::from_nibbles_unchecked([0x04]); + let b4 = BranchNodeCompact::new(0b1, 0, 0, vec![], Some(B256::random())); + let n5 = Nibbles::from_nibbles_unchecked([0x05]); + let b5 = BranchNodeCompact::new(0b1, 0, 0, vec![], Some(B256::random())); + + { + let updates1 = vec![(n2, None), (n1, Some(b1)), (n4, Some(b4))]; + store + .store_account_branches(updates1.clone()) + .expect("write"); + + let tx = store.env.tx().expect("ro tx"); + let mut cur = tx.cursor_dup_read::().expect("cursor"); + + for (nibble, branch) in updates1 { + let v = cur + .seek_by_key_subkey(StoredNibbles::from(nibble), B0) + .expect("seek") + .expect("exists"); + assert_eq!(v.block_number, B0); + assert_eq!(v.value.0, branch); + } + } + + { + // Second call + let updates2 = vec![(n5, Some(b5)), (n3, Some(b3))]; + store + .store_account_branches(updates2.clone()) + .expect("write"); + + let tx = store.env.tx().expect("ro tx"); + let mut cur = tx.cursor_dup_read::().expect("cursor"); + + for (nibble, branch) in updates2 { + let v = cur + .seek_by_key_subkey(StoredNibbles::from(nibble), B0) + .expect("seek") + .expect("exists"); + assert_eq!(v.block_number, B0); + assert_eq!(v.value.0, branch); + } + } + } + + #[test] + fn test_store_storage_branches_writes_versioned_values() { + let dir = TempDir::new().unwrap(); + let store = MdbxProofsStorage::new(dir.path()).expect("env"); + + let hashed_address = B256::random(); + let nibble = Nibbles::from_nibbles_unchecked([0x12, 0x34]); + let branch_node = BranchNodeCompact::new(0b1, 0, 0, vec![], Some(B256::random())); + let items = vec![(nibble, Some(branch_node.clone()))]; + + store + .store_storage_branches(hashed_address, items) + .expect("write"); + + let tx = store.env.tx().expect("ro tx"); + let mut cur = tx.cursor_dup_read::().expect("cursor"); + + let key = StorageTrieKey::new(hashed_address, StoredNibbles::from(nibble)); + let vv = cur + .seek_by_key_subkey(key, B0) + .expect("seek") + .expect("entry exists"); + + assert_eq!(vv.block_number, B0); + assert_eq!(vv.value.0, Some(branch_node)); + } + + #[test] + fn store_storage_branches_multiple_items_unsorted() { + let dir = TempDir::new().unwrap(); + let store = MdbxProofsStorage::new(dir.path()).expect("env"); + + let hashed_address = B256::random(); + let n1 = Nibbles::from_nibbles_unchecked([0x01]); + let b1 = BranchNodeCompact::new(0b1, 0, 0, vec![], Some(B256::random())); + let n2 = Nibbles::from_nibbles_unchecked([0x02]); + let n3 = Nibbles::from_nibbles_unchecked([0x03]); + let b3 = BranchNodeCompact::new(0b1, 0, 0, vec![], Some(B256::random())); + + let items = vec![(n2, None), (n1, Some(b1)), (n3, Some(b3))]; + store + .store_storage_branches(hashed_address, items.clone()) + .expect("write"); + + let tx = store.env.tx().expect("ro tx"); + let mut cur = tx.cursor_dup_read::().expect("cursor"); + + for (nibble, branch) in items { + let key = StorageTrieKey::new(hashed_address, StoredNibbles::from(nibble)); + let v = cur + .seek_by_key_subkey(key, B0) + .expect("seek") + .expect("exists"); + assert_eq!(v.block_number, B0); + assert_eq!(v.value.0, branch); + } + } + + #[test] + fn store_storage_branches_multiple_calls() { + let dir = TempDir::new().unwrap(); + let store = MdbxProofsStorage::new(dir.path()).expect("env"); + + let hashed_address = B256::random(); + let n1 = Nibbles::from_nibbles_unchecked([0x01]); + let b1 = BranchNodeCompact::new(0b1, 0, 0, vec![], Some(B256::random())); + let n2 = Nibbles::from_nibbles_unchecked([0x02]); + let n3 = Nibbles::from_nibbles_unchecked([0x03]); + let b3 = BranchNodeCompact::new(0b1, 0, 0, vec![], Some(B256::random())); + let n4 = Nibbles::from_nibbles_unchecked([0x04]); + let b4 = BranchNodeCompact::new(0b1, 0, 0, vec![], Some(B256::random())); + let n5 = Nibbles::from_nibbles_unchecked([0x05]); + let b5 = BranchNodeCompact::new(0b1, 0, 0, vec![], Some(B256::random())); + + { + let items1 = vec![(n2, None), (n1, Some(b1)), (n5, Some(b5))]; + store + .store_storage_branches(hashed_address, items1.clone()) + .expect("write"); + + let tx = store.env.tx().expect("ro tx"); + let mut cur = tx.cursor_dup_read::().expect("cursor"); + + for (nibble, branch) in items1 { + let key = StorageTrieKey::new(hashed_address, StoredNibbles::from(nibble)); + let v = cur + .seek_by_key_subkey(key, B0) + .expect("seek") + .expect("exists"); + assert_eq!(v.block_number, B0); + assert_eq!(v.value.0, branch); + } + } + + { + // Second call + let items2 = vec![(n4, Some(b4)), (n3, Some(b3))]; + store + .store_storage_branches(hashed_address, items2.clone()) + .expect("write"); + + let tx = store.env.tx().expect("ro tx"); + let mut cur = tx.cursor_dup_read::().expect("cursor"); + + for (nibble, branch) in items2 { + let key = StorageTrieKey::new(hashed_address, StoredNibbles::from(nibble)); + let v = cur + .seek_by_key_subkey(key, B0) + .expect("seek") + .expect("exists"); + assert_eq!(v.block_number, B0); + assert_eq!(v.value.0, branch); + } + } + } + + #[test] + fn test_store_trie_updates_comprehensive() { + let dir = TempDir::new().unwrap(); + let store = MdbxProofsStorage::new(dir.path()).expect("env"); + + // Sample block number + const BLOCK: BlockWithParent = + BlockWithParent::new(B256::ZERO, NumHash::new(42, B256::ZERO)); + + // Sample addresses and keys + let addr1 = B256::from([0x11; 32]); + let addr2 = B256::from([0x22; 32]); + let slot1 = B256::from([0xA1; 32]); + let slot2 = B256::from([0xA2; 32]); + + // Sample accounts + let acc1 = Account { + nonce: 1, + balance: U256::from(100), + ..Default::default() + }; + + // Sample storage values + let val1 = U256::from(1234u64); + let val2 = U256::from(5678u64); + + // Sample trie paths + let account_path1 = Nibbles::from_nibbles_unchecked(vec![0, 1, 2, 3]); + let account_path2 = Nibbles::from_nibbles_unchecked(vec![4, 5, 6, 7]); + let removed_account_path = Nibbles::from_nibbles_unchecked(vec![7, 8, 9]); + + let account_node1 = BranchNodeCompact::default(); + let account_node2 = BranchNodeCompact::default(); + + let storage_path1 = Nibbles::from_nibbles_unchecked(vec![1, 2, 3, 4]); + let storage_path2 = Nibbles::from_nibbles_unchecked(vec![8, 9, 0, 1]); + + let storage_node1 = BranchNodeCompact::default(); + let storage_node2 = BranchNodeCompact::default(); + + // Construct test BlockStateDiff + let mut block_state_diff_trie_updates = TrieUpdates::default(); + let mut block_state_diff_post_state = HashedPostState::default(); + + // Add account trie nodes + block_state_diff_trie_updates + .account_nodes + .insert(account_path1, account_node1); + block_state_diff_trie_updates + .account_nodes + .insert(account_path2, account_node2); + block_state_diff_trie_updates + .removed_nodes + .insert(removed_account_path); + + // Add storage trie nodes for two addresses + let mut storage_nodes1 = StorageTrieUpdates::default(); + storage_nodes1 + .storage_nodes + .insert(storage_path1, storage_node1); + block_state_diff_trie_updates + .storage_tries + .insert(addr1, storage_nodes1); + + let mut storage_nodes2 = StorageTrieUpdates::default(); + storage_nodes2 + .storage_nodes + .insert(storage_path2, storage_node2); + block_state_diff_trie_updates + .storage_tries + .insert(addr2, storage_nodes2); + + // Add hashed accounts (one Some, one None) + block_state_diff_post_state + .accounts + .insert(addr1, Some(acc1)); + block_state_diff_post_state.accounts.insert(addr2, None); // Deletion + + // Add storage slots for both addresses + let mut storage1 = HashedStorage::default(); + storage1.storage.insert(slot1, val1); + block_state_diff_post_state.storages.insert(addr1, storage1); + + let mut storage2 = HashedStorage::default(); + storage2.storage.insert(slot2, val2); + block_state_diff_post_state.storages.insert(addr2, storage2); + + // Store everything + let block_state_diff = BlockStateDiff { + sorted_trie_updates: block_state_diff_trie_updates.into_sorted(), + sorted_post_state: block_state_diff_post_state.into_sorted(), + }; + store + .store_trie_updates(BLOCK, block_state_diff) + .expect("store"); + + // Verify account trie nodes + { + let tx = store.env.tx().expect("tx"); + let mut cur = tx.new_cursor::().expect("cursor"); + + // Check first node + let vv1 = cur + .seek_by_key_subkey(account_path1.into(), BLOCK.block.number) + .expect("seek") + .expect("exists"); + assert_eq!(vv1.block_number, BLOCK.block.number); + assert!(vv1.value.0.is_some()); + + // Check second node + let vv2 = cur + .seek_by_key_subkey(account_path2.into(), BLOCK.block.number) + .expect("seek") + .expect("exists"); + assert_eq!(vv2.block_number, BLOCK.block.number); + assert!(vv2.value.0.is_some()); + + // Check removed node + let vv3 = cur + .seek_by_key_subkey(removed_account_path.into(), BLOCK.block.number) + .expect("seek") + .expect("exists"); + assert_eq!(vv3.block_number, BLOCK.block.number); + assert!(vv3.value.0.is_none(), "Expected node deletion"); + } + + // Verify storage trie nodes + { + let tx = store.env.tx().expect("tx"); + let mut cur = tx.new_cursor::().expect("cursor"); + + // Check node for addr1 + let key1 = StorageTrieKey::new(addr1, storage_path1.into()); + let vv1 = cur + .seek_by_key_subkey(key1, BLOCK.block.number) + .expect("seek") + .expect("exists"); + assert_eq!(vv1.block_number, BLOCK.block.number); + assert!(vv1.value.0.is_some()); + + // Check node for addr2 + let key2 = StorageTrieKey::new(addr2, storage_path2.into()); + let vv2 = cur + .seek_by_key_subkey(key2, BLOCK.block.number) + .expect("seek") + .expect("exists"); + assert_eq!(vv2.block_number, BLOCK.block.number); + assert!(vv2.value.0.is_some()); + } + + // Verify hashed accounts + { + let tx = store.env.tx().expect("tx"); + let mut cur = tx.new_cursor::().expect("cursor"); + + // Check account1 (exists) + let vv1 = cur + .seek_by_key_subkey(addr1, BLOCK.block.number) + .expect("seek") + .expect("exists"); + assert_eq!(vv1.block_number, BLOCK.block.number); + assert_eq!(vv1.value.0, Some(acc1)); + + // Check account2 (deletion) + let vv2 = cur + .seek_by_key_subkey(addr2, BLOCK.block.number) + .expect("seek") + .expect("exists"); + assert_eq!(vv2.block_number, BLOCK.block.number); + assert!(vv2.value.0.is_none(), "Expected account deletion"); + } + + // Verify hashed storages + { + let tx = store.env.tx().expect("tx"); + let mut cur = tx.new_cursor::().expect("cursor"); + + // Check storage for addr1 + let key1 = HashedStorageKey::new(addr1, slot1); + let vv1 = cur + .seek_by_key_subkey(key1, BLOCK.block.number) + .expect("seek") + .expect("exists"); + assert_eq!(vv1.block_number, BLOCK.block.number); + let inner1 = vv1.value.0.as_ref().expect("Some(StorageValue)"); + assert_eq!(inner1.0, val1); + + // Check storage for addr2 + let key2 = HashedStorageKey::new(addr2, slot2); + let vv2 = cur + .seek_by_key_subkey(key2, BLOCK.block.number) + .expect("seek") + .expect("exists"); + assert_eq!(vv2.block_number, BLOCK.block.number); + let inner2 = vv2.value.0.as_ref().expect("Some(StorageValue)"); + assert_eq!(inner2.0, val2); + } + + // Verify BlockChangeSet entries + { + let tx = store.env.tx().expect("tx"); + let mut cur = tx.new_cursor::().expect("cursor"); + let entries: Vec<_> = cur.walk(Some(BLOCK.block.number)).expect("walk").collect(); + assert_eq!(entries.len(), 1, "Expected 1 BlockChangeSet entry"); + } + + // check the latest block number in proof window + { + let tx = store.env.tx().expect("tx"); + let mut proof_window_cursor = tx.new_cursor::().expect("cursor"); + let latest_block = proof_window_cursor + .seek(ProofWindowKey::LatestBlock) + .expect("seek") + .expect("exists"); + assert_eq!(latest_block.1.number(), BLOCK.block.number); + assert_eq!(*latest_block.1.hash(), BLOCK.block.hash); + } + } + + #[test] + fn store_trie_updates_out_of_order_rejects() { + let dir = tempfile::TempDir::new().unwrap(); + let store = MdbxProofsStorage::new(dir.path()).expect("env"); + + // set latest to some hash H1 + let existing_block = BlockWithParent::new(B256::random(), NumHash::new(1, B256::random())); + store + .set_earliest_block_number(existing_block.block.number, existing_block.block.hash) + .expect("set"); + + // incoming block whose parent != existing latest + let bad_parent = B256::from([0xFF; 32]); + let bad_block = BlockWithParent::new(bad_parent, NumHash::new(2, B256::ZERO)); + let diff = BlockStateDiff::default(); + + let res = store.store_trie_updates(bad_block, diff); + assert!(matches!( + res, + Err(MorphProofsStorageError::OutOfOrder { .. }) + )); + // verify nothing written: proof window still unchanged + let latest = store.get_latest_block_number().expect("get latest"); + assert_eq!(latest.unwrap().1, existing_block.block.hash); + } + + #[test] + fn store_trie_updates_multiple_blocks_append_versions() { + let dir = tempfile::TempDir::new().unwrap(); + let store = MdbxProofsStorage::new(dir.path()).expect("env"); + + let addr = B256::from([0x21; 32]); + // block A (parent = ZERO) + let block_a = BlockWithParent::new(B256::ZERO, NumHash::new(1, B256::random())); + let mut diff_a_post_state = HashedPostState::default(); + diff_a_post_state + .accounts + .insert(addr, Some(Account::default())); + + let diff_a = BlockStateDiff { + sorted_trie_updates: TrieUpdatesSorted::default(), + sorted_post_state: diff_a_post_state.into_sorted(), + }; + store.store_trie_updates(block_a, diff_a).expect("store A"); + + // block B (parent = hash of A) + let block_b = BlockWithParent::new(block_a.block.hash, NumHash::new(2, B256::random())); + let mut diff_b_post_state = HashedPostState::default(); + diff_b_post_state.accounts.insert( + addr, + Some(Account { + nonce: 5, + ..Default::default() + }), + ); + + let diff_b = BlockStateDiff { + sorted_trie_updates: TrieUpdatesSorted::default(), + sorted_post_state: diff_b_post_state.into_sorted(), + }; + store.store_trie_updates(block_b, diff_b).expect("store B"); + + // verify we can retrieve entries for both block numbers + let tx = store.env.tx().expect("tx"); + let mut cur = tx.new_cursor::().expect("cursor"); + let v_a = cur + .seek_by_key_subkey(addr, block_a.block.number) + .expect("seek") + .expect("exists"); + let v_b = cur + .seek_by_key_subkey(addr, block_b.block.number) + .expect("seek") + .expect("exists"); + assert_eq!(v_a.block_number, block_a.block.number); + assert_eq!(v_b.block_number, block_b.block.number); + } + + #[test] + fn test_store_trie_updates_empty_collections() { + let dir = TempDir::new().unwrap(); + let store = MdbxProofsStorage::new(dir.path()).expect("env"); + + const BLOCK: BlockWithParent = + BlockWithParent::new(B256::ZERO, NumHash::new(42, B256::ZERO)); + + // Create BlockStateDiff with empty collections + let block_state_diff = BlockStateDiff::default(); + + // This should work without errors + store + .store_trie_updates(BLOCK, block_state_diff) + .expect("store"); + + // Verify nothing was written (should be empty) + let tx = store.env.tx().expect("tx"); + + let mut cur1 = tx.new_cursor::().expect("cursor"); + assert!( + cur1.next_dup_val().expect("first").is_none(), + "Account trie should be empty" + ); + + let mut cur2 = tx.new_cursor::().expect("cursor"); + assert!( + cur2.next_dup_val().expect("first").is_none(), + "Storage trie should be empty" + ); + + let mut cur3 = tx.new_cursor::().expect("cursor"); + assert!( + cur3.next_dup_val().expect("first").is_none(), + "Hashed accounts should be empty" + ); + + let mut cur4 = tx.new_cursor::().expect("cursor"); + assert!( + cur4.next_dup_val().expect("first").is_none(), + "Hashed storage should be empty" + ); + + let mut cur5 = tx.new_cursor::().expect("cursor"); + assert!( + cur5.next().expect("first").is_some(), + "Pruning index SHOULD populate the change set even for empty diffs" + ); + } + + #[test] + fn fetch_trie_updates_missing_changeset_returns_error() { + let dir = TempDir::new().unwrap(); + let store = MdbxProofsStorage::new(dir.path()).expect("env"); + + let res = store.fetch_trie_updates(99); + assert!(matches!( + res, + Err(MorphProofsStorageError::NoChangeSetForBlock(99)) + )); + } + + #[test] + fn fetch_trie_updates_empty_changeset() { + let dir = TempDir::new().unwrap(); + let store = MdbxProofsStorage::new(dir.path()).expect("env"); + + let block = BlockWithParent::new(B256::ZERO, NumHash::new(1, B256::random())); + let diff = BlockStateDiff::default(); + + store.store_trie_updates(block, diff).expect("store"); + let got = store.fetch_trie_updates(1).expect("fetch"); + assert!(got.sorted_trie_updates.account_nodes_ref().is_empty()); + assert!(got.sorted_trie_updates.storage_tries_ref().is_empty()); + assert!(got.sorted_post_state.accounts.is_empty()); + assert!(got.sorted_post_state.storages.is_empty()); + } + + #[test] + fn fetch_trie_updates_missing_account_history_entry_returns_error() { + let dir = TempDir::new().unwrap(); + let store = MdbxProofsStorage::new(dir.path()).expect("env"); + + // prepare ChangeSet that references StoredNibbles for account key + // (insert ChangeSet into BlockChangeSet directly using tx) + { + let tx = store.env.tx_mut().unwrap(); + let mut cur = tx.cursor_write::().unwrap(); + cur.insert( + 1, + &ChangeSet { + account_trie_keys: vec![StoredNibbles::default()], + ..Default::default() + }, + ) + .unwrap(); + tx.commit().unwrap(); + } + + let res = store.fetch_trie_updates(1); + assert!(matches!( + res, + Err(MorphProofsStorageError::MissingAccountTrieHistory(..)) + )); + } + + #[test] + fn fetch_trie_updates_account_history_seek_returns_later_block_treated_as_missing() { + let dir = TempDir::new().unwrap(); + let store = MdbxProofsStorage::new(dir.path()).expect("env"); + + // manually insert account history and ChangeSet for block 1 referencing same key + { + let tx = store.env.tx_mut().unwrap(); + let mut acc_cur = tx.cursor_write::().unwrap(); + acc_cur + .insert( + StoredNibbles::from(Nibbles::from_nibbles_unchecked([0x1])), + &VersionedValue::new(2, MaybeDeleted(Some(BranchNodeCompact::default()))), + ) + .unwrap(); + + let mut cur = tx.cursor_write::().unwrap(); + cur.insert( + 1, + &ChangeSet { + account_trie_keys: vec![StoredNibbles::from(Nibbles::from_nibbles_unchecked( + [0x1], + ))], + ..Default::default() + }, + ) + .unwrap(); + tx.commit().unwrap(); + } + + // fetch block 1 -> seek will find block 2 but block_number != 1 so expect + // MissingAccountTrieHistory + let res = store.fetch_trie_updates(1); + assert!(matches!( + res, + Err(MorphProofsStorageError::MissingAccountTrieHistory(..)) + )); + } + + #[test] + fn fetch_trie_updates_missing_storage_history_entry_returns_error() { + let dir = TempDir::new().unwrap(); + let store = MdbxProofsStorage::new(dir.path()).expect("env"); + + // prepare ChangeSet that references StorageTrieKey for storage trie + // (insert ChangeSet into BlockChangeSet directly using tx) + { + let tx = store.env.tx_mut().unwrap(); + let mut cur = tx.cursor_write::().unwrap(); + cur.insert( + 1, + &ChangeSet { + storage_trie_keys: vec![StorageTrieKey::new( + B256::from([0u8; 32]), + StoredNibbles::default(), + )], + ..Default::default() + }, + ) + .unwrap(); + tx.commit().unwrap(); + } + + let res = store.fetch_trie_updates(1); + assert!(matches!( + res, + Err(MorphProofsStorageError::MissingStorageTrieHistory(..)) + )); + } + + #[test] + fn fetch_trie_updates_storage_history_seek_returns_later_block_treated_as_missing() { + let dir = TempDir::new().unwrap(); + let store = MdbxProofsStorage::new(dir.path()).expect("env"); + + // manually insert storage history and ChangeSet for block 1 referencing same key + { + let tx = store.env.tx_mut().unwrap(); + let mut stor_cur = tx.cursor_write::().unwrap(); + stor_cur + .insert( + StorageTrieKey::new( + B256::from([0u8; 32]), + StoredNibbles::from(Nibbles::from_nibbles_unchecked([0x1])), + ), + &VersionedValue::new(2, MaybeDeleted(Some(BranchNodeCompact::default()))), + ) + .unwrap(); + + let mut cur = tx.cursor_write::().unwrap(); + cur.insert( + 1, + &ChangeSet { + storage_trie_keys: vec![StorageTrieKey::new( + B256::from([0u8; 32]), + StoredNibbles::from(Nibbles::from_nibbles_unchecked([0x1])), + )], + ..Default::default() + }, + ) + .unwrap(); + tx.commit().unwrap(); + } + + // fetch block 1 -> seek will find block 2 but block_number != 1 so expect + // MissingStorageTrieHistory + let res = store.fetch_trie_updates(1); + assert!(matches!( + res, + Err(MorphProofsStorageError::MissingStorageTrieHistory(..)) + )); + } + + #[test] + fn fetch_trie_updates_missing_hashed_account_entry_returns_error() { + let dir = TempDir::new().unwrap(); + let store = MdbxProofsStorage::new(dir.path()).expect("env"); + + // prepare ChangeSet that references hashed account address + // (insert ChangeSet into BlockChangeSet directly using tx) + { + let tx = store.env.tx_mut().unwrap(); + let mut cur = tx.cursor_write::().unwrap(); + cur.insert( + 1, + &ChangeSet { + hashed_account_keys: vec![B256::from([0u8; 32])], + ..Default::default() + }, + ) + .unwrap(); + tx.commit().unwrap(); + } + + let res = store.fetch_trie_updates(1); + assert!(matches!( + res, + Err(MorphProofsStorageError::MissingHashedAccountHistory(..)) + )); + } + + #[test] + fn fetch_trie_updates_hashed_account_seek_returns_later_block_treated_as_missing() { + let dir = TempDir::new().unwrap(); + let store = MdbxProofsStorage::new(dir.path()).expect("env"); + + // manually insert hashed account history and ChangeSet for block 1 referencing same key + { + let tx = store.env.tx_mut().unwrap(); + let mut acc_cur = tx.cursor_write::().unwrap(); + acc_cur + .insert( + B256::from([0u8; 32]), + &VersionedValue::new(2, MaybeDeleted(Some(Account::default()))), + ) + .unwrap(); + + let mut cur = tx.cursor_write::().unwrap(); + cur.insert( + 1, + &ChangeSet { + hashed_account_keys: vec![B256::from([0u8; 32])], + ..Default::default() + }, + ) + .unwrap(); + tx.commit().unwrap(); + } + + // fetch block 1 -> seek will find block 2 but block_number != 1 so expect + // MissingHashedAccountHistory + let res = store.fetch_trie_updates(1); + assert!(matches!( + res, + Err(MorphProofsStorageError::MissingHashedAccountHistory(..)) + )); + } + + #[test] + fn fetch_trie_updates_missing_hashed_storage_entry_returns_error() { + let dir = TempDir::new().unwrap(); + let store = MdbxProofsStorage::new(dir.path()).expect("env"); + + // prepare ChangeSet that references hashed storage key + // (insert ChangeSet into BlockChangeSet directly using tx) + { + let tx = store.env.tx_mut().unwrap(); + let mut cur = tx.cursor_write::().unwrap(); + cur.insert( + 1, + &ChangeSet { + hashed_storage_keys: vec![HashedStorageKey::new( + B256::from([0u8; 32]), + B256::from([0u8; 32]), + )], + ..Default::default() + }, + ) + .unwrap(); + tx.commit().unwrap(); + } + + let res = store.fetch_trie_updates(1); + assert!(matches!( + res, + Err(MorphProofsStorageError::MissingHashedStorageHistory { .. }) + )); + } + + #[test] + fn fetch_trie_updates_hashed_storage_seek_returns_later_block_treated_as_missing() { + let dir = TempDir::new().unwrap(); + let store = MdbxProofsStorage::new(dir.path()).expect("env"); + + // manually insert hashed storage history and ChangeSet for block 1 referencing same key + { + let tx = store.env.tx_mut().unwrap(); + let mut stor_cur = tx.cursor_write::().unwrap(); + stor_cur + .insert( + HashedStorageKey::new(B256::from([0u8; 32]), B256::from([0u8; 32])), + &VersionedValue::new(2, MaybeDeleted(Some(StorageValue::new(U256::ZERO)))), + ) + .unwrap(); + + let mut cur = tx.cursor_write::().unwrap(); + cur.insert( + 1, + &ChangeSet { + hashed_storage_keys: vec![HashedStorageKey::new( + B256::from([0u8; 32]), + B256::from([0u8; 32]), + )], + ..Default::default() + }, + ) + .unwrap(); + tx.commit().unwrap(); + } + + // fetch block 1 -> seek will find block 2 but block_number != 1 so expect + // MissingHashedStorageHistory + let res = store.fetch_trie_updates(1); + assert!(matches!( + res, + Err(MorphProofsStorageError::MissingHashedStorageHistory { .. }) + )); + } + + #[test] + fn fetch_trie_updates_basic() { + let dir = TempDir::new().unwrap(); + let store = MdbxProofsStorage::new(dir.path()).expect("env"); + + // Build a block with mixed changes (accounts, trie nodes, hashed storages) + let block = BlockWithParent::new(B256::ZERO, NumHash::new(1, B256::random())); + + // prepare data + let addr1 = B256::from([0x11; 32]); + let addr2 = B256::from([0x22; 32]); + let slot1 = B256::from([0xA1; 32]); + let slot2 = B256::from([0xA2; 32]); + + let acc1 = Account { + nonce: 1, + balance: U256::from(100), + ..Default::default() + }; + + let val1 = U256::from(1234u64); + let val2 = U256::from(5678u64); + + let account_path1 = Nibbles::from_nibbles_unchecked(vec![0, 1, 2, 3]); + let account_path2 = Nibbles::from_nibbles_unchecked(vec![4, 5, 6, 7]); + let account_node1 = BranchNodeCompact { + root_hash: Some(B256::random()), + ..Default::default() + }; + let account_node2 = BranchNodeCompact { + root_hash: Some(B256::random()), + ..Default::default() + }; + + let storage_path1 = Nibbles::from_nibbles_unchecked(vec![1, 2, 3, 4]); + let storage_node1 = BranchNodeCompact { + root_hash: Some(B256::random()), + ..Default::default() + }; + + // Construct BlockStateDiff + let mut block_state_diff_trie_updates = TrieUpdates::default(); + block_state_diff_trie_updates + .account_nodes + .insert(account_path1, account_node1); + block_state_diff_trie_updates + .account_nodes + .insert(account_path2, account_node2); + // storage trie for addr1 + let mut storage_nodes1 = StorageTrieUpdates::default(); + storage_nodes1 + .storage_nodes + .insert(storage_path1, storage_node1); + block_state_diff_trie_updates + .storage_tries + .insert(addr1, storage_nodes1); + + // hashed accounts: addr1 -> Some, addr2 -> None + let mut block_state_diff_post_state = HashedPostState::default(); + block_state_diff_post_state + .accounts + .insert(addr1, Some(acc1)); + block_state_diff_post_state.accounts.insert(addr2, None); + + // hashed storages + let mut storage1 = HashedStorage::default(); + storage1.storage.insert(slot1, val1); + block_state_diff_post_state.storages.insert(addr1, storage1); + + let mut storage2 = HashedStorage::default(); + storage2.storage.insert(slot2, val2); + block_state_diff_post_state.storages.insert(addr2, storage2); + + // store then fetch + let block_state_diff = BlockStateDiff { + sorted_trie_updates: block_state_diff_trie_updates.into_sorted(), + sorted_post_state: block_state_diff_post_state.into_sorted(), + }; + store + .store_trie_updates(block, block_state_diff.clone()) + .expect("store"); + let got = store.fetch_trie_updates(1).expect("fetch"); + + // verify trie updates + assert_eq!( + got.sorted_trie_updates.account_nodes_ref(), + block_state_diff.sorted_trie_updates.account_nodes_ref(), + ); + assert_eq!( + got.sorted_trie_updates.storage_tries_ref(), + block_state_diff.sorted_trie_updates.storage_tries_ref(), + ); + + // verify post state + assert_eq!( + got.sorted_post_state.accounts, + block_state_diff.sorted_post_state.accounts + ); + assert_eq!( + got.sorted_post_state.storages, + block_state_diff.sorted_post_state.storages + ); + } + + #[test] + fn test_prune_earliest_state_single_entry() { + let dir = TempDir::new().unwrap(); + let store = MdbxProofsStorage::new(dir.path()).expect("env"); + let block = BlockWithParent::new(B256::ZERO, NumHash::new(1, B256::random())); + store.set_earliest_block_number(0, B256::ZERO).unwrap(); + + // Insert a single entry to be pruned + let addr = B256::random(); + let mut state_diff_post_state = HashedPostState::default(); + state_diff_post_state + .accounts + .insert(addr, Some(Account::default())); + let state_diff = BlockStateDiff { + sorted_post_state: state_diff_post_state.into_sorted(), + ..Default::default() + }; + store.store_trie_updates(block, state_diff).unwrap(); + + // Prune the entry - pass empty diff since we're just removing data + let next_block = BlockWithParent::new(block.block.hash, NumHash::new(2, B256::random())); + store.prune_earliest_state(next_block).unwrap(); + + // Verify the entry was pruned + let tx = store.env.tx().unwrap(); + let mut cur = tx.new_cursor::().unwrap(); + + // The survivor at block 1 should remain + let val = cur + .seek_by_key_subkey(addr, block.block.number) + .unwrap() + .expect("Survivor should exist"); + assert_eq!(val.block_number, 1); + + let mut pruning_cur = tx.new_cursor::().unwrap(); + assert!( + pruning_cur + .seek_exact(block.block.number) + .unwrap() + .is_none() + ); + + // Verify earliest block was updated + let earliest = store.get_earliest_block_number().unwrap(); + assert_eq!(earliest, Some((2, next_block.block.hash))); + } + + #[test] + fn test_prune_earliest_state_multiple_entries_same_block() { + let dir = TempDir::new().unwrap(); + let store = MdbxProofsStorage::new(dir.path()).expect("env"); + let block = BlockWithParent::new(B256::ZERO, NumHash::new(1, B256::random())); + store.set_earliest_block_number(0, B256::ZERO).unwrap(); + + // Insert multiple entries for the same block + let addr1 = B256::random(); + let addr2 = B256::random(); + let mut state_diff_post_state = HashedPostState::default(); + state_diff_post_state + .accounts + .insert(addr1, Some(Account::default())); + state_diff_post_state + .accounts + .insert(addr2, Some(Account::default())); + let state_diff = BlockStateDiff { + sorted_post_state: state_diff_post_state.into_sorted(), + ..Default::default() + }; + store.store_trie_updates(block, state_diff).unwrap(); + + // Prune the entries + let next_block = BlockWithParent::new(block.block.hash, NumHash::new(2, B256::random())); + store.prune_earliest_state(next_block).unwrap(); + + // Verify the entries were pruned + let tx = store.env.tx().unwrap(); + let mut cur = tx.new_cursor::().unwrap(); + + // All entries should survive at block 1 + let v1 = cur + .seek_by_key_subkey(addr1, block.block.number) + .unwrap() + .expect("addr1 survivor"); + assert_eq!(v1.block_number, 1); + let v2 = cur + .seek_by_key_subkey(addr2, block.block.number) + .unwrap() + .expect("addr2 survivor"); + assert_eq!(v2.block_number, 1); + + let mut pruning_cur = tx.new_cursor::().unwrap(); + assert!( + pruning_cur + .seek_exact(block.block.number) + .unwrap() + .is_none() + ); + } + + #[test] + fn test_prune_earliest_state_multiple_blocks() { + let dir = TempDir::new().unwrap(); + let store = MdbxProofsStorage::new(dir.path()).expect("env"); + let block_1 = BlockWithParent::new(B256::ZERO, NumHash::new(1, B256::random())); + let block_2 = BlockWithParent::new(block_1.block.hash, NumHash::new(2, B256::random())); + let block_3 = BlockWithParent::new(block_2.block.hash, NumHash::new(3, B256::random())); + store.set_earliest_block_number(0, B256::ZERO).unwrap(); + + // Insert entries for multiple blocks + let addr1 = B256::random(); + let addr2 = B256::random(); + let mut state_diff1_post_state = HashedPostState::default(); + state_diff1_post_state + .accounts + .insert(addr1, Some(Account::default())); + let state_diff1 = BlockStateDiff { + sorted_post_state: state_diff1_post_state.into_sorted(), + ..Default::default() + }; + store.store_trie_updates(block_1, state_diff1).unwrap(); + + let mut state_diff2_post_state = HashedPostState::default(); + state_diff2_post_state + .accounts + .insert(addr2, Some(Account::default())); + let state_diff2 = BlockStateDiff { + sorted_post_state: state_diff2_post_state.into_sorted(), + ..Default::default() + }; + store.store_trie_updates(block_2, state_diff2).unwrap(); + + // Prune up to block 3 (should remove blocks 1 and 2) + store.prune_earliest_state(block_3).unwrap(); + + // Verify the entries were pruned + let tx = store.env.tx().unwrap(); + let mut cur = tx.new_cursor::().unwrap(); + + // addr1 survivor at block 1 must exist + let v1 = cur + .seek_by_key_subkey(addr1, 1) + .unwrap() + .expect("addr1 survivor"); + assert_eq!(v1.block_number, 1); + + // addr2 survivor at block 2 must exist + let v2 = cur + .seek_by_key_subkey(addr2, 2) + .unwrap() + .expect("addr2 survivor"); + assert_eq!(v2.block_number, 2); + + let mut pruning_cur = tx.new_cursor::().unwrap(); + assert!(pruning_cur.seek_exact(1).unwrap().is_none()); + assert!(pruning_cur.seek_exact(2).unwrap().is_none()); + } + + #[test] + fn test_prune_earliest_state_no_op() { + let dir = TempDir::new().unwrap(); + let store = MdbxProofsStorage::new(dir.path()).expect("env"); + store.set_earliest_block_number(1, B256::random()).unwrap(); + + // Attempt to prune with a new earliest block that is not newer + let block_1 = BlockWithParent::new(B256::ZERO, NumHash::new(1, B256::random())); + let block_0 = BlockWithParent::new(B256::ZERO, NumHash::new(0, B256::random())); + store.prune_earliest_state(block_1).unwrap(); + store.prune_earliest_state(block_0).unwrap(); + + // Nothing should have been pruned, this call should not panic or error + } + + #[test] + fn test_prune_earliest_state_no_entries_to_prune() { + let dir = TempDir::new().unwrap(); + let store = MdbxProofsStorage::new(dir.path()).expect("env"); + store.set_earliest_block_number(1, B256::random()).unwrap(); + + // Prune a range where no entries exist + let block_10 = BlockWithParent::new(B256::ZERO, NumHash::new(10, B256::random())); + store.prune_earliest_state(block_10).unwrap(); + + // Nothing should have been pruned, this call should not panic or error + } + + #[test] + fn test_prune_earliest_state_with_diff_insertion() { + let dir = TempDir::new().unwrap(); + let store = MdbxProofsStorage::new(dir.path()).expect("env"); + store.set_earliest_block_number(0, B256::ZERO).unwrap(); + + // Insert entries for blocks 1 and 2 + let addr1 = B256::random(); + let addr2 = B256::random(); + let acc1 = Account { + nonce: 1, + balance: U256::from(100), + ..Default::default() + }; + let acc2 = Account { + nonce: 2, + balance: U256::from(200), + ..Default::default() + }; + + let block_1 = BlockWithParent::new(B256::ZERO, NumHash::new(1, B256::random())); + let mut state_diff1_post_state = HashedPostState::default(); + state_diff1_post_state.accounts.insert(addr1, Some(acc1)); + let state_diff1 = BlockStateDiff { + sorted_post_state: state_diff1_post_state.into_sorted(), + ..Default::default() + }; + store.store_trie_updates(block_1, state_diff1).unwrap(); + + let block_2 = BlockWithParent::new(block_1.block.hash, NumHash::new(2, B256::random())); + let mut state_diff2_post_state = HashedPostState::default(); + state_diff2_post_state.accounts.insert(addr2, Some(acc2)); + let state_diff2 = BlockStateDiff { + sorted_post_state: state_diff2_post_state.into_sorted(), + ..Default::default() + }; + store.store_trie_updates(block_2, state_diff2).unwrap(); + + // Now prune to block 3 + let block_3 = BlockWithParent::new(block_2.block.hash, NumHash::new(3, B256::random())); + store.prune_earliest_state(block_3).unwrap(); + + let tx = store.env.tx().unwrap(); + let mut cur = tx.new_cursor::().unwrap(); + + // Verify that blocks 1 and 2 entries were NOT pruned (they are survivors for their + // respective keys) + let vv1 = cur + .seek_by_key_subkey(addr1, 0) + .unwrap() + .expect("addr1 should survive"); + assert_eq!(vv1.block_number, 1); + assert_eq!(vv1.value.0, Some(acc1)); + + let vv2 = cur + .seek_by_key_subkey(addr2, 0) + .unwrap() + .expect("addr2 should survive"); + assert_eq!(vv2.block_number, 2); + assert_eq!(vv2.value.0, Some(acc2)); + + // Verify change sets for blocks 1 and 2 were removed + let mut pruning_cur = tx.new_cursor::().unwrap(); + assert!(pruning_cur.seek_exact(1).unwrap().is_none()); + assert!(pruning_cur.seek_exact(2).unwrap().is_none()); + + // Verify earliest block was updated + let earliest = store.get_earliest_block_number().unwrap(); + assert_eq!(earliest, Some((3, block_3.block.hash))); + } + + #[test] + fn test_prune_earliest_state_with_removed_nodes() { + let dir = TempDir::new().unwrap(); + let store = MdbxProofsStorage::new(dir.path()).expect("env"); + store.set_earliest_block_number(0, B256::ZERO).unwrap(); + + // Create some trie nodes in blocks 1, 2, 3 + let path1 = Nibbles::from_nibbles_unchecked([0x01, 0x02]); + let path2 = Nibbles::from_nibbles_unchecked([0x03, 0x04]); + let node1 = BranchNodeCompact::new(0b1, 0, 0, vec![], Some(B256::random())); + let node2 = BranchNodeCompact::new(0b10, 0, 0, vec![], Some(B256::random())); + + let block_1 = BlockWithParent::new(B256::ZERO, NumHash::new(1, B256::random())); + let mut diff1_trie_updates = TrieUpdates::default(); + diff1_trie_updates.account_nodes.insert(path1, node1); + let diff1 = BlockStateDiff { + sorted_trie_updates: diff1_trie_updates.into_sorted(), + ..Default::default() + }; + store.store_trie_updates(block_1, diff1).unwrap(); + + let block_2 = BlockWithParent::new(block_1.block.hash, NumHash::new(2, B256::random())); + let mut diff2_trie_updates = TrieUpdates::default(); + diff2_trie_updates + .account_nodes + .insert(path2, node2.clone()); + let diff2 = BlockStateDiff { + sorted_trie_updates: diff2_trie_updates.into_sorted(), + ..Default::default() + }; + store.store_trie_updates(block_2, diff2).unwrap(); + + // In block 3, path1 is deleted (stored as None in the database) + // This happens when we store trie updates with path1 mapped to None + let block_3 = BlockWithParent::new(block_2.block.hash, NumHash::new(3, B256::random())); + // Simulate storing a deletion by directly writing to DB + store + .env + .update(|tx| { + let mut cursor = tx.new_cursor::()?; + let vv = VersionedValue { + block_number: 3, + value: MaybeDeleted(None), + }; + cursor.upsert(StoredNibbles::from(path1), &vv)?; + + // Record in change set + let mut change_set_cursor = tx.new_cursor::()?; + change_set_cursor.upsert( + 3, + &ChangeSet { + account_trie_keys: vec![StoredNibbles::from(path1)], + storage_trie_keys: vec![], + hashed_account_keys: vec![], + hashed_storage_keys: vec![], + }, + )?; + + // Update proof window + let mut proof_window_cursor = tx.new_cursor::()?; + proof_window_cursor.upsert( + ProofWindowKey::LatestBlock, + &BlockNumberHash::new(3, block_3.block.hash), + )?; + Ok::<(), DatabaseError>(()) + }) + .unwrap() + .unwrap(); + + // Now prune to block 5, with the new initial state: + // - path1 should be in removed_nodes (it was deleted in block 3) + // - path2 should be included with its value (it still exists from block 2) + let block_5 = BlockWithParent::new(B256::random(), NumHash::new(5, B256::random())); + store.prune_earliest_state(block_5).unwrap(); + + // Verify that all entries for path1 before block 5 were removed + let tx = store.env.tx().unwrap(); + let mut cur = tx.cursor_dup_read::().unwrap(); + + // path1 at block 1 should be gone; seeking 1 finds survivor (tombstone at 3) + if let Some(v) = cur + .seek_by_key_subkey(StoredNibbles::from(path1), 1) + .unwrap() + { + assert!(v.block_number >= 3, "path1 at block 1 should be pruned"); + } + + // path1 at block 1 should be gone. + // path1 survivor at block 3 (tombstone) should ALSO be gone now (optimization). + // So seeking for path1 should return None. + assert!( + cur.seek_by_key_subkey(StoredNibbles::from(path1), 0) + .unwrap() + .is_none(), + "path1 should be completely removed including tombstone" + ); + + // path2 entries should be pruned (blocks < 5) + // Survivor for path2 is at block 2. + let v2 = cur + .seek_by_key_subkey(StoredNibbles::from(path2), 0) + .unwrap() + .expect("path2 survivor"); + assert_eq!(v2.block_number, 2); + assert_eq!(v2.value.0, Some(node2)); + } + + #[test] + fn test_prune_earliest_state_overlapping_keys() { + let dir = TempDir::new().unwrap(); + let store = MdbxProofsStorage::new(dir.path()).expect("env"); + store.set_earliest_block_number(0, B256::ZERO).unwrap(); + + // Use overlapping key (addr1 updated in blocks 1 and 2) + let addr1 = B256::random(); + let acc1 = Account { + nonce: 1, + balance: U256::from(100), + ..Default::default() + }; + let acc2 = Account { + nonce: 2, + balance: U256::from(200), + ..Default::default() + }; + + let block_1 = BlockWithParent::new(B256::ZERO, NumHash::new(1, B256::random())); + let mut diff1_post_state = HashedPostState::default(); + diff1_post_state.accounts.insert(addr1, Some(acc1)); + let diff1 = BlockStateDiff { + sorted_post_state: diff1_post_state.into_sorted(), + ..Default::default() + }; + store.store_trie_updates(block_1, diff1).unwrap(); + + let block_2 = BlockWithParent::new(block_1.block.hash, NumHash::new(2, B256::random())); + let mut diff2_post_state = HashedPostState::default(); + diff2_post_state.accounts.insert(addr1, Some(acc2)); + let diff2 = BlockStateDiff { + sorted_post_state: diff2_post_state.into_sorted(), + ..Default::default() + }; + store.store_trie_updates(block_2, diff2).unwrap(); + + // Prune to block 3 + let block_3 = BlockWithParent::new(block_2.block.hash, NumHash::new(3, B256::random())); + store.prune_earliest_state(block_3).unwrap(); + + let tx = store.env.tx().unwrap(); + let mut cur = tx.new_cursor::().unwrap(); + + // Verify survivor (acc2 at block 2) remains + let vv1 = cur + .seek_by_key_subkey(addr1, 0) + .unwrap() + .expect("addr1 should having surviving state"); + assert_eq!(vv1.block_number, 2); + assert_eq!(vv1.value.0, Some(acc2)); + } + + #[test] + fn test_prune_earliest_state_comprehensive() { + let dir = TempDir::new().unwrap(); + let store = MdbxProofsStorage::new(dir.path()).expect("env"); + store.set_earliest_block_number(0, B256::ZERO).unwrap(); + + // Setup complex scenario with accounts, storage, and trie nodes + let addr1 = B256::random(); + let slot1 = B256::random(); + let path1 = Nibbles::from_nibbles_unchecked([0x01]); + let storage_path1 = Nibbles::from_nibbles_unchecked([0x03]); + + let acc1 = Account { + nonce: 1, + balance: U256::from(100), + ..Default::default() + }; + let node1 = BranchNodeCompact::new(0b1, 0, 0, vec![], Some(B256::random())); + let storage_node1 = BranchNodeCompact::new(0b10, 0, 0, vec![], Some(B256::random())); + + // Block 1: Insert account, trie node, and storage for addr1 + let block_1 = BlockWithParent::new(B256::ZERO, NumHash::new(1, B256::random())); + + let mut diff1_trie_updates = TrieUpdates::default(); + let mut diff1_post_state = HashedPostState::default(); + + diff1_post_state.accounts.insert(addr1, Some(acc1)); + diff1_trie_updates + .account_nodes + .insert(path1, node1.clone()); + let mut storage1 = HashedStorage::default(); + storage1.storage.insert(slot1, U256::from(1234)); + diff1_post_state.storages.insert(addr1, storage1.clone()); + let mut storage_updates1 = StorageTrieUpdates::default(); + storage_updates1 + .storage_nodes + .insert(storage_path1, storage_node1.clone()); + diff1_trie_updates + .storage_tries + .insert(addr1, storage_updates1.clone()); + + let diff_1 = BlockStateDiff { + sorted_post_state: diff1_post_state.into_sorted(), + sorted_trie_updates: diff1_trie_updates.into_sorted(), + }; + store.store_trie_updates(block_1, diff_1).unwrap(); + + // Block 2: Update account (overwriting addr1) + let acc2 = Account { + nonce: 2, + balance: U256::from(200), + ..Default::default() + }; + let block_2 = BlockWithParent::new(block_1.block.hash, NumHash::new(2, B256::random())); + + let mut diff2_post_state = HashedPostState::default(); + diff2_post_state.accounts.insert(addr1, Some(acc2)); + let diff2 = BlockStateDiff { + sorted_post_state: diff2_post_state.into_sorted(), + ..Default::default() + }; + store.store_trie_updates(block_2, diff2).unwrap(); + + // Prune to block 3 + let block_3 = BlockWithParent::new(block_2.block.hash, NumHash::new(3, B256::random())); + store.prune_earliest_state(block_3).unwrap(); + + let tx = store.env.tx().unwrap(); + + // Verify account history - acc1 at block 1 pruned + let mut acc_cur = tx.new_cursor::().unwrap(); + // Survivor acc2 at block 2 remains + let acc_vv = acc_cur + .seek_by_key_subkey(addr1, 0) + .unwrap() + .expect("Survivor at block 2"); + assert_eq!(acc_vv.value.0, Some(acc2)); + assert_eq!(acc_vv.block_number, 2); + + // Verify account trie history - path1 at block 1 remains because it's the latest for that + // key + let mut trie_cur = tx.cursor_dup_read::().unwrap(); + let trie_vv = trie_cur + .seek_by_key_subkey(StoredNibbles::from(path1), 0) + .unwrap() + .expect("Survivor at block 1"); + assert_eq!(trie_vv.value.0, Some(node1)); + assert_eq!(trie_vv.block_number, 1); + + // Verify storage history - slot1 at block 1 remains + let mut storage_cur = tx.new_cursor::().unwrap(); + let storage_key = HashedStorageKey::new(addr1, slot1); + let storage_vv = storage_cur + .seek_by_key_subkey(storage_key, 0) + .unwrap() + .expect("Survivor at block 1"); + assert_eq!(storage_vv.value.0.as_ref().unwrap().0, U256::from(1234)); + assert_eq!(storage_vv.block_number, 1); + + // Verify storage trie history + let mut storage_trie_cur = tx.cursor_dup_read::().unwrap(); + let storage_trie_key = StorageTrieKey::new(addr1, StoredNibbles::from(storage_path1)); + let storage_trie_vv = storage_trie_cur + .seek_by_key_subkey(storage_trie_key, 0) + .unwrap() + .expect("Survivor at block 1"); + assert_eq!(storage_trie_vv.value.0, Some(storage_node1)); + assert_eq!(storage_trie_vv.block_number, 1); + + // Verify change sets pruned + let mut change_cur = tx.new_cursor::().unwrap(); + assert!(change_cur.seek_exact(1).unwrap().is_none()); + assert!(change_cur.seek_exact(2).unwrap().is_none()); + } + + #[test] + fn test_prune_earliest_state_churn_create_delete_recreate() { + let dir = TempDir::new().unwrap(); + let store = MdbxProofsStorage::new(dir.path()).expect("env"); + store.set_earliest_block_number(0, B256::ZERO).unwrap(); + + let addr = B256::random(); + + // Block 1: Create + let b1 = BlockWithParent::new(B256::ZERO, NumHash::new(1, B256::random())); + let acc1 = Account { + nonce: 1, + ..Default::default() + }; + let mut diff1 = HashedPostState::default(); + diff1.accounts.insert(addr, Some(acc1)); + store + .store_trie_updates( + b1, + BlockStateDiff { + sorted_post_state: diff1.into_sorted(), + ..Default::default() + }, + ) + .unwrap(); + + // Block 2: Delete + let b2 = BlockWithParent::new(b1.block.hash, NumHash::new(2, B256::random())); + let mut diff2 = HashedPostState::default(); + diff2.accounts.insert(addr, None); + store + .store_trie_updates( + b2, + BlockStateDiff { + sorted_post_state: diff2.into_sorted(), + ..Default::default() + }, + ) + .unwrap(); + + // Block 3: Recreate + let b3 = BlockWithParent::new(b2.block.hash, NumHash::new(3, B256::random())); + let acc3 = Account { + nonce: 3, + ..Default::default() + }; + let mut diff3 = HashedPostState::default(); + diff3.accounts.insert(addr, Some(acc3)); + store + .store_trie_updates( + b3, + BlockStateDiff { + sorted_post_state: diff3.into_sorted(), + ..Default::default() + }, + ) + .unwrap(); + + // Prune to Block 3 + store.prune_earliest_state(b3).unwrap(); + + let tx = store.env.tx().unwrap(); + let mut cur = tx.new_cursor::().unwrap(); + + // Block 1 (Older than 3) should be deleted. + // Since Block 3 exists, seek(1) will land on Block 3. + if let Some(val) = cur.seek_by_key_subkey(addr, 1).unwrap() { + assert!( + val.block_number >= 3, + "Block 1 should use be pruned, found {}", + val.block_number + ); + } + + // Block 2 (Older than 3) should be deleted + if let Some(val) = cur.seek_by_key_subkey(addr, 2).unwrap() { + assert!( + val.block_number >= 3, + "Block 2 should use be pruned, found {}", + val.block_number + ); + } + + // Block 3 (Survivor) should exist + let val = cur + .seek_by_key_subkey(addr, 3) + .unwrap() + .expect("Block 3 should survive"); + assert_eq!(val.block_number, 3); + assert_eq!(val.value.0, Some(acc3)); + } + + #[test] + fn test_prune_earliest_state_returns_correct_counts() { + let dir = TempDir::new().unwrap(); + let store = MdbxProofsStorage::new(dir.path()).expect("env"); + store.set_earliest_block_number(0, B256::ZERO).unwrap(); + + let addr = B256::random(); + + // Block 1: Insert + let b1 = BlockWithParent::new(B256::ZERO, NumHash::new(1, B256::random())); + let acc1 = Account { + nonce: 1, + ..Default::default() + }; + let mut diff1 = HashedPostState::default(); + diff1.accounts.insert(addr, Some(acc1)); + store + .store_trie_updates( + b1, + BlockStateDiff { + sorted_post_state: diff1.into_sorted(), + ..Default::default() + }, + ) + .unwrap(); + + // Block 2: Update + let b2 = BlockWithParent::new(b1.block.hash, NumHash::new(2, B256::random())); + let acc2 = Account { + nonce: 2, + ..Default::default() + }; + let mut diff2 = HashedPostState::default(); + diff2.accounts.insert(addr, Some(acc2)); + store + .store_trie_updates( + b2, + BlockStateDiff { + sorted_post_state: diff2.into_sorted(), + ..Default::default() + }, + ) + .unwrap(); + + // Prune to Block 2. + // Survivor is at Block 2. + // Block 1 should be deleted. + // Count should be 1. + let counts = store.prune_earliest_state(b2).unwrap(); + + assert_eq!(counts.hashed_accounts_written_total, 1); + assert_eq!(counts.account_trie_updates_written_total, 0); + } + + #[test] + fn test_prune_earliest_state_empty_window_updates_pointer() { + let dir = TempDir::new().unwrap(); + let store = MdbxProofsStorage::new(dir.path()).expect("env"); + store.set_earliest_block_number(0, B256::ZERO).unwrap(); + + let target = BlockWithParent::new(B256::random(), NumHash::new(5, B256::random())); + + // Prune empty + store.prune_earliest_state(target).unwrap(); + + let earliest = store.get_earliest_block_number().unwrap(); + assert_eq!(earliest, Some((5, target.block.hash))); + } + + #[test] + fn test_prune_earliest_state_uninitialized_guard() { + let dir = TempDir::new().unwrap(); + let store = MdbxProofsStorage::new(dir.path()).expect("env"); + + // Earliest not set + let target = BlockWithParent::new(B256::random(), NumHash::new(5, B256::random())); + + let counts = store.prune_earliest_state(target).unwrap(); + assert_eq!(counts, WriteCounts::default()); + + // Check earliest is still None + assert_eq!(store.get_earliest_block_number().unwrap(), None); + } + + #[test] + fn test_block_change_set_crud_operations() { + let dir = TempDir::new().unwrap(); + let store = MdbxProofsStorage::new(dir.path()).expect("env"); + let tx = store.env.tx_mut().expect("rw tx"); + let mut cursor = tx.cursor_write::().expect("cursor"); + + let block_1 = 42u64; + let block_2 = 43u64; + + let entry1 = ChangeSet { + account_trie_keys: vec![StoredNibbles::default()], + storage_trie_keys: vec![], + hashed_account_keys: vec![B256::ZERO], + hashed_storage_keys: vec![], + }; + let entry2 = ChangeSet { + account_trie_keys: vec![], + storage_trie_keys: vec![StorageTrieKey::new(B256::ZERO, StoredNibbles::default())], + hashed_account_keys: vec![], + hashed_storage_keys: vec![HashedStorageKey::new(B256::ZERO, B256::ZERO)], + }; + + // Insert entries + cursor.insert(block_1, &entry1).unwrap(); + cursor.insert(block_2, &entry2).unwrap(); + + // Read entries + let mut walker = cursor.walk(Some(block_1)).unwrap(); + let mut entries = vec![walker.next().unwrap().unwrap().1]; + if let Some(Ok((_, val))) = walker.next() { + entries.push(val); + } + entries.sort(); + let mut expected = vec![entry1.clone(), entry2.clone()]; + expected.sort(); + assert_eq!(entries, expected); + + // Delete entry1 + let mut walker = cursor.walk(Some(block_1)).unwrap(); + while let Some(Ok((_, val))) = walker.next() { + if val == entry1 { + walker.delete_current().unwrap(); + break; + } + } + + // Verify delete + let mut walker = cursor.walk(Some(block_1)).unwrap(); + assert_eq!(walker.next().unwrap().unwrap().1, entry2); + assert!(walker.next().is_none()); + } + + #[test] + fn store_trie_updates_deleted_account_trie() { + let dir = TempDir::new().unwrap(); + let store = MdbxProofsStorage::new(dir.path()).expect("env"); + + const BLOCK: BlockWithParent = + BlockWithParent::new(B256::ZERO, NumHash::new(7, B256::ZERO)); + + // Prepare a BlockStateDiff that removes an account trie node at `acc_path` + let acc_path = Nibbles::from_nibbles_unchecked([0x0A, 0x0B, 0x0C]); + let mut diff_trie_updates = TrieUpdates::default(); + diff_trie_updates.removed_nodes.insert(acc_path); + let diff = BlockStateDiff { + sorted_trie_updates: diff_trie_updates.into_sorted(), + sorted_post_state: HashedPostStateSorted::default(), + }; + store.store_trie_updates(BLOCK, diff).expect("store"); + + // Verify deletion was written at BLOCK + let tx = store.env.tx().expect("tx"); + let mut cur = tx.new_cursor::().expect("cursor"); + let vv = cur + .seek_by_key_subkey(StoredNibbles::from(acc_path), BLOCK.block.number) + .expect("seek") + .expect("exists"); + assert_eq!(vv.block_number, BLOCK.block.number); + assert!(vv.value.0.is_none(), "expected account trie deletion"); + } + + #[test] + fn store_trie_updates_deleted_storage_trie() { + let dir = TempDir::new().unwrap(); + let store = MdbxProofsStorage::new(dir.path()).expect("env"); + + const BLOCK: BlockWithParent = + BlockWithParent::new(B256::ZERO, NumHash::new(8, B256::ZERO)); + + // Prepare a BlockStateDiff that removes a storage trie node for `addr` at `st_path` + let addr = B256::from([0xAB; 32]); + let st_path = Nibbles::from_nibbles_unchecked([0x01, 0x02, 0x03]); + + let mut diff_trie_updates = TrieUpdates::default(); + let mut st_updates = reth_trie::updates::StorageTrieUpdates::default(); + // mark this storage trie node as removed + st_updates.removed_nodes.insert(st_path); + diff_trie_updates.storage_tries.insert(addr, st_updates); + let diff = BlockStateDiff { + sorted_trie_updates: diff_trie_updates.into_sorted(), + sorted_post_state: HashedPostStateSorted::default(), + }; + store.store_trie_updates(BLOCK, diff).expect("store"); + + // Verify deletion was written at BLOCK + let tx = store.env.tx().expect("tx"); + let mut cur = tx.new_cursor::().expect("cursor"); + let key = StorageTrieKey::new(addr, StoredNibbles::from(st_path)); + let vv = cur + .seek_by_key_subkey(key, BLOCK.block.number) + .expect("seek") + .expect("exists"); + assert_eq!(vv.block_number, BLOCK.block.number); + assert!(vv.value.0.is_none(), "expected storage trie deletion"); + } + + #[test] + fn store_trie_updates_wiped_storage_trie_nodes() { + let dir = TempDir::new().unwrap(); + let store = MdbxProofsStorage::new(dir.path()).expect("env"); + + let addr_wiped = B256::from([0x10; 32]); + let addr_live = B256::from([0xF0; 32]); + + // Seed some storage-trie nodes at block 0 for the address that will be wiped. + let p1 = Nibbles::from_nibbles_unchecked([0x01, 0x02]); + let p2 = Nibbles::from_nibbles_unchecked([0x0A, 0x0B, 0x0C]); + let n1 = BranchNodeCompact::default(); + let n2 = BranchNodeCompact::default(); + + store + .store_storage_branches(addr_wiped, vec![(p1, Some(n1)), (p2, Some(n2))]) + .expect("seed wiped addr trie nodes"); + + // Build a BlockStateDiff that wipes addr_wiped's storage trie, and + // also adds a normal storage-trie node for addr_live. + const BLOCK: BlockWithParent = + BlockWithParent::new(B256::ZERO, NumHash::new(123, B256::ZERO)); + let mut diff_trie_updates = TrieUpdates::default(); + + // Wipe for addr_wiped + let mut wiped_updates = StorageTrieUpdates::default(); + wiped_updates.set_deleted(true); + diff_trie_updates + .storage_tries + .insert(addr_wiped, wiped_updates); + + // Normal update for addr_live + let live_path = Nibbles::from_nibbles_unchecked([0xEE, 0xFF]); + let live_node = BranchNodeCompact::default(); + let mut live_updates = StorageTrieUpdates::default(); + live_updates.storage_nodes.insert(live_path, live_node); + diff_trie_updates + .storage_tries + .insert(addr_live, live_updates); + + // Execute the store + let diff = BlockStateDiff { + sorted_trie_updates: diff_trie_updates.into_sorted(), + sorted_post_state: HashedPostStateSorted::default(), + }; + store.store_trie_updates(BLOCK, diff).expect("store"); + + // Verify: for addr_wiped, each previously existing path now has a deletion tombstone at + // BLOCK. + { + let tx = store.env.tx().expect("tx"); + let mut cur = tx.new_cursor::().expect("cursor"); + + for path in [p1, p2] { + let key = StorageTrieKey::new(addr_wiped, StoredNibbles::from(path)); + let vv = cur + .seek_by_key_subkey(key, BLOCK.block.number) + .expect("seek") + .expect("exists"); + assert_eq!(vv.block_number, BLOCK.block.number); + assert!( + vv.value.0.is_none(), + "expected tombstone at wipe block for path {path:?}" + ); + } + } + + // Verify: addr_live got its normal node written at BLOCK (not a deletion). + { + let tx = store.env.tx().expect("tx"); + let mut cur = tx.new_cursor::().expect("cursor"); + + let key = StorageTrieKey::new(addr_live, StoredNibbles::from(live_path)); + let vv = cur + .seek_by_key_subkey(key, BLOCK.block.number) + .expect("seek") + .expect("exists"); + assert_eq!(vv.block_number, BLOCK.block.number); + assert!( + vv.value.0.is_some(), + "expected normal node for non-wiped address at BLOCK" + ); + } + } + + /// Mirror of `tests/lib.rs::test_store_trie_updates_with_wiped_storage_and_new_slots` for + /// the storage *trie* path. When `StorageTrieUpdates::is_deleted` is true AND + /// `storage_nodes` is non-empty (the shape revm/reth produce when a contract is destroyed + /// and recreated with a fresh trie in the same block), the wipe branch must tombstone every + /// pre-existing path for the address AND persist the new post-recreation nodes — with new + /// nodes winning on path collision. + #[test] + fn store_trie_updates_wiped_storage_trie_with_new_nodes() { + let dir = TempDir::new().unwrap(); + let store = MdbxProofsStorage::new(dir.path()).expect("env"); + + let addr = B256::from([0x77; 32]); + let pre_path_dropped = Nibbles::from_nibbles_unchecked([0x01, 0x02]); + let pre_path_overwritten = Nibbles::from_nibbles_unchecked([0x0A, 0x0B]); + let new_path_kept = Nibbles::from_nibbles_unchecked([0xCC, 0xDD]); + + store + .store_storage_branches( + addr, + vec![ + (pre_path_dropped, Some(BranchNodeCompact::default())), + (pre_path_overwritten, Some(BranchNodeCompact::default())), + ], + ) + .expect("seed"); + + const BLOCK: BlockWithParent = + BlockWithParent::new(B256::ZERO, NumHash::new(123, B256::ZERO)); + let mut diff_trie_updates = TrieUpdates::default(); + let mut wiped_with_new = StorageTrieUpdates::default(); + wiped_with_new.set_deleted(true); + wiped_with_new + .storage_nodes + .insert(new_path_kept, BranchNodeCompact::default()); + wiped_with_new + .storage_nodes + .insert(pre_path_overwritten, BranchNodeCompact::default()); + diff_trie_updates.storage_tries.insert(addr, wiped_with_new); + + let diff = BlockStateDiff { + sorted_trie_updates: diff_trie_updates.into_sorted(), + sorted_post_state: HashedPostStateSorted::default(), + }; + store.store_trie_updates(BLOCK, diff).expect("store"); + + let tx = store.env.tx().expect("tx"); + let mut cur = tx.new_cursor::().expect("cursor"); + + let dropped_key = StorageTrieKey::new(addr, StoredNibbles::from(pre_path_dropped)); + let dropped_vv = cur + .seek_by_key_subkey(dropped_key, BLOCK.block.number) + .expect("seek") + .expect("tombstone exists"); + assert_eq!(dropped_vv.block_number, BLOCK.block.number); + assert!( + dropped_vv.value.0.is_none(), + "pre-existing path with no new node must tombstone" + ); + + let overwritten_key = StorageTrieKey::new(addr, StoredNibbles::from(pre_path_overwritten)); + let overwritten_vv = cur + .seek_by_key_subkey(overwritten_key, BLOCK.block.number) + .expect("seek") + .expect("overlay exists"); + assert_eq!(overwritten_vv.block_number, BLOCK.block.number); + assert!( + overwritten_vv.value.0.is_some(), + "collision path must reflect the new node, not the wipe tombstone", + ); + + let new_key = StorageTrieKey::new(addr, StoredNibbles::from(new_path_kept)); + let new_vv = cur + .seek_by_key_subkey(new_key, BLOCK.block.number) + .expect("seek") + .expect("new node exists"); + assert_eq!(new_vv.block_number, BLOCK.block.number); + assert!( + new_vv.value.0.is_some(), + "new path must be persisted, not dropped by the wipe" + ); + } + + #[test] + fn store_trie_updates_wiped_storage() { + let dir = TempDir::new().unwrap(); + let store = MdbxProofsStorage::new(dir.path()).expect("env"); + + // We'll pre-seed storage at block 0, then issue a wipe at BLOCK. + const BLOCK: BlockWithParent = + BlockWithParent::new(B256::ZERO, NumHash::new(42, B256::ZERO)); + + let addr = B256::from([0x55; 32]); + let s1 = B256::from([0x01; 32]); + let s2 = B256::from([0x02; 32]); + let v1 = U256::from(111u64); + let v2 = U256::from(222u64); + + // Seed prior storage (block_number = 0 in store_hashed_storages) + store + .store_hashed_storages(addr, vec![(s1, v1), (s2, v2)]) + .expect("seed"); + + // Build BlockStateDiff that marks this address as wiped at BLOCK + let mut diff_post_state = HashedPostState::default(); + + let wiped = reth_trie::HashedStorage::new(true); + + diff_post_state.storages.insert(addr, wiped); + + // Execute + let diff = BlockStateDiff { + sorted_trie_updates: TrieUpdatesSorted::default(), + sorted_post_state: diff_post_state.into_sorted(), + }; + store.store_trie_updates(BLOCK, diff).expect("store"); + + // Verify: for each pre-existing slot, there should be a tombstone (MaybeDeleted(None)) at + // BLOCK. + let tx = store.env.tx().expect("tx"); + let mut cur = tx.new_cursor::().expect("cursor"); + + for slot in [s1, s2] { + let key = HashedStorageKey::new(addr, slot); + let vv = cur + .seek_by_key_subkey(key, BLOCK.block.number) + .expect("seek") + .expect("exists"); + assert_eq!(vv.block_number, BLOCK.block.number); + assert!( + vv.value.0.is_none(), + "expected deletion tombstone for slot {:?} at block {}", + slot, + BLOCK.block.number, + ); + } + } + + #[test] + fn store_trie_updates_wiped_and_non_wiped_mixed_order() { + let dir = TempDir::new().unwrap(); + let store = MdbxProofsStorage::new(dir.path()).expect("env"); + + // Choose addresses so that wiped < non_wiped in sort order (your impl sorts by address) + let addr_wiped = B256::from([0x01; 32]); // will sort first + let addr_live = B256::from([0xF0; 32]); // will sort later + + // Slots & values + let ws1 = B256::from([0xA1; 32]); + let ws2 = B256::from([0xA2; 32]); + let wv1 = U256::from(111u64); + let wv2 = U256::from(222u64); + + let ls1 = B256::from([0xB1; 32]); + let lv1_old = U256::from(333u64); + let lv1_new = U256::from(999u64); // will be written at BLOCK + + // Seed prior storage at block 0 for BOTH addresses + store + .store_hashed_storages(addr_wiped, vec![(ws1, wv1), (ws2, wv2)]) + .expect("seed wiped addr"); + store + .store_hashed_storages(addr_live, vec![(ls1, lv1_old)]) + .expect("seed live addr"); + + // Build diff: wiped first (by address sort), then non-wiped with a write + const BLOCK: BlockWithParent = + BlockWithParent::new(B256::ZERO, NumHash::new(77, B256::ZERO)); + let mut diff_post_state = HashedPostState::default(); + + // Wiped storage for addr_wiped + let wiped = reth_trie::HashedStorage::new(true); + diff_post_state.storages.insert(addr_wiped, wiped); + + // Non-wiped storage for addr_live (append new value) + let mut live = reth_trie::HashedStorage::default(); + live.storage.insert(ls1, lv1_new); + diff_post_state.storages.insert(addr_live, live); + + // Execute + let diff = BlockStateDiff { + sorted_trie_updates: TrieUpdatesSorted::default(), + sorted_post_state: diff_post_state.into_sorted(), + }; + store.store_trie_updates(BLOCK, diff).expect("store"); + + // Verify: wiped address got tombstones at BLOCK for each pre-existing slot + { + let tx = store.env.tx().expect("tx"); + let mut cur = tx.new_cursor::().expect("cursor"); + for slot in [ws1, ws2] { + let key = HashedStorageKey::new(addr_wiped, slot); + let vv = cur + .seek_by_key_subkey(key, BLOCK.block.number) + .expect("seek") + .expect("exists"); + assert_eq!(vv.block_number, BLOCK.block.number); + assert!( + vv.value.0.is_none(), + "expected deletion tombstone for wiped slot {:?} at block {}", + slot, + BLOCK.block.number, + ); + } + } + + // Verify: non-wiped address got the new value at BLOCK (not a deletion) + { + let tx = store.env.tx().expect("tx"); + let mut cur = tx.new_cursor::().expect("cursor"); + let key = HashedStorageKey::new(addr_live, ls1); + let vv = cur + .seek_by_key_subkey(key, BLOCK.block.number) + .expect("seek") + .expect("exists"); + assert_eq!(vv.block_number, BLOCK.block.number); + let inner = vv.value.0.as_ref().expect("Some(StorageValue)"); + assert_eq!( + inner.0, lv1_new, + "expected updated value for non-wiped address" + ); + } + } + + #[test] + fn test_proof_window() { + let dir = TempDir::new().unwrap(); + let store = MdbxProofsStorage::new(dir.path()).expect("env"); + + // Test initial state (no values set) + let initial_value = store.get_earliest_block_number().expect("get earliest"); + assert_eq!(initial_value, None); + + // Test setting the value + let block_number = 42u64; + let hash = B256::random(); + store + .set_earliest_block_number(block_number, hash) + .expect("set earliest"); + + // Verify value was stored correctly + let retrieved = store.get_earliest_block_number().expect("get earliest"); + assert_eq!(retrieved, Some((block_number, hash))); + + // Test updating with new values + let new_block_number = 100u64; + let new_hash = B256::random(); + store + .set_earliest_block_number(new_block_number, new_hash) + .expect("update earliest"); + + // Verify update worked + let updated = store + .get_earliest_block_number() + .expect("get updated earliest"); + assert_eq!(updated, Some((new_block_number, new_hash))); + + // Verify that latest_block falls back to earliest when not set + let latest = store.get_latest_block_number().expect("get latest"); + assert_eq!( + latest, + Some((new_block_number, new_hash)), + "Latest block should fall back to earliest when not explicitly set" + ); + } + + #[test] + fn replace_updates_prunes_and_adds_new_chain() { + let dir = tempfile::TempDir::new().unwrap(); + let store = MdbxProofsStorage::new(dir.path()).expect("env"); + + // Test address and helper to make diffs with distinct nonces. + let addr = B256::from([0xAB; 32]); + let make_diff = |nonce: u64| { + let mut d_post_state = HashedPostState::default(); + d_post_state.accounts.insert( + addr, + Some(Account { + nonce, + ..Default::default() + }), + ); + BlockStateDiff { + sorted_trie_updates: TrieUpdatesSorted::default(), + sorted_post_state: d_post_state.into_sorted(), + } + }; + + // --- Build initial canonical chain: 1 -> 2 -> 3 --- + let b1 = BlockWithParent::new(B256::ZERO, NumHash::new(1, B256::random())); + let b2 = BlockWithParent::new(b1.block.hash, NumHash::new(2, B256::random())); + let b3 = BlockWithParent::new(b2.block.hash, NumHash::new(3, B256::random())); + + store + .store_trie_updates(b1, make_diff(10)) + .expect("store b1"); + store + .store_trie_updates(b2, make_diff(20)) + .expect("store b2"); + store + .store_trie_updates(b3, make_diff(30)) + .expect("store b3"); + + // Sanity: entries for 1,2,3 exist with expected nonces. + { + let tx = store.env.tx().expect("tx"); + let mut cur = tx.new_cursor::().expect("cursor"); + let v1 = cur + .seek_by_key_subkey(addr, 1) + .expect("seek") + .expect("exists"); + let v2 = cur + .seek_by_key_subkey(addr, 2) + .expect("seek") + .expect("exists"); + let v3 = cur + .seek_by_key_subkey(addr, 3) + .expect("seek") + .expect("exists"); + assert_eq!(v1.value.0.unwrap().nonce, 10); + assert_eq!(v2.value.0.unwrap().nonce, 20); + assert_eq!(v3.value.0.unwrap().nonce, 30); + } + + // --- Reorg at LCA = 2: prune >2, then add 3' and 4' --- + let b3p = BlockWithParent::new(b2.block.hash, NumHash::new(3, B256::random())); // 3' + let b4p = BlockWithParent::new(b3p.block.hash, NumHash::new(4, B256::random())); // 4' + + // Build blocks_to_add vec. + let blocks_to_add = vec![(b3p, make_diff(300)), (b4p, make_diff(400))]; + + store + .replace_updates(BlockNumHash::new(2, b2.block.hash), blocks_to_add) + .expect("replace_updates succeeds"); + + // --- Verify post-conditions --- + + // 1) HashedAccountHistory: blocks 1,2 remain; block 3 was replaced (nonce=300); block 4 + // exists (nonce=400). + { + let tx = store.env.tx().expect("tx"); + let mut cur = tx.new_cursor::().expect("cursor"); + + let v1 = cur + .seek_by_key_subkey(addr, 1) + .expect("seek") + .expect("exists"); + assert_eq!(v1.value.0.unwrap().nonce, 10); + + let v2 = cur + .seek_by_key_subkey(addr, 2) + .expect("seek") + .expect("exists"); + assert_eq!(v2.value.0.unwrap().nonce, 20); + + // Old block 3 (nonce=30) should have been pruned; we should now have a fresh entry at 3 + // with nonce=300. + let v3_new = cur + .seek_by_key_subkey(addr, 3) + .expect("seek") + .expect("replaced exists"); + assert_eq!( + v3_new.value.0.unwrap().nonce, + 300, + "block 3 should be replaced by new chain" + ); + + // New block 4 should exist. + let v4 = cur + .seek_by_key_subkey(addr, 4) + .expect("seek") + .expect("exists"); + assert_eq!(v4.value.0.unwrap().nonce, 400); + } + + // 2) BlockChangeSet contains exactly entries for block numbers {1,2,3,4}. + { + let tx = store.env.tx().expect("tx"); + let mut cur = tx.new_cursor::().expect("cursor"); + let mut seen = std::collections::BTreeSet::new(); + let mut it = cur.walk(Some(1)).expect("walk"); + while let Some(Ok((bn, _))) = it.next() { + seen.insert(bn); + } + let expected: std::collections::BTreeSet = [1u64, 2, 3, 4].into_iter().collect(); + assert_eq!( + seen, expected, + "BlockChangeSet should reflect pruned+new chain" + ); + } + } + + #[test] + fn test_unwind_history_basic() { + let dir = TempDir::new().unwrap(); + let store = MdbxProofsStorage::new(dir.path()).expect("env"); + + let addr = B256::random(); + let make_diff = |nonce: u64| { + let mut d_post_state = HashedPostState::default(); + d_post_state.accounts.insert( + addr, + Some(Account { + nonce, + ..Default::default() + }), + ); + BlockStateDiff { + sorted_trie_updates: TrieUpdatesSorted::default(), + sorted_post_state: d_post_state.into_sorted(), + } + }; + + // Build chain: blocks 1 -> 2 -> 3 -> 4 + let b0 = NumHash::new(0, B256::random()); + let b1 = BlockWithParent::new(b0.hash, NumHash::new(1, B256::random())); + let b2 = BlockWithParent::new(b1.block.hash, NumHash::new(2, B256::random())); + let b3 = BlockWithParent::new(b2.block.hash, NumHash::new(3, B256::random())); + let b4 = BlockWithParent::new(b3.block.hash, NumHash::new(4, B256::random())); + + store + .set_earliest_block_number_hash(b0.number, b0.hash) + .expect("set earliest"); + store + .store_trie_updates(b1, make_diff(10)) + .expect("store b1"); + store + .store_trie_updates(b2, make_diff(20)) + .expect("store b2"); + store + .store_trie_updates(b3, make_diff(30)) + .expect("store b3"); + store + .store_trie_updates(b4, make_diff(40)) + .expect("store b4"); + + // Unwind to block 2 + store.unwind_history(b2).expect("unwind"); + + // Verify: blocks 1 and 2 remain, blocks 3 and 4 are removed + let tx = store.env.tx().expect("tx"); + let mut cur = tx.new_cursor::().expect("cursor"); + + assert!( + cur.seek_by_key_subkey(addr, 1).unwrap().is_some(), + "Block 1 should remain" + ); + assert!( + cur.seek_by_key_subkey(addr, 2).unwrap().is_none(), + "Block 2 should be removed" + ); + assert!( + cur.seek_by_key_subkey(addr, 3).unwrap().is_none(), + "Block 3 should be removed" + ); + assert!( + cur.seek_by_key_subkey(addr, 4).unwrap().is_none(), + "Block 4 should be removed" + ); + + // Verify ProofWindow LatestBlock is updated + let mut proof_window_cur = tx.cursor_read::().expect("cursor"); + let latest = proof_window_cur + .seek_exact(ProofWindowKey::LatestBlock) + .expect("seek") + .expect("latest exists"); + assert_eq!(latest.1.number(), 1); + assert_eq!(*latest.1.hash(), b2.parent); + } + + #[test] + fn test_unwind_history_to_earliest() { + let dir = TempDir::new().unwrap(); + let store = MdbxProofsStorage::new(dir.path()).expect("env"); + + let addr = B256::random(); + let make_diff = |nonce: u64| { + let mut d_post_state = HashedPostState::default(); + d_post_state.accounts.insert( + addr, + Some(Account { + nonce, + ..Default::default() + }), + ); + BlockStateDiff { + sorted_trie_updates: TrieUpdatesSorted::default(), + sorted_post_state: d_post_state.into_sorted(), + } + }; + + // Build chain: blocks 1 -> 2 -> 3 + let b1 = BlockWithParent::new(B256::ZERO, NumHash::new(1, B256::random())); + let b2 = BlockWithParent::new(b1.block.hash, NumHash::new(2, B256::random())); + let b3 = BlockWithParent::new(b2.block.hash, NumHash::new(3, B256::random())); + + store + .set_earliest_block_number_hash(b1.block.number, b1.block.hash) + .expect("set earliest"); + store + .store_trie_updates(b2, make_diff(20)) + .expect("store b2"); + store + .store_trie_updates(b3, make_diff(30)) + .expect("store b3"); + + // Unwind to block b1 + let res = store.unwind_history(b1); + // should fail as we cannot unwind past earliest block + assert!(res.is_err(), "unwind to earliest block should error"); + assert!(matches!( + res.unwrap_err(), + MorphProofsStorageError::UnwindBeyondEarliest { .. } + )); + } + + #[test] + fn test_unwind_history_with_storage() { + let dir = TempDir::new().unwrap(); + let store = MdbxProofsStorage::new(dir.path()).expect("env"); + + let addr = B256::random(); + let slot = B256::random(); + + let make_diff = |nonce: u64, slot_value: u64| { + let mut d_post_state = HashedPostState::default(); + d_post_state.accounts.insert( + addr, + Some(Account { + nonce, + ..Default::default() + }), + ); + let mut storage = HashedStorage::default(); + storage.storage.insert(slot, U256::from(slot_value)); + d_post_state.storages.insert(addr, storage); + BlockStateDiff { + sorted_trie_updates: TrieUpdatesSorted::default(), + sorted_post_state: d_post_state.into_sorted(), + } + }; + + // Build chain with storage changes + let b0 = BlockWithParent::new(B256::ZERO, NumHash::new(0, B256::random())); + let b1 = BlockWithParent::new(b0.block.hash, NumHash::new(1, B256::random())); + let b2 = BlockWithParent::new(b1.block.hash, NumHash::new(2, B256::random())); + let b3 = BlockWithParent::new(b2.block.hash, NumHash::new(3, B256::random())); + + store + .set_earliest_block_number_hash(b0.block.number, b0.block.hash) + .expect("set earliest"); + store + .store_trie_updates(b1, make_diff(10, 100)) + .expect("store b1"); + store + .store_trie_updates(b2, make_diff(20, 200)) + .expect("store b2"); + store + .store_trie_updates(b3, make_diff(30, 300)) + .expect("store b3"); + + // Unwind to block 1 + store.unwind_history(b2).expect("unwind"); + + // Verify account history + let tx = store.env.tx().expect("tx"); + let mut acc_cur = tx.new_cursor::().expect("cursor"); + assert!( + acc_cur.seek_by_key_subkey(addr, 1).unwrap().is_some(), + "Block 1 should remain" + ); + assert!( + acc_cur.seek_by_key_subkey(addr, 2).unwrap().is_none(), + "Block 2 should be removed" + ); + assert!( + acc_cur.seek_by_key_subkey(addr, 3).unwrap().is_none(), + "Block 3 should be removed" + ); + + // Verify storage history + let mut storage_cur = tx.new_cursor::().expect("cursor"); + let storage_key = HashedStorageKey::new(addr, slot); + assert!( + storage_cur + .seek_by_key_subkey(storage_key.clone(), 1) + .unwrap() + .is_some(), + "Storage at block 1 should remain" + ); + assert!( + storage_cur + .seek_by_key_subkey(storage_key.clone(), 2) + .unwrap() + .is_none(), + "Storage at block 2 should be removed" + ); + assert!( + storage_cur + .seek_by_key_subkey(storage_key, 3) + .unwrap() + .is_none(), + "Storage at block 3 should be removed" + ); + } + + #[test] + fn test_unwind_history_with_trie_nodes() { + let dir = TempDir::new().unwrap(); + let store = MdbxProofsStorage::new(dir.path()).expect("env"); + + let path1 = Nibbles::from_nibbles_unchecked([0x01]); + let path2 = Nibbles::from_nibbles_unchecked([0x02]); + let node1 = BranchNodeCompact::new(0b1, 0, 0, vec![], Some(B256::random())); + let node2 = BranchNodeCompact::new(0b10, 0, 0, vec![], Some(B256::random())); + + let make_diff = |path: Nibbles, node: BranchNodeCompact| { + let mut d_trie_updates = TrieUpdates::default(); + d_trie_updates.account_nodes.insert(path, node); + BlockStateDiff { + sorted_trie_updates: d_trie_updates.into_sorted(), + sorted_post_state: HashedPostStateSorted::default(), + } + }; + + // Build chain with trie updates + let b0 = NumHash::new(0, B256::random()); + let b1 = BlockWithParent::new(b0.hash, NumHash::new(1, B256::random())); + let b2 = BlockWithParent::new(b1.block.hash, NumHash::new(2, B256::random())); + let b3 = BlockWithParent::new(b2.block.hash, NumHash::new(3, B256::random())); + + store + .set_earliest_block_number_hash(b0.number, b0.hash) + .expect("set earliest"); + store + .store_trie_updates(b1, make_diff(path1, node1)) + .expect("store b1"); + store + .store_trie_updates(b2, make_diff(path2, node2.clone())) + .expect("store b2"); + store + .store_trie_updates(b3, make_diff(path1, node2)) + .expect("store b3"); + + // Unwind to block 1 + store.unwind_history(b2).expect("unwind"); + + // Verify trie node history + let tx = store.env.tx().expect("tx"); + let mut trie_cur = tx.cursor_dup_read::().expect("cursor"); + + assert!( + trie_cur + .seek_by_key_subkey(StoredNibbles::from(path1), 1) + .unwrap() + .is_some(), + "Trie node at block 1 should remain" + ); + assert!( + trie_cur + .seek_by_key_subkey(StoredNibbles::from(path2), 2) + .unwrap() + .is_none(), + "Trie node at block 2 should be removed" + ); + assert!( + trie_cur + .seek_by_key_subkey(StoredNibbles::from(path1), 3) + .unwrap() + .is_none(), + "Trie node at block 3 should be removed" + ); + } + + #[test] + fn test_unwind_history_comprehensive() { + let dir = TempDir::new().unwrap(); + let store = MdbxProofsStorage::new(dir.path()).expect("env"); + + // Setup comprehensive scenario with accounts, storage, and trie nodes + let addr1 = B256::random(); + let addr2 = B256::random(); + let slot1 = B256::random(); + let slot2 = B256::random(); + let path1 = Nibbles::from_nibbles_unchecked([0x01]); + let path2 = Nibbles::from_nibbles_unchecked([0x02]); + let storage_path1 = Nibbles::from_nibbles_unchecked([0x03]); + + let acc1 = Account { + nonce: 1, + balance: U256::from(100), + ..Default::default() + }; + let acc2 = Account { + nonce: 2, + balance: U256::from(200), + ..Default::default() + }; + let node1 = BranchNodeCompact::new(0b1, 0, 0, vec![], Some(B256::random())); + let node2 = BranchNodeCompact::new(0b10, 0, 0, vec![], Some(B256::random())); + let storage_node1 = BranchNodeCompact::new(0b100, 0, 0, vec![], Some(B256::random())); + + // Block 0: Set earliest block + let b0 = NumHash::new(0, B256::random()); + store + .set_earliest_block_number_hash(b0.number, b0.hash) + .expect("set earliest"); + + // Block 1: Insert multiple types of data + let b1 = BlockWithParent::new(b0.hash, NumHash::new(1, B256::random())); + let mut diff1_trie_updates = TrieUpdates::default(); + let mut diff1_post_state = HashedPostState::default(); + diff1_post_state.accounts.insert(addr1, Some(acc1)); + diff1_trie_updates.account_nodes.insert(path1, node1); + let mut storage1 = HashedStorage::default(); + storage1.storage.insert(slot1, U256::from(1111)); + diff1_post_state.storages.insert(addr1, storage1); + let mut storage_updates1 = StorageTrieUpdates::default(); + storage_updates1 + .storage_nodes + .insert(storage_path1, storage_node1); + diff1_trie_updates + .storage_tries + .insert(addr1, storage_updates1); + let diff1 = BlockStateDiff { + sorted_trie_updates: diff1_trie_updates.into_sorted(), + sorted_post_state: diff1_post_state.into_sorted(), + }; + store.store_trie_updates(b1, diff1).expect("store b1"); + + // Block 2: More updates + let b2 = BlockWithParent::new(b1.block.hash, NumHash::new(2, B256::random())); + let mut diff2_trie_updates = TrieUpdates::default(); + let mut diff2_post_state = HashedPostState::default(); + diff2_post_state.accounts.insert(addr2, Some(acc2)); + diff2_trie_updates.account_nodes.insert(path2, node2); + let mut storage2 = HashedStorage::default(); + storage2.storage.insert(slot2, U256::from(2222)); + diff2_post_state.storages.insert(addr2, storage2); + let diff2 = BlockStateDiff { + sorted_trie_updates: diff2_trie_updates.into_sorted(), + sorted_post_state: diff2_post_state.into_sorted(), + }; + store.store_trie_updates(b2, diff2).expect("store b2"); + + // Block 3: Additional updates + let b3 = BlockWithParent::new(b2.block.hash, NumHash::new(3, B256::random())); + let mut diff3_post_state = HashedPostState::default(); + diff3_post_state.accounts.insert(addr1, Some(acc2)); // update addr1 + let diff3 = BlockStateDiff { + sorted_trie_updates: TrieUpdatesSorted::default(), + sorted_post_state: diff3_post_state.into_sorted(), + }; + store.store_trie_updates(b3, diff3).expect("store b3"); + + // Unwind to block 1 + store.unwind_history(b2).expect("unwind"); + + let tx = store.env.tx().expect("tx"); + + // Verify account history: block 1 remains, blocks 2 and 3 removed + let mut acc_cur = tx.new_cursor::().expect("cursor"); + assert!(acc_cur.seek_by_key_subkey(addr1, 1).unwrap().is_some()); + assert!(acc_cur.seek_by_key_subkey(addr2, 2).unwrap().is_none()); + assert!(acc_cur.seek_by_key_subkey(addr1, 3).unwrap().is_none()); + + // Verify trie history + let mut trie_cur = tx.cursor_dup_read::().expect("cursor"); + assert!( + trie_cur + .seek_by_key_subkey(StoredNibbles::from(path1), 1) + .unwrap() + .is_some() + ); + assert!( + trie_cur + .seek_by_key_subkey(StoredNibbles::from(path2), 2) + .unwrap() + .is_none() + ); + + // Verify storage history + let mut storage_cur = tx.new_cursor::().expect("cursor"); + assert!( + storage_cur + .seek_by_key_subkey(HashedStorageKey::new(addr1, slot1), 1) + .unwrap() + .is_some() + ); + assert!( + storage_cur + .seek_by_key_subkey(HashedStorageKey::new(addr2, slot2), 2) + .unwrap() + .is_none() + ); + + // Verify storage trie history + let mut storage_trie_cur = tx.cursor_dup_read::().expect("cursor"); + assert!( + storage_trie_cur + .seek_by_key_subkey( + StorageTrieKey::new(addr1, StoredNibbles::from(storage_path1)), + 1 + ) + .unwrap() + .is_some() + ); + + // Verify ProofWindow LatestBlock is updated + let mut proof_window_cur = tx.cursor_read::().expect("cursor"); + let latest = proof_window_cur + .seek_exact(ProofWindowKey::LatestBlock) + .expect("seek") + .expect("latest exists"); + assert_eq!(latest.1.number(), 1); + assert_eq!(*latest.1.hash(), b1.block.hash); + + // Verify change sets are removed for blocks > 1 + let mut change_cur = tx.new_cursor::().expect("cursor"); + assert!( + change_cur.seek_exact(1).unwrap().is_some(), + "Block 1 changeset should remain" + ); + assert!( + change_cur.seek_exact(2).unwrap().is_none(), + "Block 2 changeset should be removed" + ); + assert!( + change_cur.seek_exact(3).unwrap().is_none(), + "Block 3 changeset should be removed" + ); + } + + #[test] + fn test_unwind_history_empty_chain() { + let dir = TempDir::new().unwrap(); + let store = MdbxProofsStorage::new(dir.path()).expect("env"); + + // Try to unwind when there's nothing stored yet + let unwind_to = BlockWithParent::new(B256::ZERO, NumHash::new(0, B256::ZERO)); + let result = store.unwind_history(unwind_to); + + // Should succeed (no-op) + assert!(result.is_ok(), "Unwinding empty chain should succeed"); + } + + #[test] + fn test_unwind_history_idempotent() { + let dir = TempDir::new().unwrap(); + let store = MdbxProofsStorage::new(dir.path()).expect("env"); + + let addr = B256::random(); + let make_diff = |nonce: u64| { + let mut d_post_state = HashedPostState::default(); + d_post_state.accounts.insert( + addr, + Some(Account { + nonce, + ..Default::default() + }), + ); + BlockStateDiff { + sorted_trie_updates: TrieUpdatesSorted::default(), + sorted_post_state: d_post_state.into_sorted(), + } + }; + + // Build chain: blocks 1 -> 2 -> 3 + let b0 = NumHash::new(0, B256::random()); + let b1 = BlockWithParent::new(b0.hash, NumHash::new(1, B256::random())); + let b2 = BlockWithParent::new(b1.block.hash, NumHash::new(2, B256::random())); + let b3 = BlockWithParent::new(b2.block.hash, NumHash::new(3, B256::random())); + + store + .set_earliest_block_number_hash(b0.number, b0.hash) + .expect("set earliest"); + store + .store_trie_updates(b1, make_diff(10)) + .expect("store b1"); + store + .store_trie_updates(b2, make_diff(20)) + .expect("store b2"); + store + .store_trie_updates(b3, make_diff(30)) + .expect("store b3"); + + // Unwind to block 1 + store.unwind_history(b2).expect("first unwind"); + + // Unwind again to the same block (should be idempotent) + store.unwind_history(b2).expect("second unwind"); + + // Verify state is still correct + let tx = store.env.tx().expect("tx"); + let mut cur = tx.new_cursor::().expect("cursor"); + + assert!( + cur.seek_by_key_subkey(addr, 1).unwrap().is_some(), + "Block 1 should remain" + ); + assert!( + cur.seek_by_key_subkey(addr, 2).unwrap().is_none(), + "Block 2 should be removed" + ); + assert!( + cur.seek_by_key_subkey(addr, 3).unwrap().is_none(), + "Block 3 should be removed" + ); + } + + #[test] + fn test_unwind_history_beyond_latest() { + let dir = TempDir::new().unwrap(); + let store = MdbxProofsStorage::new(dir.path()).expect("env"); + + let addr = B256::random(); + let make_diff = |nonce: u64| { + let mut d_post_state = HashedPostState::default(); + d_post_state.accounts.insert( + addr, + Some(Account { + nonce, + ..Default::default() + }), + ); + BlockStateDiff { + sorted_trie_updates: TrieUpdatesSorted::default(), + sorted_post_state: d_post_state.into_sorted(), + } + }; + + // Build chain: blocks 1 -> 2 -> 3 + let b0 = NumHash::new(0, B256::random()); + let b1 = BlockWithParent::new(b0.hash, NumHash::new(1, B256::random())); + let b2 = BlockWithParent::new(b1.block.hash, NumHash::new(2, B256::random())); + let b3 = BlockWithParent::new(b2.block.hash, NumHash::new(3, B256::random())); + let b4 = BlockWithParent::new(b3.block.hash, NumHash::new(4, B256::random())); + let b5 = BlockWithParent::new(b4.block.hash, NumHash::new(5, B256::random())); + + store + .set_earliest_block_number_hash(b0.number, b0.hash) + .expect("set earliest"); + store + .store_trie_updates(b1, make_diff(10)) + .expect("store b1"); + store + .store_trie_updates(b2, make_diff(20)) + .expect("store b2"); + store + .store_trie_updates(b3, make_diff(30)) + .expect("store b3"); + + // Unwind to block 1 + store.unwind_history(b5).expect("first unwind"); + + // Verify state is still correct + let tx = store.env.tx().expect("tx"); + let mut cur = tx.new_cursor::().expect("cursor"); + + assert!( + cur.seek_by_key_subkey(addr, 1).unwrap().is_some(), + "Block 1 should remain" + ); + assert!( + cur.seek_by_key_subkey(addr, 2).unwrap().is_some(), + "Block 2 should remain" + ); + assert!( + cur.seek_by_key_subkey(addr, 3).unwrap().is_some(), + "Block 3 should remain" + ); + + let mut proof_window_cur = tx.cursor_read::().expect("cursor"); + let latest = proof_window_cur + .seek_exact(ProofWindowKey::LatestBlock) + .expect("seek") + .expect("latest exists"); + assert_eq!(latest.1.number(), b3.block.number); + assert_eq!(*latest.1.hash(), b3.block.hash); + } +} diff --git a/crates/proofs/src/error.rs b/crates/proofs/src/error.rs new file mode 100644 index 00000000..03a3aee6 --- /dev/null +++ b/crates/proofs/src/error.rs @@ -0,0 +1,233 @@ +//! Errors interfacing with [`MorphProofsStore`](crate::MorphProofsStore) type. + +use std::sync::Arc; + +use alloy_primitives::B256; +use reth_codecs::DecompressError; +use reth_db::DatabaseError; +use reth_execution_errors::BlockExecutionError; +use reth_provider::ProviderError; +use reth_trie_common::Nibbles; +use thiserror::Error; + +/// Error type for storage operations +#[derive(Debug, Clone, Error)] +pub enum MorphProofsStorageError { + /// The proof DB belongs to another canonical chain. + #[error("proof database chain identity mismatch: {0}")] + ChainIdentityMismatch(&'static str), + /// The proof DB schema cannot be read by this binary. + #[error("proof database schema mismatch: expected {expected}, found {actual}")] + SchemaMismatch { + /// Schema expected by this binary. + expected: u32, + /// Schema persisted in the database. + actual: u32, + }, + /// Durable metadata is missing or malformed. + #[error("corrupt proof database metadata: {0}")] + CorruptMetadata(&'static str), + /// Only part of the durable identity metadata is present. + #[error( + "proof database identity metadata is incomplete; manually delete the proof database before reinitializing" + )] + IncompleteIdentityMetadata, + /// Proof/history data exists without the metadata needed to validate its identity. + #[error( + "proof database contains proof/history data but identity metadata is missing; manually delete the proof database before reinitializing" + )] + ProofDataWithoutIdentityMetadata, + /// No blocks found + #[error("No blocks found")] + NoBlocksFound, + /// Parent block number is less than earliest stored block number + #[error("Parent block number is less than earliest stored block number")] + UnknownParent, + /// Block is out of order + #[error( + "Block {block_number} is out of order (parent: {parent_block_hash}, latest stored block hash: {latest_block_hash})" + )] + OutOfOrder { + /// The block number being inserted + block_number: u64, + /// The parent hash of the block being inserted + parent_block_hash: B256, + /// block hash of the latest stored block + latest_block_hash: B256, + }, + /// Block update failed since parent state + #[error( + "Cannot execute block updates for block {block_number} without parent state {parent_block_number} (latest stored block number: {latest_block_number})" + )] + MissingParentBlock { + /// The block number being executed + block_number: u64, + /// The parent state of the block being executed + parent_block_number: u64, + /// Latest stored block number + latest_block_number: u64, + }, + /// State root mismatch + #[error( + "State root mismatch for block {block_number} (have: {current_state_hash}, expected: {expected_state_hash})" + )] + StateRootMismatch { + /// Block number + block_number: u64, + /// Have state root + current_state_hash: B256, + /// Expected state root + expected_state_hash: B256, + }, + /// No change set for block + #[error("No change set found for block {0}")] + NoChangeSetForBlock(u64), + /// Missing account trie history for a specific path at a specific block number + #[error("Missing account trie history for path {0:?} at block {1}")] + MissingAccountTrieHistory(Nibbles, u64), + /// Missing storage trie history for a specific address and path at a specific block number + #[error("Missing storage trie history for address {0:?}, path {1:?} at block {2}")] + MissingStorageTrieHistory(B256, Nibbles, u64), + /// Missing hashed account history for a specific key at a specific block number + #[error("Missing hashed account history for key {0:?} at block {1}")] + MissingHashedAccountHistory(B256, u64), + /// Missing hashed storage history for a specific address and key at a specific block number + #[error( + "Missing hashed storage history for address {hashed_address:?}, key {hashed_storage_key:?} at block {block_number}" + )] + MissingHashedStorageHistory { + /// The hashed address + hashed_address: B256, + /// The hashed storage key + hashed_storage_key: B256, + /// The block number + block_number: u64, + }, + /// Attempted to unwind to a block beyond the earliest stored block + #[error( + "Attempted to unwind to block {unwind_block_number} beyond earliest stored block {earliest_block_number}" + )] + UnwindBeyondEarliest { + /// The block number being unwound to + unwind_block_number: u64, + /// The earliest stored block number + earliest_block_number: u64, + }, + /// Error occurred while interacting with the database. + #[error(transparent)] + DatabaseError(DatabaseError), + /// Error occurred while trying to acquire a lock. + #[error("failed lock attempt")] + TryLockError, + /// Error occurred during block execution. + #[error(transparent)] + ExecutionError(Arc), + /// Error occurred while interacting with the provider. + #[error(transparent)] + ProviderError(Arc), + /// Initialization detected inconsistent state between proofs storage and source DB. + #[error( + "Initialization Proofs storage detected inconsistent state. Storage does not match source DB. \ + Please clear proofs data and retry initialization." + )] + InitializeStorageInconsistentState, + /// Batch session used after its underlying transaction has been committed or aborted. + #[error("Batch session used after its underlying transaction was closed")] + BatchSessionClosed, +} + +impl From for MorphProofsStorageError { + fn from(error: BlockExecutionError) -> Self { + Self::ExecutionError(Arc::new(error)) + } +} + +impl From for MorphProofsStorageError { + fn from(error: ProviderError) -> Self { + Self::ProviderError(Arc::new(error)) + } +} + +impl From for DatabaseError { + fn from(error: MorphProofsStorageError) -> Self { + match error { + MorphProofsStorageError::DatabaseError(err) => err, + _ => Self::Custom(Arc::new(error)), + } + } +} + +impl From for MorphProofsStorageError { + fn from(error: DatabaseError) -> Self { + if let DatabaseError::Custom(ref err) = error + && let Some(err) = err.downcast_ref::() + { + return err.clone(); + } + Self::DatabaseError(error) + } +} + +impl From for MorphProofsStorageError { + fn from(error: DecompressError) -> Self { + Self::DatabaseError(DatabaseError::Other(format!("decompress error: {error}"))) + } +} + +/// Result type for storage operations +pub type MorphProofsStorageResult = Result; + +#[cfg(test)] +mod tests { + use reth_execution_errors::BlockValidationError; + + use super::*; + + #[test] + fn test_morph_proofs_store_error_to_db_error() { + let original_error = MorphProofsStorageError::NoBlocksFound; + + let db_error = DatabaseError::from(original_error); + assert!(matches!(db_error, DatabaseError::Custom(_))); + + let converted_error = MorphProofsStorageError::from(db_error); + assert!(matches!( + converted_error, + MorphProofsStorageError::NoBlocksFound + )) + } + + #[test] + fn test_db_error_to_morph_proofs_store_error() { + let original_error = DatabaseError::Decode; + + let morph_proofs_store_error = MorphProofsStorageError::from(original_error); + assert!(matches!( + morph_proofs_store_error, + MorphProofsStorageError::DatabaseError(DatabaseError::Decode) + )); + + let converted_error = DatabaseError::from(morph_proofs_store_error); + assert!(matches!(converted_error, DatabaseError::Decode)) + } + + #[test] + fn test_conversion_from_block_execution_error() { + let block_execution_error = + BlockExecutionError::Validation(BlockValidationError::IncrementBalanceFailed); + + let morph_proofs_store_error = MorphProofsStorageError::from(block_execution_error); + assert!( + matches!(morph_proofs_store_error, MorphProofsStorageError::ExecutionError(err) if matches!(*err, BlockExecutionError::Validation(BlockValidationError::IncrementBalanceFailed))) + ) + } + + #[test] + fn test_conversion_from_provider_error() { + let provider_error = ProviderError::SenderRecoveryError; + let morph_proofs_store_error = MorphProofsStorageError::from(provider_error); + assert!( + matches!(morph_proofs_store_error, MorphProofsStorageError::ProviderError(err) if matches!(*err, ProviderError::SenderRecoveryError)) + ) + } +} diff --git a/crates/proofs/src/in_memory.rs b/crates/proofs/src/in_memory.rs new file mode 100644 index 00000000..08ac6787 --- /dev/null +++ b/crates/proofs/src/in_memory.rs @@ -0,0 +1,1138 @@ +//! In-memory implementation of [`MorphProofsStore`] for testing purposes + +use std::{collections::BTreeMap, sync::Arc}; + +use alloy_eips::{BlockNumHash, NumHash, eip1898::BlockWithParent}; +use alloy_primitives::{B256, U256}; +use parking_lot::RwLock; +use reth_db::DatabaseError; +use reth_primitives_traits::Account; +use reth_trie::{ + hashed_cursor::{HashedCursor, HashedStorageCursor}, + trie_cursor::{TrieCursor, TrieStorageCursor}, +}; +use reth_trie_common::{ + BranchNodeCompact, HashedPostStateSorted, Nibbles, StoredNibbles, updates::TrieUpdatesSorted, +}; + +use crate::{ + BlockStateDiff, MorphProofsStorageError, MorphProofsStorageResult, MorphProofsStore, + api::{ + InitialStateAnchor, InitialStateStatus, MorphProofsBatchSession, MorphProofsBatchStore, + MorphProofsInitialStateStore, WriteCounts, + }, + db::{HashedStorageKey, StorageTrieKey}, +}; + +/// In-memory implementation of [`MorphProofsStore`] for testing purposes +#[derive(Debug, Clone)] +pub struct InMemoryProofsStorage { + /// Shared state across all instances + inner: Arc>, +} + +#[derive(Debug, Default)] +struct InMemoryStorageInner { + /// Account trie branches: (`block_number`, path) -> `branch_node` + account_branches: BTreeMap<(u64, Nibbles), Option>, + + /// Storage trie branches: (`block_number`, `hashed_address`, path) -> `branch_node` + storage_branches: BTreeMap<(u64, B256, Nibbles), Option>, + + /// Hashed accounts: (`block_number`, `hashed_address`) -> account + hashed_accounts: BTreeMap<(u64, B256), Option>, + + /// Hashed storages: (`block_number`, `hashed_address`, `hashed_slot`) -> value + hashed_storages: BTreeMap<(u64, B256, B256), U256>, + + /// Trie updates by block number + trie_updates: BTreeMap, + + /// Post state by block number + post_states: BTreeMap, + + /// Earliest block number and hash + earliest_block: Option<(u64, B256)>, + + /// The anchor block (initial state) of the store. + anchor_block: Option<(u64, B256)>, +} + +impl InMemoryStorageInner { + fn store_trie_updates( + &mut self, + block_number: u64, + block_state_diff: BlockStateDiff, + ) -> WriteCounts { + let mut result = WriteCounts::default(); + + // Store account branch nodes + for (path, branch) in block_state_diff.sorted_trie_updates.account_nodes_ref() { + self.account_branches + .insert((block_number, *path), branch.clone()); + result.account_trie_updates_written_total += 1; + } + + // Store storage branch nodes and removals + for (address, storage_trie_updates) in + block_state_diff.sorted_trie_updates.storage_tries_ref() + { + // Store storage branch nodes + for (path, branch) in storage_trie_updates.storage_nodes_ref() { + self.storage_branches + .insert((block_number, *address, *path), branch.clone()); + result.storage_trie_updates_written_total += 1; + } + } + + for (address, account) in &block_state_diff.sorted_post_state.accounts { + self.hashed_accounts + .insert((block_number, *address), *account); + result.hashed_accounts_written_total += 1; + } + + for (hashed_address, storage) in &block_state_diff.sorted_post_state.storages { + // Handle wiped storage: iterate all existing values and mark them as deleted + // This is an expensive operation and should never happen for blocks going forward. + if storage.wiped { + // Collect latest values for each slot up to the current block + let mut slot_to_latest: std::collections::BTreeMap = + std::collections::BTreeMap::new(); + + for ((block, address, slot), value) in &self.hashed_storages { + if *block < block_number && *address == *hashed_address { + if let Some((existing_block, _)) = slot_to_latest.get(slot) { + if *block > *existing_block { + slot_to_latest.insert(*slot, (*block, *value)); + } + } else { + slot_to_latest.insert(*slot, (*block, *value)); + } + } + } + + // Store zero values for all non-zero slots to mark them as deleted + for (slot, (_, value)) in slot_to_latest { + if !value.is_zero() { + self.hashed_storages + .insert((block_number, *hashed_address, slot), U256::ZERO); + result.hashed_storages_written_total += 1; + } + } + } + // Persist any new slots that arrive in the same block as the wipe (e.g. SELFDESTRUCT + // + same-block re-CREATE2 produces `wiped = true` together with fresh + // `storage_slots_ref()` for the recreated contract). Mirrors the MDBX backend. + for (slot, value) in storage.storage_slots_ref() { + self.hashed_storages + .insert((block_number, *hashed_address, *slot), *value); + result.hashed_storages_written_total += 1; + } + } + + self.trie_updates + .insert(block_number, block_state_diff.sorted_trie_updates.clone()); + self.post_states + .insert(block_number, block_state_diff.sorted_post_state); + + result + } +} + +impl Default for InMemoryProofsStorage { + fn default() -> Self { + Self::new() + } +} + +impl InMemoryProofsStorage { + /// Create a new in-memory Morph proofs storage instance. + pub fn new() -> Self { + Self { + inner: Arc::new(RwLock::new(InMemoryStorageInner::default())), + } + } + + fn get_latest_account_trie_key(&self) -> MorphProofsStorageResult> { + let inner = self.inner.read(); + Ok(inner + .account_branches + .range(( + std::ops::Bound::Included((0, Nibbles::default())), + std::ops::Bound::Excluded((1, Nibbles::default())), + )) + .next_back() + .map(|((_, nibbles), _)| StoredNibbles::from(*nibbles))) + } + + fn get_latest_storage_trie_key(&self) -> MorphProofsStorageResult> { + let inner = self.inner.read(); + Ok(inner + .storage_branches + .range(( + std::ops::Bound::Included((0, B256::ZERO, Nibbles::default())), + std::ops::Bound::Excluded((1, B256::ZERO, Nibbles::default())), + )) + .next_back() + .map(|((_, address, nibbles), _)| { + StorageTrieKey::new(*address, StoredNibbles::from(*nibbles)) + })) + } + + fn get_latest_hashed_account_key(&self) -> MorphProofsStorageResult> { + let inner = self.inner.read(); + Ok(inner + .hashed_accounts + .range(( + std::ops::Bound::Included((0, B256::ZERO)), + std::ops::Bound::Excluded((1, B256::ZERO)), + )) + .next_back() + .map(|((_, address), _)| *address)) + } + + fn get_latest_hashed_storage_key(&self) -> MorphProofsStorageResult> { + let inner = self.inner.read(); + Ok(inner + .hashed_storages + .range(( + std::ops::Bound::Included((0, B256::ZERO, B256::ZERO)), + std::ops::Bound::Excluded((1, B256::ZERO, B256::ZERO)), + )) + .next_back() + .map(|((_, address, slot), _)| HashedStorageKey::new(*address, *slot))) + } +} + +/// In-memory implementation of [`TrieCursor`]. +#[derive(Debug)] +pub struct InMemoryTrieCursor { + /// inner storage reference + inner: Arc>, + /// Hashed address being queried + hashed_address: Option, + /// max block number for the cursor + max_block_number: u64, + /// Current position in the iteration (-1 means not positioned yet) + position: isize, + + /// Whether the entries have been populated + is_populated: bool, + /// Sorted entries that match the query parameters + entries: Vec<(Nibbles, BranchNodeCompact)>, +} + +impl InMemoryTrieCursor { + const fn new( + inner: Arc>, + hashed_address: Option, + max_block_number: u64, + ) -> Self { + Self { + inner, + hashed_address, + max_block_number, + position: -1, + + is_populated: false, + entries: Vec::new(), + } + } + + fn ensure_entries_populated(&mut self) -> Result<(), DatabaseError> { + if self.is_populated { + return Ok(()); + } + + let storage = self + .inner + .try_read() + .ok_or(MorphProofsStorageError::TryLockError)?; + + // Common logic: collect latest values for each path + let mut path_to_latest: std::collections::BTreeMap< + Nibbles, + (u64, Option), + > = std::collections::BTreeMap::new(); + + let mut collected_entries: Vec<(Nibbles, BranchNodeCompact)> = + if let Some(addr) = self.hashed_address { + // Storage trie cursor + for ((block, address, path), branch) in &storage.storage_branches { + if *block <= self.max_block_number && *address == addr { + if let Some((existing_block, _)) = path_to_latest.get(path) { + if *block > *existing_block { + path_to_latest.insert(*path, (*block, branch.clone())); + } + } else { + path_to_latest.insert(*path, (*block, branch.clone())); + } + } + } + + path_to_latest + .into_iter() + .filter_map(|(path, (_, branch))| branch.map(|b| (path, b))) + .collect() + } else { + // Account trie cursor + for ((block, path), branch) in &storage.account_branches { + if *block <= self.max_block_number { + if let Some((existing_block, _)) = path_to_latest.get(path) { + if *block > *existing_block { + path_to_latest.insert(*path, (*block, branch.clone())); + } + } else { + path_to_latest.insert(*path, (*block, branch.clone())); + } + } + } + + path_to_latest + .into_iter() + .filter_map(|(path, (_, branch))| branch.map(|b| (path, b))) + .collect() + }; + + // Sort by path for consistent ordering + collected_entries.sort_by_key(|(a, _)| *a); + self.entries = collected_entries; + self.is_populated = true; + Ok(()) + } +} + +impl TrieCursor for InMemoryTrieCursor { + fn seek_exact( + &mut self, + path: Nibbles, + ) -> Result, DatabaseError> { + self.ensure_entries_populated()?; + + if let Some(pos) = self.entries.iter().position(|(p, _)| *p == path) { + self.position = pos as isize; + Ok(Some(self.entries[pos].clone())) + } else { + Ok(None) + } + } + + fn seek( + &mut self, + path: Nibbles, + ) -> Result, DatabaseError> { + self.ensure_entries_populated()?; + + if let Some(pos) = self.entries.iter().position(|(p, _)| *p >= path) { + self.position = pos as isize; + Ok(Some(self.entries[pos].clone())) + } else { + Ok(None) + } + } + + fn next(&mut self) -> Result, DatabaseError> { + self.ensure_entries_populated()?; + + self.position += 1; + if self.position >= 0 && (self.position as usize) < self.entries.len() { + Ok(Some(self.entries[self.position as usize].clone())) + } else { + Ok(None) + } + } + + fn current(&mut self) -> Result, DatabaseError> { + self.ensure_entries_populated()?; + + if self.position >= 0 && (self.position as usize) < self.entries.len() { + Ok(Some(self.entries[self.position as usize].0)) + } else { + Ok(None) + } + } + + fn reset(&mut self) { + self.position = -1; + } +} + +impl TrieStorageCursor for InMemoryTrieCursor { + fn set_hashed_address(&mut self, hashed_address: B256) { + self.hashed_address = Some(hashed_address); + self.is_populated = false; + self.entries.clear(); + self.reset(); + } +} + +/// In-memory implementation of [`HashedCursor`] for storage slots +#[derive(Debug)] +pub struct InMemoryStorageCursor { + /// inner storage reference + inner: Arc>, + /// hashed address for which the cursor is iterating + hashed_address: B256, + /// max block number for the cursor + max_block_number: u64, + /// Current position in the iteration (-1 means not positioned yet) + position: isize, + + /// Whether the entries have been populated + is_populated: bool, + /// Sorted entries that match the query parameters + entries: Vec<(B256, U256)>, +} + +impl InMemoryStorageCursor { + const fn new( + storage: Arc>, + hashed_address: B256, + max_block_number: u64, + ) -> Self { + Self { + inner: storage, + hashed_address, + max_block_number, + position: -1, + + is_populated: false, + entries: Vec::new(), + } + } + + fn ensure_entries_populated(&mut self) -> Result<(), DatabaseError> { + if self.is_populated { + return Ok(()); + } + + let storage = self + .inner + .try_read() + .ok_or(MorphProofsStorageError::TryLockError)?; + + // Collect latest values for each slot + let mut slot_to_latest: std::collections::BTreeMap = + std::collections::BTreeMap::new(); + + for ((block, address, slot), value) in &storage.hashed_storages { + if *block <= self.max_block_number && *address == self.hashed_address { + if let Some((existing_block, _)) = slot_to_latest.get(slot) { + if *block > *existing_block { + slot_to_latest.insert(*slot, (*block, *value)); + } + } else { + slot_to_latest.insert(*slot, (*block, *value)); + } + } + } + + // Filter out zero values - they represent deleted/empty storage slots + let mut entries: Vec<(B256, U256)> = slot_to_latest + .into_iter() + .filter_map(|(slot, (_, value))| { + if value.is_zero() { + None + } else { + Some((slot, value)) + } + }) + .collect(); + + entries.sort_by_key(|(slot, _)| *slot); + self.entries = entries; + self.is_populated = true; + Ok(()) + } +} + +impl HashedCursor for InMemoryStorageCursor { + type Value = U256; + + fn seek(&mut self, key: B256) -> Result, DatabaseError> { + self.ensure_entries_populated()?; + + if let Some(pos) = self.entries.iter().position(|(k, _)| *k >= key) { + self.position = pos as isize; + Ok(Some(self.entries[pos])) + } else { + Ok(None) + } + } + + fn next(&mut self) -> Result, DatabaseError> { + self.ensure_entries_populated()?; + + self.position += 1; + if self.position >= 0 && (self.position as usize) < self.entries.len() { + Ok(Some(self.entries[self.position as usize])) + } else { + Ok(None) + } + } + + fn reset(&mut self) { + self.position = -1; + } +} + +impl HashedStorageCursor for InMemoryStorageCursor { + fn is_storage_empty(&mut self) -> Result { + Ok(self.seek(B256::ZERO)?.is_none()) + } + + fn set_hashed_address(&mut self, hashed_address: B256) { + self.hashed_address = hashed_address; + self.is_populated = false; + self.entries.clear(); + self.reset(); + } +} + +/// In-memory implementation of [`HashedCursor`] for accounts +#[derive(Debug)] +pub struct InMemoryAccountCursor { + /// Current position in the iteration (-1 means not positioned yet) + position: isize, + /// Sorted entries that match the query parameters + entries: Vec<(B256, Account)>, +} + +impl InMemoryAccountCursor { + fn new(storage: &InMemoryStorageInner, max_block_number: u64) -> Self { + // Collect latest accounts for each address + let mut addr_to_latest: std::collections::BTreeMap)> = + std::collections::BTreeMap::new(); + + for ((block, address), account) in &storage.hashed_accounts { + if *block <= max_block_number { + if let Some((existing_block, _)) = addr_to_latest.get(address) { + if *block > *existing_block { + addr_to_latest.insert(*address, (*block, *account)); + } + } else { + addr_to_latest.insert(*address, (*block, *account)); + } + } + } + + let mut entries: Vec<(B256, Account)> = addr_to_latest + .into_iter() + .filter_map(|(address, (_, account))| account.map(|acc| (address, acc))) + .collect(); + + entries.sort_by_key(|(address, _)| *address); + + Self { + position: -1, + entries, + } + } +} + +impl HashedCursor for InMemoryAccountCursor { + type Value = Account; + + fn seek(&mut self, key: B256) -> Result, DatabaseError> { + if let Some(pos) = self.entries.iter().position(|(k, _)| *k >= key) { + self.position = pos as isize; + Ok(Some(self.entries[pos])) + } else { + Ok(None) + } + } + + fn next(&mut self) -> Result, DatabaseError> { + self.position += 1; + if self.position >= 0 && (self.position as usize) < self.entries.len() { + Ok(Some(self.entries[self.position as usize])) + } else { + Ok(None) + } + } + + fn reset(&mut self) { + // no reset needed + } +} + +impl MorphProofsStore for InMemoryProofsStorage { + type StorageTrieCursor<'tx> = InMemoryTrieCursor; + type AccountTrieCursor<'tx> = InMemoryTrieCursor; + type StorageCursor<'tx> = InMemoryStorageCursor; + type AccountHashedCursor<'tx> = InMemoryAccountCursor; + type Tx<'tx> + = () + where + Self: 'tx; + + fn get_earliest_block_number(&self) -> MorphProofsStorageResult> { + let inner = self.inner.read(); + Ok(inner.earliest_block) + } + + fn get_latest_block_number(&self) -> MorphProofsStorageResult> { + let inner = self.inner.read(); + // Find the latest block number from trie_updates + let latest_block = inner.trie_updates.keys().max().copied(); + Ok(latest_block.map_or(inner.earliest_block, |block| Some((block, B256::ZERO)))) + } + + fn get_earliest_block_number_with_tx<'tx, 'db>( + &self, + _tx: &'tx Self::Tx<'db>, + ) -> MorphProofsStorageResult> + where + Self: 'db, + 'db: 'tx, + { + self.get_earliest_block_number() + } + + fn get_latest_block_number_with_tx<'tx, 'db>( + &self, + _tx: &'tx Self::Tx<'db>, + ) -> MorphProofsStorageResult> + where + Self: 'db, + 'db: 'tx, + { + self.get_latest_block_number() + } + + fn storage_trie_cursor<'tx>( + &'tx self, + hashed_address: B256, + max_block_number: u64, + ) -> MorphProofsStorageResult> { + Ok(InMemoryTrieCursor::new( + Arc::clone(&self.inner), + Some(hashed_address), + max_block_number, + )) + } + + fn account_trie_cursor<'tx>( + &'tx self, + max_block_number: u64, + ) -> MorphProofsStorageResult> { + Ok(InMemoryTrieCursor::new( + Arc::clone(&self.inner), + None, + max_block_number, + )) + } + + fn storage_hashed_cursor<'tx>( + &'tx self, + hashed_address: B256, + max_block_number: u64, + ) -> MorphProofsStorageResult> { + Ok(InMemoryStorageCursor::new( + Arc::clone(&self.inner), + hashed_address, + max_block_number, + )) + } + + fn account_hashed_cursor<'tx>( + &'tx self, + max_block_number: u64, + ) -> MorphProofsStorageResult> { + let inner = self + .inner + .try_read() + .ok_or(MorphProofsStorageError::TryLockError)?; + Ok(InMemoryAccountCursor::new(&inner, max_block_number)) + } + + fn ro_tx<'tx>(&'tx self) -> MorphProofsStorageResult> { + Ok(()) + } + + fn storage_trie_cursor_with_tx<'tx, 'db>( + &self, + _tx: &'tx Self::Tx<'db>, + hashed_address: B256, + max_block_number: u64, + ) -> MorphProofsStorageResult> + where + Self: 'db, + 'db: 'tx, + { + self.storage_trie_cursor(hashed_address, max_block_number) + } + + fn account_trie_cursor_with_tx<'tx, 'db>( + &self, + _tx: &'tx Self::Tx<'db>, + max_block_number: u64, + ) -> MorphProofsStorageResult> + where + Self: 'db, + 'db: 'tx, + { + self.account_trie_cursor(max_block_number) + } + + fn storage_hashed_cursor_with_tx<'tx, 'db>( + &self, + _tx: &'tx Self::Tx<'db>, + hashed_address: B256, + max_block_number: u64, + ) -> MorphProofsStorageResult> + where + Self: 'db, + 'db: 'tx, + { + self.storage_hashed_cursor(hashed_address, max_block_number) + } + + fn account_hashed_cursor_with_tx<'tx, 'db>( + &self, + _tx: &'tx Self::Tx<'db>, + max_block_number: u64, + ) -> MorphProofsStorageResult> + where + Self: 'db, + 'db: 'tx, + { + self.account_hashed_cursor(max_block_number) + } + + fn store_trie_updates( + &self, + block_ref: BlockWithParent, + block_state_diff: BlockStateDiff, + ) -> MorphProofsStorageResult { + let mut inner = self.inner.write(); + + Ok(inner.store_trie_updates(block_ref.block.number, block_state_diff)) + } + + fn fetch_trie_updates(&self, block_number: u64) -> MorphProofsStorageResult { + let inner = self.inner.read(); + + let trie_updates = inner + .trie_updates + .get(&block_number) + .cloned() + .unwrap_or_default(); + let post_state = inner + .post_states + .get(&block_number) + .cloned() + .unwrap_or_default(); + + Ok(BlockStateDiff { + sorted_trie_updates: trie_updates, + sorted_post_state: post_state, + }) + } + + fn prune_earliest_state( + &self, + new_earliest_block_ref: BlockWithParent, + ) -> MorphProofsStorageResult { + let mut write_counts = WriteCounts::default(); + let mut inner = self.inner.write(); + let new_earliest = new_earliest_block_ref.block.number; + + // 1. Account Branches + // Identify keys to move (blocks <= new_earliest but > 0) + let account_keys: Vec<_> = inner + .account_branches + .keys() + .filter(|(block, _)| *block > 0 && *block <= new_earliest) + .copied() + .collect(); + + for key in account_keys { + if let Some(branch) = inner.account_branches.remove(&key) { + // Determine if we should update the base state (block 0) + // In a simpler model without diff calculation, we just overwrite block 0 + // with the latest version found in the pruned range. + // Since keys are sorted by block, this logic naturally keeps the latest if we + // iterate in order, but here we are just grabbing all of them. + // For correctness in BTreeMap iteration order (which is sorted): + inner.account_branches.insert((0, key.1), branch); + write_counts.account_trie_updates_written_total += 1; + } + } + + // 2. Storage Branches + let storage_keys: Vec<_> = inner + .storage_branches + .keys() + .filter(|(block, _, _)| *block > 0 && *block <= new_earliest) + .copied() + .collect(); + + for key in storage_keys { + if let Some(branch) = inner.storage_branches.remove(&key) { + inner.storage_branches.insert((0, key.1, key.2), branch); + write_counts.storage_trie_updates_written_total += 1; + } + } + + // 3. Hashed Accounts + let acc_keys: Vec<_> = inner + .hashed_accounts + .keys() + .filter(|(block, _)| *block > 0 && *block <= new_earliest) + .copied() + .collect(); + + for key in acc_keys { + if let Some(acc) = inner.hashed_accounts.remove(&key) { + inner.hashed_accounts.insert((0, key.1), acc); + write_counts.hashed_accounts_written_total += 1; + } + } + + // 4. Hashed Storages + let stor_keys: Vec<_> = inner + .hashed_storages + .keys() + .filter(|(block, _, _)| *block > 0 && *block <= new_earliest) + .copied() + .collect(); + + for key in stor_keys { + if let Some(val) = inner.hashed_storages.remove(&key) { + inner.hashed_storages.insert((0, key.1, key.2), val); + write_counts.hashed_storages_written_total += 1; + } + } + + // Update earliest block pointer + if let Some((_, hash)) = inner.earliest_block { + inner.earliest_block = Some((new_earliest, hash)); + } + + // 5. Cleanup Metadata + inner.trie_updates.retain(|block, _| *block > new_earliest); + inner.post_states.retain(|block, _| *block > new_earliest); + + Ok(write_counts) + } + + fn unwind_history(&self, unwind_upto_block: BlockWithParent) -> MorphProofsStorageResult<()> { + let mut inner = self.inner.write(); + let unwind_upto_block_number = unwind_upto_block.block.number - 1; + + // Remove all updates after unwind_upto_block_number + inner + .trie_updates + .retain(|block, _| *block <= unwind_upto_block_number); + inner + .post_states + .retain(|block, _| *block <= unwind_upto_block_number); + inner + .account_branches + .retain(|(block, _), _| *block <= unwind_upto_block_number); + inner + .storage_branches + .retain(|(block, _, _), _| *block <= unwind_upto_block_number); + inner + .hashed_accounts + .retain(|(block, _), _| *block <= unwind_upto_block_number); + inner + .hashed_storages + .retain(|(block, _, _), _| *block <= unwind_upto_block_number); + + Ok(()) + } + + fn replace_updates( + &self, + latest_common_block: BlockNumHash, + blocks_to_add: Vec<(BlockWithParent, BlockStateDiff)>, + ) -> MorphProofsStorageResult<()> { + let mut inner = self.inner.write(); + let latest_common_block_number = latest_common_block.number; + + // Remove all updates after latest_common_block_number + inner + .trie_updates + .retain(|block, _| *block <= latest_common_block_number); + inner + .post_states + .retain(|block, _| *block <= latest_common_block_number); + inner + .account_branches + .retain(|(block, _), _| *block <= latest_common_block_number); + inner + .storage_branches + .retain(|(block, _, _), _| *block <= latest_common_block_number); + inner + .hashed_accounts + .retain(|(block, _), _| *block <= latest_common_block_number); + inner + .hashed_storages + .retain(|(block, _, _), _| *block <= latest_common_block_number); + + for (block, block_state_diff) in blocks_to_add { + inner.store_trie_updates(block.block.number, block_state_diff); + } + + Ok(()) + } + + fn set_earliest_block_number( + &self, + block_number: u64, + hash: B256, + ) -> MorphProofsStorageResult<()> { + let mut inner = self.inner.write(); + inner.earliest_block = Some((block_number, hash)); + Ok(()) + } +} + +/// Degenerate batch session for [`InMemoryProofsStorage`]: writes go directly to shared +/// storage and are visible to subsequent reads. There is no commit/abort distinction — +/// the in-memory store does not provide cross-block atomicity. +#[derive(Debug)] +pub struct InMemoryBatchSession<'a> { + storage: &'a InMemoryProofsStorage, +} + +impl MorphProofsBatchSession for InMemoryBatchSession<'_> { + type StorageTrieCursor<'a> + = InMemoryTrieCursor + where + Self: 'a; + type AccountTrieCursor<'a> + = InMemoryTrieCursor + where + Self: 'a; + type StorageCursor<'a> + = InMemoryStorageCursor + where + Self: 'a; + type AccountHashedCursor<'a> + = InMemoryAccountCursor + where + Self: 'a; + + fn get_earliest_block_number(&self) -> MorphProofsStorageResult> { + self.storage.get_earliest_block_number() + } + + fn get_latest_block_number(&self) -> MorphProofsStorageResult> { + self.storage.get_latest_block_number() + } + + fn storage_trie_cursor( + &self, + hashed_address: B256, + max_block_number: u64, + ) -> MorphProofsStorageResult> { + self.storage + .storage_trie_cursor(hashed_address, max_block_number) + } + + fn account_trie_cursor( + &self, + max_block_number: u64, + ) -> MorphProofsStorageResult> { + self.storage.account_trie_cursor(max_block_number) + } + + fn storage_hashed_cursor( + &self, + hashed_address: B256, + max_block_number: u64, + ) -> MorphProofsStorageResult> { + self.storage + .storage_hashed_cursor(hashed_address, max_block_number) + } + + fn account_hashed_cursor( + &self, + max_block_number: u64, + ) -> MorphProofsStorageResult> { + self.storage.account_hashed_cursor(max_block_number) + } + + fn store_trie_updates( + &mut self, + block_ref: BlockWithParent, + block_state_diff: BlockStateDiff, + ) -> MorphProofsStorageResult { + self.storage.store_trie_updates(block_ref, block_state_diff) + } +} + +impl MorphProofsBatchStore for InMemoryProofsStorage { + type BatchSession<'a> + = InMemoryBatchSession<'a> + where + Self: 'a; + + fn with_batch_session(&self, f: F) -> MorphProofsStorageResult + where + F: FnOnce(&mut Self::BatchSession<'_>) -> MorphProofsStorageResult, + { + let mut session = InMemoryBatchSession { storage: self }; + f(&mut session) + } +} + +impl MorphProofsInitialStateStore for InMemoryProofsStorage { + fn initial_state_anchor(&self) -> MorphProofsStorageResult { + let inner = self.inner.read(); + + let Some((block_num, block_hash)) = inner.anchor_block else { + return Ok(InitialStateAnchor::default()); + }; + + let completed = inner.earliest_block.is_some(); + + Ok(InitialStateAnchor { + block: Some(NumHash::new(block_num, block_hash)), + status: if completed { + InitialStateStatus::Completed + } else { + InitialStateStatus::InProgress + }, + latest_account_trie_key: self.get_latest_account_trie_key()?, + latest_storage_trie_key: self.get_latest_storage_trie_key()?, + latest_hashed_account_key: self.get_latest_hashed_account_key()?, + latest_hashed_storage_key: self.get_latest_hashed_storage_key()?, + }) + } + + fn set_initial_state_anchor(&self, anchor: BlockNumHash) -> MorphProofsStorageResult<()> { + let mut inner = self.inner.write(); + inner.anchor_block = Some((anchor.number, anchor.hash)); + Ok(()) + } + + fn store_account_branches( + &self, + updates: Vec<(Nibbles, Option)>, + ) -> MorphProofsStorageResult<()> { + let mut inner = self.inner.write(); + + for (path, branch) in updates { + inner.account_branches.insert((0, path), branch); + } + + Ok(()) + } + + fn store_storage_branches( + &self, + hashed_address: B256, + items: Vec<(Nibbles, Option)>, + ) -> MorphProofsStorageResult<()> { + let mut inner = self.inner.write(); + + for (path, branch) in items { + inner + .storage_branches + .insert((0, hashed_address, path), branch); + } + + Ok(()) + } + + fn store_hashed_accounts( + &self, + accounts: Vec<(B256, Option)>, + ) -> MorphProofsStorageResult<()> { + let mut inner = self.inner.write(); + + for (address, account) in accounts { + inner.hashed_accounts.insert((0, address), account); + } + + Ok(()) + } + + fn store_hashed_storages( + &self, + hashed_address: B256, + storages: Vec<(B256, U256)>, + ) -> MorphProofsStorageResult<()> { + let mut inner = self.inner.write(); + + for (slot, value) in storages { + inner + .hashed_storages + .insert((0, hashed_address, slot), value); + } + + Ok(()) + } + + fn commit_initial_state(&self) -> MorphProofsStorageResult { + let mut inner = self.inner.write(); + if let Some((number, hash)) = inner.anchor_block { + inner.earliest_block = Some((number, hash)); + Ok(BlockNumHash::new(number, hash)) + } else { + Err(MorphProofsStorageError::NoBlocksFound) + } + } +} + +#[cfg(test)] +mod tests { + use alloy_eips::NumHash; + use alloy_primitives::U256; + use reth_primitives_traits::Account; + + use super::*; + use crate::MorphProofsStorageError; + + #[test] + fn test_in_memory_storage_basic_operations() -> Result<(), MorphProofsStorageError> { + let storage = InMemoryProofsStorage::new(); + + // Test setting earliest block + let block_hash = B256::random(); + storage.set_earliest_block_number(1, block_hash)?; + let earliest = storage.get_earliest_block_number()?; + assert_eq!(earliest, Some((1, block_hash))); + + // Test storing and retrieving accounts + let account = Account { + nonce: 1, + balance: U256::from(100), + bytecode_hash: None, + }; + let hashed_address = B256::random(); + + storage.store_hashed_accounts(vec![(hashed_address, Some(account))])?; + + let _cursor = storage.account_hashed_cursor(10)?; + // Note: cursor testing would require more complex setup with proper seek/next operations + + Ok(()) + } + + #[test] + fn test_trie_updates_storage() -> Result<(), MorphProofsStorageError> { + let storage = InMemoryProofsStorage::new(); + + let sorted_trie_updates = TrieUpdatesSorted::default(); + let sorted_post_state = HashedPostStateSorted::default(); + let block_state_diff = BlockStateDiff { + sorted_trie_updates: sorted_trie_updates.clone(), + sorted_post_state: sorted_post_state.clone(), + }; + + const BLOCK: BlockWithParent = + BlockWithParent::new(B256::ZERO, NumHash::new(5, B256::ZERO)); + storage.store_trie_updates(BLOCK, block_state_diff)?; + + let retrieved_diff = storage.fetch_trie_updates(BLOCK.block.number)?; + assert_eq!(retrieved_diff.sorted_trie_updates, sorted_trie_updates); + assert_eq!(retrieved_diff.sorted_post_state, sorted_post_state); + + Ok(()) + } +} diff --git a/crates/proofs/src/initialize.rs b/crates/proofs/src/initialize.rs new file mode 100644 index 00000000..b61277bc --- /dev/null +++ b/crates/proofs/src/initialize.rs @@ -0,0 +1,1357 @@ +//! Initialization job for proofs storage. Handles storing the existing state into the proofs +//! storage. + +use std::{collections::HashMap, time::Instant}; + +use alloy_eips::BlockNumHash; +use alloy_primitives::{B256, U256}; +use derive_more::Constructor; +use reth_db::{ + DatabaseError, + cursor::{DbCursorRO, DbDupCursorRO}, + tables, + transaction::DbTx, +}; +use reth_primitives_traits::{Account, StorageEntry}; +use reth_trie_common::{ + BranchNodeCompact, Nibbles, StorageTrieEntry, StoredNibbles, StoredNibblesSubKey, +}; +use tracing::{debug, info}; + +use crate::{ + MorphProofsStorageError, MorphProofsStore, + api::{InitialStateAnchor, InitialStateStatus, MorphProofsInitialStateStore}, + db::{HashedStorageKey, StorageTrieKey}, +}; + +/// Batch size threshold for storing entries during initialization +const INITIALIZE_STORAGE_THRESHOLD: usize = 100000; + +/// Threshold for logging progress during initialization +const INITIALIZE_LOG_THRESHOLD: usize = 100000; + +/// Initialization job for external storage. +#[derive(Debug, Constructor)] +pub struct InitializationJob { + storage: S, + tx: Tx, +} + +/// Macro to generate simple cursor iterators for tables +macro_rules! define_simple_cursor_iter { + ($iter_name:ident, $table:ty, $key_type:ty, $value_type:ty) => { + struct $iter_name(C); + + impl $iter_name { + const fn new(cursor: C) -> Self { + Self(cursor) + } + } + + impl> Iterator for $iter_name { + type Item = Result<($key_type, $value_type), DatabaseError>; + + fn next(&mut self) -> Option { + self.0.next().transpose() + } + } + }; +} + +/// Macro to generate duplicate cursor iterators for tables with custom logic +macro_rules! define_dup_cursor_iter { + ($iter_name:ident, $table:ty, $key_type:ty, $value_type:ty) => { + struct $iter_name(C); + + impl $iter_name { + const fn new(cursor: C) -> Self { + Self(cursor) + } + } + + impl + DbCursorRO<$table>> Iterator for $iter_name { + type Item = Result<($key_type, $value_type), DatabaseError>; + + fn next(&mut self) -> Option { + // First try to get the next duplicate value + if let Some(res) = self.0.next_dup().transpose() { + return Some(res); + } + + // If no more duplicates, find the next key with values + let Some(Ok((next_key, _))) = self.0.next_no_dup().transpose() else { + // If no more entries, return None + return None; + }; + + // If found, seek to the first duplicate for this key + return self.0.seek(next_key).transpose(); + } + } + }; +} + +// Generate iterators for all 4 table types +define_simple_cursor_iter!(HashedAccountsInit, tables::HashedAccounts, B256, Account); +define_dup_cursor_iter!( + HashedStoragesInit, + tables::HashedStorages, + B256, + StorageEntry +); +define_simple_cursor_iter!( + AccountsTrieInit, + tables::AccountsTrie, + StoredNibbles, + BranchNodeCompact +); +define_dup_cursor_iter!( + StoragesTrieInit, + tables::StoragesTrie, + B256, + StorageTrieEntry +); + +/// Trait to estimate the progress of a initialization job based on the key. +trait CompletionEstimatable { + // Returns a progress estimate as a percentage (0.0 to 1.0) + fn estimate_progress(&self) -> f64; +} + +impl CompletionEstimatable for B256 { + fn estimate_progress(&self) -> f64 { + // use the first 3 bytes as a progress estimate + let progress = self.0[..3].to_vec(); + let mut val: u64 = 0; + for nibble in &progress { + val = (val << 8) | *nibble as u64; + } + val as f64 / (256u64.pow(3)) as f64 + } +} + +impl CompletionEstimatable for StoredNibbles { + fn estimate_progress(&self) -> f64 { + // use the first 6 nibbles as a progress estimate + let progress_nibbles = if self.0.is_empty() { + Nibbles::new() + } else { + self.0.slice(0..(self.0.len().min(6))) + }; + let mut val: u64 = 0; + for nibble in progress_nibbles.iter() { + val = (val << 4) | nibble as u64; + } + val as f64 / (16u64.pow(progress_nibbles.len() as u32)) as f64 + } +} + +impl + InitializationJob +{ + /// Initialize a table from a source iterator to a storage function. Handles batching and + /// logging. + fn initialize< + I: Iterator> + InitTable, + Key: CompletionEstimatable + 'static, + Value: 'static, + >( + &self, + name: &str, + source: I, + storage_threshold: usize, + log_threshold: usize, + ) -> Result { + info!(name = %name, "Starting initialization"); + let start_time = Instant::now(); + + let mut source = source.peekable(); + let Some(first_entry) = source.peek() else { + debug!(target: "reth::cli", "No entries to store for table"); + return Ok(0); + }; + let initial_progress = match first_entry { + Ok(i) => i.0.estimate_progress(), + Err(e) => Err(e.clone())?, + }; + + let storage = &self.storage; + let source_size_hint = source.size_hint().0; + let mut batch = Vec::with_capacity(source_size_hint.min(storage_threshold)); + let mut total_entries: usize = 0; + + for entry in source { + batch.push(entry?); + total_entries += 1; + + if total_entries.is_multiple_of(log_threshold) { + let progress = batch.last().expect("non-empty batch").0.estimate_progress(); + let elapsed = start_time.elapsed(); + let elapsed_secs = elapsed.as_secs_f64(); + + let progress_per_second = if elapsed_secs.is_normal() { + (progress - initial_progress) / elapsed_secs + } else { + 0.0 + }; + let estimated_total_time = if progress_per_second.is_normal() { + (1.0 - progress) / progress_per_second + } else { + 0.0 + }; + let progress_pct = progress * 100.0; + info!( + name = %name, + total_entries, + progress_pct = format_args!("{:.2}", progress_pct), + estimated_total_time = format_args!("{:.1}", estimated_total_time), + "Processed entries" + ); + } + + if batch.len() >= storage_threshold { + info!(name = %name, total_entries = total_entries, "Storing entries"); + I::store_entries(storage, batch)?; + batch = Vec::with_capacity( + (source_size_hint.saturating_sub(total_entries)).min(storage_threshold), + ); + } + } + + if !batch.is_empty() { + info!(name = %name, "Storing final entries"); + I::store_entries(storage, batch)?; + } + + info!(name = %name, total_entries = total_entries, "initialization complete"); + Ok(total_entries as u64) + } + + /// Initialize hashed accounts data + fn initialize_hashed_accounts( + &self, + start_key: Option, + ) -> Result<(), MorphProofsStorageError> { + let mut start_cursor = self.tx.cursor_read::()?; + + if let Some(latest) = start_key { + start_cursor + .seek(latest)? + .filter(|(k, _)| *k == latest) + .ok_or(MorphProofsStorageError::InitializeStorageInconsistentState)?; + } + + let source = HashedAccountsInit::new(start_cursor); + self.initialize( + "hashed accounts", + source, + INITIALIZE_STORAGE_THRESHOLD, + INITIALIZE_LOG_THRESHOLD, + )?; + + Ok(()) + } + + /// Initialize hashed storage data + fn initialize_hashed_storages( + &self, + start_key: Option, + ) -> Result<(), MorphProofsStorageError> { + let mut start_cursor = self.tx.cursor_dup_read::()?; + + if let Some(latest) = start_key { + start_cursor + .seek_by_key_subkey(latest.hashed_address, latest.hashed_storage_key)? + .filter(|v| v.key == latest.hashed_storage_key) + .ok_or(MorphProofsStorageError::InitializeStorageInconsistentState)?; + } + + let source = HashedStoragesInit::new(start_cursor); + self.initialize( + "hashed storage", + source, + INITIALIZE_STORAGE_THRESHOLD, + INITIALIZE_LOG_THRESHOLD, + )?; + + Ok(()) + } + + /// Initialize accounts trie data + fn initialize_accounts_trie( + &self, + start_key: Option, + ) -> Result<(), MorphProofsStorageError> { + let mut start_cursor = self.tx.cursor_read::()?; + + if let Some(latest_key) = start_key { + start_cursor + .seek(latest_key.clone())? + .filter(|(k, _)| *k == latest_key) + .ok_or(MorphProofsStorageError::InitializeStorageInconsistentState)?; + } + + let source = AccountsTrieInit::new(start_cursor); + self.initialize( + "accounts trie", + source, + INITIALIZE_STORAGE_THRESHOLD, + INITIALIZE_LOG_THRESHOLD, + )?; + + Ok(()) + } + + /// Initialize storage trie data + fn initialize_storages_trie( + &self, + start_key: Option, + ) -> Result<(), MorphProofsStorageError> { + let mut start_cursor = self.tx.cursor_dup_read::()?; + + if let Some(latest_key) = start_key { + start_cursor + .seek_by_key_subkey( + latest_key.hashed_address, + StoredNibblesSubKey::from(latest_key.path.0), + )? + .filter(|v| v.nibbles.0 == latest_key.path.0) + .ok_or(MorphProofsStorageError::InitializeStorageInconsistentState)?; + } + + let source = StoragesTrieInit::new(start_cursor); + self.initialize( + "storage trie", + source, + INITIALIZE_STORAGE_THRESHOLD, + INITIALIZE_LOG_THRESHOLD, + )?; + + Ok(()) + } + + /// Run complete initialization of all preimage data + fn initialize_trie(&self, anchor: InitialStateAnchor) -> Result<(), MorphProofsStorageError> { + self.initialize_hashed_accounts(anchor.latest_hashed_account_key)?; + self.initialize_hashed_storages(anchor.latest_hashed_storage_key)?; + self.initialize_storages_trie(anchor.latest_storage_trie_key)?; + self.initialize_accounts_trie(anchor.latest_account_trie_key)?; + Ok(()) + } + + fn validate_anchor_block( + &self, + anchor: &InitialStateAnchor, + best_number: u64, + best_hash: B256, + ) -> Result<(), MorphProofsStorageError> { + let block = anchor + .block + .ok_or(MorphProofsStorageError::InitializeStorageInconsistentState)?; + + if block.number != best_number || block.hash != best_hash { + return Err(MorphProofsStorageError::InitializeStorageInconsistentState); + } + + Ok(()) + } + + /// Run the initialization job. + pub fn run(&self, best_number: u64, best_hash: B256) -> Result<(), MorphProofsStorageError> { + let anchor = self.storage.initial_state_anchor()?; + + match anchor.status { + InitialStateStatus::Completed => return Ok(()), + InitialStateStatus::NotStarted => { + self.storage + .set_initial_state_anchor(BlockNumHash::new(best_number, best_hash))?; + } + InitialStateStatus::InProgress => { + self.validate_anchor_block(&anchor, best_number, best_hash)?; + } + } + + self.initialize_trie(anchor)?; + self.storage.commit_initial_state()?; + + Ok(()) + } +} + +/// Handles storing entries for a particular KV-pair type. +trait InitTable { + /// Key of target table. + type Key: CompletionEstimatable + 'static; + /// Value of target table. + type Value: 'static; + + /// Writes given entries to given storage. + fn store_entries( + store: &impl MorphProofsInitialStateStore, + entries: impl IntoIterator, + ) -> Result<(), MorphProofsStorageError>; +} + +impl InitTable for HashedAccountsInit { + type Key = B256; + type Value = Account; + + /// Save mapping of hashed addresses to accounts to storage. + fn store_entries( + store: &impl MorphProofsInitialStateStore, + entries: impl IntoIterator, + ) -> Result<(), MorphProofsStorageError> { + store.store_hashed_accounts( + entries + .into_iter() + .map(|(address, account)| (address, Some(account))) + .collect(), + )?; + Ok(()) + } +} + +impl InitTable for HashedStoragesInit { + type Key = B256; + type Value = StorageEntry; + + /// Save mapping of hashed addresses to storage entries to storage. + fn store_entries( + store: &impl MorphProofsInitialStateStore, + entries: impl IntoIterator, + ) -> Result<(), MorphProofsStorageError> { + let entries_iter = entries.into_iter(); + let mut by_address: HashMap> = + HashMap::with_capacity(entries_iter.size_hint().0); + + for (address, entry) in entries_iter { + by_address + .entry(address) + .or_default() + .push((entry.key, entry.value)); + } + store.store_hashed_storages_bulk(by_address.into_iter().collect())?; + + Ok(()) + } +} + +impl InitTable for AccountsTrieInit { + type Key = StoredNibbles; + type Value = BranchNodeCompact; + + /// Save mapping of account trie paths to branch nodes to storage. + fn store_entries( + store: &impl MorphProofsInitialStateStore, + entries: impl IntoIterator, + ) -> Result<(), MorphProofsStorageError> { + store.store_account_branches( + entries + .into_iter() + .map(|(path, branch)| (path.0, Some(branch))) + .collect(), + )?; + + Ok(()) + } +} + +impl InitTable for StoragesTrieInit { + type Key = B256; + type Value = StorageTrieEntry; + + /// Save mapping of hashed addresses to storage trie entries to storage. + fn store_entries( + store: &impl MorphProofsInitialStateStore, + entries: impl IntoIterator, + ) -> Result<(), MorphProofsStorageError> { + let entries_iter = entries.into_iter(); + let mut by_address: HashMap)>> = + HashMap::with_capacity(entries_iter.size_hint().0); + + for (hashed_address, storage_entry) in entries_iter { + by_address + .entry(hashed_address) + .or_default() + .push((storage_entry.nibbles.0, Some(storage_entry.node))); + } + store.store_storage_branches_bulk(by_address.into_iter().collect())?; + + Ok(()) + } +} + +#[cfg(test)] +mod tests { + macro_rules! proof_storage_init_tests { + ($mod_name:ident, $storage:ident) => { + mod $mod_name { + use std::sync::Arc; + + use alloy_primitives::{Address, U256, keccak256}; + use reth_db::{ + Database, cursor::DbCursorRW, test_utils::create_test_rw_db, + transaction::DbTxMut, + }; + use reth_primitives_traits::Account; + use reth_trie::{ + BranchNodeCompact, StorageTrieEntry, StoredNibbles, StoredNibblesSubKey, + TrieMask, hashed_cursor::HashedCursor, trie_cursor::TrieCursor, + }; + use tempfile::TempDir; + + use super::super::*; + use crate::$storage; + + /// Helper function to create a key + fn k(b: u8) -> B256 { + let mut bytes = [0u8; 32]; + bytes[0] = b; + B256::from(bytes) + } + + /// Helper function to create a test branch node + fn create_test_branch_node() -> BranchNodeCompact { + let mut state_mask = TrieMask::default(); + state_mask.set_bit(0); + state_mask.set_bit(1); + + BranchNodeCompact { + state_mask, + tree_mask: TrieMask::default(), + hash_mask: TrieMask::default(), + hashes: Arc::new(vec![]), + root_hash: None, + } + } + + #[test] + fn test_initialize_hashed_accounts() { + let db = create_test_rw_db(); + let dir = TempDir::new().unwrap(); + let storage = Arc::new($storage::new(dir.path()).expect("env")); + + // Insert test accounts into database + let tx = db.tx_mut().unwrap(); + let mut cursor = tx.cursor_write::().unwrap(); + + let mut accounts = vec![ + ( + keccak256(Address::repeat_byte(0x01)), + Account { + nonce: 1, + balance: U256::from(100), + bytecode_hash: None, + }, + ), + ( + keccak256(Address::repeat_byte(0x02)), + Account { + nonce: 2, + balance: U256::from(200), + bytecode_hash: None, + }, + ), + ( + keccak256(Address::repeat_byte(0x03)), + Account { + nonce: 3, + balance: U256::from(300), + bytecode_hash: None, + }, + ), + ]; + + // Sort accounts by address for cursor.append (which requires sorted order) + accounts.sort_by_key(|(addr, _)| *addr); + + for (addr, account) in &accounts { + cursor.append(*addr, account).unwrap(); + } + drop(cursor); + tx.commit().unwrap(); + + // Run initialization + let tx = db.tx().unwrap(); + let job = InitializationJob::new(Arc::clone(&storage), tx); + job.initialize_hashed_accounts(None).unwrap(); + + // Verify data was stored (will be in sorted order) + let mut account_cursor = storage.account_hashed_cursor(100).unwrap(); + let mut count = 0; + while let Some((key, account)) = account_cursor.next().unwrap() { + // Find matching account in our test data + let expected = accounts.iter().find(|(addr, _)| *addr == key).unwrap(); + assert_eq!((key, account), *expected); + count += 1; + } + assert_eq!(count, 3); + } + + #[test] + fn test_initialize_hashed_storage() { + let db = create_test_rw_db(); + let dir = TempDir::new().unwrap(); + let storage = Arc::new($storage::new(dir.path()).expect("env")); + + // Insert test storage into database + let tx = db.tx_mut().unwrap(); + let mut cursor = tx.cursor_dup_write::().unwrap(); + + let addr1 = keccak256(Address::repeat_byte(0x01)); + let addr2 = keccak256(Address::repeat_byte(0x02)); + + let storage_entries = vec![ + ( + addr1, + StorageEntry { + key: keccak256(B256::repeat_byte(0x10)), + value: U256::from(100), + }, + ), + ( + addr1, + StorageEntry { + key: keccak256(B256::repeat_byte(0x20)), + value: U256::from(200), + }, + ), + ( + addr2, + StorageEntry { + key: keccak256(B256::repeat_byte(0x30)), + value: U256::from(300), + }, + ), + ]; + + for (addr, entry) in &storage_entries { + cursor.upsert(*addr, entry).unwrap(); + } + drop(cursor); + tx.commit().unwrap(); + + // Run initialization + let tx = db.tx().unwrap(); + let job = InitializationJob::new(Arc::clone(&storage), tx); + job.initialize_hashed_storages(None).unwrap(); + + // Verify data was stored for addr1 + let mut storage_cursor = storage.storage_hashed_cursor(addr1, 100).unwrap(); + let mut found = vec![]; + while let Some((key, value)) = storage_cursor.next().unwrap() { + found.push((key, value)); + } + assert_eq!(found.len(), 2); + assert_eq!( + found[0], + (storage_entries[0].1.key, storage_entries[0].1.value) + ); + assert_eq!( + found[1], + (storage_entries[1].1.key, storage_entries[1].1.value) + ); + + // Verify data was stored for addr2 + let mut storage_cursor = storage.storage_hashed_cursor(addr2, 100).unwrap(); + let mut found = vec![]; + while let Some((key, value)) = storage_cursor.next().unwrap() { + found.push((key, value)); + } + assert_eq!(found.len(), 1); + assert_eq!( + found[0], + (storage_entries[2].1.key, storage_entries[2].1.value) + ); + } + + #[test] + fn test_initialize_accounts_trie() { + let db = create_test_rw_db(); + let dir = TempDir::new().unwrap(); + let storage = Arc::new($storage::new(dir.path()).expect("env")); + + // Insert test trie nodes into database + let tx = db.tx_mut().unwrap(); + let mut cursor = tx.cursor_write::().unwrap(); + + let branch = create_test_branch_node(); + let nodes = vec![ + ( + StoredNibbles(Nibbles::from_nibbles_unchecked(vec![1])), + branch.clone(), + ), + ( + StoredNibbles(Nibbles::from_nibbles_unchecked(vec![2])), + branch.clone(), + ), + ( + StoredNibbles(Nibbles::from_nibbles_unchecked(vec![3])), + branch, + ), + ]; + + for (path, node) in &nodes { + cursor.append(path.clone(), node).unwrap(); + } + drop(cursor); + tx.commit().unwrap(); + + // Run initialization + let tx = db.tx().unwrap(); + let job = InitializationJob::new(Arc::clone(&storage), tx); + job.initialize_accounts_trie(None).unwrap(); + + // Verify data was stored + let mut trie_cursor = storage.account_trie_cursor(100).unwrap(); + let mut count = 0; + while let Some((path, _node)) = trie_cursor.next().unwrap() { + assert_eq!(path, nodes[count].0.0); + count += 1; + } + assert_eq!(count, 3); + } + + #[test] + fn test_initialize_storages_trie() { + let db = create_test_rw_db(); + let dir = TempDir::new().unwrap(); + let storage = Arc::new($storage::new(dir.path()).expect("env")); + + // Insert test storage trie nodes into database + let tx = db.tx_mut().unwrap(); + let mut cursor = tx.cursor_dup_write::().unwrap(); + + let branch = create_test_branch_node(); + let addr1 = keccak256(Address::repeat_byte(0x01)); + let addr2 = keccak256(Address::repeat_byte(0x02)); + + let nodes = vec![ + ( + addr1, + StorageTrieEntry { + nibbles: StoredNibblesSubKey(Nibbles::from_nibbles_unchecked( + vec![1], + )), + node: branch.clone(), + }, + ), + ( + addr1, + StorageTrieEntry { + nibbles: StoredNibblesSubKey(Nibbles::from_nibbles_unchecked( + vec![2], + )), + node: branch.clone(), + }, + ), + ( + addr2, + StorageTrieEntry { + nibbles: StoredNibblesSubKey(Nibbles::from_nibbles_unchecked( + vec![3], + )), + node: branch, + }, + ), + ]; + + for (addr, entry) in &nodes { + cursor.upsert(*addr, entry).unwrap(); + } + drop(cursor); + tx.commit().unwrap(); + + // Run initialization + let tx = db.tx().unwrap(); + let job = InitializationJob::new(Arc::clone(&storage), tx); + job.initialize_storages_trie(None).unwrap(); + + // Verify data was stored for addr1 + let mut trie_cursor = storage.storage_trie_cursor(addr1, 100).unwrap(); + let mut found = vec![]; + while let Some((path, _node)) = trie_cursor.next().unwrap() { + found.push(path); + } + assert_eq!(found.len(), 2); + assert_eq!(found[0], nodes[0].1.nibbles.0); + assert_eq!(found[1], nodes[1].1.nibbles.0); + + // Verify data was stored for addr2 + let mut trie_cursor = storage.storage_trie_cursor(addr2, 100).unwrap(); + let mut found = vec![]; + while let Some((path, _node)) = trie_cursor.next().unwrap() { + found.push(path); + } + assert_eq!(found.len(), 1); + assert_eq!(found[0], nodes[2].1.nibbles.0); + } + + #[test] + fn test_full_initialize_run() { + let db = create_test_rw_db(); + let dir = TempDir::new().unwrap(); + let storage = Arc::new($storage::new(dir.path()).expect("env")); + + // Insert some test data + let tx = db.tx_mut().unwrap(); + + // Add accounts + let mut cursor = tx.cursor_write::().unwrap(); + let addr = keccak256(Address::repeat_byte(0x01)); + cursor + .append( + addr, + &Account { + nonce: 1, + balance: U256::from(100), + bytecode_hash: None, + }, + ) + .unwrap(); + drop(cursor); + + // Add storage + let mut cursor = tx.cursor_dup_write::().unwrap(); + cursor + .upsert( + addr, + &StorageEntry { + key: keccak256(B256::repeat_byte(0x10)), + value: U256::from(100), + }, + ) + .unwrap(); + drop(cursor); + + // Add account trie + let mut cursor = tx.cursor_write::().unwrap(); + cursor + .append( + StoredNibbles(Nibbles::from_nibbles_unchecked(vec![1])), + &create_test_branch_node(), + ) + .unwrap(); + drop(cursor); + + // Add storage trie + let mut cursor = tx.cursor_dup_write::().unwrap(); + cursor + .upsert( + addr, + &StorageTrieEntry { + nibbles: StoredNibblesSubKey(Nibbles::from_nibbles_unchecked( + vec![1], + )), + node: create_test_branch_node(), + }, + ) + .unwrap(); + drop(cursor); + + tx.commit().unwrap(); + + // Run full initialization + let tx = db.tx().unwrap(); + let job = InitializationJob::new(Arc::clone(&storage), tx); + let best_number = 100; + let best_hash = B256::repeat_byte(0x42); + + // Should be None initially + assert_eq!(storage.initial_state_anchor().unwrap().block, None); + assert_eq!(storage.get_earliest_block_number().unwrap(), None); + + job.run(best_number, best_hash).unwrap(); + + // Should be set after initialization + assert_eq!( + storage.get_earliest_block_number().unwrap(), + Some((best_number, best_hash)) + ); + + // Verify data was initialized + let mut account_cursor = storage.account_hashed_cursor(100).unwrap(); + assert!(account_cursor.next().unwrap().is_some()); + + let mut storage_cursor = storage.storage_hashed_cursor(addr, 100).unwrap(); + assert!(storage_cursor.next().unwrap().is_some()); + + let mut trie_cursor = storage.account_trie_cursor(100).unwrap(); + assert!(trie_cursor.next().unwrap().is_some()); + + let mut storage_trie_cursor = storage.storage_trie_cursor(addr, 100).unwrap(); + assert!(storage_trie_cursor.next().unwrap().is_some()); + } + + #[test] + fn test_initialize_run_skips_if_already_done() { + let db = create_test_rw_db(); + let dir = TempDir::new().unwrap(); + let storage = Arc::new($storage::new(dir.path()).expect("env")); + + // set and commit initial state anchor + storage + .set_initial_state_anchor(BlockNumHash::new(50, B256::repeat_byte(0x01))) + .expect("set anchor"); + storage.commit_initial_state().expect("commit anchor"); + + let tx = db.tx().unwrap(); + let job = InitializationJob::new(Arc::clone(&storage), tx); + + // Run initialization - should skip + job.run(100, B256::repeat_byte(0x42)).unwrap(); + + // Should still have the old anchor + let anchor_block = storage + .initial_state_anchor() + .expect("get anchor") + .block + .expect("block"); + assert_eq!( + Some((anchor_block.number, anchor_block.hash)), + Some((50, B256::repeat_byte(0x01))) + ); + + // Should still have the old earliest block + assert_eq!( + storage.get_earliest_block_number().unwrap(), + Some((50, B256::repeat_byte(0x01))) + ); + } + + #[test] + fn test_initialize_resumes_hashed_accounts_with_no_dups() { + let db = create_test_rw_db(); + let dir = TempDir::new().unwrap(); + let store = Arc::new($storage::new(dir.path()).expect("env")); + + store + .set_initial_state_anchor(BlockNumHash::new(0, B256::default())) + .expect("set anchor"); + + // Phase 1 in source: k1, k2 + let k1 = k(1); + let k2 = k(2); + { + let tx = db.tx_mut().unwrap(); + let mut cur = tx.cursor_write::().unwrap(); + cur.append( + k1, + &Account { + nonce: 1, + balance: U256::from(100), + bytecode_hash: None, + }, + ) + .unwrap(); + cur.append( + k2, + &Account { + nonce: 2, + balance: U256::from(200), + bytecode_hash: None, + }, + ) + .unwrap(); + tx.commit().unwrap(); + } + + // Initialization #1 + { + let tx = db.tx().unwrap(); + let job = InitializationJob::new(Arc::clone(&store), tx); + job.initialize_hashed_accounts(None).unwrap(); + } + + // Resume point must be k2 (max) + assert_eq!( + store + .initial_state_anchor() + .expect("get anchor") + .latest_hashed_account_key, + Some(k2) + ); + + // Phase 2 in source: k3, k4 + let k3 = k(3); + let k4 = k(4); + { + let tx = db.tx_mut().unwrap(); + let mut cur = tx.cursor_write::().unwrap(); + cur.append( + k3, + &Account { + nonce: 3, + balance: U256::from(300), + bytecode_hash: None, + }, + ) + .unwrap(); + cur.append( + k4, + &Account { + nonce: 4, + balance: U256::from(400), + bytecode_hash: None, + }, + ) + .unwrap(); + tx.commit().unwrap(); + } + + // Initialization #2 (restart) + { + let tx = db.tx().unwrap(); + let job = InitializationJob::new(Arc::clone(&store), tx); + job.initialize_hashed_accounts(Some(k2)).unwrap(); + } + + // Now resume point must be k4 + assert_eq!( + store + .initial_state_anchor() + .expect("get anchor") + .latest_hashed_account_key, + Some(k4) + ); + + // Verify order + no dupes by iterating proofs cursor + let mut cur = store.account_hashed_cursor(0).unwrap(); + let mut got = Vec::new(); + while let Some((key, acct)) = cur.next().unwrap() { + got.push((key, acct)); + } + + // Expect exactly 4, in increasing key order. + assert_eq!(got.len(), 4); + assert_eq!(got[0].0, k1); + assert_eq!(got[1].0, k2); + assert_eq!(got[2].0, k3); + assert_eq!(got[3].0, k4); + + // No dupes + for w in got.windows(2) { + assert!(w[0].0 < w[1].0); + } + } + + #[test] + fn test_initialize_resumes_hashed_storages_with_no_dups() { + let db = create_test_rw_db(); + let dir = TempDir::new().unwrap(); + let store = Arc::new($storage::new(dir.path()).expect("env")); + + store + .set_initial_state_anchor(BlockNumHash::new(0, B256::default())) + .expect("set anchor"); + + let a1 = k(0x10); + let a2 = k(0x20); + + let s11 = k(0x01); + let s12 = k(0x02); + let s21 = k(0x03); + let s22 = k(0x04); + + // Phase 1 source: + // a1: s11,s12 + // a2: s21 + { + let tx = db.tx_mut().unwrap(); + let mut cur = tx.cursor_dup_write::().unwrap(); + cur.upsert( + a1, + &StorageEntry { + key: s11, + value: U256::from(11), + }, + ) + .unwrap(); + cur.upsert( + a1, + &StorageEntry { + key: s12, + value: U256::from(12), + }, + ) + .unwrap(); + cur.upsert( + a2, + &StorageEntry { + key: s21, + value: U256::from(21), + }, + ) + .unwrap(); + tx.commit().unwrap(); + } + + // Initialization #1 + { + let tx = db.tx().unwrap(); + let job = InitializationJob::new(Arc::clone(&store), tx); + job.initialize_hashed_storages(None).unwrap(); + } + + // Latest key must be (a2, s21) because a2 > a1 + let last1 = store + .initial_state_anchor() + .expect("get anchor") + .latest_hashed_storage_key + .expect("ok"); + assert_eq!(last1.hashed_address, a2); + assert_eq!(last1.hashed_storage_key, s21); + + // Phase 2 source: add s22 to a2 + { + let tx = db.tx_mut().unwrap(); + let mut cur = tx.cursor_dup_write::().unwrap(); + cur.upsert( + a2, + &StorageEntry { + key: s22, + value: U256::from(22), + }, + ) + .unwrap(); + tx.commit().unwrap(); + } + + // Initialization #2 + { + let tx = db.tx().unwrap(); + let job = InitializationJob::new(Arc::clone(&store), tx); + job.initialize_hashed_storages(Some(HashedStorageKey::new(a2, s21))) + .unwrap(); + } + + // Latest key now must be (a2, s22) + let last2 = store + .initial_state_anchor() + .expect("get anchor") + .latest_hashed_storage_key + .expect("ok"); + assert_eq!(last2.hashed_address, a2); + assert_eq!(last2.hashed_storage_key, s22); + + // Verify no dupes by iterating per-address + { + let mut c = store.storage_hashed_cursor(a1, 0).unwrap(); + let mut got = Vec::new(); + while let Some((slot, val)) = c.next().unwrap() { + got.push((slot, val)); + } + assert_eq!(got.len(), 2); + assert_eq!(got[0].0, s11); + assert_eq!(got[1].0, s12); + } + { + let mut c = store.storage_hashed_cursor(a2, 0).unwrap(); + let mut got = Vec::new(); + while let Some((slot, val)) = c.next().unwrap() { + got.push((slot, val)); + } + assert_eq!(got.len(), 2); + assert_eq!(got[0].0, s21); + assert_eq!(got[1].0, s22); + } + } + + #[test] + fn test_initialize_resumes_accounts_trie_with_no_dups() { + let db = create_test_rw_db(); + let dir = TempDir::new().unwrap(); + let store = Arc::new($storage::new(dir.path()).expect("env")); + + store + .set_initial_state_anchor(BlockNumHash::new(0, B256::default())) + .expect("set anchor"); + + let p1 = StoredNibbles(Nibbles::from_nibbles_unchecked(vec![1])); + let p2 = StoredNibbles(Nibbles::from_nibbles_unchecked(vec![2])); + let p3 = StoredNibbles(Nibbles::from_nibbles_unchecked(vec![3])); + let p4 = StoredNibbles(Nibbles::from_nibbles_unchecked(vec![4])); + + // Phase 1 source: p1,p2 + { + let tx = db.tx_mut().unwrap(); + let mut cur = tx.cursor_write::().unwrap(); + cur.append(p1.clone(), &create_test_branch_node()).unwrap(); + cur.append(p2.clone(), &create_test_branch_node()).unwrap(); + tx.commit().unwrap(); + } + + // Initialization #1 + { + let tx = db.tx().unwrap(); + let job = InitializationJob::new(Arc::clone(&store), tx); + job.initialize_accounts_trie(None).unwrap(); + } + + assert_eq!( + store + .initial_state_anchor() + .expect("get anchor") + .latest_account_trie_key, + Some(p2.clone()) + ); + + // Phase 2 source: p3,p4 + { + let tx = db.tx_mut().unwrap(); + let mut cur = tx.cursor_write::().unwrap(); + cur.append(p3.clone(), &create_test_branch_node()).unwrap(); + cur.append(p4.clone(), &create_test_branch_node()).unwrap(); + tx.commit().unwrap(); + } + + // Initialization #2 + { + let tx = db.tx().unwrap(); + let job = InitializationJob::new(Arc::clone(&store), tx); + job.initialize_accounts_trie(Some(p2.clone())).unwrap(); + } + + assert_eq!( + store + .initial_state_anchor() + .expect("get anchor") + .latest_account_trie_key, + Some(p4.clone()) + ); + + // Verify 4 ordered, no dupes + let mut c = store.account_trie_cursor(0).unwrap(); + let mut got = Vec::new(); + while let Some((path, _node)) = c.next().unwrap() { + got.push(path); + } + assert_eq!(got.len(), 4); + assert_eq!(got[0], p1.0); + assert_eq!(got[1], p2.0); + assert_eq!(got[2], p3.0); + assert_eq!(got[3], p4.0); + } + + #[test] + fn test_initialize_resumes_storages_trie_with_no_dups() { + let db = create_test_rw_db(); + let dir = TempDir::new().unwrap(); + let store = Arc::new($storage::new(dir.path()).expect("env")); + + store + .set_initial_state_anchor(BlockNumHash::new(0, B256::default())) + .expect("set anchor"); + + let a1 = k(0x10); + let a2 = k(0x20); + + let n1 = StoredNibblesSubKey(Nibbles::from_nibbles_unchecked(vec![1])); + let n2 = StoredNibblesSubKey(Nibbles::from_nibbles_unchecked(vec![2])); + let n3 = StoredNibblesSubKey(Nibbles::from_nibbles_unchecked(vec![3])); + + // Phase 1 source: (a1,n1), (a2,n2) + { + let tx = db.tx_mut().unwrap(); + let mut cur = tx.cursor_dup_write::().unwrap(); + cur.upsert( + a1, + &StorageTrieEntry { + nibbles: n1.clone(), + node: create_test_branch_node(), + }, + ) + .unwrap(); + cur.upsert( + a2, + &StorageTrieEntry { + nibbles: n2.clone(), + node: create_test_branch_node(), + }, + ) + .unwrap(); + tx.commit().unwrap(); + } + + // Initialization #1 + { + let tx = db.tx().unwrap(); + let job = InitializationJob::new(Arc::clone(&store), tx); + job.initialize_storages_trie(None).unwrap(); + } + + // Latest must be (a2, n2) because a2 > a1 + let last1 = store + .initial_state_anchor() + .expect("get anchor") + .latest_storage_trie_key + .expect("ok"); + assert_eq!(last1.hashed_address, a2); + assert_eq!(last1.path.0, n2.0); + + // Phase 2 source: add (a2,n3) + { + let tx = db.tx_mut().unwrap(); + let mut cur = tx.cursor_dup_write::().unwrap(); + cur.upsert( + a2, + &StorageTrieEntry { + nibbles: n3.clone(), + node: create_test_branch_node(), + }, + ) + .unwrap(); + tx.commit().unwrap(); + } + + // Initialization #2 + { + let tx = db.tx().unwrap(); + let job = InitializationJob::new(Arc::clone(&store), tx); + job.initialize_storages_trie(Some(StorageTrieKey::new( + a2, + StoredNibbles::from(n2.0), + ))) + .unwrap(); + } + + // Latest must now be (a2,n3) + let last2 = store + .initial_state_anchor() + .expect("get anchor") + .latest_storage_trie_key + .expect("ok"); + assert_eq!(last2.hashed_address, a2); + assert_eq!(last2.path.0, n3.0); + + // Verify per-address no dupes and stable ordering + { + let mut c = store.storage_trie_cursor(a1, 0).unwrap(); + + let mut got = Vec::new(); + + // next returns the rest + while let Some((path, _node)) = c.next().unwrap() { + got.push(path); + } + + assert_eq!(got, vec![n1.0]); + } + { + let mut c = store.storage_trie_cursor(a2, 0).unwrap(); + + let mut got = Vec::new(); + + // next returns the rest + while let Some((path, _node)) = c.next().unwrap() { + got.push(path); + } + assert_eq!(got, vec![n2.0, n3.0]); + } + } + } + }; + } + + proof_storage_init_tests!(mdbx_tests, MdbxProofsStorage); +} diff --git a/crates/proofs/src/lib.rs b/crates/proofs/src/lib.rs new file mode 100644 index 00000000..438ec198 --- /dev/null +++ b/crates/proofs/src/lib.rs @@ -0,0 +1,61 @@ +//! Bounded historical MPT storage for Morph `eth_getProof`. +//! +//! The versioned-trie implementation is adapted from Base's proof-history +//! storage at commit `b2673bbd927cb34d7cfad4d448bfbd5bd30eae88` under MIT. +//! Morph intentionally exposes only the MDBX production backend. + +#![cfg_attr(not(test), warn(unused_crate_dependencies))] + +pub mod api; +pub use api::{ + BlockStateDiff, InitialStateAnchor, InitialStateStatus, MorphProofsBatchSession, + MorphProofsBatchStore, MorphProofsInitialStateStore, MorphProofsStore, OperationDurations, + StorageBranchEntries, WriteCounts, +}; + +pub mod initialize; +pub use initialize::InitializationJob; + +#[cfg(any(test, feature = "test-utils"))] +pub mod in_memory; +#[cfg(any(test, feature = "test-utils"))] +pub use in_memory::{ + InMemoryAccountCursor, InMemoryBatchSession, InMemoryProofsStorage, InMemoryStorageCursor, + InMemoryTrieCursor, +}; + +pub mod db; +pub use db::{ + MdbxAccountCursor, MdbxBatchSession, MdbxProofsStorage, MdbxProofsStorageOptions, + MdbxStorageCursor, MdbxTrieCursor, ProofDbIdentity, ProofWindowBounds, +}; + +pub mod metrics; +pub use metrics::{ + MorphProofsHashedAccountCursor, MorphProofsHashedStorageCursor, MorphProofsStorage, + MorphProofsTrieCursor, ProofHistoryMetrics, +}; + +pub mod proof; +pub mod provider; + +mod batch_provider; +pub use batch_provider::MorphProofsBatchStateProviderRef; + +pub mod live; + +pub mod cursor; +pub mod cursor_factory; +pub use cursor_factory::{ + MorphProofsBatchHashedAccountCursorFactory, MorphProofsBatchTrieCursorFactory, + MorphProofsHashedAccountCursorFactory, MorphProofsTrieCursorFactory, +}; + +pub mod error; +pub use error::{MorphProofsStorageError, MorphProofsStorageResult}; + +mod prune; +pub use prune::{ + MorphProofStoragePruner, MorphProofStoragePrunerResult, MorphProofStoragePrunerTask, + PrunerError, PrunerOutput, +}; diff --git a/crates/proofs/src/live.rs b/crates/proofs/src/live.rs new file mode 100644 index 00000000..fafc1023 --- /dev/null +++ b/crates/proofs/src/live.rs @@ -0,0 +1,432 @@ +//! Live trie collector for external proofs storage. + +use std::{sync::Arc, time::Instant}; + +use alloy_eips::{BlockNumHash, NumHash, eip1898::BlockWithParent}; +use derive_more::Constructor; +use reth_evm::{ConfigureEvm, execute::Executor}; +use reth_primitives_traits::{AlloyBlockHeader, BlockTy, NodePrimitives, RecoveredBlock}; +use reth_provider::{ + DatabaseProviderFactory, HashedPostStateProvider, StateProviderFactory, StateRootProvider, +}; +use reth_revm::database::StateProviderDatabase; +use reth_trie_common::{HashedPostStateSorted, updates::TrieUpdatesSorted}; +use tracing::{info, warn}; + +use crate::{ + BlockStateDiff, MorphProofsStorage, MorphProofsStorageError, MorphProofsStore, + api::{MorphProofsBatchSession, MorphProofsBatchStore, OperationDurations, WriteCounts}, + batch_provider::MorphProofsBatchStateProviderRef, + metrics::ProofHistoryMetrics, + provider::MorphProofsStateProviderRef, +}; + +/// Live trie collector for external proofs storage. +#[derive(Debug, Constructor)] +pub struct LiveTrieCollector<'tx, Evm, Provider, PreimageStore> +where + Evm: ConfigureEvm, + Provider: DatabaseProviderFactory + StateProviderFactory, +{ + evm_config: Evm, + provider: Provider, + storage: &'tx MorphProofsStorage, +} + +impl<'tx, Evm, Provider, Store> LiveTrieCollector<'tx, Evm, Provider, Store> +where + Evm: ConfigureEvm, + Provider: DatabaseProviderFactory + StateProviderFactory, + Store: 'tx + MorphProofsStore + Clone + 'static, +{ + fn record_storage_metrics( + operation_durations: &OperationDurations, + write_counts: Option<&WriteCounts>, + ) { + ProofHistoryMetrics::record_block(operation_durations, write_counts); + } + + /// Execute a block and store the updates in the storage. + pub fn execute_and_store_block_updates( + &self, + block: &RecoveredBlock>, + ) -> Result<(), MorphProofsStorageError> { + let mut operation_durations = OperationDurations::default(); + + let start = Instant::now(); + // ensure that we have the state of the parent block + let (Some((earliest, _)), Some((latest, _))) = ( + self.storage.get_earliest_block_number()?, + self.storage.get_latest_block_number()?, + ) else { + return Err(MorphProofsStorageError::NoBlocksFound); + }; + + let parent_block_number = block.number() - 1; + if parent_block_number < earliest { + return Err(MorphProofsStorageError::UnknownParent); + } + + if parent_block_number > latest { + return Err(MorphProofsStorageError::MissingParentBlock { + block_number: block.number(), + parent_block_number, + latest_block_number: latest, + }); + } + + let block_ref = BlockWithParent::new( + block.parent_hash(), + NumHash::new(block.number(), block.hash()), + ); + + let state_provider = MorphProofsStateProviderRef::new( + self.provider.latest()?, + self.storage, + parent_block_number, + ); + + let db = StateProviderDatabase::new(&state_provider); + let block_executor = self.evm_config.batch_executor(db); + + let execution_result = block_executor.execute(&(*block).clone())?; + + operation_durations.execution_duration_seconds = start.elapsed(); + + let hashed_state = state_provider.hashed_post_state(&execution_result.state); + let (state_root, trie_updates) = + state_provider.state_root_with_updates(hashed_state.clone())?; + + operation_durations.state_root_duration_seconds = + start.elapsed() - operation_durations.execution_duration_seconds; + + if state_root != block.state_root() { + return Err(MorphProofsStorageError::StateRootMismatch { + block_number: block.number(), + current_state_hash: state_root, + expected_state_hash: block.state_root(), + }); + } + + let update_result = self.storage.store_trie_updates( + block_ref, + BlockStateDiff { + sorted_trie_updates: trie_updates.into_sorted(), + sorted_post_state: hashed_state.into_sorted(), + }, + )?; + + operation_durations.total_duration_seconds = start.elapsed(); + operation_durations.write_duration_seconds = operation_durations.total_duration_seconds + - operation_durations.state_root_duration_seconds + - operation_durations.execution_duration_seconds; + + Self::record_storage_metrics(&operation_durations, Some(&update_result)); + + info!( + block_number = block.number(), + block_hash = ?block.hash(), + parent_block_number, + parent_block_hash = ?block.parent_hash(), + earliest_proof_before = earliest, + latest_proof_before = latest, + latest_proof_after = block.number(), + state_root = ?state_root, + ?operation_durations, + ?update_result, + "Block executed and trie updates stored successfully", + ); + + Ok(()) + } + + /// Store trie updates for a given block. + pub fn store_block_updates( + &self, + block: BlockWithParent, + sorted_trie_updates: TrieUpdatesSorted, + sorted_post_state: HashedPostStateSorted, + ) -> Result<(), MorphProofsStorageError> { + let start = Instant::now(); + let mut operation_durations = OperationDurations::default(); + + let storage_result = match self.storage.store_trie_updates( + block, + BlockStateDiff { + sorted_trie_updates, + sorted_post_state, + }, + ) { + Ok(res) => res, + Err(MorphProofsStorageError::OutOfOrder { + block_number, + latest_block_hash, + parent_block_hash, + }) => { + warn!( + block_number, + ?latest_block_hash, + ?parent_block_hash, + "Skipping out of order block updates" + ); + return Ok(()); + } + Err(e) => return Err(e), + }; + + let write_duration = start.elapsed(); + operation_durations.total_duration_seconds = write_duration; + operation_durations.write_duration_seconds = write_duration; + + Self::record_storage_metrics(&operation_durations, Some(&storage_result)); + + info!( + block_number = block.block.number, + ?operation_durations, + ?storage_result, + "Trie updates stored successfully", + ); + + Ok(()) + } + + /// Handles chain reorganizations by replacing block updates after a common ancestor. + /// + /// This method removes all block updates after the latest common ancestor (the block before + /// the first block in `new_blocks`) and replaces them with the updates from the provided new + /// chain. + /// + /// # Arguments + /// + /// * `new_blocks` - A vector of references to `RecoveredBlock` instances representing the new + /// blocks to be added to the trie storage. + pub fn unwind_and_store_block_updates( + &self, + block_updates: Vec<( + BlockWithParent, + Arc, + Arc, + )>, + ) -> Result<(), MorphProofsStorageError> { + if block_updates.is_empty() { + return Ok(()); + } + + let start = Instant::now(); + let mut operation_durations = OperationDurations::default(); + let first = &block_updates[0].0; + let latest_common_block = + BlockNumHash::new(first.block.number.saturating_sub(1), first.parent); + let mut block_trie_updates: Vec<(BlockWithParent, BlockStateDiff)> = + Vec::with_capacity(block_updates.len()); + + for (block, trie_updates, hashed_state) in &block_updates { + block_trie_updates.push(( + *block, + BlockStateDiff { + sorted_trie_updates: (**trie_updates).clone(), + sorted_post_state: (**hashed_state).clone(), + }, + )); + } + + self.storage + .replace_updates(latest_common_block, block_trie_updates)?; + let write_duration = start.elapsed(); + operation_durations.total_duration_seconds = write_duration; + operation_durations.write_duration_seconds = write_duration; + + Self::record_storage_metrics(&operation_durations, None); + + info!( + start_block_number = block_updates.first().map(|(b, _, _)| b.block.number), + end_block_number = block_updates.last().map(|(b, _, _)| b.block.number), + ?operation_durations, + "Trie updates rewound and stored successfully", + ); + Ok(()) + } + + /// Remove account, storage and trie updates from historical storage for all blocks from + /// the specified block (inclusive). + pub fn unwind_history(&self, to: BlockWithParent) -> Result<(), MorphProofsStorageError> { + if let Err(error) = self.storage.unwind_history(to) { + ProofHistoryMetrics::record_unwind_error(); + return Err(error); + } + if let Some((earliest, _)) = self.storage.get_earliest_block_number()? + && let Some((latest, _)) = self.storage.get_latest_block_number()? + { + ProofHistoryMetrics::set_window(earliest, latest); + } + Ok(()) + } +} + +/// One block to process inside a batch session: either pre-computed cached trie data (fast +/// path) or a fully recovered block that needs full execution against the session's +/// transaction-local state (cold catch-up path). +#[derive(Debug)] +pub enum BatchBlock { + /// Pre-computed cached trie data; only writes happen. + Cached { + /// Block reference being written. + block_with_parent: BlockWithParent, + /// Pre-computed trie updates from cached notification data. + sorted_trie_updates: Arc, + /// Pre-computed hashed post-state from cached notification data. + sorted_post_state: Arc, + }, + /// Full block requiring execution against session-local state. + Execute(Box>>), +} + +impl<'tx, Evm, Provider, Store> LiveTrieCollector<'tx, Evm, Provider, Store> +where + Evm: ConfigureEvm, + Provider: DatabaseProviderFactory + StateProviderFactory, + Store: 'tx + MorphProofsBatchStore + Clone + 'static, +{ + /// Execute and write a batch of blocks inside a single underlying transaction. + /// Reads during execution of block N observe writes from blocks earlier in the batch, + /// enabling cold catch-up where parent state is staged but not yet committed. The + /// entire batch commits atomically on success and aborts on the first error. + pub fn execute_and_store_batch( + &self, + blocks: Vec>, + ) -> Result<(), MorphProofsStorageError> { + if blocks.is_empty() { + return Ok(()); + } + + let start = Instant::now(); + let mut total_writes = WriteCounts::default(); + let mut block_count: u64 = 0; + let mut last_block_number: u64 = 0; + + self.storage.with_batch_session(|session| { + let (Some((earliest, _)), Some((_, _))) = ( + session.get_earliest_block_number()?, + session.get_latest_block_number()?, + ) else { + return Err(MorphProofsStorageError::NoBlocksFound); + }; + + for entry in blocks { + match entry { + BatchBlock::Cached { + block_with_parent, + sorted_trie_updates, + sorted_post_state, + } => { + let counts = session.store_trie_updates( + block_with_parent, + BlockStateDiff { + sorted_trie_updates: (*sorted_trie_updates).clone(), + sorted_post_state: (*sorted_post_state).clone(), + }, + )?; + total_writes += counts; + block_count += 1; + last_block_number = block_with_parent.block.number; + } + BatchBlock::Execute(block) => { + let counts = self.execute_one_in_session(session, &block, earliest)?; + total_writes += counts; + block_count += 1; + last_block_number = block.number(); + } + } + } + + Ok(()) + })?; + + let total = start.elapsed(); + ProofHistoryMetrics::record_block( + &OperationDurations { + total_duration_seconds: total, + ..Default::default() + }, + Some(&total_writes), + ); + if let Some((earliest, _)) = self.storage.get_earliest_block_number()? { + ProofHistoryMetrics::set_window(earliest, last_block_number); + } + + info!( + block_count, + last_block_number, + ?total_writes, + duration = ?total, + "Batch executed and committed", + ); + + Ok(()) + } + + fn execute_one_in_session( + &self, + session: &mut S, + block: &RecoveredBlock>, + earliest: u64, + ) -> Result + where + S: MorphProofsBatchSession, + { + let latest_in_session = session + .get_latest_block_number()? + .ok_or(MorphProofsStorageError::NoBlocksFound)? + .0; + + let parent_block_number = block.number() - 1; + if parent_block_number < earliest { + return Err(MorphProofsStorageError::UnknownParent); + } + if parent_block_number > latest_in_session { + return Err(MorphProofsStorageError::MissingParentBlock { + block_number: block.number(), + parent_block_number, + latest_block_number: latest_in_session, + }); + } + + let block_ref = BlockWithParent::new( + block.parent_hash(), + NumHash::new(block.number(), block.hash()), + ); + + let state_provider = MorphProofsBatchStateProviderRef::new( + self.provider.latest()?, + session, + parent_block_number, + ); + + let db = StateProviderDatabase::new(&state_provider); + let block_executor = self.evm_config.batch_executor(db); + + let execution_result = block_executor.execute(&(*block).clone())?; + + let hashed_state = state_provider.hashed_post_state(&execution_result.state); + let (state_root, trie_updates) = + state_provider.state_root_with_updates(hashed_state.clone())?; + + if state_root != block.state_root() { + return Err(MorphProofsStorageError::StateRootMismatch { + block_number: block.number(), + current_state_hash: state_root, + expected_state_hash: block.state_root(), + }); + } + + drop(state_provider); + + session.store_trie_updates( + block_ref, + BlockStateDiff { + sorted_trie_updates: trie_updates.into_sorted(), + sorted_post_state: hashed_state.into_sorted(), + }, + ) + } +} diff --git a/crates/proofs/src/metrics.rs b/crates/proofs/src/metrics.rs new file mode 100644 index 00000000..21211c9e --- /dev/null +++ b/crates/proofs/src/metrics.rs @@ -0,0 +1,97 @@ +//! Metrics for bounded historical-proof storage. + +use reth_metrics::{ + Metrics, + metrics::{Counter, Gauge, Histogram}, +}; + +use crate::{OperationDurations, WriteCounts, cursor}; + +/// The production storage type. Kept as an alias so callers remain backend-agnostic +/// without introducing another production backend. +pub type MorphProofsStorage = S; + +/// Trie cursor used by proof generation. +pub type MorphProofsTrieCursor = cursor::MorphProofsTrieCursor; +/// Hashed account cursor used by proof generation. +pub type MorphProofsHashedAccountCursor = cursor::MorphProofsHashedAccountCursor; +/// Hashed storage cursor used by proof generation. +pub type MorphProofsHashedStorageCursor = cursor::MorphProofsHashedStorageCursor; + +/// Proof-history storage and lifecycle metrics. +#[derive(Metrics, Clone)] +#[metrics(scope = "morph.proofs")] +pub struct ProofHistoryMetrics { + /// Total time spent processing a block. + pub block_total_duration_seconds: Histogram, + /// Time spent executing a block when catch-up re-execution is required. + pub block_execution_duration_seconds: Histogram, + /// Time spent calculating a state root. + pub block_state_root_duration_seconds: Histogram, + /// Time spent persisting a block. + pub block_write_duration_seconds: Histogram, + /// Number of account-trie updates written. + pub account_trie_updates_written_total: Counter, + /// Number of storage-trie updates written. + pub storage_trie_updates_written_total: Counter, + /// Number of account leaves written. + pub hashed_accounts_written_total: Counter, + /// Number of storage leaves written. + pub hashed_storages_written_total: Counter, + /// Earliest block currently served by proof history. + pub earliest_block: Gauge, + /// Latest block currently served by proof history. + pub latest_block: Gauge, + /// Number of prune failures. + pub prune_errors_total: Counter, + /// Number of unwind failures. + pub unwind_errors_total: Counter, +} + +impl ProofHistoryMetrics { + /// Records latency and write-count measurements for one processed block or batch. + pub fn record_block(durations: &OperationDurations, counts: Option<&WriteCounts>) { + let metrics = Self::default(); + metrics + .block_total_duration_seconds + .record(durations.total_duration_seconds.as_secs_f64()); + metrics + .block_execution_duration_seconds + .record(durations.execution_duration_seconds.as_secs_f64()); + metrics + .block_state_root_duration_seconds + .record(durations.state_root_duration_seconds.as_secs_f64()); + metrics + .block_write_duration_seconds + .record(durations.write_duration_seconds.as_secs_f64()); + if let Some(counts) = counts { + metrics + .account_trie_updates_written_total + .increment(counts.account_trie_updates_written_total); + metrics + .storage_trie_updates_written_total + .increment(counts.storage_trie_updates_written_total); + metrics + .hashed_accounts_written_total + .increment(counts.hashed_accounts_written_total); + metrics + .hashed_storages_written_total + .increment(counts.hashed_storages_written_total); + } + } + + /// Updates the gauges for the inclusive durable proof window. + pub fn set_window(earliest: u64, latest: u64) { + let metrics = Self::default(); + metrics.earliest_block.set(earliest as f64); + metrics.latest_block.set(latest as f64); + } + + pub(crate) fn record_prune_error() { + Self::default().prune_errors_total.increment(1); + } + + pub(crate) fn record_unwind_error() { + Self::default().unwind_errors_total.increment(1); + } +} diff --git a/crates/proofs/src/proof.rs b/crates/proofs/src/proof.rs new file mode 100644 index 00000000..5a1adeb0 --- /dev/null +++ b/crates/proofs/src/proof.rs @@ -0,0 +1,667 @@ +//! Provides proof operation implementations for [`MorphProofsStorage`]. + +use alloy_primitives::{ + Address, B256, Bytes, keccak256, + map::{B256Map, HashMap}, +}; +use reth_db::DatabaseError; +use reth_execution_errors::{StateProofError, StateRootError, StorageRootError, TrieWitnessError}; +use reth_trie::{ + StateRoot, StorageRoot, TrieType, + hashed_cursor::HashedPostStateCursorFactory, + metrics::TrieRootMetrics, + proof::{self, Proof}, + trie_cursor::InMemoryTrieCursorFactory, + witness::TrieWitness, +}; +use reth_trie_common::{ + AccountProof, ExecutionWitnessMode, HashedPostState, HashedPostStateSorted, HashedStorage, + MultiProof, MultiProofTargets, StorageMultiProof, StorageProof, TrieInput, + updates::TrieUpdates, +}; + +use crate::{ + MorphProofsHashedAccountCursorFactory, MorphProofsStorage, MorphProofsStore, + MorphProofsTrieCursorFactory, +}; + +/// Build the trie + hashed cursor factories sharing one read transaction at the given block. +const fn from_tx<'tx, 'db, S>( + storage: &'db MorphProofsStorage, + tx: &'tx as MorphProofsStore>::Tx<'db>, + block_number: u64, +) -> ( + MorphProofsTrieCursorFactory<'tx, 'db, S>, + MorphProofsHashedAccountCursorFactory<'tx, 'db, S>, +) +where + S: MorphProofsStore + 'db, + 'db: 'tx, +{ + ( + MorphProofsTrieCursorFactory::new(storage, tx, block_number), + MorphProofsHashedAccountCursorFactory::new(storage, tx, block_number), + ) +} + +/// Extends [`Proof`] with operations specific for working with [`MorphProofsStorage`]. +pub trait DatabaseProof<'tx, S: MorphProofsStore + 'tx> { + /// Generates the state proof for target account based on [`TrieInput`]. + fn overlay_account_proof( + storage: &'tx MorphProofsStorage, + block_number: u64, + input: TrieInput, + address: Address, + slots: &[B256], + ) -> Result; + + /// Generates an account proof while reusing an existing read transaction. + fn overlay_account_proof_with_tx<'cursor>( + storage: &'tx MorphProofsStorage, + tx: &'cursor as MorphProofsStore>::Tx<'tx>, + block_number: u64, + input: TrieInput, + address: Address, + slots: &[B256], + ) -> Result + where + 'tx: 'cursor; + + /// Generates the state [`MultiProof`] for target hashed account and storage keys. + fn overlay_multiproof( + storage: &'tx MorphProofsStorage, + block_number: u64, + input: TrieInput, + targets: MultiProofTargets, + ) -> Result; + + /// Generates a state multiproof while reusing an existing read transaction. + fn overlay_multiproof_with_tx<'cursor>( + storage: &'tx MorphProofsStorage, + tx: &'cursor as MorphProofsStore>::Tx<'tx>, + block_number: u64, + input: TrieInput, + targets: MultiProofTargets, + ) -> Result + where + 'tx: 'cursor; +} + +impl<'tx, S> DatabaseProof<'tx, S> + for Proof< + MorphProofsTrieCursorFactory<'tx, 'tx, S>, + MorphProofsHashedAccountCursorFactory<'tx, 'tx, S>, + > +where + S: MorphProofsStore + 'tx + Clone, +{ + fn overlay_account_proof( + storage: &'tx MorphProofsStorage, + block_number: u64, + input: TrieInput, + address: Address, + slots: &[B256], + ) -> Result { + let tx = storage.ro_tx().map_err(Into::::into)?; + Self::overlay_account_proof_with_tx(storage, &tx, block_number, input, address, slots) + } + + fn overlay_account_proof_with_tx<'cursor>( + storage: &'tx MorphProofsStorage, + tx: &'cursor as MorphProofsStore>::Tx<'tx>, + block_number: u64, + input: TrieInput, + address: Address, + slots: &[B256], + ) -> Result + where + 'tx: 'cursor, + { + let nodes_sorted = input.nodes.into_sorted(); + let state_sorted = input.state.into_sorted(); + let (trie_factory, hashed_factory) = from_tx(storage, tx, block_number); + Proof::new(trie_factory.clone(), hashed_factory.clone()) + .with_trie_cursor_factory(InMemoryTrieCursorFactory::new(trie_factory, &nodes_sorted)) + .with_hashed_cursor_factory(HashedPostStateCursorFactory::new( + hashed_factory, + &state_sorted, + )) + .with_prefix_sets_mut(input.prefix_sets) + .account_proof(address, slots) + } + + fn overlay_multiproof( + storage: &'tx MorphProofsStorage, + block_number: u64, + input: TrieInput, + targets: MultiProofTargets, + ) -> Result { + let tx = storage.ro_tx().map_err(Into::::into)?; + Self::overlay_multiproof_with_tx(storage, &tx, block_number, input, targets) + } + + fn overlay_multiproof_with_tx<'cursor>( + storage: &'tx MorphProofsStorage, + tx: &'cursor as MorphProofsStore>::Tx<'tx>, + block_number: u64, + input: TrieInput, + targets: MultiProofTargets, + ) -> Result + where + 'tx: 'cursor, + { + let nodes_sorted = input.nodes.into_sorted(); + let state_sorted = input.state.into_sorted(); + let (trie_factory, hashed_factory) = from_tx(storage, tx, block_number); + Proof::new(trie_factory.clone(), hashed_factory.clone()) + .with_trie_cursor_factory(InMemoryTrieCursorFactory::new(trie_factory, &nodes_sorted)) + .with_hashed_cursor_factory(HashedPostStateCursorFactory::new( + hashed_factory, + &state_sorted, + )) + .with_prefix_sets_mut(input.prefix_sets) + .multiproof(targets) + } +} + +/// Extends [`StorageProof`] with operations specific for working with [`MorphProofsStorage`]. +pub trait DatabaseStorageProof<'tx, S: MorphProofsStore + 'tx> { + /// Generates the storage proof for target slot based on [`TrieInput`]. + fn overlay_storage_proof( + storage: &'tx MorphProofsStorage, + block_number: u64, + address: Address, + slot: B256, + storage: HashedStorage, + ) -> Result; + + /// Generates the storage multiproof for target slots based on [`TrieInput`]. + fn overlay_storage_multiproof( + storage: &'tx MorphProofsStorage, + block_number: u64, + address: Address, + slots: &[B256], + storage: HashedStorage, + ) -> Result; +} + +impl<'tx, S> DatabaseStorageProof<'tx, S> + for proof::StorageProof< + 'static, + MorphProofsTrieCursorFactory<'tx, 'tx, S>, + MorphProofsHashedAccountCursorFactory<'tx, 'tx, S>, + > +where + S: MorphProofsStore + 'tx + Clone, +{ + fn overlay_storage_proof( + storage: &'tx MorphProofsStorage, + block_number: u64, + address: Address, + slot: B256, + hashed_storage: HashedStorage, + ) -> Result { + let hashed_address = keccak256(address); + let prefix_set = hashed_storage.construct_prefix_set(); + let state_sorted = HashedPostStateSorted::new( + Default::default(), + HashMap::from_iter([(hashed_address, hashed_storage.into_sorted())]), + ); + let tx = storage.ro_tx().map_err(Into::::into)?; + let (trie_factory, hashed_factory) = from_tx(storage, &tx, block_number); + proof::StorageProof::new(trie_factory, hashed_factory.clone(), address) + .with_hashed_cursor_factory(HashedPostStateCursorFactory::new( + hashed_factory, + &state_sorted, + )) + .with_prefix_set_mut(prefix_set) + .storage_proof(slot) + } + + fn overlay_storage_multiproof( + storage: &'tx MorphProofsStorage, + block_number: u64, + address: Address, + slots: &[B256], + hashed_storage: HashedStorage, + ) -> Result { + let hashed_address = keccak256(address); + let targets = slots.iter().map(keccak256).collect(); + let prefix_set = hashed_storage.construct_prefix_set(); + let state_sorted = HashedPostStateSorted::new( + Default::default(), + HashMap::from_iter([(hashed_address, hashed_storage.into_sorted())]), + ); + let tx = storage.ro_tx().map_err(Into::::into)?; + let (trie_factory, hashed_factory) = from_tx(storage, &tx, block_number); + proof::StorageProof::new(trie_factory, hashed_factory.clone(), address) + .with_hashed_cursor_factory(HashedPostStateCursorFactory::new( + hashed_factory, + &state_sorted, + )) + .with_prefix_set_mut(prefix_set) + .storage_multiproof(targets) + } +} + +/// Extends [`StateRoot`] with operations specific for working with [`MorphProofsStorage`]. +pub trait DatabaseStateRoot<'tx, S: MorphProofsStore + 'tx + Clone>: Sized { + /// Calculate the state root for this [`HashedPostState`]. + /// Internally, this method retrieves prefixsets and uses them + /// to calculate incremental state root. + /// + /// # Returns + /// + /// The state root for this [`HashedPostState`]. + fn overlay_root( + storage: &'tx MorphProofsStorage, + block_number: u64, + post_state: HashedPostState, + ) -> Result; + + /// Calculates the state root for this [`HashedPostState`] and returns it alongside trie + /// updates. See [`Self::overlay_root`] for more info. + fn overlay_root_with_updates( + storage: &'tx MorphProofsStorage, + block_number: u64, + post_state: HashedPostState, + ) -> Result<(B256, TrieUpdates), StateRootError>; + + /// Calculates the state root for provided [`HashedPostState`] using cached intermediate nodes. + fn overlay_root_from_nodes( + storage: &'tx MorphProofsStorage, + block_number: u64, + input: TrieInput, + ) -> Result; + + /// Calculates the state root and trie updates for provided [`HashedPostState`] using + /// cached intermediate nodes. + fn overlay_root_from_nodes_with_updates( + storage: &'tx MorphProofsStorage, + block_number: u64, + input: TrieInput, + ) -> Result<(B256, TrieUpdates), StateRootError>; +} + +impl<'tx, S> DatabaseStateRoot<'tx, S> + for StateRoot< + MorphProofsTrieCursorFactory<'tx, 'tx, S>, + MorphProofsHashedAccountCursorFactory<'tx, 'tx, S>, + > +where + S: MorphProofsStore + 'tx + Clone, +{ + fn overlay_root( + storage: &'tx MorphProofsStorage, + block_number: u64, + post_state: HashedPostState, + ) -> Result { + let prefix_sets = post_state.construct_prefix_sets().freeze(); + let state_sorted = post_state.into_sorted(); + let tx = storage.ro_tx().map_err(Into::::into)?; + let (trie_factory, hashed_factory) = from_tx(storage, &tx, block_number); + StateRoot::new( + trie_factory, + HashedPostStateCursorFactory::new(hashed_factory, &state_sorted), + ) + .with_prefix_sets(prefix_sets) + .root() + } + + fn overlay_root_with_updates( + storage: &'tx MorphProofsStorage, + block_number: u64, + post_state: HashedPostState, + ) -> Result<(B256, TrieUpdates), StateRootError> { + let prefix_sets = post_state.construct_prefix_sets().freeze(); + let state_sorted = post_state.into_sorted(); + let tx = storage.ro_tx().map_err(Into::::into)?; + let (trie_factory, hashed_factory) = from_tx(storage, &tx, block_number); + StateRoot::new( + trie_factory, + HashedPostStateCursorFactory::new(hashed_factory, &state_sorted), + ) + .with_prefix_sets(prefix_sets) + .root_with_updates() + } + + fn overlay_root_from_nodes( + storage: &'tx MorphProofsStorage, + block_number: u64, + input: TrieInput, + ) -> Result { + let state_sorted = input.state.into_sorted(); + let nodes_sorted = input.nodes.into_sorted(); + let tx = storage.ro_tx().map_err(Into::::into)?; + let (trie_factory, hashed_factory) = from_tx(storage, &tx, block_number); + StateRoot::new( + InMemoryTrieCursorFactory::new(trie_factory, &nodes_sorted), + HashedPostStateCursorFactory::new(hashed_factory, &state_sorted), + ) + .with_prefix_sets(input.prefix_sets.freeze()) + .root() + } + + fn overlay_root_from_nodes_with_updates( + storage: &'tx MorphProofsStorage, + block_number: u64, + input: TrieInput, + ) -> Result<(B256, TrieUpdates), StateRootError> { + let state_sorted = input.state.into_sorted(); + let nodes_sorted = input.nodes.into_sorted(); + let tx = storage.ro_tx().map_err(Into::::into)?; + let (trie_factory, hashed_factory) = from_tx(storage, &tx, block_number); + StateRoot::new( + InMemoryTrieCursorFactory::new(trie_factory, &nodes_sorted), + HashedPostStateCursorFactory::new(hashed_factory, &state_sorted), + ) + .with_prefix_sets(input.prefix_sets.freeze()) + .root_with_updates() + } +} + +/// Extends [`StorageRoot`] with operations specific for working with [`MorphProofsStorage`]. +pub trait DatabaseStorageRoot<'tx, S: MorphProofsStore + 'tx + Clone> { + /// Calculates the storage root for provided [`HashedStorage`]. + fn overlay_root( + storage: &'tx MorphProofsStorage, + block_number: u64, + address: Address, + hashed_storage: HashedStorage, + ) -> Result; +} + +impl<'tx, S> DatabaseStorageRoot<'tx, S> + for StorageRoot< + MorphProofsTrieCursorFactory<'tx, 'tx, S>, + MorphProofsHashedAccountCursorFactory<'tx, 'tx, S>, + > +where + S: MorphProofsStore + 'tx + Clone, +{ + fn overlay_root( + storage: &'tx MorphProofsStorage, + block_number: u64, + address: Address, + hashed_storage: HashedStorage, + ) -> Result { + let prefix_set = hashed_storage.construct_prefix_set().freeze(); + let state_sorted = + HashedPostState::from_hashed_storage(keccak256(address), hashed_storage).into_sorted(); + let tx = storage.ro_tx().map_err(Into::::into)?; + let (trie_factory, hashed_factory) = from_tx(storage, &tx, block_number); + StorageRoot::new( + trie_factory, + HashedPostStateCursorFactory::new(hashed_factory, &state_sorted), + address, + prefix_set, + TrieRootMetrics::new(TrieType::Custom("morph_historical_proofs_storage")), + ) + .root() + } +} + +/// Extends [`TrieWitness`] with operations specific for working with [`MorphProofsStorage`]. +pub trait DatabaseTrieWitness<'tx, S: MorphProofsStore + 'tx + Clone> { + /// Generates the trie witness for the target state based on [`TrieInput`]. + fn overlay_witness( + storage: &'tx MorphProofsStorage, + block_number: u64, + input: TrieInput, + target: HashedPostState, + mode: ExecutionWitnessMode, + ) -> Result, TrieWitnessError>; + + /// Generates the trie witness for the target state, reusing `tx`. + fn overlay_witness_with_tx<'cursor>( + storage: &'tx MorphProofsStorage, + tx: &'cursor as MorphProofsStore>::Tx<'tx>, + block_number: u64, + input: TrieInput, + target: HashedPostState, + mode: ExecutionWitnessMode, + ) -> Result, TrieWitnessError> + where + 'tx: 'cursor; +} + +impl<'tx, S> DatabaseTrieWitness<'tx, S> + for TrieWitness< + MorphProofsTrieCursorFactory<'tx, 'tx, S>, + MorphProofsHashedAccountCursorFactory<'tx, 'tx, S>, + > +where + S: MorphProofsStore + 'tx + Clone, +{ + fn overlay_witness( + storage: &'tx MorphProofsStorage, + block_number: u64, + input: TrieInput, + target: HashedPostState, + mode: ExecutionWitnessMode, + ) -> Result, TrieWitnessError> { + let tx = storage.ro_tx().map_err(|error| { + let error = Into::::into(error); + StateProofError::from(error) + })?; + Self::overlay_witness_with_tx(storage, &tx, block_number, input, target, mode) + } + + fn overlay_witness_with_tx<'cursor>( + storage: &'tx MorphProofsStorage, + tx: &'cursor as MorphProofsStore>::Tx<'tx>, + block_number: u64, + input: TrieInput, + target: HashedPostState, + mode: ExecutionWitnessMode, + ) -> Result, TrieWitnessError> + where + 'tx: 'cursor, + { + let nodes_sorted = input.nodes.into_sorted(); + let state_sorted = input.state.into_sorted(); + let (trie_factory, hashed_factory) = from_tx(storage, tx, block_number); + TrieWitness::new(trie_factory.clone(), hashed_factory.clone()) + .with_trie_cursor_factory(InMemoryTrieCursorFactory::new(trie_factory, &nodes_sorted)) + .with_hashed_cursor_factory(HashedPostStateCursorFactory::new( + hashed_factory, + &state_sorted, + )) + .with_prefix_sets_mut(input.prefix_sets) + .always_include_root_node() + .with_execution_witness_mode(mode) + .compute(target) + } +} + +#[cfg(test)] +mod tests { + use std::sync::Arc; + + use alloy_eips::{BlockNumHash, NumHash, eip1898::BlockWithParent}; + use alloy_primitives::{Address, B256, U256, keccak256}; + use reth_primitives_traits::Account; + use reth_trie::EMPTY_ROOT_HASH; + + use super::*; + use crate::{ + BlockStateDiff, InMemoryProofsStorage, MdbxProofsStorage, MorphProofsInitialStateStore, + MorphProofsStore, + }; + + fn assert_account_and_storage_proof(storage: S) + where + S: MorphProofsStore + MorphProofsInitialStateStore + Clone, + { + let genesis_hash = B256::repeat_byte(0x01); + storage + .set_initial_state_anchor(BlockNumHash::new(0, genesis_hash)) + .unwrap(); + storage.commit_initial_state().unwrap(); + + let address = Address::repeat_byte(0x11); + let slot = B256::repeat_byte(0x22); + let value = U256::from(42); + let account = Account { + nonce: 7, + balance: U256::from(1_000_000), + bytecode_hash: None, + }; + let hashed_address = keccak256(address); + let post_state = HashedPostState::default() + .with_accounts([(hashed_address, Some(account))]) + .with_storages([( + hashed_address, + HashedStorage::from_iter(false, [(keccak256(slot), value)]), + )]); + + let (state_root, trie_updates) = + StateRoot::overlay_root_with_updates(&storage, 0, post_state.clone()).unwrap(); + let block_hash = B256::repeat_byte(0x02); + storage + .store_trie_updates( + BlockWithParent::new(genesis_hash, NumHash::new(1, block_hash)), + BlockStateDiff { + sorted_trie_updates: trie_updates.into_sorted(), + sorted_post_state: post_state.into_sorted(), + }, + ) + .unwrap(); + + let proof = + Proof::overlay_account_proof(&storage, 1, TrieInput::default(), address, &[slot]) + .unwrap(); + + assert_eq!(proof.info, Some(account)); + assert_eq!(proof.storage_proofs.len(), 1); + assert_eq!(proof.storage_proofs[0].value, value); + proof.verify(state_root).unwrap(); + } + + #[test] + fn account_and_storage_proof_verify_in_memory() { + assert_account_and_storage_proof(InMemoryProofsStorage::new()); + } + + #[test] + fn account_and_storage_proof_verify_in_mdbx() { + let directory = tempfile::tempdir().unwrap(); + let storage = Arc::new(MdbxProofsStorage::new(directory.path()).unwrap()); + assert_account_and_storage_proof(storage); + } + + #[test] + fn proof_snapshot_survives_concurrent_prune_boundary_move() { + let directory = tempfile::tempdir().unwrap(); + let storage = Arc::new(MdbxProofsStorage::new(directory.path()).unwrap()); + let genesis_hash = B256::repeat_byte(0x01); + storage + .set_initial_state_anchor(BlockNumHash::new(0, genesis_hash)) + .unwrap(); + storage.commit_initial_state().unwrap(); + + let address = Address::repeat_byte(0x11); + let post_state = HashedPostState::default().with_accounts([( + keccak256(address), + Some(Account { + nonce: 1, + balance: U256::from(1), + bytecode_hash: None, + }), + )]); + let (_, trie_updates) = + StateRoot::overlay_root_with_updates(&storage, 0, post_state.clone()).unwrap(); + let block_one = + BlockWithParent::new(genesis_hash, NumHash::new(1, B256::repeat_byte(0x02))); + storage + .store_trie_updates( + block_one, + BlockStateDiff { + sorted_trie_updates: trie_updates.into_sorted(), + sorted_post_state: post_state.into_sorted(), + }, + ) + .unwrap(); + + let request_tx = storage.ro_tx().unwrap(); + assert_eq!( + storage + .get_earliest_block_number_with_tx(&request_tx) + .unwrap() + .unwrap() + .0, + 0 + ); + + storage.prune_earliest_state(block_one).unwrap(); + assert_eq!(storage.get_earliest_block_number().unwrap().unwrap().0, 1); + assert_eq!( + storage + .get_earliest_block_number_with_tx(&request_tx) + .unwrap() + .unwrap() + .0, + 0, + "request snapshot must retain the validated proof window" + ); + + let proof = Proof::overlay_account_proof_with_tx( + &storage, + &request_tx, + 0, + TrieInput::default(), + address, + &[], + ) + .unwrap(); + assert!(proof.info.is_none()); + proof.verify(EMPTY_ROOT_HASH).unwrap(); + } + + #[test] + fn exclusion_proofs_verify_at_historical_block() { + let storage = InMemoryProofsStorage::new(); + let genesis_hash = B256::repeat_byte(0x01); + storage + .set_initial_state_anchor(BlockNumHash::new(0, genesis_hash)) + .unwrap(); + storage.commit_initial_state().unwrap(); + + let address = Address::repeat_byte(0x11); + let post_state = HashedPostState::default().with_accounts([( + keccak256(address), + Some(Account { + nonce: 1, + balance: U256::from(1), + bytecode_hash: None, + }), + )]); + let (state_root, trie_updates) = + StateRoot::overlay_root_with_updates(&storage, 0, post_state.clone()).unwrap(); + storage + .store_trie_updates( + BlockWithParent::new(genesis_hash, NumHash::new(1, B256::repeat_byte(0x02))), + BlockStateDiff { + sorted_trie_updates: trie_updates.into_sorted(), + sorted_post_state: post_state.into_sorted(), + }, + ) + .unwrap(); + + let missing_address = Address::repeat_byte(0x33); + let missing_slot = B256::repeat_byte(0x44); + let proof = Proof::overlay_account_proof( + &storage, + 1, + TrieInput::default(), + missing_address, + &[missing_slot], + ) + .unwrap(); + + assert!(proof.info.is_none()); + assert_eq!(proof.storage_proofs.len(), 1); + assert_eq!(proof.storage_proofs[0].value, U256::ZERO); + proof.verify(state_root).unwrap(); + } +} diff --git a/crates/proofs/src/provider.rs b/crates/proofs/src/provider.rs new file mode 100644 index 00000000..2d9e2925 --- /dev/null +++ b/crates/proofs/src/provider.rs @@ -0,0 +1,340 @@ +//! Provider for external proofs storage + +use std::fmt::Debug; + +use alloy_primitives::keccak256; +use parking_lot::{MappedMutexGuard, Mutex, MutexGuard}; +use reth_primitives_traits::{Account, Bytecode}; +use reth_provider::{ + AccountReader, BlockHashReader, BytecodeReader, HashedPostStateProvider, ProviderError, + ProviderResult, StateProofProvider, StateProvider, StateRootProvider, StorageRootProvider, +}; +use reth_revm::{ + db::BundleState, + primitives::{Address, B256, Bytes, StorageValue, alloy_primitives::BlockNumber}, +}; +use reth_trie::{ + StateRoot, StorageRoot, + hashed_cursor::HashedCursor, + proof::{self, Proof}, + witness::TrieWitness, +}; +use reth_trie_common::{ + AccountProof, ExecutionWitnessMode, HashedPostState, HashedStorage, KeccakKeyHasher, + MultiProof, MultiProofTargets, StorageMultiProof, StorageProof, TrieInput, + updates::TrieUpdates, +}; + +use crate::{ + MorphProofsStorage, MorphProofsStorageError, MorphProofsStore, + proof::{ + DatabaseProof, DatabaseStateRoot, DatabaseStorageProof, DatabaseStorageRoot, + DatabaseTrieWitness, + }, +}; + +/// State provider for external proofs storage. +pub struct MorphProofsStateProviderRef<'a, Storage: MorphProofsStore> { + /// Historical state provider for non-state related tasks. + latest: Box, + + /// Storage provider for state lookups. + storage: &'a MorphProofsStorage, + + /// Max block number that can be used for state lookups. + block_number: BlockNumber, + + /// Lazily-acquired read-only transaction shared across all EVM state reads. + /// + /// Acquired once on the first [`basic_account`](AccountReader::basic_account) or + /// [`storage`](StateProvider::storage) call and reused for the lifetime of this provider, + /// so that all EVM state reads within a single execution context share one database snapshot + /// and avoid per-call transaction-acquisition contention. + lazy_tx: Mutex>>, +} + +impl<'a, Storage: MorphProofsStore> MorphProofsStateProviderRef<'a, Storage> { + /// Creates a new state provider. + pub fn new( + latest: Box, + storage: &'a MorphProofsStorage, + block_number: BlockNumber, + ) -> Self { + Self { + latest, + storage, + block_number, + lazy_tx: Mutex::new(None), + } + } + + /// Creates a state provider pinned to an existing proof-database read transaction. + pub fn new_with_tx( + latest: Box, + storage: &'a MorphProofsStorage, + block_number: BlockNumber, + tx: Storage::Tx<'a>, + ) -> Self { + Self { + latest, + storage, + block_number, + lazy_tx: Mutex::new(Some(tx)), + } + } + + fn ensure_tx(&self) -> ProviderResult>> { + let mut guard = self.lazy_tx.lock(); + if guard.is_none() { + *guard = Some(self.storage.ro_tx().map_err(Into::::into)?); + } + + Ok(MutexGuard::map(guard, |tx| { + tx.as_mut().expect("read-only transaction initialized") + })) + } +} + +impl<'a, Storage> Debug for MorphProofsStateProviderRef<'a, Storage> +where + Storage: MorphProofsStore + 'a + Debug, +{ + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + f.debug_struct("MorphProofsStateProviderRef") + .field("storage", &self.storage) + .field("block_number", &self.block_number) + .finish() + } +} + +impl<'a, Storage: MorphProofsStore + Clone> MorphProofsStateProviderRef<'a, Storage> { + fn storage_by_hashed_key( + &self, + address: Address, + hashed_key: B256, + ) -> ProviderResult> { + let tx = self.ensure_tx()?; + Ok(self + .storage + .storage_hashed_cursor_with_tx(&tx, keccak256(address.0), self.block_number) + .map_err(Into::::into)? + .seek(hashed_key) + .map_err(Into::::into)? + .and_then(|(key, val)| (key == hashed_key).then_some(val))) + } +} + +impl From for ProviderError { + fn from(error: MorphProofsStorageError) -> Self { + Self::other(error) + } +} + +impl<'a, Storage: MorphProofsStore> BlockHashReader for MorphProofsStateProviderRef<'a, Storage> { + fn block_hash(&self, number: BlockNumber) -> ProviderResult> { + self.latest.block_hash(number) + } + + fn canonical_hashes_range( + &self, + start: BlockNumber, + end: BlockNumber, + ) -> ProviderResult> { + self.latest.canonical_hashes_range(start, end) + } +} + +impl<'a, Storage: MorphProofsStore + Clone> StateRootProvider + for MorphProofsStateProviderRef<'a, Storage> +{ + fn state_root(&self, state: HashedPostState) -> ProviderResult { + Ok(StateRoot::overlay_root( + self.storage, + self.block_number, + state, + )?) + } + + fn state_root_from_nodes(&self, input: TrieInput) -> ProviderResult { + Ok(StateRoot::overlay_root_from_nodes( + self.storage, + self.block_number, + input, + )?) + } + + fn state_root_with_updates( + &self, + state: HashedPostState, + ) -> ProviderResult<(B256, TrieUpdates)> { + Ok(StateRoot::overlay_root_with_updates( + self.storage, + self.block_number, + state, + )?) + } + + fn state_root_from_nodes_with_updates( + &self, + input: TrieInput, + ) -> ProviderResult<(B256, TrieUpdates)> { + Ok(StateRoot::overlay_root_from_nodes_with_updates( + self.storage, + self.block_number, + input, + )?) + } +} + +impl<'a, Storage: MorphProofsStore + Clone> StorageRootProvider + for MorphProofsStateProviderRef<'a, Storage> +{ + fn storage_root(&self, address: Address, storage: HashedStorage) -> ProviderResult { + StorageRoot::overlay_root(self.storage, self.block_number, address, storage) + .map_err(|err| ProviderError::Database(err.into())) + } + + fn storage_proof( + &self, + address: Address, + slot: B256, + storage: HashedStorage, + ) -> ProviderResult { + proof::StorageProof::overlay_storage_proof( + self.storage, + self.block_number, + address, + slot, + storage, + ) + .map_err(ProviderError::from) + } + + fn storage_multiproof( + &self, + address: Address, + slots: &[B256], + storage: HashedStorage, + ) -> ProviderResult { + proof::StorageProof::overlay_storage_multiproof( + self.storage, + self.block_number, + address, + slots, + storage, + ) + .map_err(ProviderError::from) + } +} + +impl<'a, Storage: MorphProofsStore + Clone> StateProofProvider + for MorphProofsStateProviderRef<'a, Storage> +{ + fn proof( + &self, + input: TrieInput, + address: Address, + slots: &[B256], + ) -> ProviderResult { + let tx = self.ensure_tx()?; + Proof::overlay_account_proof_with_tx( + self.storage, + &tx, + self.block_number, + input, + address, + slots, + ) + .map_err(ProviderError::from) + } + + fn multiproof( + &self, + input: TrieInput, + targets: MultiProofTargets, + ) -> ProviderResult { + let tx = self.ensure_tx()?; + Proof::overlay_multiproof_with_tx(self.storage, &tx, self.block_number, input, targets) + .map_err(ProviderError::from) + } + + fn witness( + &self, + input: TrieInput, + target: HashedPostState, + mode: ExecutionWitnessMode, + ) -> ProviderResult> { + let tx = self.ensure_tx()?; + TrieWitness::overlay_witness_with_tx( + self.storage, + &tx, + self.block_number, + input, + target, + mode, + ) + .map_err(ProviderError::from) + .map(|hm| hm.into_values().collect()) + } +} + +impl<'a, Storage: MorphProofsStore> HashedPostStateProvider + for MorphProofsStateProviderRef<'a, Storage> +{ + fn hashed_post_state(&self, bundle_state: &BundleState) -> HashedPostState { + HashedPostState::from_bundle_state::(bundle_state.state()) + } +} + +impl<'a, Storage: MorphProofsStore> AccountReader for MorphProofsStateProviderRef<'a, Storage> { + fn basic_account(&self, address: &Address) -> ProviderResult> { + let hashed_key = keccak256(address.0); + let tx = self.ensure_tx()?; + Ok(self + .storage + .account_hashed_cursor_with_tx(&tx, self.block_number) + .map_err(Into::::into)? + .seek(hashed_key) + .map_err(Into::::into)? + .and_then(|(key, account)| (key == hashed_key).then_some(account))) + } +} + +impl<'a, Storage> StateProvider for MorphProofsStateProviderRef<'a, Storage> +where + Storage: MorphProofsStore + Clone, +{ + fn storage(&self, address: Address, storage_key: B256) -> ProviderResult> { + let hashed_key = keccak256(storage_key); + self.storage_by_hashed_key(address, hashed_key) + } +} + +impl<'a, Storage: MorphProofsStore> BytecodeReader for MorphProofsStateProviderRef<'a, Storage> { + fn bytecode_by_hash(&self, code_hash: &B256) -> ProviderResult> { + self.latest.bytecode_by_hash(code_hash) + } +} + +#[cfg(test)] +mod tests { + use reth_provider::noop::NoopProvider; + + use super::*; + use crate::InMemoryProofsStorage; + + #[test] + fn test_morph_proofs_state_provider_ref_debug() { + let latest: Box = Box::::default(); + let storage: crate::MorphProofsStorage = + InMemoryProofsStorage::new(); + let block_number = 42u64; + + let provider = MorphProofsStateProviderRef::new(latest, &storage, block_number); + + assert_eq!( + format!("{provider:?}"), + "MorphProofsStateProviderRef { storage: InMemoryProofsStorage { inner: RwLock { data: InMemoryStorageInner { account_branches: {}, storage_branches: {}, hashed_accounts: {}, hashed_storages: {}, trie_updates: {}, post_states: {}, earliest_block: None, anchor_block: None } } }, block_number: 42 }" + ); + } +} diff --git a/crates/proofs/src/prune/error.rs b/crates/proofs/src/prune/error.rs new file mode 100644 index 00000000..56992c93 --- /dev/null +++ b/crates/proofs/src/prune/error.rs @@ -0,0 +1,102 @@ +use std::{ + fmt, + fmt::{Display, Formatter}, + time::Duration, +}; + +use reth_provider::ProviderError; +use strum::Display; +use thiserror::Error; + +use crate::{MorphProofsStorageError, api::WriteCounts}; + +/// Result of [`MorphProofStoragePruner::run`](crate::MorphProofStoragePruner::run) execution. +pub type MorphProofStoragePrunerResult = Result; + +/// Successful prune summary. +#[derive(Debug, Clone, Default, Eq, PartialEq)] +pub struct PrunerOutput { + /// Total elapsed wall time for this run (fetch + apply). + pub duration: Duration, + /// Earliest block at the start of the run. + pub start_block: u64, + /// New earliest block at the end of the run. + pub end_block: u64, + /// Number of entries updated/removed per table. + pub write_counts: WriteCounts, +} + +impl Display for PrunerOutput { + fn fmt(&self, f: &mut Formatter<'_>) -> fmt::Result { + let blocks = self.end_block.saturating_sub(self.start_block); + let total_entries = self.write_counts.hashed_accounts_written_total + + self.write_counts.hashed_storages_written_total + + self.write_counts.account_trie_updates_written_total + + self.write_counts.storage_trie_updates_written_total; + write!( + f, + "Pruned {}→{} ({} blocks), entries={}, elapsed={:.3}s", + self.start_block, + self.end_block, + blocks, + total_entries, + self.duration.as_secs_f64(), + ) + } +} + +impl PrunerOutput { + /// extend the current [`PrunerOutput`] with another [`PrunerOutput`] + pub fn extend_ref(&mut self, other: Self) { + self.duration += other.duration; + // take the earliest start block + if self.start_block > other.start_block { + self.start_block = other.start_block; + } + // take the latest end block + if self.end_block < other.end_block { + self.end_block = other.end_block; + } + self.write_counts += other.write_counts; + } +} + +/// Error returned by the pruner. +#[derive(Debug, Error, Display)] +pub enum PrunerError { + /// Wrapped error from the underlying `MorphProofStorage` layer. + Storage(#[from] MorphProofsStorageError), + + /// Wrapped error from the reth db provider. + Provider(#[from] ProviderError), + + /// Block not found in the underlying reth storage provider. + BlockNotFound(u64), + + /// The pruner timed out before finishing the prune + TimedOut(Duration), +} + +#[cfg(test)] +mod tests { + use std::time::Duration; + + use super::PrunerOutput; + use crate::api::WriteCounts; + + #[test] + fn test_pruner_output_display() { + let pruner_output = PrunerOutput { + duration: Duration::from_secs(10), + start_block: 1, + end_block: 2, + write_counts: WriteCounts::new(1, 2, 3, 4), + }; + let formatted_pruner_output = format!("{pruner_output}"); + + assert_eq!( + formatted_pruner_output, + "Pruned 1→2 (1 blocks), entries=10, elapsed=10.000s" + ); + } +} diff --git a/crates/proofs/src/prune/metrics.rs b/crates/proofs/src/prune/metrics.rs new file mode 100644 index 00000000..2ddc2a46 --- /dev/null +++ b/crates/proofs/src/prune/metrics.rs @@ -0,0 +1,54 @@ +//! Pruner metrics. + +use reth_metrics::{ + Metrics, + metrics::{Gauge, Histogram}, +}; + +use crate::PrunerOutput; + +#[derive(Metrics, Clone)] +#[metrics(scope = "morph.proofs.pruner")] +pub(super) struct PrunerMetrics { + /// Pruning duration. + total_duration_seconds: Histogram, + /// Number of pruned blocks. + pruned_blocks: Gauge, + /// Number of account trie updates written. + account_trie_updates_written: Gauge, + /// Number of storage trie updates written. + storage_trie_updates_written: Gauge, + /// Number of hashed accounts written. + hashed_accounts_written: Gauge, + /// Number of hashed storages written. + hashed_storages_written: Gauge, +} + +impl PrunerMetrics { + pub(super) fn record_prune_result(result: PrunerOutput) { + let blocks_pruned = result.end_block.saturating_sub(result.start_block); + if blocks_pruned == 0 { + return; + } + + let metrics = Self::default(); + metrics + .total_duration_seconds + .record(result.duration.as_secs_f64()); + metrics.pruned_blocks.set(blocks_pruned as f64); + + let counts = &result.write_counts; + metrics + .account_trie_updates_written + .set(counts.account_trie_updates_written_total as f64); + metrics + .storage_trie_updates_written + .set(counts.storage_trie_updates_written_total as f64); + metrics + .hashed_accounts_written + .set(counts.hashed_accounts_written_total as f64); + metrics + .hashed_storages_written + .set(counts.hashed_storages_written_total as f64); + } +} diff --git a/crates/proofs/src/prune/mod.rs b/crates/proofs/src/prune/mod.rs new file mode 100644 index 00000000..4a5781ae --- /dev/null +++ b/crates/proofs/src/prune/mod.rs @@ -0,0 +1,12 @@ +//! Proof storage pruner for removing stale trie data. + +mod error; +pub use error::{MorphProofStoragePrunerResult, PrunerError, PrunerOutput}; + +mod pruner; +pub use pruner::MorphProofStoragePruner; + +mod metrics; + +mod task; +pub use task::MorphProofStoragePrunerTask; diff --git a/crates/proofs/src/prune/pruner.rs b/crates/proofs/src/prune/pruner.rs new file mode 100644 index 00000000..4f64a787 --- /dev/null +++ b/crates/proofs/src/prune/pruner.rs @@ -0,0 +1,735 @@ +use std::cmp; + +use alloy_eips::{BlockNumHash, eip1898::BlockWithParent}; +use reth_provider::BlockHashReader; +use tokio::time::Instant; +use tracing::{error, info, trace}; + +use crate::{ + MorphProofsStorage, MorphProofsStore, ProofHistoryMetrics, + prune::{ + error::{MorphProofStoragePrunerResult, PrunerError, PrunerOutput}, + metrics::PrunerMetrics, + }, +}; + +/// Prunes the proof storage by calling `prune_earliest_state` on the storage provider. +#[derive(Debug)] +pub struct MorphProofStoragePruner { + /// Database provider for the prune + provider: MorphProofsStorage

, + /// Reader to fetch block hash by block number + block_hash_reader: H, + /// Keep at least these many recent blocks. + retention_blocks: u64, + /// Maximum number of blocks to prune in one database transaction + prune_batch_size: u64, +} + +impl MorphProofStoragePruner { + /// Create a new pruner. + pub const fn new( + provider: MorphProofsStorage

, + block_hash_reader: H, + retention_blocks: u64, + prune_batch_size: u64, + ) -> Self { + assert!( + prune_batch_size > 0, + "prune batch size must be greater than zero" + ); + Self { + provider, + block_hash_reader, + retention_blocks, + prune_batch_size, + } + } + + const fn target_earliest_block(&self, latest_block: u64) -> u64 { + latest_block.saturating_sub(self.retention_blocks) + } +} + +impl MorphProofStoragePruner +where + P: MorphProofsStore, + H: BlockHashReader, +{ + fn run_inner(&self) -> MorphProofStoragePrunerResult { + let Some(((earliest_block, _), (latest_block, _))) = self + .provider + .get_earliest_block_number()? + .zip(self.provider.get_latest_block_number()?) + else { + trace!(target: "trie::pruner", "No earliest or latest block in the proof storage"); + return Ok(PrunerOutput::default()); + }; + + let target_earliest_block = self.target_earliest_block(latest_block); + info!( + target: "trie::pruner", + earliest_block, + latest_block, + target_earliest_block, + retention_blocks = self.retention_blocks, + prune_batch_size = self.prune_batch_size, + "Calculated proof storage pruning target", + ); + if earliest_block >= target_earliest_block { + trace!(target: "trie::pruner", "Nothing to prune"); + return Ok(PrunerOutput::default()); + } + + info!( + target: "trie::pruner", + from_block = earliest_block, + to_block = target_earliest_block, + latest_block, + retention_blocks = self.retention_blocks, + "Starting pruning proof storage", + ); + + let mut current_earliest_block = earliest_block; + let mut prune_output = PrunerOutput { + start_block: earliest_block, + end_block: target_earliest_block, + ..Default::default() + }; + + // Prune in batches + while current_earliest_block < target_earliest_block { + // Calculate the end of this batch + let batch_end_block = cmp::min( + current_earliest_block.saturating_add(self.prune_batch_size), + target_earliest_block, + ); + info!( + target: "trie::pruner", + start_block = current_earliest_block, + end_block = batch_end_block, + target_earliest_block, + batch_size = batch_end_block.saturating_sub(current_earliest_block), + "Starting proof storage prune batch", + ); + + let batch_output = self.prune_batch(current_earliest_block, batch_end_block)?; + + prune_output.extend_ref(batch_output); + + // Update loop state + current_earliest_block = batch_end_block; + } + + Ok(prune_output) + } + + /// Prunes a single batch of blocks. + fn prune_batch(&self, start_block: u64, end_block: u64) -> Result { + let batch_start_time = Instant::now(); + info!( + target: "trie::pruner", + start_block, + end_block, + "Resolving proof storage prune batch block hashes", + ); + if end_block == 0 { + trace!( + target: "trie::pruner", + start_block, + end_block, + "Skipping proof storage prune batch at genesis block", + ); + return Ok(PrunerOutput { + duration: batch_start_time.elapsed(), + start_block, + end_block, + ..Default::default() + }); + } + + // Fetch the block hash for the new earliest block of this batch. + // + // The parent hash is intentionally not fetched: `prune_earliest_state` only reads + // `block.number` and `block.hash` from the supplied `BlockWithParent`. Fetching the + // parent hash here was wasted I/O proportional to the number of batches. + let new_earliest_block_hash = self + .block_hash_reader + .block_hash(end_block) + .inspect_err(|err| { + error!( + target: "trie::pruner", + block = end_block, + ?err, + "Failed to fetch block hash for new earliest block during pruning" + ) + })? + .ok_or(PrunerError::BlockNotFound(end_block))?; + + let block_with_parent = BlockWithParent { + parent: Default::default(), + block: BlockNumHash { + number: end_block, + hash: new_earliest_block_hash, + }, + }; + + info!( + target: "trie::pruner", + start_block, + end_block, + block_hash = ?new_earliest_block_hash, + "Resolved proof storage prune batch block hashes", + ); + + info!( + target: "trie::pruner", + start_block, + end_block, + "Applying proof storage prune batch", + ); + let write_counts = self.provider.prune_earliest_state(block_with_parent)?; + + let duration = batch_start_time.elapsed(); + let batch_output = PrunerOutput { + duration, + start_block, + end_block, + write_counts, + }; + + PrunerMetrics::record_prune_result(batch_output.clone()); + + info!( + target: "trie::pruner", + ?batch_output, + "Finished pruning batch of proof storage", + ); + Ok(batch_output) + } + + /// Run the pruner + pub fn run(&self) { + let res = self.try_run(); + match res { + Err(e) => { + error!(target: "trie::pruner", err=%e, "Pruner failed"); + } + Ok(res) => { + info!(target: "trie::pruner", result = %res, "Finished pruning proof storage"); + } + } + } + + /// Runs the pruner and returns failures to command-style callers. + pub fn try_run(&self) -> MorphProofStoragePrunerResult { + let result = self.run_inner(); + match &result { + Ok(_) => { + if let Ok(Some((earliest, _))) = self.provider.get_earliest_block_number() + && let Ok(Some((latest, _))) = self.provider.get_latest_block_number() + { + ProofHistoryMetrics::set_window(earliest, latest); + } + } + Err(_) => ProofHistoryMetrics::record_prune_error(), + } + result + } +} + +#[cfg(test)] +mod tests { + use alloy_eips::{BlockHashOrNumber, NumHash}; + use alloy_primitives::{B256, BlockNumber, keccak256}; + use mockall::mock; + use reth_storage_errors::provider::ProviderResult; + + use super::*; + + mock! ( + #[derive(Debug)] + pub BlockHashReader {} + + impl BlockHashReader for BlockHashReader { + fn block_hash(&self, number: BlockNumber) -> ProviderResult>; + + fn convert_block_hash( + &self, + _hash_or_number: BlockHashOrNumber, + ) -> ProviderResult>; + + fn canonical_hashes_range( + &self, + _start: BlockNumber, + _end: BlockNumber, + ) -> ProviderResult>; + } + ); + + fn b256(n: u64) -> B256 { + keccak256(n.to_be_bytes()) + } + + /// Build a block-with-parent for number `n` with deterministic hash. + fn block(n: u64, parent: B256) -> BlockWithParent { + BlockWithParent::new(parent, NumHash::new(n, b256(n))) + } + + macro_rules! proof_storage_pruner_tests { + ($mod_name:ident, $storage:ident) => { + mod $mod_name { + use std::sync::Arc; + + use alloy_primitives::{B256, U256}; + use reth_primitives_traits::Account; + use reth_trie::{ + BranchNodeCompact, HashedPostState, HashedStorage, Nibbles, + hashed_cursor::HashedCursor, + trie_cursor::TrieCursor, + updates::{StorageTrieUpdates, TrieUpdates, TrieUpdatesSorted}, + }; + use tempfile::TempDir; + + use super::{super::*, MockBlockHashReader, b256, block}; + use crate::{BlockStateDiff, $storage}; + + #[tokio::test] + async fn run_inner_and_and_verify_updated_state() { + // --- env/store --- + let dir = TempDir::new().unwrap(); + let store: MorphProofsStorage> = + MorphProofsStorage::from(Arc::new($storage::new(dir.path()).expect("env"))); + + store + .set_earliest_block_number(0, B256::ZERO) + .expect("set earliest"); + + // --- entities --- + // accounts + let a1 = B256::from([0xA1; 32]); + let a2 = B256::from([0xA2; 32]); + let a3 = B256::from([0xA3; 32]); // introduced later + + // one storage address with 3 slots + let stor_addr = B256::from([0x10; 32]); + let s1 = B256::from([0xB1; 32]); + let s2 = B256::from([0xB2; 32]); + let s3 = B256::from([0xB3; 32]); + + // account-trie paths (p1 gets removed by block 3; p2 remains; p3 added later) + let p1 = Nibbles::from_nibbles_unchecked([0x01, 0x02]); + let p2 = Nibbles::from_nibbles_unchecked([0x03, 0x04]); + let p3 = Nibbles::from_nibbles_unchecked([0x05, 0x06]); + + let node_p1 = + BranchNodeCompact::new(0b1, 0, 0, vec![], Some(B256::from([0x11; 32]))); + let node_p2 = + BranchNodeCompact::new(0b10, 0, 0, vec![], Some(B256::from([0x22; 32]))); + let node_p3 = + BranchNodeCompact::new(0b11, 0, 0, vec![], Some(B256::from([0x33; 32]))); + + // storage-trie paths (st1 removed by block 3; st2 remains; st3 added later) + let st1 = Nibbles::from_nibbles_unchecked([0x0A]); + let st2 = Nibbles::from_nibbles_unchecked([0x0B]); + let st3 = Nibbles::from_nibbles_unchecked([0x0C]); + + let node_st2 = + BranchNodeCompact::new(0b101, 0, 0, vec![], Some(B256::from([0x44; 32]))); + let node_st3 = + BranchNodeCompact::new(0b110, 0, 0, vec![], Some(B256::from([0x55; 32]))); + + // --- write 5 blocks manually --- + let mut parent = B256::ZERO; + + // Block 1: add a1,a2; s1=100, s2=200; add p1, st1 + { + let b1 = block(1, parent); + + let mut d_trie_updates = TrieUpdates::default(); + let mut d_post_state = HashedPostState::default(); + + d_post_state.accounts.insert( + a1, + Some(Account { + nonce: 1, + balance: U256::from(1_001), + ..Default::default() + }), + ); + d_post_state.accounts.insert( + a2, + Some(Account { + nonce: 1, + balance: U256::from(1_002), + ..Default::default() + }), + ); + + let mut hs = HashedStorage::default(); + hs.storage.insert(s1, U256::from(100)); + hs.storage.insert(s2, U256::from(200)); + d_post_state.storages.insert(stor_addr, hs); + + d_trie_updates.account_nodes.insert(p1, node_p1); + let e = d_trie_updates.storage_tries.entry(stor_addr).or_default(); + e.storage_nodes.insert(st1, BranchNodeCompact::default()); + + let d = BlockStateDiff { + sorted_post_state: d_post_state.into_sorted(), + sorted_trie_updates: d_trie_updates.into_sorted(), + }; + store.store_trie_updates(b1, d).expect("b1"); + parent = b256(1); + } + + // Block 2: update a2; add a3; s2=220, s3=300; add p2, st2 + { + let b2 = block(2, parent); + + let mut d_trie_updates = TrieUpdates::default(); + let mut d_post_state = HashedPostState::default(); + + d_post_state.accounts.insert( + a2, + Some(Account { + nonce: 2, + balance: U256::from(2_002), + ..Default::default() + }), + ); + d_post_state.accounts.insert( + a3, + Some(Account { + nonce: 1, + balance: U256::from(1_003), + ..Default::default() + }), + ); + + let mut hs = HashedStorage::default(); + hs.storage.insert(s2, U256::from(220)); + hs.storage.insert(s3, U256::from(300)); + d_post_state.storages.insert(stor_addr, hs); + + d_trie_updates.account_nodes.insert(p2, node_p2.clone()); + let e = d_trie_updates.storage_tries.entry(stor_addr).or_default(); + e.storage_nodes.insert(st2, node_st2.clone()); + + let d = BlockStateDiff { + sorted_post_state: d_post_state.into_sorted(), + sorted_trie_updates: d_trie_updates.into_sorted(), + }; + store.store_trie_updates(b2, d).expect("b2"); + parent = b256(2); + } + + // Block 3: delete a1; leave a2,a3; remove p1; remove st1 (storage-trie) + { + let b3 = block(3, parent); + + let mut d_trie_updates = TrieUpdates::default(); + let mut d_post_state = HashedPostState::default(); + + // delete a1, keep a2 & a3 values unchanged for this block + d_post_state.accounts.insert(a1, None); + + // remove account trie node p1 + d_trie_updates.removed_nodes.insert(p1); + + // remove storage-trie node st1 + let mut st_upd = StorageTrieUpdates::default(); + st_upd.removed_nodes.insert(st1); + d_trie_updates.storage_tries.insert(stor_addr, st_upd); + + let d = BlockStateDiff { + sorted_post_state: d_post_state.into_sorted(), + sorted_trie_updates: d_trie_updates.into_sorted(), + }; + store.store_trie_updates(b3, d).expect("b3"); + parent = b256(3); + } + + // Block 4 (kept): update a2; s1=140; add p3, st3 + { + let b4 = block(4, parent); + + let mut d_trie_updates = TrieUpdates::default(); + let mut d_post_state = HashedPostState::default(); + + d_post_state.accounts.insert( + a2, + Some(Account { + nonce: 3, + balance: U256::from(3_002), + ..Default::default() + }), + ); + + let mut hs = HashedStorage::default(); + hs.storage.insert(s1, U256::from(140)); + d_post_state.storages.insert(stor_addr, hs); + d_trie_updates.account_nodes.insert(p3, node_p3.clone()); + let e = d_trie_updates.storage_tries.entry(stor_addr).or_default(); + e.storage_nodes.insert(st3, node_st3.clone()); + + let d = BlockStateDiff { + sorted_post_state: d_post_state.into_sorted(), + sorted_trie_updates: d_trie_updates.into_sorted(), + }; + store.store_trie_updates(b4, d).expect("b4"); + parent = b256(4); + } + + // Block 5 (kept): update a3; s3=330 + { + let b5 = block(5, parent); + + let mut d_post_state = HashedPostState::default(); + + d_post_state.accounts.insert( + a3, + Some(Account { + nonce: 2, + balance: U256::from(2_003), + ..Default::default() + }), + ); + + let mut hs = HashedStorage::default(); + hs.storage.insert(s3, U256::from(330)); + d_post_state.storages.insert(stor_addr, hs); + + let d = BlockStateDiff { + sorted_post_state: d_post_state.into_sorted(), + sorted_trie_updates: TrieUpdatesSorted::default(), + }; + store.store_trie_updates(b5, d).expect("b5"); + } + + // sanity: earliest=0, latest=5 + { + let e = store + .get_earliest_block_number() + .expect("earliest") + .expect("some"); + let l = store + .get_latest_block_number() + .expect("latest") + .expect("some"); + assert_eq!(e.0, 0); + assert_eq!(l.0, 5); + } + + // --- prune: remove the first 3 blocks, keep 4 and 5 + // new_earliest = 5-1 = 4 + let mut block_hash_reader = MockBlockHashReader::new(); + block_hash_reader + .expect_block_hash() + .withf(move |block_num| *block_num == 4) + .returning(move |_| Ok(Some(b256(4)))); + + let pruner = + MorphProofStoragePruner::new(store.clone(), block_hash_reader, 1, 1000); + let out = pruner.run_inner().expect("pruner ok"); + assert_eq!(out.start_block, 0); + assert_eq!( + out.end_block, 4, + "pruned up to 4 (inclusive); new earliest is 4" + ); + + // proof window moved: earliest=4, latest=5 + { + let e = store + .get_earliest_block_number() + .expect("earliest") + .expect("some"); + let l = store + .get_latest_block_number() + .expect("latest") + .expect("some"); + assert_eq!(e.0, 4); + assert_eq!(e.1, b256(4)); + assert_eq!(l.0, 5); + assert_eq!(l.1, b256(5)); + } + + // --- DB checks + let mut acc_cur = store.account_hashed_cursor(4).expect("acc cur"); + let mut stor_cur = store.storage_hashed_cursor(stor_addr, 4).expect("stor cur"); + let mut acc_trie_cur = store.account_trie_cursor(4).expect("acc trie cur"); + let mut stor_trie_cur = store + .storage_trie_cursor(stor_addr, 4) + .expect("stor trie cur"); + + // Check these histories have been removed + let pruned_hashed_account = a1; + let pruned_trie_accounts = p1; + let pruned_trie_storage = st1; + + assert_ne!( + acc_cur + .seek(pruned_hashed_account) + .expect("seek") + .unwrap() + .0, + pruned_hashed_account, + "deleted account must not exist in earliest snapshot" + ); + assert_ne!( + acc_trie_cur + .seek(pruned_trie_accounts) + .expect("seek") + .unwrap() + .0, + pruned_trie_accounts, + "deleted account trie must not exist in earliest snapshot" + ); + assert_ne!( + stor_trie_cur + .seek(pruned_trie_storage) + .expect("seek") + .unwrap() + .0, + pruned_trie_storage, + "deleted storage trie must not exist in earliest snapshot" + ); + + // Check these histories have been updated - till block 4 + let updated_hashed_accounts = vec![ + ( + a2, + Account { + nonce: 3, + balance: U256::from(3_002), + ..Default::default() + }, + ), + ( + a3, + Account { + nonce: 1, + balance: U256::from(1_003), + ..Default::default() + }, + ), + ]; + let updated_hashed_storage = vec![ + (s1, U256::from(140)), + (s2, U256::from(220)), + (s3, U256::from(300)), + ]; + let updated_trie_accounts = vec![(p2, node_p2), (p3, node_p3)]; + let updated_trie_storage = vec![(st2, node_st2), (st3, node_st3)]; + + for (key, val) in updated_hashed_accounts { + let (k, vv) = acc_cur.seek(key).expect("seek").unwrap(); + assert_eq!(key, k, "key must exist"); + assert_eq!(val, vv, "value must be updated"); + } + + for (key, val) in updated_hashed_storage { + let (k, vv) = stor_cur.seek(key).expect("seek").unwrap(); + assert_eq!(key, k, "key must exist"); + assert_eq!(val, vv, "value must be updated"); + } + + for (key, val) in updated_trie_accounts { + let (k, vv) = acc_trie_cur.seek(key).expect("seek").unwrap(); + assert_eq!(key, k, "key must exist"); + assert_eq!(val, vv, "value must be updated"); + } + for (key, val) in updated_trie_storage { + let (k, vv) = stor_trie_cur.seek(key).expect("seek").unwrap(); + assert_eq!(key, k, "key must exist"); + assert_eq!(val, vv, "value must be updated"); + } + } + + // Both latest and earliest blocks are None -> early return default; DB untouched. + #[tokio::test] + async fn run_inner_where_latest_block_is_none() { + let dir = TempDir::new().unwrap(); + let store: MorphProofsStorage> = + MorphProofsStorage::from(Arc::new($storage::new(dir.path()).expect("env"))); + + let earliest = store.get_earliest_block_number().unwrap(); + let latest = store.get_latest_block_number().unwrap(); + assert!(earliest.is_none()); + assert!(latest.is_none()); + + let block_hash_reader = MockBlockHashReader::new(); + let pruner = MorphProofStoragePruner::new(store, block_hash_reader, 10, 1000); + let out = pruner.run_inner().expect("ok"); + assert_eq!( + out, + PrunerOutput::default(), + "should early-return default output" + ); + } + + // The earliest block is None, but the latest block exists -> early return default. + #[tokio::test] + async fn run_inner_earliest_none_real_db() { + let dir = TempDir::new().unwrap(); + let store: MorphProofsStorage> = + MorphProofsStorage::from(Arc::new($storage::new(dir.path()).expect("env"))); + + // Write a single block to set *latest* only. + store + .store_trie_updates(block(3, B256::ZERO), BlockStateDiff::default()) + .expect("store b1"); + + let earliest = store.get_earliest_block_number().unwrap(); + let latest = store.get_latest_block_number().unwrap(); + assert!(earliest.is_none(), "earliest must remain None"); + assert_eq!(latest.unwrap().0, 3); + + let block_hash_reader = MockBlockHashReader::new(); + let pruner = MorphProofStoragePruner::new(store, block_hash_reader, 1, 1000); + let out = pruner.run_inner().expect("ok"); + assert_eq!( + out, + PrunerOutput::default(), + "should early-return default output" + ); + } + + // interval < min_block_interval -> "Nothing to prune" path; default output. + #[tokio::test] + async fn run_inner_interval_too_small_real_db() { + let dir = TempDir::new().unwrap(); + let store: MorphProofsStorage> = + MorphProofsStorage::from(Arc::new($storage::new(dir.path()).expect("env"))); + + // Set earliest=4 explicitly + let earliest_num = 4u64; + let h4 = b256(4); + store + .set_earliest_block_number(earliest_num, h4) + .expect("set earliest"); + + // Set latest=5 by storing block 5 + let b5 = block(5, h4); + store + .store_trie_updates(b5, BlockStateDiff::default()) + .expect("store b5"); + + // Sanity: earliest=4, latest=5 => interval=1 + let e = store.get_earliest_block_number().unwrap().unwrap(); + let l = store.get_latest_block_number().unwrap().unwrap(); + assert_eq!(e.0, 4); + assert_eq!(l.0, 5); + + // Require min_block_interval=2 (or greater) so interval < min + let block_hash_reader = MockBlockHashReader::new(); + let pruner = MorphProofStoragePruner::new(store, block_hash_reader, 2, 1000); + let out = pruner.run_inner().expect("ok"); + assert_eq!(out, PrunerOutput::default(), "no pruning should occur"); + } + } + }; + } + + proof_storage_pruner_tests!(mdbx_tests, MdbxProofsStorage); +} diff --git a/crates/proofs/src/prune/task.rs b/crates/proofs/src/prune/task.rs new file mode 100644 index 00000000..d9f4fb84 --- /dev/null +++ b/crates/proofs/src/prune/task.rs @@ -0,0 +1,80 @@ +use std::sync::Arc; + +use reth_provider::BlockHashReader; +use reth_tasks::shutdown::GracefulShutdown; +use tokio::{ + time, + time::{Duration, MissedTickBehavior}, +}; +use tracing::{info, warn}; + +use crate::{MorphProofsStorage, MorphProofsStore, prune::MorphProofStoragePruner}; + +/// Number of blocks pruned per MDBX write transaction. +/// +/// Each batch is its own write tx, so the per-tx overhead (page allocation, freelist mgmt, +/// fsync at commit) amortizes over this many blocks. The previous value of 200 caused tx +/// overhead to dominate catch-up pruning runs; 2000 amortizes the fixed cost ~10x while +/// keeping per-tx dirty-page sets and free-page reclamation lag bounded. +const PRUNE_BATCH_SIZE: u64 = 2000; + +/// Periodic pruner task: constructs the pruner and runs it every interval. +#[derive(Debug)] +pub struct MorphProofStoragePrunerTask { + pruner: MorphProofStoragePruner, + retention_blocks: u64, + task_run_interval: Duration, +} + +impl MorphProofStoragePrunerTask +where + P: MorphProofsStore + Send + Sync + 'static, + H: BlockHashReader + Send + Sync + 'static, +{ + /// Initialize a new [`MorphProofStoragePrunerTask`] + pub const fn new( + provider: MorphProofsStorage

, + hash_reader: H, + retention_blocks: u64, + task_run_interval: Duration, + ) -> Self { + let pruner = + MorphProofStoragePruner::new(provider, hash_reader, retention_blocks, PRUNE_BATCH_SIZE); + Self { + pruner, + retention_blocks, + task_run_interval, + } + } + + /// Run forever (until `cancel`), executing one prune pass per `task_run_interval`. + pub async fn run(self, mut signal: GracefulShutdown) { + info!( + target: "trie::pruner_task", + retention_blocks = self.retention_blocks, + interval_secs = self.task_run_interval.as_secs(), + "Starting pruner task" + ); + + let mut interval = time::interval(self.task_run_interval); + interval.set_missed_tick_behavior(MissedTickBehavior::Delay); + + let pruner = Arc::new(self.pruner); + loop { + tokio::select! { + _ = &mut signal => { + info!(target: "trie::pruner_task", "Pruner task cancelled; exiting"); + break; + } + _ = interval.tick() => { + let pruner = Arc::clone(&pruner); + if let Err(e) = tokio::task::spawn_blocking(move || pruner.run()).await { + warn!(target: "trie::pruner_task", err=%e, "Pruner blocking task failed"); + } + } + } + } + + info!(target: "trie::pruner_task", "Pruner task stopped"); + } +} diff --git a/crates/proofs/tests/identity.rs b/crates/proofs/tests/identity.rs new file mode 100644 index 00000000..c45cb71e --- /dev/null +++ b/crates/proofs/tests/identity.rs @@ -0,0 +1,24 @@ +use alloy_primitives::B256; +use morph_proofs::{MdbxProofsStorage, MorphProofsStorageError, ProofDbIdentity}; + +#[test] +fn opening_a_store_persists_and_validates_chain_identity() { + let directory = tempfile::tempdir().unwrap(); + let identity = ProofDbIdentity::new(2818, B256::repeat_byte(0x11)); + + let store = MdbxProofsStorage::open(directory.path(), identity).unwrap(); + assert_eq!(store.proof_window().unwrap(), None); + drop(store); + + MdbxProofsStorage::open(directory.path(), identity).unwrap(); + let error = MdbxProofsStorage::open( + directory.path(), + ProofDbIdentity::new(2819, identity.genesis_hash), + ) + .unwrap_err(); + + assert!(matches!( + error, + MorphProofsStorageError::ChainIdentityMismatch("chain_id") + )); +} diff --git a/crates/rpc/Cargo.toml b/crates/rpc/Cargo.toml index 356a7cb7..e7539dc9 100644 --- a/crates/rpc/Cargo.toml +++ b/crates/rpc/Cargo.toml @@ -16,6 +16,7 @@ morph-chainspec.workspace = true morph-reference-index.workspace = true morph-revm = { workspace = true, features = ["rpc"] } morph-evm = { workspace = true, features = ["rpc"] } +morph-proofs.workspace = true # Reth reth-chainspec.workspace = true @@ -23,9 +24,13 @@ reth-node-api.workspace = true reth-node-builder.workspace = true reth-primitives-traits.workspace = true reth-rpc.workspace = true +reth-rpc-api.workspace = true reth-rpc-convert.workspace = true reth-rpc-eth-api.workspace = true reth-rpc-eth-types.workspace = true +reth-rpc-server-types.workspace = true +reth-metrics.workspace = true +metrics.workspace = true reth-evm.workspace = true reth-provider.workspace = true reth-revm.workspace = true @@ -45,6 +50,7 @@ alloy-eips.workspace = true # JSON-RPC jsonrpsee.workspace = true +async-trait.workspace = true # Serialization serde.workspace = true diff --git a/crates/rpc/src/eth/mod.rs b/crates/rpc/src/eth/mod.rs index cb7331a1..9e325b62 100644 --- a/crates/rpc/src/eth/mod.rs +++ b/crates/rpc/src/eth/mod.rs @@ -32,6 +32,7 @@ use reth_tasks::{ use std::{fmt, marker::PhantomData, sync::Arc, time::Duration}; pub mod call; +pub mod proofs; pub mod receipt; pub mod transaction; diff --git a/crates/rpc/src/eth/proofs.rs b/crates/rpc/src/eth/proofs.rs new file mode 100644 index 00000000..f63c6ca2 --- /dev/null +++ b/crates/rpc/src/eth/proofs.rs @@ -0,0 +1,146 @@ +//! Historical EIP-1186 proof RPC override. + +use std::time::Instant; + +use alloy_eips::BlockId; +use alloy_primitives::Address; +use alloy_rpc_types_eth::EIP1186AccountProofResponse; +use alloy_serde::JsonStorageKey; +use async_trait::async_trait; +use jsonrpsee::{ + core::RpcResult, + proc_macros::rpc, + types::{ErrorCode, ErrorObject}, +}; +use morph_proofs::{MorphProofsStorage, MorphProofsStore}; +use reth_metrics::{ + Metrics, + metrics::{Counter, Histogram}, +}; +use reth_provider::StateProofProvider; +use reth_rpc_api::eth::helpers::FullEthApi; +use reth_rpc_eth_types::EthApiError; + +use crate::state::MorphProofStateProviderFactory; + +/// Maximum number of storage keys accepted by one `eth_getProof` request. +pub const MAX_PROOF_KEYS: usize = 1024; + +/// Validates the storage-key count before any database work starts. +#[derive(Debug)] +pub struct ProofKeyLimit; + +impl ProofKeyLimit { + /// Rejects requests above [`MAX_PROOF_KEYS`]. + pub fn check(keys_len: usize) -> Result<(), ErrorObject<'static>> { + if keys_len > MAX_PROOF_KEYS { + return Err(ErrorObject::owned( + ErrorCode::InvalidParams.code(), + format!("too many storage keys: max {MAX_PROOF_KEYS}, got {keys_len}"), + None::<()>, + )); + } + Ok(()) + } +} + +#[derive(Metrics, Clone)] +#[metrics(scope = "morph.rpc.proofs")] +struct ProofRpcMetrics { + /// Total `eth_getProof` requests. + requests_total: Counter, + /// Successful `eth_getProof` responses. + successful_responses_total: Counter, + /// Failed `eth_getProof` responses. + failures_total: Counter, + /// Successful request latency in seconds. + latency_seconds: Histogram, +} + +#[cfg_attr(not(test), rpc(server, namespace = "eth"))] +#[cfg_attr(test, rpc(server, client, namespace = "eth"))] +pub trait EthProofApiOverride { + /// Returns EIP-1186 account and storage proofs at a retained canonical block. + #[method(name = "getProof")] + async fn get_proof( + &self, + address: Address, + keys: Vec, + block: Option, + ) -> RpcResult; +} + +/// `eth_getProof` implementation backed exclusively by Morph proof history. +#[derive(Debug)] +pub struct EthProofApiExt { + state_provider_factory: MorphProofStateProviderFactory, +} + +impl EthProofApiExt +where + Eth: FullEthApi + Send + Sync + 'static, + P: MorphProofsStore + Clone + 'static, +{ + /// Creates the historical proof RPC override. + pub const fn new(eth_api: Eth, storage: MorphProofsStorage

) -> Self { + Self { + state_provider_factory: MorphProofStateProviderFactory::new(eth_api, storage), + } + } +} + +#[async_trait] +impl EthProofApiOverrideServer for EthProofApiExt +where + Eth: FullEthApi + Send + Sync + 'static, + P: MorphProofsStore + Clone + 'static, +{ + async fn get_proof( + &self, + address: Address, + keys: Vec, + block: Option, + ) -> RpcResult { + ProofKeyLimit::check(keys.len())?; + + let metrics = ProofRpcMetrics::default(); + let start = Instant::now(); + metrics.requests_total.increment(1); + let storage_keys = keys.iter().map(JsonStorageKey::as_b256).collect::>(); + + let result = (|| { + let proof = self + .state_provider_factory + .state_provider(block) + .map_err(|error| ErrorObject::from(EthApiError::from(error)))? + .proof(Default::default(), address, &storage_keys) + .map_err(|error| ErrorObject::from(EthApiError::from(error)))?; + Ok(proof.into_eip1186_response(keys)) + })(); + + match &result { + Ok(_) => { + metrics + .latency_seconds + .record(start.elapsed().as_secs_f64()); + metrics.successful_responses_total.increment(1); + } + Err(_) => metrics.failures_total.increment(1), + } + result + } +} + +#[cfg(test)] +mod tests { + use jsonrpsee::types::ErrorCode; + + use super::{MAX_PROOF_KEYS, ProofKeyLimit}; + + #[test] + fn enforces_get_proof_key_limit() { + assert!(ProofKeyLimit::check(MAX_PROOF_KEYS).is_ok()); + let error = ProofKeyLimit::check(MAX_PROOF_KEYS + 1).expect_err("must reject"); + assert_eq!(error.code(), ErrorCode::InvalidParams.code()); + } +} diff --git a/crates/rpc/src/lib.rs b/crates/rpc/src/lib.rs index e7efe377..eff5e953 100644 --- a/crates/rpc/src/lib.rs +++ b/crates/rpc/src/lib.rs @@ -6,10 +6,13 @@ pub mod error; pub mod eth; pub mod eth_config; pub mod morph; +pub mod proof_status; +pub mod state; pub mod types; pub use error::MorphEthApiError; pub use eth::{MorphEthApi, MorphEthApiBuilder, MorphRpcConverter, MorphRpcTypes}; pub use eth_config::{MorphEthConfigApiServer, MorphEthConfigHandler}; pub use morph::{MorphRpc, MorphRpcHandler, MorphRpcServer, ReferenceQueryArgs}; +pub use proof_status::{ProofStatusApiExt, ProofStatusApiOverrideServer, ProofsSyncStatus}; pub use types::*; diff --git a/crates/rpc/src/proof_status.rs b/crates/rpc/src/proof_status.rs new file mode 100644 index 00000000..23424a65 --- /dev/null +++ b/crates/rpc/src/proof_status.rs @@ -0,0 +1,130 @@ +//! Proof-history synchronization status RPC. + +use async_trait::async_trait; +use jsonrpsee::{core::RpcResult, proc_macros::rpc}; +use morph_proofs::{MorphProofsStorage, MorphProofsStorageResult, MorphProofsStore}; +use reth_rpc_server_types::result::internal_rpc_err; +use serde::{Deserialize, Serialize}; + +/// Inclusive proof-history range currently available to RPC requests. +#[derive(Debug, Serialize, Deserialize, Clone, PartialEq, Eq)] +pub struct ProofsSyncStatus { + /// Earliest retained canonical block. + pub earliest: Option, + /// Latest durably indexed canonical block. + pub latest: Option, +} + +#[cfg_attr(not(test), rpc(server, namespace = "debug"))] +#[cfg_attr(test, rpc(server, client, namespace = "debug"))] +pub trait ProofStatusApiOverride { + /// Returns the current durable proof-history range. + #[method(name = "proofsSyncStatus")] + async fn proofs_sync_status(&self) -> RpcResult; +} + +/// `debug_proofsSyncStatus` backed by the proof database. +#[derive(Debug)] +pub struct ProofStatusApiExt

{ + storage: MorphProofsStorage

, +} + +impl

ProofStatusApiExt

{ + /// Creates a proof-status handler. + pub const fn new(storage: MorphProofsStorage

) -> Self { + Self { storage } + } +} + +fn proofs_sync_status_with_tx<'tx, 'db, P>( + storage: &P, + tx: &'tx P::Tx<'db>, +) -> MorphProofsStorageResult +where + P: MorphProofsStore + 'db, + 'db: 'tx, +{ + let earliest = storage + .get_earliest_block_number_with_tx(tx)? + .map(|(number, _)| number); + let latest = storage + .get_latest_block_number_with_tx(tx)? + .map(|(number, _)| number); + Ok(ProofsSyncStatus { earliest, latest }) +} + +#[async_trait] +impl

ProofStatusApiOverrideServer for ProofStatusApiExt

+where + P: MorphProofsStore + Clone + Send + Sync + 'static, +{ + async fn proofs_sync_status(&self) -> RpcResult { + // Keep both bounds on the same MDBX snapshot. A concurrent prune may advance the live + // earliest block, but it must not produce a status range assembled from two generations. + let tx = self + .storage + .ro_tx() + .map_err(|error| internal_rpc_err(error.to_string()))?; + proofs_sync_status_with_tx(&self.storage, &tx) + .map_err(|error| internal_rpc_err(error.to_string())) + } +} + +#[cfg(test)] +mod tests { + use std::sync::Arc; + + use alloy_eips::{BlockNumHash, NumHash, eip1898::BlockWithParent}; + use alloy_primitives::B256; + use morph_proofs::{ + BlockStateDiff, MdbxProofsStorage, MorphProofsInitialStateStore, MorphProofsStore, + }; + + use super::{ProofsSyncStatus, proofs_sync_status_with_tx}; + + #[test] + fn empty_status_uses_json_nulls() { + let json = serde_json::to_value(ProofsSyncStatus { + earliest: None, + latest: None, + }) + .expect("serialize status"); + assert!(json["earliest"].is_null()); + assert!(json["latest"].is_null()); + } + + #[test] + fn status_bounds_share_one_snapshot_during_prune() { + let directory = tempfile::tempdir().expect("tempdir"); + let storage = Arc::new(MdbxProofsStorage::new(directory.path()).expect("open storage")); + let genesis_hash = B256::repeat_byte(0x01); + storage + .set_initial_state_anchor(BlockNumHash::new(0, genesis_hash)) + .expect("set initial anchor"); + storage + .commit_initial_state() + .expect("commit initial state"); + + let block_one = + BlockWithParent::new(genesis_hash, NumHash::new(1, B256::repeat_byte(0x02))); + storage + .store_trie_updates(block_one, BlockStateDiff::default()) + .expect("store block one"); + + let request_tx = storage.ro_tx().expect("open status snapshot"); + storage + .prune_earliest_state(block_one) + .expect("advance live proof window"); + + let request_status = proofs_sync_status_with_tx(&storage, &request_tx) + .expect("read status from pinned snapshot"); + assert_eq!(request_status.earliest, Some(0)); + assert_eq!(request_status.latest, Some(1)); + + let current_tx = storage.ro_tx().expect("open current snapshot"); + let current_status = + proofs_sync_status_with_tx(&storage, ¤t_tx).expect("read current status"); + assert_eq!(current_status.earliest, Some(1)); + assert_eq!(current_status.latest, Some(1)); + } +} diff --git a/crates/rpc/src/state.rs b/crates/rpc/src/state.rs new file mode 100644 index 00000000..657fc7dc --- /dev/null +++ b/crates/rpc/src/state.rs @@ -0,0 +1,185 @@ +//! State-provider routing for bounded historical proofs. + +use alloy_eips::BlockId; +use alloy_primitives::B256; +use morph_proofs::{MorphProofsStorage, MorphProofsStore, provider::MorphProofsStateProviderRef}; +use reth_provider::{ + BlockHashReader, BlockIdReader, ProviderError, ProviderResult, StateProvider, + StateProviderFactory, +}; +use reth_rpc_api::eth::helpers::FullEthApi; +use reth_rpc_eth_types::EthApiError; +use thiserror::Error; + +/// A request cannot be served by the current durable proof window. +#[derive(Debug, Error, PartialEq, Eq)] +enum ProofWindowError { + #[error("historical proof database is not initialized")] + Uninitialized, + #[error("block {requested} is outside the historical proof window [{earliest}, {latest}]")] + Outside { + requested: u64, + earliest: u64, + latest: u64, + }, + #[error( + "historical proof database is not canonical at block {block}: stored {stored}, canonical {canonical:?}" + )] + CanonicalMismatch { + block: u64, + stored: B256, + canonical: Option, + }, +} + +fn validate_canonical_anchor( + block: u64, + stored: B256, + canonical: Option, +) -> Result<(), ProofWindowError> { + if canonical != Some(stored) { + return Err(ProofWindowError::CanonicalMismatch { + block, + stored, + canonical, + }); + } + Ok(()) +} + +fn validate_window( + requested: u64, + earliest: Option, + latest: Option, +) -> Result<(), ProofWindowError> { + let (Some(earliest), Some(latest)) = (earliest, latest) else { + return Err(ProofWindowError::Uninitialized); + }; + if requested < earliest || requested > latest { + return Err(ProofWindowError::Outside { + requested, + earliest, + latest, + }); + } + Ok(()) +} + +/// Builds providers that source account/storage state only from Morph proof history. +#[derive(Debug)] +pub struct MorphProofStateProviderFactory { + eth_api: Eth, + storage: MorphProofsStorage

, +} + +impl MorphProofStateProviderFactory { + /// Creates a proof state-provider factory. + pub const fn new(eth_api: Eth, storage: MorphProofsStorage

) -> Self { + Self { eth_api, storage } + } +} + +impl<'a, Eth, P> MorphProofStateProviderFactory +where + Eth: FullEthApi + Send + Sync + 'static, + P: MorphProofsStore + Clone + 'a, +{ + /// Returns a provider only when `block_id` is inside the durable proof window. + pub fn state_provider( + &'a self, + block_id: Option, + ) -> ProviderResult> { + let block_id = block_id.unwrap_or_default(); + let block_number = self + .eth_api + .provider() + .block_number_for_id(block_id)? + .ok_or(EthApiError::HeaderNotFound(block_id)) + .map_err(ProviderError::other)?; + if let BlockId::Hash(requested) = block_id { + let canonical_hash = self.eth_api.provider().block_hash(block_number)?; + if canonical_hash != Some(requested.block_hash) { + return Err(ProviderError::other(EthApiError::HeaderNotFound(block_id))); + } + } + + // Pin window validation and all proof cursors to one MDBX snapshot. Otherwise a prune + // committed between the bounds check and proof generation could remap version zero to a + // newer baseline and produce a proof for the wrong state root. + let proof_tx = self + .storage + .ro_tx() + .map_err(|error| ProviderError::Database(error.into()))?; + let earliest = self + .storage + .get_earliest_block_number_with_tx(&proof_tx) + .map_err(|error| ProviderError::Database(error.into()))? + .map(|(number, _)| number); + let latest = self + .storage + .get_latest_block_number_with_tx(&proof_tx) + .map_err(|error| ProviderError::Database(error.into()))?; + validate_window(block_number, earliest, latest.map(|(number, _)| number)) + .map_err(ProviderError::other)?; + + let (latest_number, latest_hash) = + latest.ok_or_else(|| ProviderError::other(ProofWindowError::Uninitialized))?; + let canonical_latest_hash = self.eth_api.provider().block_hash(latest_number)?; + validate_canonical_anchor(latest_number, latest_hash, canonical_latest_hash) + .map_err(ProviderError::other)?; + + // Bytecode is content-addressed and block hashes are canonical-chain data, so a latest + // provider is sufficient for those auxiliary reads. Account/storage trie reads below are + // always routed to proof history; no Reth historical overlay is constructed. + let auxiliary = self.eth_api.provider().latest()?; + Ok(Box::new(MorphProofsStateProviderRef::new_with_tx( + auxiliary, + &self.storage, + block_number, + proof_tx, + ))) + } +} + +#[cfg(test)] +mod tests { + use alloy_primitives::B256; + + use super::{ProofWindowError, validate_canonical_anchor, validate_window}; + + #[test] + fn rejects_an_empty_window() { + assert_eq!( + validate_window(10, None, None), + Err(ProofWindowError::Uninitialized) + ); + } + + #[test] + fn accepts_only_inclusive_window_bounds() { + assert!(validate_window(10, Some(10), Some(20)).is_ok()); + assert!(validate_window(20, Some(10), Some(20)).is_ok()); + assert!(matches!( + validate_window(9, Some(10), Some(20)), + Err(ProofWindowError::Outside { .. }) + )); + assert!(matches!( + validate_window(21, Some(10), Some(20)), + Err(ProofWindowError::Outside { .. }) + )); + } + + #[test] + fn fails_closed_while_proof_history_is_on_a_stale_fork() { + let stored = B256::repeat_byte(0x11); + assert!(validate_canonical_anchor(20, stored, Some(stored)).is_ok()); + assert!(matches!( + validate_canonical_anchor(20, stored, Some(B256::repeat_byte(0x22))), + Err(ProofWindowError::CanonicalMismatch { block: 20, .. }) + )); + assert!(matches!( + validate_canonical_anchor(20, stored, None), + Err(ProofWindowError::CanonicalMismatch { block: 20, .. }) + )); + } +} diff --git a/local-test/README.md b/local-test/README.md index e07d35a7..bc48e407 100644 --- a/local-test/README.md +++ b/local-test/README.md @@ -98,7 +98,7 @@ To wipe chain data and start syncing from scratch: ./local-test/reset.sh hoodi --yes # Reset hoodi (no confirmation) ``` -This removes reth's canonical DB, RocksDB history indices, static files, ExEx state, Morph reference index state, and `node-data/data/` for the specified network. Config files (genesis, keys), `reth.toml`, discovery secrets, and logs are preserved. +This removes reth's canonical DB, RocksDB history indices, static files, ExEx state, Morph reference index state, historical proof MDBX state, and `node-data/data/` for the specified network. Config files (genesis, keys), `reth.toml`, discovery secrets, and logs are preserved. ## Storage & engine tuning diff --git a/local-test/reset.sh b/local-test/reset.sh index babc3e9d..68298e4a 100755 --- a/local-test/reset.sh +++ b/local-test/reset.sh @@ -21,6 +21,7 @@ echo " - ${RETH_DATA_DIR}/rocksdb" echo " - ${RETH_DATA_DIR}/static_files" echo " - ${RETH_DATA_DIR}/exex" echo " - ${RETH_DATA_DIR}/morph" +echo " - ${RETH_DATA_DIR}/historical-proofs" echo " - ${NODE_HOME}/data" echo echo "This keeps:" @@ -45,6 +46,7 @@ rm -rf \ "${RETH_DATA_DIR}/static_files" \ "${RETH_DATA_DIR}/exex" \ "${RETH_DATA_DIR}/morph" \ + "${RETH_DATA_DIR}/historical-proofs" \ "${NODE_HOME}/data" mkdir -p "${RETH_DATA_DIR}" "${NODE_HOME}/data" diff --git a/local-test/reth-start.sh b/local-test/reth-start.sh index 3634f579..2cbc3c0a 100755 --- a/local-test/reth-start.sh +++ b/local-test/reth-start.sh @@ -37,7 +37,6 @@ args=( --authrpc.jwtsecret "${JWT_SECRET}" --log.file.directory "$(dirname "${RETH_LOG_FILE}")" --log.file.filter info - --rpc.eth-proof-window 1209600 # Local testing: explicitly avoid external IP probes such as icanhazip.com. --nat none # Batch MDBX writes so they don't compete with Tendermint's LevelDB fsyncs From a2537ffb96299525bf6f39514deb087236ce81f7 Mon Sep 17 00:00:00 2001 From: panos-xyz Date: Wed, 15 Jul 2026 17:39:33 +0800 Subject: [PATCH 6/6] refactor(proofs): require storage v2 and retain seven days --- README.md | 2 +- bin/morph-reth/src/proofs.rs | 6 +- crates/node/src/args.rs | 6 +- crates/proofs-exex/src/lib.rs | 6 +- crates/proofs/src/initialize.rs | 181 +++++++++++++++++--------------- crates/proofs/src/lib.rs | 3 + 6 files changed, 110 insertions(+), 94 deletions(-) diff --git a/README.md b/README.md index 697f15ff..0dd12cf8 100644 --- a/README.md +++ b/README.md @@ -123,7 +123,7 @@ openssl rand -hex 32 > jwt.hex | `--morph.max-tx-per-block` | None (unlimited) | Maximum number of transactions per block | | `--proofs-history` | false | Enable historical `eth_getProof` and proof-history accumulation | | `--proofs-history.storage-path` | `/historical-proofs` | Override the proof MDBX directory | -| `--proofs-history.window` | 1296000 | Number of canonical blocks retained | +| `--proofs-history.window` | 604800 | Number of canonical blocks retained (7 days at 1s/block) | | `--proofs-history.verification-interval` | 0 | Re-execute every Nth indexed block; 0 disables verification | | `--rpc.eth-proof-window` | 0 | Reth overlay limit used only while proof history is disabled | diff --git a/bin/morph-reth/src/proofs.rs b/bin/morph-reth/src/proofs.rs index e7865c07..5b9b9994 100644 --- a/bin/morph-reth/src/proofs.rs +++ b/bin/morph-reth/src/proofs.rs @@ -5,10 +5,10 @@ use std::{path::PathBuf, sync::Arc}; use alloy_consensus::BlockHeader; use clap::{Parser, Subcommand}; use morph_chainspec::{MorphChainSpec, MorphChainSpecParser}; -use morph_node::{MorphNode, args::DEFAULT_PROOFS_HISTORY_WINDOW}; +use morph_node::MorphNode; use morph_proofs::{ - InitializationJob, MdbxProofsStorage, MorphProofStoragePruner, MorphProofsStorage, - MorphProofsStore, ProofDbIdentity, + DEFAULT_PROOFS_HISTORY_WINDOW, InitializationJob, MdbxProofsStorage, MorphProofStoragePruner, + MorphProofsStorage, MorphProofsStore, ProofDbIdentity, }; use reth_chainspec::EthChainSpec; use reth_cli_commands::common::{AccessRights, Environment, EnvironmentArgs}; diff --git a/crates/node/src/args.rs b/crates/node/src/args.rs index 16846daa..b68657c0 100644 --- a/crates/node/src/args.rs +++ b/crates/node/src/args.rs @@ -3,15 +3,13 @@ use std::path::PathBuf; use clap::Args; +use morph_proofs::DEFAULT_PROOFS_HISTORY_WINDOW; /// Default maximum transaction payload bytes per block (120KB). /// /// This matches Morph's go-ethereum configuration. pub const MORPH_DEFAULT_MAX_TX_PAYLOAD_BYTES: u64 = 122_880; -/// Default proof-history retention window: 30 days at a two-second block time. -pub const DEFAULT_PROOFS_HISTORY_WINDOW: u64 = 1_296_000; - /// Morph-specific CLI arguments. /// /// These arguments extend the standard reth CLI with Morph-specific options @@ -101,7 +99,7 @@ mod tests { assert_eq!(args.max_tx_per_block, None); assert!(!args.proofs_history); assert_eq!(args.proofs_history_storage_path, None); - assert_eq!(args.proofs_history_window, 1_296_000); + assert_eq!(args.proofs_history_window, DEFAULT_PROOFS_HISTORY_WINDOW); assert_eq!(args.proofs_history_verification_interval, 0); } diff --git a/crates/proofs-exex/src/lib.rs b/crates/proofs-exex/src/lib.rs index e1ad5d98..03b51315 100644 --- a/crates/proofs-exex/src/lib.rs +++ b/crates/proofs-exex/src/lib.rs @@ -12,7 +12,8 @@ use alloy_consensus::BlockHeader; use alloy_eips::eip1898::BlockWithParent; use futures::TryStreamExt; use morph_proofs::{ - MorphProofStoragePrunerTask, MorphProofsBatchStore, MorphProofsStorage, + DEFAULT_PROOFS_HISTORY_WINDOW, MorphProofStoragePrunerTask, MorphProofsBatchStore, + MorphProofsStorage, live::{BatchBlock, LiveTrieCollector}, metrics::ProofHistoryMetrics, }; @@ -35,9 +36,6 @@ const DEFAULT_MAX_PRUNE_BLOCKS_STARTUP: u64 = 1_000; /// How many blocks to process in a single sync turn before yielding. const SYNC_BLOCKS_PER_TURN: usize = 50; -/// Default proofs history window: 1 month of blocks at 2s block time -const DEFAULT_PROOFS_HISTORY_WINDOW: u64 = 1_296_000; - /// Default interval between proof-storage prune runs. Default is 15 seconds. const DEFAULT_PRUNE_INTERVAL: Duration = Duration::from_secs(15); diff --git a/crates/proofs/src/initialize.rs b/crates/proofs/src/initialize.rs index b61277bc..4f906dfd 100644 --- a/crates/proofs/src/initialize.rs +++ b/crates/proofs/src/initialize.rs @@ -14,7 +14,8 @@ use reth_db::{ }; use reth_primitives_traits::{Account, StorageEntry}; use reth_trie_common::{ - BranchNodeCompact, Nibbles, StorageTrieEntry, StoredNibbles, StoredNibblesSubKey, + BranchNodeCompact, Nibbles, PackedStorageTrieEntry, PackedStoredNibbles, + PackedStoredNibblesSubKey, StoredNibbles, }; use tracing::{debug, info}; @@ -30,7 +31,7 @@ const INITIALIZE_STORAGE_THRESHOLD: usize = 100000; /// Threshold for logging progress during initialization const INITIALIZE_LOG_THRESHOLD: usize = 100000; -/// Initialization job for external storage. +/// Initialization job for external storage backed by Reth Storage V2 tables. #[derive(Debug, Constructor)] pub struct InitializationJob { storage: S, @@ -91,7 +92,7 @@ macro_rules! define_dup_cursor_iter { }; } -// Generate iterators for all 4 table types +// Generate iterators for the hashed-state and Storage V2 trie tables. define_simple_cursor_iter!(HashedAccountsInit, tables::HashedAccounts, B256, Account); define_dup_cursor_iter!( HashedStoragesInit, @@ -100,16 +101,16 @@ define_dup_cursor_iter!( StorageEntry ); define_simple_cursor_iter!( - AccountsTrieInit, - tables::AccountsTrie, - StoredNibbles, + PackedAccountsTrieInit, + tables::PackedAccountsTrie, + PackedStoredNibbles, BranchNodeCompact ); define_dup_cursor_iter!( - StoragesTrieInit, - tables::StoragesTrie, + PackedStoragesTrieInit, + tables::PackedStoragesTrie, B256, - StorageTrieEntry + PackedStorageTrieEntry ); /// Trait to estimate the progress of a initialization job based on the key. @@ -130,13 +131,13 @@ impl CompletionEstimatable for B256 { } } -impl CompletionEstimatable for StoredNibbles { +impl CompletionEstimatable for Nibbles { fn estimate_progress(&self) -> f64 { // use the first 6 nibbles as a progress estimate - let progress_nibbles = if self.0.is_empty() { - Nibbles::new() + let progress_nibbles = if self.is_empty() { + Self::new() } else { - self.0.slice(0..(self.0.len().min(6))) + self.slice(0..(self.len().min(6))) }; let mut val: u64 = 0; for nibble in progress_nibbles.iter() { @@ -146,6 +147,12 @@ impl CompletionEstimatable for StoredNibbles { } } +impl CompletionEstimatable for PackedStoredNibbles { + fn estimate_progress(&self) -> f64 { + self.0.estimate_progress() + } +} + impl InitializationJob { @@ -282,19 +289,19 @@ impl &self, start_key: Option, ) -> Result<(), MorphProofsStorageError> { - let mut start_cursor = self.tx.cursor_read::()?; + let mut start_cursor = self.tx.cursor_read::()?; if let Some(latest_key) = start_key { + let packed_key = PackedStoredNibbles::from(latest_key.0); start_cursor - .seek(latest_key.clone())? - .filter(|(k, _)| *k == latest_key) + .seek(packed_key.clone())? + .filter(|(key, _)| *key == packed_key) .ok_or(MorphProofsStorageError::InitializeStorageInconsistentState)?; } - let source = AccountsTrieInit::new(start_cursor); self.initialize( "accounts trie", - source, + PackedAccountsTrieInit::new(start_cursor), INITIALIZE_STORAGE_THRESHOLD, INITIALIZE_LOG_THRESHOLD, )?; @@ -307,22 +314,19 @@ impl &self, start_key: Option, ) -> Result<(), MorphProofsStorageError> { - let mut start_cursor = self.tx.cursor_dup_read::()?; + let mut start_cursor = self.tx.cursor_dup_read::()?; if let Some(latest_key) = start_key { + let packed_subkey = PackedStoredNibblesSubKey::from(latest_key.path.0); start_cursor - .seek_by_key_subkey( - latest_key.hashed_address, - StoredNibblesSubKey::from(latest_key.path.0), - )? - .filter(|v| v.nibbles.0 == latest_key.path.0) + .seek_by_key_subkey(latest_key.hashed_address, packed_subkey)? + .filter(|value| value.nibbles.0 == latest_key.path.0) .ok_or(MorphProofsStorageError::InitializeStorageInconsistentState)?; } - let source = StoragesTrieInit::new(start_cursor); self.initialize( "storage trie", - source, + PackedStoragesTrieInit::new(start_cursor), INITIALIZE_STORAGE_THRESHOLD, INITIALIZE_LOG_THRESHOLD, )?; @@ -436,11 +440,10 @@ impl InitTable for HashedStoragesInit { } } -impl InitTable for AccountsTrieInit { - type Key = StoredNibbles; +impl InitTable for PackedAccountsTrieInit { + type Key = PackedStoredNibbles; type Value = BranchNodeCompact; - /// Save mapping of account trie paths to branch nodes to storage. fn store_entries( store: &impl MorphProofsInitialStateStore, entries: impl IntoIterator, @@ -451,16 +454,14 @@ impl InitTable for AccountsTrieInit { .map(|(path, branch)| (path.0, Some(branch))) .collect(), )?; - Ok(()) } } -impl InitTable for StoragesTrieInit { +impl InitTable for PackedStoragesTrieInit { type Key = B256; - type Value = StorageTrieEntry; + type Value = PackedStorageTrieEntry; - /// Save mapping of hashed addresses to storage trie entries to storage. fn store_entries( store: &impl MorphProofsInitialStateStore, entries: impl IntoIterator, @@ -476,7 +477,6 @@ impl InitTable for StoragesTrieInit { .push((storage_entry.nibbles.0, Some(storage_entry.node))); } store.store_storage_branches_bulk(by_address.into_iter().collect())?; - Ok(()) } } @@ -495,8 +495,9 @@ mod tests { }; use reth_primitives_traits::Account; use reth_trie::{ - BranchNodeCompact, StorageTrieEntry, StoredNibbles, StoredNibblesSubKey, - TrieMask, hashed_cursor::HashedCursor, trie_cursor::TrieCursor, + BranchNodeCompact, PackedStorageTrieEntry, PackedStoredNibbles, + PackedStoredNibblesSubKey, TrieMask, hashed_cursor::HashedCursor, + trie_cursor::TrieCursor, }; use tempfile::TempDir; @@ -673,20 +674,20 @@ mod tests { // Insert test trie nodes into database let tx = db.tx_mut().unwrap(); - let mut cursor = tx.cursor_write::().unwrap(); + let mut cursor = tx.cursor_write::().unwrap(); let branch = create_test_branch_node(); let nodes = vec![ ( - StoredNibbles(Nibbles::from_nibbles_unchecked(vec![1])), + PackedStoredNibbles::from(Nibbles::from_nibbles_unchecked(vec![1])), branch.clone(), ), ( - StoredNibbles(Nibbles::from_nibbles_unchecked(vec![2])), + PackedStoredNibbles::from(Nibbles::from_nibbles_unchecked(vec![2])), branch.clone(), ), ( - StoredNibbles(Nibbles::from_nibbles_unchecked(vec![3])), + PackedStoredNibbles::from(Nibbles::from_nibbles_unchecked(vec![3])), branch, ), ]; @@ -720,7 +721,7 @@ mod tests { // Insert test storage trie nodes into database let tx = db.tx_mut().unwrap(); - let mut cursor = tx.cursor_dup_write::().unwrap(); + let mut cursor = tx.cursor_dup_write::().unwrap(); let branch = create_test_branch_node(); let addr1 = keccak256(Address::repeat_byte(0x01)); @@ -729,28 +730,28 @@ mod tests { let nodes = vec![ ( addr1, - StorageTrieEntry { - nibbles: StoredNibblesSubKey(Nibbles::from_nibbles_unchecked( - vec![1], - )), + PackedStorageTrieEntry { + nibbles: PackedStoredNibblesSubKey::from( + Nibbles::from_nibbles_unchecked(vec![1]), + ), node: branch.clone(), }, ), ( addr1, - StorageTrieEntry { - nibbles: StoredNibblesSubKey(Nibbles::from_nibbles_unchecked( - vec![2], - )), + PackedStorageTrieEntry { + nibbles: PackedStoredNibblesSubKey::from( + Nibbles::from_nibbles_unchecked(vec![2]), + ), node: branch.clone(), }, ), ( addr2, - StorageTrieEntry { - nibbles: StoredNibblesSubKey(Nibbles::from_nibbles_unchecked( - vec![3], - )), + PackedStorageTrieEntry { + nibbles: PackedStoredNibblesSubKey::from( + Nibbles::from_nibbles_unchecked(vec![3]), + ), node: branch, }, ), @@ -825,24 +826,24 @@ mod tests { drop(cursor); // Add account trie - let mut cursor = tx.cursor_write::().unwrap(); + let mut cursor = tx.cursor_write::().unwrap(); cursor .append( - StoredNibbles(Nibbles::from_nibbles_unchecked(vec![1])), + PackedStoredNibbles::from(Nibbles::from_nibbles_unchecked(vec![1])), &create_test_branch_node(), ) .unwrap(); drop(cursor); // Add storage trie - let mut cursor = tx.cursor_dup_write::().unwrap(); + let mut cursor = tx.cursor_dup_write::().unwrap(); cursor .upsert( addr, - &StorageTrieEntry { - nibbles: StoredNibblesSubKey(Nibbles::from_nibbles_unchecked( - vec![1], - )), + &PackedStorageTrieEntry { + nibbles: PackedStoredNibblesSubKey::from( + Nibbles::from_nibbles_unchecked(vec![1]), + ), node: create_test_branch_node(), }, ) @@ -1175,9 +1176,17 @@ mod tests { // Phase 1 source: p1,p2 { let tx = db.tx_mut().unwrap(); - let mut cur = tx.cursor_write::().unwrap(); - cur.append(p1.clone(), &create_test_branch_node()).unwrap(); - cur.append(p2.clone(), &create_test_branch_node()).unwrap(); + let mut cur = tx.cursor_write::().unwrap(); + cur.append( + PackedStoredNibbles::from(p1.0.clone()), + &create_test_branch_node(), + ) + .unwrap(); + cur.append( + PackedStoredNibbles::from(p2.0.clone()), + &create_test_branch_node(), + ) + .unwrap(); tx.commit().unwrap(); } @@ -1199,9 +1208,17 @@ mod tests { // Phase 2 source: p3,p4 { let tx = db.tx_mut().unwrap(); - let mut cur = tx.cursor_write::().unwrap(); - cur.append(p3.clone(), &create_test_branch_node()).unwrap(); - cur.append(p4.clone(), &create_test_branch_node()).unwrap(); + let mut cur = tx.cursor_write::().unwrap(); + cur.append( + PackedStoredNibbles::from(p3.0.clone()), + &create_test_branch_node(), + ) + .unwrap(); + cur.append( + PackedStoredNibbles::from(p4.0.clone()), + &create_test_branch_node(), + ) + .unwrap(); tx.commit().unwrap(); } @@ -1246,26 +1263,26 @@ mod tests { let a1 = k(0x10); let a2 = k(0x20); - let n1 = StoredNibblesSubKey(Nibbles::from_nibbles_unchecked(vec![1])); - let n2 = StoredNibblesSubKey(Nibbles::from_nibbles_unchecked(vec![2])); - let n3 = StoredNibblesSubKey(Nibbles::from_nibbles_unchecked(vec![3])); + let n1 = Nibbles::from_nibbles_unchecked(vec![1]); + let n2 = Nibbles::from_nibbles_unchecked(vec![2]); + let n3 = Nibbles::from_nibbles_unchecked(vec![3]); // Phase 1 source: (a1,n1), (a2,n2) { let tx = db.tx_mut().unwrap(); - let mut cur = tx.cursor_dup_write::().unwrap(); + let mut cur = tx.cursor_dup_write::().unwrap(); cur.upsert( a1, - &StorageTrieEntry { - nibbles: n1.clone(), + &PackedStorageTrieEntry { + nibbles: PackedStoredNibblesSubKey::from(n1.clone()), node: create_test_branch_node(), }, ) .unwrap(); cur.upsert( a2, - &StorageTrieEntry { - nibbles: n2.clone(), + &PackedStorageTrieEntry { + nibbles: PackedStoredNibblesSubKey::from(n2.clone()), node: create_test_branch_node(), }, ) @@ -1287,16 +1304,16 @@ mod tests { .latest_storage_trie_key .expect("ok"); assert_eq!(last1.hashed_address, a2); - assert_eq!(last1.path.0, n2.0); + assert_eq!(last1.path.0, n2); // Phase 2 source: add (a2,n3) { let tx = db.tx_mut().unwrap(); - let mut cur = tx.cursor_dup_write::().unwrap(); + let mut cur = tx.cursor_dup_write::().unwrap(); cur.upsert( a2, - &StorageTrieEntry { - nibbles: n3.clone(), + &PackedStorageTrieEntry { + nibbles: PackedStoredNibblesSubKey::from(n3.clone()), node: create_test_branch_node(), }, ) @@ -1310,7 +1327,7 @@ mod tests { let job = InitializationJob::new(Arc::clone(&store), tx); job.initialize_storages_trie(Some(StorageTrieKey::new( a2, - StoredNibbles::from(n2.0), + StoredNibbles::from(n2.clone()), ))) .unwrap(); } @@ -1322,7 +1339,7 @@ mod tests { .latest_storage_trie_key .expect("ok"); assert_eq!(last2.hashed_address, a2); - assert_eq!(last2.path.0, n3.0); + assert_eq!(last2.path.0, n3); // Verify per-address no dupes and stable ordering { @@ -1335,7 +1352,7 @@ mod tests { got.push(path); } - assert_eq!(got, vec![n1.0]); + assert_eq!(got, vec![n1]); } { let mut c = store.storage_trie_cursor(a2, 0).unwrap(); @@ -1346,7 +1363,7 @@ mod tests { while let Some((path, _node)) = c.next().unwrap() { got.push(path); } - assert_eq!(got, vec![n2.0, n3.0]); + assert_eq!(got, vec![n2, n3]); } } } diff --git a/crates/proofs/src/lib.rs b/crates/proofs/src/lib.rs index 438ec198..953632df 100644 --- a/crates/proofs/src/lib.rs +++ b/crates/proofs/src/lib.rs @@ -16,6 +16,9 @@ pub use api::{ pub mod initialize; pub use initialize::InitializationJob; +/// Default proof-history retention window: 7 days at a one-second block time. +pub const DEFAULT_PROOFS_HISTORY_WINDOW: u64 = 604_800; + #[cfg(any(test, feature = "test-utils"))] pub mod in_memory; #[cfg(any(test, feature = "test-utils"))]