From 5fa204fc3098c583c7cfb36aac57a1e7f9854f0a Mon Sep 17 00:00:00 2001 From: Rohit Goswami Date: Sun, 15 Nov 2026 23:58:04 +0000 Subject: [PATCH 1/3] fix: do not overflow the tile split or a perfect subtree A tree wider than 2^63 made split_point panic in debug and loop in release. A perfect subtree whose span does not fit in a u64 is refused. --- changelog.d/+split-point.bugfix.md | 2 ++ crates/rfc9162-tiles/src/tree.rs | 48 ++++++++++++++++++++++++++---- 2 files changed, 45 insertions(+), 5 deletions(-) create mode 100644 changelog.d/+split-point.bugfix.md diff --git a/changelog.d/+split-point.bugfix.md b/changelog.d/+split-point.bugfix.md new file mode 100644 index 0000000..eb8543c --- /dev/null +++ b/changelog.d/+split-point.bugfix.md @@ -0,0 +1,2 @@ +The tile split no longer overflows when a tree is wider than 2^63. +A perfect subtree whose span does not fit in a u64 is refused instead of being accepted or panicking. diff --git a/crates/rfc9162-tiles/src/tree.rs b/crates/rfc9162-tiles/src/tree.rs index e302889..ace287e 100644 --- a/crates/rfc9162-tiles/src/tree.rs +++ b/crates/rfc9162-tiles/src/tree.rs @@ -368,8 +368,21 @@ impl TileHashReader { /// The root of the perfect subtree over leaves `[start, start + 2^height)`, /// which must be aligned and inside the tree. pub fn perfect(&self, start: u64, height: u32) -> Result { + // A shift of 64 or more panics, and a span that does not fit in a + // u64 is not a subtree of any tree this reader can name. + if height >= u64::BITS { + return Err(Error::OutOfRange(format!( + "perfect subtree at {start} of height {height} does not fit in a u64" + ))); + } let span = 1u64 << height; - if start % span != 0 || start.saturating_add(span) > self.size { + let Some(end) = start.checked_add(span) else { + return Err(Error::OutOfRange(format!( + "perfect subtree at {start} of height {height} at size {}", + self.size + ))); + }; + if start % span != 0 || end > self.size { return Err(Error::OutOfRange(format!( "perfect subtree at {start} of height {height} at size {}", self.size @@ -475,13 +488,17 @@ impl TileHashReader { /// The largest power of two strictly below `width`, where RFC 9162 divides a /// tree of that width. +/// +/// A width of 0 or 1 is not a tree this function divides; callers split only +/// when the width is at least 2. Widths above `2^63` used to overflow the +/// doubling loop, which panics in debug and does not terminate in release. #[must_use] pub fn split_point(width: u64) -> u64 { - let mut split = 1u64; - while split * 2 < width { - split *= 2; + if width <= 2 { + return 1; } - split + // Largest power of two that is <= width-1. + 1u64 << (63 - (width - 1).leading_zeros()) } #[cfg(test)] @@ -495,6 +512,27 @@ mod tests { assert_eq!(split_point(4), 2); assert_eq!(split_point(5), 4); assert_eq!(split_point(257), 256); + assert_eq!(split_point(1u64 << 63), 1u64 << 62); + assert_eq!(split_point((1u64 << 63) + 1), 1u64 << 63); + assert_eq!(split_point(u64::MAX), 1u64 << 63); + } + + #[test] + fn a_wide_tree_and_a_tall_subtree_are_refused_without_overflow() { + let store = crate::store::MemoryStore::new(); + let wide = TileHashReader::new(store, (1u64 << 63) + 1, [0u8; 32]); + assert!(wide.inclusion_proof(0).is_err()); + + let store = crate::store::MemoryStore::new(); + let reader = TileHashReader::new(store, u64::MAX, [0u8; 32]); + let tall = reader.perfect(0, 64).unwrap_err(); + assert!(tall.to_string().contains("does not fit in a u64"), "{tall}"); + // start + 2^63 does not fit, so the bounds check refuses before any tile read. + let overflow = reader.perfect(1u64 << 63, 63).unwrap_err(); + assert!( + overflow.to_string().contains("perfect subtree"), + "{overflow}" + ); } #[test] From a2ad31496851afea2c4163a10f5d070a61350984 Mon Sep 17 00:00:00 2001 From: Rohit Goswami Date: Sun, 15 Nov 2026 23:58:04 +0000 Subject: [PATCH 2/3] fix: refuse a sidecar that does not fit its length prefixes The u16 pair count and key length were truncated, so a wide key set was written as a sidecar no reader could decode. A bundle wider than 256 entries is refused because the offset is one byte. --- changelog.d/+sidecar-bounds.bugfix.md | 2 + crates/rfc9162-tiles/src/index.rs | 69 +++++++++++++++++++++------ crates/rfc9162-tiles/src/log.rs | 8 ++-- crates/rfc9162-tiles/tests/index.rs | 5 +- 4 files changed, 65 insertions(+), 19 deletions(-) create mode 100644 changelog.d/+sidecar-bounds.bugfix.md diff --git a/changelog.d/+sidecar-bounds.bugfix.md b/changelog.d/+sidecar-bounds.bugfix.md new file mode 100644 index 0000000..54fadfe --- /dev/null +++ b/changelog.d/+sidecar-bounds.bugfix.md @@ -0,0 +1,2 @@ +A sidecar whose pair count or key length does not fit in its u16 prefix is refused, instead of being written with the length truncated. +A bundle of more than 256 entries is refused, because the entry offset is one byte. diff --git a/crates/rfc9162-tiles/src/index.rs b/crates/rfc9162-tiles/src/index.rs index ff5d9cd..5d11624 100644 --- a/crates/rfc9162-tiles/src/index.rs +++ b/crates/rfc9162-tiles/src/index.rs @@ -49,8 +49,14 @@ pub struct Sidecar { impl Sidecar { /// The sidecar over a bundle's entries with the log's key function. - #[must_use] - pub fn build>(entries: &[E], keys: &KeyFn) -> Self { + /// A bundle wider than 256 entries is refused: the offset is one byte. + pub fn build>(entries: &[E], keys: &KeyFn) -> Result { + if entries.len() > usize::from(u8::MAX) + 1 { + return Err(Error::OutOfRange(format!( + "bundle of {} entries exceeds 256", + entries.len() + ))); + } let mut set: BTreeSet<(Vec, u8)> = BTreeSet::new(); for (offset, entry) in entries.iter().enumerate() { let entry = entry.as_ref(); @@ -63,7 +69,7 @@ impl Sidecar { let distinct: BTreeSet<&[u8]> = pairs.iter().map(|(k, _)| k.as_slice()).collect(); let distinct: Vec<&[u8]> = distinct.into_iter().collect(); let filter = Bloom::build(&distinct, BITS_PER_KEY); - Self { pairs, filter } + Ok(Self { pairs, filter }) } /// Offsets of the entries carrying `key`, ascending. @@ -89,20 +95,31 @@ impl Sidecar { )) } - /// The sidecar as bytes. - #[must_use] - pub fn to_bytes(&self) -> Vec { + /// The sidecar as bytes. A pair count or a key that does not fit in the + /// u16 length prefix is refused, rather than written truncated. + pub fn to_bytes(&self) -> Result> { let filter = self.filter.to_bytes(); + let count = u16::try_from(self.pairs.len()).map_err(|_| { + Error::Format(format!( + "sidecar has {} pairs, and the format holds 65535", + self.pairs.len() + )) + })?; + let filter_len = u32::try_from(filter.len()) + .map_err(|_| Error::Format("sidecar filter exceeds 4 GiB".to_string()))?; let mut out = vec![VERSION]; - out.extend_from_slice(&(self.pairs.len() as u16).to_be_bytes()); - out.extend_from_slice(&(filter.len() as u32).to_be_bytes()); + out.extend_from_slice(&count.to_be_bytes()); + out.extend_from_slice(&filter_len.to_be_bytes()); out.extend_from_slice(&filter); for (key, offset) in &self.pairs { - out.extend_from_slice(&(key.len() as u16).to_be_bytes()); + let len = u16::try_from(key.len()).map_err(|_| { + Error::Format(format!("sidecar key of {} bytes exceeds 65535", key.len())) + })?; + out.extend_from_slice(&len.to_be_bytes()); out.extend_from_slice(key); out.push(*offset); } - out + Ok(out) } /// The sidecar `to_bytes` wrote. @@ -257,10 +274,10 @@ mod tests { .map(|i| format!("{}:{i}", i % 7).into_bytes()) .collect(); let keys: KeyFn = Box::new(|e| vec![e.split(|b| *b == b':').next().unwrap().to_vec()]); - let sidecar = Sidecar::build(&entries, &keys); + let sidecar = Sidecar::build(&entries, &keys).unwrap(); // 256 identity keys and 256 caller keys, each pair once. assert_eq!(sidecar.pairs.len(), 512); - let bytes = sidecar.to_bytes(); + let bytes = sidecar.to_bytes().unwrap(); assert_eq!(Sidecar::from_bytes("s", &bytes).unwrap(), sidecar); for (offset, entry) in entries.iter().enumerate() { assert_eq!(sidecar.lookup(&identity(entry)), vec![offset as u8]); @@ -278,8 +295,8 @@ mod tests { #[test] fn a_manifest_names_the_rows_holding_a_key() { - let sidecar = Sidecar::build(&[b"a".to_vec(), b"b".to_vec()], &no_keys()); - let row = Manifest::row(0, &sidecar, &sidecar.to_bytes()); + let sidecar = Sidecar::build(&[b"a".to_vec(), b"b".to_vec()], &no_keys()).unwrap(); + let row = Manifest::row(0, &sidecar, &sidecar.to_bytes().unwrap()); assert_eq!(row.keys, 2); let manifest = Manifest { version: 1, @@ -291,4 +308,28 @@ mod tests { assert_eq!(manifest.candidates(&[0u8; 32]).count(), 0); assert_eq!(manifest.candidates(&[0xffu8; 32]).count(), 0); } + + #[test] + fn a_sidecar_that_does_not_fit_its_length_prefixes_is_refused() { + let wide: Vec> = (0..257u16).map(|i| i.to_be_bytes().to_vec()).collect(); + assert!(Sidecar::build(&wide, &no_keys()).is_err()); + + let pairs: Vec<(Vec, u8)> = (0..65_536u32) + .map(|i| (i.to_be_bytes().to_vec(), 0)) + .collect(); + let owned: Vec> = pairs.iter().map(|(key, _)| key.clone()).collect(); + let sidecar = Sidecar { + pairs, + filter: Bloom::build(&owned, BITS_PER_KEY), + }; + let err = sidecar.to_bytes().unwrap_err(); + assert!(err.to_string().contains("65535"), "{err}"); + + let long = Sidecar { + pairs: vec![(vec![1u8; 65_536], 0)], + filter: Bloom::build(&[b"x".as_slice()], BITS_PER_KEY), + }; + let err = long.to_bytes().unwrap_err(); + assert!(err.to_string().contains("65535"), "{err}"); + } } diff --git a/crates/rfc9162-tiles/src/log.rs b/crates/rfc9162-tiles/src/log.rs index fff03ca..437524e 100644 --- a/crates/rfc9162-tiles/src/log.rs +++ b/crates/rfc9162-tiles/src/log.rs @@ -271,8 +271,8 @@ impl Writer { return Ok(()); } for (index, bundle) in sealed { - let sidecar = Sidecar::build(bundle, &self.keys); - let bytes = sidecar.to_bytes(); + let sidecar = Sidecar::build(bundle, &self.keys)?; + let bytes = sidecar.to_bytes()?; self.store.put(&index_path(*index), &bytes)?; self.manifest .bundles @@ -372,8 +372,8 @@ fn repair_index(store: &impl TileStore, keys: &KeyFn, frontier: &Frontier) -> Re reason: "missing".to_string(), })?; let entries = decode_bundle(&path, &bytes)?; - let sidecar = Sidecar::build(&entries, keys); - let bytes = sidecar.to_bytes(); + let sidecar = Sidecar::build(&entries, keys)?; + let bytes = sidecar.to_bytes()?; if store.get(&index_path(index))?.as_deref() != Some(bytes.as_slice()) { store.put(&index_path(index), &bytes)?; } diff --git a/crates/rfc9162-tiles/tests/index.rs b/crates/rfc9162-tiles/tests/index.rs index 3070fc6..1f8a2d6 100644 --- a/crates/rfc9162-tiles/tests/index.rs +++ b/crates/rfc9162-tiles/tests/index.rs @@ -75,7 +75,10 @@ fn a_rebuilt_sidecar_is_the_written_one() { let reader = Reader::open(store.clone(), keys()).unwrap(); for n in 0..2u64 { - let rebuilt = Sidecar::build(&reader.bundle(n).unwrap(), &keys()).to_bytes(); + let rebuilt = Sidecar::build(&reader.bundle(n).unwrap(), &keys()) + .unwrap() + .to_bytes() + .unwrap(); assert_eq!(rebuilt, written[n as usize], "sidecar {n}"); } From e0decec575b1dbee8083eb516e91b47df3ee63a0 Mon Sep 17 00:00:00 2001 From: Rohit Goswami Date: Sun, 15 Nov 2026 23:58:04 +0000 Subject: [PATCH 3/3] test: cross-check edge-size proofs with a naive tree Sizes 0, 1, 2 and either side of a power of two are compared with a second implementation of the RFC 9162 recursion, and a flipped, truncated or extended proof is rejected. --- changelog.d/+proof-shapes.misc.md | 1 + crates/rfc9162/tests/naive_tree.rs | 151 +++++++++++++++++++++++++++++ 2 files changed, 152 insertions(+) create mode 100644 changelog.d/+proof-shapes.misc.md create mode 100644 crates/rfc9162/tests/naive_tree.rs diff --git a/changelog.d/+proof-shapes.misc.md b/changelog.d/+proof-shapes.misc.md new file mode 100644 index 0000000..a05226d --- /dev/null +++ b/changelog.d/+proof-shapes.misc.md @@ -0,0 +1 @@ +Inclusion and consistency proofs at sizes 0, 1, 2 and either side of a power of two are checked against a second tree, and a flipped, truncated or extended proof is rejected. diff --git a/crates/rfc9162/tests/naive_tree.rs b/crates/rfc9162/tests/naive_tree.rs new file mode 100644 index 0000000..8170e79 --- /dev/null +++ b/crates/rfc9162/tests/naive_tree.rs @@ -0,0 +1,151 @@ +//! The RFC 9162 tree, written out beside the crate and checked against it. +//! +//! Leaf and node hashes are the crate's. The split, the root and both proofs +//! are this file's, so a mistake in the crate's recursion does not hide in a +//! test that calls the same function twice. Sizes cover the empty tree, one +//! leaf, two leaves, and the two shapes on either side of a power of two. + +use rfc9162::{ + consistency_proof, hex, inclusion_proof, leaf_hash, node_hash, root, verify_consistency, + verify_inclusion, +}; + +fn split(width: usize) -> usize { + let mut k = 1usize; + while k * 2 < width { + k *= 2; + } + k +} + +fn empty_root() -> [u8; 32] { + let text = b"e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855"; + let mut out = [0u8; 32]; + for (slot, pair) in out.iter_mut().zip(text.chunks_exact(2)) { + let pair = std::str::from_utf8(pair).unwrap(); + *slot = u8::from_str_radix(pair, 16).unwrap(); + } + out +} + +fn naive_root(leaves: &[[u8; 32]]) -> [u8; 32] { + match leaves.len() { + 0 => empty_root(), + 1 => leaves[0], + n => { + let k = split(n); + node_hash(&naive_root(&leaves[..k]), &naive_root(&leaves[k..])) + } + } +} + +fn naive_inclusion(leaves: &[[u8; 32]], index: usize) -> Vec<[u8; 32]> { + let n = leaves.len(); + if n == 1 { + return Vec::new(); + } + let k = split(n); + if index < k { + let mut path = naive_inclusion(&leaves[..k], index); + path.push(naive_root(&leaves[k..])); + path + } else { + let mut path = naive_inclusion(&leaves[k..], index - k); + path.push(naive_root(&leaves[..k])); + path + } +} + +fn naive_consistency(leaves: &[[u8; 32]], old: usize, known: bool) -> Vec<[u8; 32]> { + if old == leaves.len() { + return if known { + Vec::new() + } else { + vec![naive_root(leaves)] + }; + } + let k = split(leaves.len()); + if old <= k { + let mut path = naive_consistency(&leaves[..k], old, known); + path.push(naive_root(&leaves[k..])); + path + } else { + let mut path = naive_consistency(&leaves[k..], old - k, false); + path.push(naive_root(&leaves[..k])); + path + } +} + +fn leaves(n: usize) -> Vec<[u8; 32]> { + (0..n) + .map(|i| leaf_hash(format!("edge-{i}").as_bytes())) + .collect() +} + +#[test] +fn edge_sizes_match_a_naive_tree() { + for size in [0usize, 1, 2, 3, 4, 5, 8, 9, 16, 17] { + let set = leaves(size); + assert_eq!(root(&set), naive_root(&set), "root of {size}"); + if size == 0 { + assert!(inclusion_proof(&set, 0).is_none()); + assert!(consistency_proof(&set, 0).is_none()); + assert!(!verify_inclusion( + &leaf_hash(b"x"), + 0, + 0, + &[], + &hex(&root(&[])) + )); + continue; + } + let head = hex(&root(&set)); + for index in 0..size { + let path = inclusion_proof(&set, index).unwrap(); + assert_eq!( + path, + naive_inclusion(&set, index), + "inclusion {size}/{index}" + ); + assert!(verify_inclusion(&set[index], index, size, &path, &head)); + if !path.is_empty() { + let mut flipped = path.clone(); + flipped[0][0] ^= 0xff; + assert!(!verify_inclusion(&set[index], index, size, &flipped, &head)); + assert!(!verify_inclusion( + &set[index], + index, + size, + &path[..path.len() - 1], + &head + )); + } + let mut extra = path.clone(); + extra.push([9u8; 32]); + assert!(!verify_inclusion(&set[index], index, size, &extra, &head)); + if size > 1 { + let other = (index + 1) % size; + assert!(!verify_inclusion(&set[index], other, size, &path, &head)); + } + } + for old in 1..=size { + let proof = consistency_proof(&set, old).unwrap(); + assert_eq!( + proof, + naive_consistency(&set, old, true), + "consistency {old} of {size}" + ); + let earlier = hex(&root(&set[..old])); + assert!(verify_consistency(old, &earlier, size, &head, &proof)); + if old != size && !proof.is_empty() { + let mut flipped = proof.clone(); + flipped[0][0] ^= 0xff; + assert!(!verify_consistency(old, &earlier, size, &head, &flipped)); + assert!(!verify_consistency(old, &earlier, size, &head, &[])); + } + } + assert!(consistency_proof(&set, 0).is_none()); + assert!(consistency_proof(&set, size + 1).is_none()); + assert!(!verify_consistency(0, &hex(&root(&[])), size, &head, &[])); + } +}