Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
1 change: 1 addition & 0 deletions changelog.d/+proof-shapes.misc.md
Original file line number Diff line number Diff line change
@@ -0,0 +1 @@
Inclusion and consistency proofs at sizes 0, 1, 2 and either side of a power of two are checked against a second tree, and a flipped, truncated or extended proof is rejected.
2 changes: 2 additions & 0 deletions changelog.d/+sidecar-bounds.bugfix.md
Original file line number Diff line number Diff line change
@@ -0,0 +1,2 @@
A sidecar whose pair count or key length does not fit in its u16 prefix is refused, instead of being written with the length truncated.
A bundle of more than 256 entries is refused, because the entry offset is one byte.
2 changes: 2 additions & 0 deletions changelog.d/+split-point.bugfix.md
Original file line number Diff line number Diff line change
@@ -0,0 +1,2 @@
The tile split no longer overflows when a tree is wider than 2^63.
A perfect subtree whose span does not fit in a u64 is refused instead of being accepted or panicking.
69 changes: 55 additions & 14 deletions crates/rfc9162-tiles/src/index.rs
Original file line number Diff line number Diff line change
Expand Up @@ -49,8 +49,14 @@ pub struct Sidecar {

impl Sidecar {
/// The sidecar over a bundle's entries with the log's key function.
#[must_use]
pub fn build<E: AsRef<[u8]>>(entries: &[E], keys: &KeyFn) -> Self {
/// A bundle wider than 256 entries is refused: the offset is one byte.
pub fn build<E: AsRef<[u8]>>(entries: &[E], keys: &KeyFn) -> Result<Self> {
if entries.len() > usize::from(u8::MAX) + 1 {
return Err(Error::OutOfRange(format!(
"bundle of {} entries exceeds 256",
entries.len()
)));
}
let mut set: BTreeSet<(Vec<u8>, u8)> = BTreeSet::new();
for (offset, entry) in entries.iter().enumerate() {
let entry = entry.as_ref();
Expand All @@ -63,7 +69,7 @@ impl Sidecar {
let distinct: BTreeSet<&[u8]> = pairs.iter().map(|(k, _)| k.as_slice()).collect();
let distinct: Vec<&[u8]> = distinct.into_iter().collect();
let filter = Bloom::build(&distinct, BITS_PER_KEY);
Self { pairs, filter }
Ok(Self { pairs, filter })
}

/// Offsets of the entries carrying `key`, ascending.
Expand All @@ -89,20 +95,31 @@ impl Sidecar {
))
}

/// The sidecar as bytes.
#[must_use]
pub fn to_bytes(&self) -> Vec<u8> {
/// The sidecar as bytes. A pair count or a key that does not fit in the
/// u16 length prefix is refused, rather than written truncated.
pub fn to_bytes(&self) -> Result<Vec<u8>> {
let filter = self.filter.to_bytes();
let count = u16::try_from(self.pairs.len()).map_err(|_| {
Error::Format(format!(
"sidecar has {} pairs, and the format holds 65535",
self.pairs.len()
))
})?;
let filter_len = u32::try_from(filter.len())
.map_err(|_| Error::Format("sidecar filter exceeds 4 GiB".to_string()))?;
let mut out = vec![VERSION];
out.extend_from_slice(&(self.pairs.len() as u16).to_be_bytes());
out.extend_from_slice(&(filter.len() as u32).to_be_bytes());
out.extend_from_slice(&count.to_be_bytes());
out.extend_from_slice(&filter_len.to_be_bytes());
out.extend_from_slice(&filter);
for (key, offset) in &self.pairs {
out.extend_from_slice(&(key.len() as u16).to_be_bytes());
let len = u16::try_from(key.len()).map_err(|_| {
Error::Format(format!("sidecar key of {} bytes exceeds 65535", key.len()))
})?;
out.extend_from_slice(&len.to_be_bytes());
out.extend_from_slice(key);
out.push(*offset);
}
out
Ok(out)
}

/// The sidecar `to_bytes` wrote.
Expand Down Expand Up @@ -257,10 +274,10 @@ mod tests {
.map(|i| format!("{}:{i}", i % 7).into_bytes())
.collect();
let keys: KeyFn = Box::new(|e| vec![e.split(|b| *b == b':').next().unwrap().to_vec()]);
let sidecar = Sidecar::build(&entries, &keys);
let sidecar = Sidecar::build(&entries, &keys).unwrap();
// 256 identity keys and 256 caller keys, each pair once.
assert_eq!(sidecar.pairs.len(), 512);
let bytes = sidecar.to_bytes();
let bytes = sidecar.to_bytes().unwrap();
assert_eq!(Sidecar::from_bytes("s", &bytes).unwrap(), sidecar);
for (offset, entry) in entries.iter().enumerate() {
assert_eq!(sidecar.lookup(&identity(entry)), vec![offset as u8]);
Expand All @@ -278,8 +295,8 @@ mod tests {

#[test]
fn a_manifest_names_the_rows_holding_a_key() {
let sidecar = Sidecar::build(&[b"a".to_vec(), b"b".to_vec()], &no_keys());
let row = Manifest::row(0, &sidecar, &sidecar.to_bytes());
let sidecar = Sidecar::build(&[b"a".to_vec(), b"b".to_vec()], &no_keys()).unwrap();
let row = Manifest::row(0, &sidecar, &sidecar.to_bytes().unwrap());
assert_eq!(row.keys, 2);
let manifest = Manifest {
version: 1,
Expand All @@ -291,4 +308,28 @@ mod tests {
assert_eq!(manifest.candidates(&[0u8; 32]).count(), 0);
assert_eq!(manifest.candidates(&[0xffu8; 32]).count(), 0);
}

#[test]
fn a_sidecar_that_does_not_fit_its_length_prefixes_is_refused() {
let wide: Vec<Vec<u8>> = (0..257u16).map(|i| i.to_be_bytes().to_vec()).collect();
assert!(Sidecar::build(&wide, &no_keys()).is_err());

let pairs: Vec<(Vec<u8>, u8)> = (0..65_536u32)
.map(|i| (i.to_be_bytes().to_vec(), 0))
.collect();
let owned: Vec<Vec<u8>> = pairs.iter().map(|(key, _)| key.clone()).collect();
let sidecar = Sidecar {
pairs,
filter: Bloom::build(&owned, BITS_PER_KEY),
};
let err = sidecar.to_bytes().unwrap_err();
assert!(err.to_string().contains("65535"), "{err}");

let long = Sidecar {
pairs: vec![(vec![1u8; 65_536], 0)],
filter: Bloom::build(&[b"x".as_slice()], BITS_PER_KEY),
};
let err = long.to_bytes().unwrap_err();
assert!(err.to_string().contains("65535"), "{err}");
}
}
8 changes: 4 additions & 4 deletions crates/rfc9162-tiles/src/log.rs
Original file line number Diff line number Diff line change
Expand Up @@ -271,8 +271,8 @@ impl<S: TileStore> Writer<S> {
return Ok(());
}
for (index, bundle) in sealed {
let sidecar = Sidecar::build(bundle, &self.keys);
let bytes = sidecar.to_bytes();
let sidecar = Sidecar::build(bundle, &self.keys)?;
let bytes = sidecar.to_bytes()?;
self.store.put(&index_path(*index), &bytes)?;
self.manifest
.bundles
Expand Down Expand Up @@ -372,8 +372,8 @@ fn repair_index(store: &impl TileStore, keys: &KeyFn, frontier: &Frontier) -> Re
reason: "missing".to_string(),
})?;
let entries = decode_bundle(&path, &bytes)?;
let sidecar = Sidecar::build(&entries, keys);
let bytes = sidecar.to_bytes();
let sidecar = Sidecar::build(&entries, keys)?;
let bytes = sidecar.to_bytes()?;
if store.get(&index_path(index))?.as_deref() != Some(bytes.as_slice()) {
store.put(&index_path(index), &bytes)?;
}
Expand Down
48 changes: 43 additions & 5 deletions crates/rfc9162-tiles/src/tree.rs
Original file line number Diff line number Diff line change
Expand Up @@ -368,8 +368,21 @@ impl<S: TileStore> TileHashReader<S> {
/// The root of the perfect subtree over leaves `[start, start + 2^height)`,
/// which must be aligned and inside the tree.
pub fn perfect(&self, start: u64, height: u32) -> Result<Hash> {
// A shift of 64 or more panics, and a span that does not fit in a
// u64 is not a subtree of any tree this reader can name.
if height >= u64::BITS {
return Err(Error::OutOfRange(format!(
"perfect subtree at {start} of height {height} does not fit in a u64"
)));
}
let span = 1u64 << height;
if start % span != 0 || start.saturating_add(span) > self.size {
let Some(end) = start.checked_add(span) else {
return Err(Error::OutOfRange(format!(
"perfect subtree at {start} of height {height} at size {}",
self.size
)));
};
if start % span != 0 || end > self.size {
return Err(Error::OutOfRange(format!(
"perfect subtree at {start} of height {height} at size {}",
self.size
Expand Down Expand Up @@ -475,13 +488,17 @@ impl<S: TileStore> TileHashReader<S> {

/// The largest power of two strictly below `width`, where RFC 9162 divides a
/// tree of that width.
///
/// A width of 0 or 1 is not a tree this function divides; callers split only
/// when the width is at least 2. Widths above `2^63` used to overflow the
/// doubling loop, which panics in debug and does not terminate in release.
#[must_use]
pub fn split_point(width: u64) -> u64 {
let mut split = 1u64;
while split * 2 < width {
split *= 2;
if width <= 2 {
return 1;
}
split
// Largest power of two that is <= width-1.
1u64 << (63 - (width - 1).leading_zeros())
}

#[cfg(test)]
Expand All @@ -495,6 +512,27 @@ mod tests {
assert_eq!(split_point(4), 2);
assert_eq!(split_point(5), 4);
assert_eq!(split_point(257), 256);
assert_eq!(split_point(1u64 << 63), 1u64 << 62);
assert_eq!(split_point((1u64 << 63) + 1), 1u64 << 63);
assert_eq!(split_point(u64::MAX), 1u64 << 63);
}

#[test]
fn a_wide_tree_and_a_tall_subtree_are_refused_without_overflow() {
let store = crate::store::MemoryStore::new();
let wide = TileHashReader::new(store, (1u64 << 63) + 1, [0u8; 32]);
assert!(wide.inclusion_proof(0).is_err());

let store = crate::store::MemoryStore::new();
let reader = TileHashReader::new(store, u64::MAX, [0u8; 32]);
let tall = reader.perfect(0, 64).unwrap_err();
assert!(tall.to_string().contains("does not fit in a u64"), "{tall}");
// start + 2^63 does not fit, so the bounds check refuses before any tile read.
let overflow = reader.perfect(1u64 << 63, 63).unwrap_err();
assert!(
overflow.to_string().contains("perfect subtree"),
"{overflow}"
);
}

#[test]
Expand Down
5 changes: 4 additions & 1 deletion crates/rfc9162-tiles/tests/index.rs
Original file line number Diff line number Diff line change
Expand Up @@ -75,7 +75,10 @@ fn a_rebuilt_sidecar_is_the_written_one() {

let reader = Reader::open(store.clone(), keys()).unwrap();
for n in 0..2u64 {
let rebuilt = Sidecar::build(&reader.bundle(n).unwrap(), &keys()).to_bytes();
let rebuilt = Sidecar::build(&reader.bundle(n).unwrap(), &keys())
.unwrap()
.to_bytes()
.unwrap();
assert_eq!(rebuilt, written[n as usize], "sidecar {n}");
}

Expand Down
151 changes: 151 additions & 0 deletions crates/rfc9162/tests/naive_tree.rs
Original file line number Diff line number Diff line change
@@ -0,0 +1,151 @@
//! The RFC 9162 tree, written out beside the crate and checked against it.
//!
//! Leaf and node hashes are the crate's. The split, the root and both proofs
//! are this file's, so a mistake in the crate's recursion does not hide in a
//! test that calls the same function twice. Sizes cover the empty tree, one
//! leaf, two leaves, and the two shapes on either side of a power of two.

use rfc9162::{
consistency_proof, hex, inclusion_proof, leaf_hash, node_hash, root, verify_consistency,
verify_inclusion,
};

fn split(width: usize) -> usize {
let mut k = 1usize;
while k * 2 < width {
k *= 2;
}
k
}

fn empty_root() -> [u8; 32] {
let text = b"e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855";
let mut out = [0u8; 32];
for (slot, pair) in out.iter_mut().zip(text.chunks_exact(2)) {
let pair = std::str::from_utf8(pair).unwrap();
*slot = u8::from_str_radix(pair, 16).unwrap();
}
out
}

fn naive_root(leaves: &[[u8; 32]]) -> [u8; 32] {
match leaves.len() {
0 => empty_root(),
1 => leaves[0],
n => {
let k = split(n);
node_hash(&naive_root(&leaves[..k]), &naive_root(&leaves[k..]))
}
}
}

fn naive_inclusion(leaves: &[[u8; 32]], index: usize) -> Vec<[u8; 32]> {
let n = leaves.len();
if n == 1 {
return Vec::new();
}
let k = split(n);
if index < k {
let mut path = naive_inclusion(&leaves[..k], index);
path.push(naive_root(&leaves[k..]));
path
} else {
let mut path = naive_inclusion(&leaves[k..], index - k);
path.push(naive_root(&leaves[..k]));
path
}
}

fn naive_consistency(leaves: &[[u8; 32]], old: usize, known: bool) -> Vec<[u8; 32]> {
if old == leaves.len() {
return if known {
Vec::new()
} else {
vec![naive_root(leaves)]
};
}
let k = split(leaves.len());
if old <= k {
let mut path = naive_consistency(&leaves[..k], old, known);
path.push(naive_root(&leaves[k..]));
path
} else {
let mut path = naive_consistency(&leaves[k..], old - k, false);
path.push(naive_root(&leaves[..k]));
path
}
}

fn leaves(n: usize) -> Vec<[u8; 32]> {
(0..n)
.map(|i| leaf_hash(format!("edge-{i}").as_bytes()))
.collect()
}

#[test]
fn edge_sizes_match_a_naive_tree() {
for size in [0usize, 1, 2, 3, 4, 5, 8, 9, 16, 17] {
let set = leaves(size);
assert_eq!(root(&set), naive_root(&set), "root of {size}");
if size == 0 {
assert!(inclusion_proof(&set, 0).is_none());
assert!(consistency_proof(&set, 0).is_none());
assert!(!verify_inclusion(
&leaf_hash(b"x"),
0,
0,
&[],
&hex(&root(&[]))
));
continue;
}
let head = hex(&root(&set));
for index in 0..size {
let path = inclusion_proof(&set, index).unwrap();
assert_eq!(
path,
naive_inclusion(&set, index),
"inclusion {size}/{index}"
);
assert!(verify_inclusion(&set[index], index, size, &path, &head));
if !path.is_empty() {
let mut flipped = path.clone();
flipped[0][0] ^= 0xff;
assert!(!verify_inclusion(&set[index], index, size, &flipped, &head));
assert!(!verify_inclusion(
&set[index],
index,
size,
&path[..path.len() - 1],
&head
));
}
let mut extra = path.clone();
extra.push([9u8; 32]);
assert!(!verify_inclusion(&set[index], index, size, &extra, &head));
if size > 1 {
let other = (index + 1) % size;
assert!(!verify_inclusion(&set[index], other, size, &path, &head));
}
}
for old in 1..=size {
let proof = consistency_proof(&set, old).unwrap();
assert_eq!(
proof,
naive_consistency(&set, old, true),
"consistency {old} of {size}"
);
let earlier = hex(&root(&set[..old]));
assert!(verify_consistency(old, &earlier, size, &head, &proof));
if old != size && !proof.is_empty() {
let mut flipped = proof.clone();
flipped[0][0] ^= 0xff;
assert!(!verify_consistency(old, &earlier, size, &head, &flipped));
assert!(!verify_consistency(old, &earlier, size, &head, &[]));
}
}
assert!(consistency_proof(&set, 0).is_none());
assert!(consistency_proof(&set, size + 1).is_none());
assert!(!verify_consistency(0, &hex(&root(&[])), size, &head, &[]));
}
}
Loading