From 4e348afc496e0471aa8c931b20247bde2d4eebf3 Mon Sep 17 00:00:00 2001 From: Rohit Goswami Date: Mon, 16 Nov 2026 02:30:59 +0000 Subject: [PATCH 1/2] test(tiles): a torn record head sizes the scan buffer to its claim scan reads a record head and resizes its buffer to the path and stored lengths the head declares before reading them, so a crash that leaves a head and nothing after it costs whatever the torn bytes claim, up to 4 GiB per open. The test appends a put head declaring 256 MiB after a good record and counts allocations through a global allocator while open_read scans it: the peak is the declared size where it should stay under 1 MiB. --- crates/rfc9162-tiles/tests/pack_scan.rs | 66 +++++++++++++++++++++++++ 1 file changed, 66 insertions(+) create mode 100644 crates/rfc9162-tiles/tests/pack_scan.rs diff --git a/crates/rfc9162-tiles/tests/pack_scan.rs b/crates/rfc9162-tiles/tests/pack_scan.rs new file mode 100644 index 0000000..494b35f --- /dev/null +++ b/crates/rfc9162-tiles/tests/pack_scan.rs @@ -0,0 +1,66 @@ +//! A torn record head costs the scan that opens a pack no more memory than +//! the bytes that are there. The test counts every allocation in this +//! binary, so it is the only test here. + +use std::alloc::{GlobalAlloc, Layout, System}; +use std::io::Write; +use std::sync::atomic::{AtomicUsize, Ordering}; + +use rfc9162_tiles::{PackStore, TileStore}; + +/// The system allocator, counting the bytes held and the most held at once. +struct Counting; + +static HELD: AtomicUsize = AtomicUsize::new(0); +static PEAK: AtomicUsize = AtomicUsize::new(0); + +unsafe impl GlobalAlloc for Counting { + unsafe fn alloc(&self, layout: Layout) -> *mut u8 { + let p = unsafe { System.alloc(layout) }; + if !p.is_null() { + let held = HELD.fetch_add(layout.size(), Ordering::Relaxed) + layout.size(); + PEAK.fetch_max(held, Ordering::Relaxed); + } + p + } + + unsafe fn dealloc(&self, p: *mut u8, layout: Layout) { + unsafe { System.dealloc(p, layout) }; + HELD.fetch_sub(layout.size(), Ordering::Relaxed); + } +} + +#[global_allocator] +static COUNTING: Counting = Counting; + +#[test] +fn a_torn_head_declaring_a_large_record_is_not_allocated() { + let dir = tempfile::tempdir().unwrap(); + let pack = dir.path().join("pack"); + PackStore::open(&pack) + .unwrap() + .put("kept", b"kept bytes") + .unwrap(); + // A crash after a put's head: kind, codec, a 4-byte path and 256 MiB + // stored and raw, with none of them written. + let declared = 256u32 << 20; + let mut head = vec![1, 0, 0, 4]; + head.extend_from_slice(&declared.to_be_bytes()); + head.extend_from_slice(&declared.to_be_bytes()); + let mut file = std::fs::OpenOptions::new() + .append(true) + .open(&pack) + .unwrap(); + file.write_all(&head).unwrap(); + drop(file); + + let before = HELD.load(Ordering::Relaxed); + PEAK.store(before, Ordering::Relaxed); + let reader = PackStore::open_read(&pack).unwrap(); + let peak = PEAK.load(Ordering::Relaxed) - before; + assert_eq!( + reader.get("kept").unwrap().as_deref(), + Some(&b"kept bytes"[..]) + ); + assert!(peak < 1 << 20, "opening the pack held {peak} bytes at once"); +} From fc1eb8f0522e61e808e7bdb79b7224507025f434 Mon Sep 17 00:00:00 2001 From: Rohit Goswami Date: Mon, 16 Nov 2026 02:31:37 +0000 Subject: [PATCH 2/2] fix(tiles): bound a pack scan's record buffer by the file length scan takes the file length when it starts and stops at a head whose declared record would run past it, before the buffer is resized, so a torn head is the end of the valid records it already was without the allocation its lengths claim. A scan runs under the change lock whenever a writer truncates after it, so nothing appended meanwhile is mistaken for a torn tail; a reader picks such records up on its next refresh as before. --- changelog.d/+pack-scan-bound.bugfix.md | 2 ++ crates/rfc9162-tiles/src/pack.rs | 10 ++++++++-- 2 files changed, 10 insertions(+), 2 deletions(-) create mode 100644 changelog.d/+pack-scan-bound.bugfix.md diff --git a/changelog.d/+pack-scan-bound.bugfix.md b/changelog.d/+pack-scan-bound.bugfix.md new file mode 100644 index 0000000..313f8ac --- /dev/null +++ b/changelog.d/+pack-scan-bound.bugfix.md @@ -0,0 +1,2 @@ +`rfc9162-tiles`: Opening or refreshing a `PackStore` stops at a record head that declares more bytes than the file holds before sizing a buffer for it. +A torn head after a crash cost an allocation of whatever length its bytes claimed, up to 4 GiB. diff --git a/crates/rfc9162-tiles/src/pack.rs b/crates/rfc9162-tiles/src/pack.rs index db010f7..4a80834 100644 --- a/crates/rfc9162-tiles/src/pack.rs +++ b/crates/rfc9162-tiles/src/pack.rs @@ -824,10 +824,11 @@ fn parse_index(body: &[u8], first: u64) -> Option<(u64, Vec<(String, Slot)>)> { /// Reads records from `valid_len` to the end of the file one at a time, /// applying each whose digest holds and stopping at the first that does -/// not: a short read, a bad digest, a path that is not UTF-8, or a kind -/// this version does not know. +/// not: a head declaring more bytes than the file has, a short read, a bad +/// digest, a path that is not UTF-8, or a kind this version does not know. fn scan(inner: &mut Inner) -> io::Result<()> { let mut at = inner.valid_len; + let end = inner.file.metadata()?.len(); inner.file.seek(SeekFrom::Start(at))?; let mut reader = io::BufReader::new(inner.file.try_clone()?); let mut head = [0u8; HEAD_LEN]; @@ -842,6 +843,11 @@ fn scan(inner: &mut Inner) -> io::Result<()> { let path_len = u16::from_be_bytes([head[2], head[3]]) as usize; let stored_len = u32::from_be_bytes([head[4], head[5], head[6], head[7]]) as usize; let raw = u32::from_be_bytes([head[8], head[9], head[10], head[11]]); + // The lengths come from bytes a crash may have torn: the buffer is + // sized only for a record the file holds. + if at + (HEAD_LEN + path_len + stored_len + DIGEST_LEN) as u64 > end { + break; + } record.clear(); record.extend_from_slice(&head); record.resize(HEAD_LEN + path_len + stored_len, 0);