Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 2 additions & 0 deletions changelog.d/+pack-scan-bound.bugfix.md
Original file line number Diff line number Diff line change
@@ -0,0 +1,2 @@
`rfc9162-tiles`: Opening or refreshing a `PackStore` stops at a record head that declares more bytes than the file holds before sizing a buffer for it.
A torn head after a crash cost an allocation of whatever length its bytes claimed, up to 4 GiB.
10 changes: 8 additions & 2 deletions crates/rfc9162-tiles/src/pack.rs
Original file line number Diff line number Diff line change
Expand Up @@ -824,10 +824,11 @@ fn parse_index(body: &[u8], first: u64) -> Option<(u64, Vec<(String, Slot)>)> {

/// Reads records from `valid_len` to the end of the file one at a time,
/// applying each whose digest holds and stopping at the first that does
/// not: a short read, a bad digest, a path that is not UTF-8, or a kind
/// this version does not know.
/// not: a head declaring more bytes than the file has, a short read, a bad
/// digest, a path that is not UTF-8, or a kind this version does not know.
fn scan(inner: &mut Inner) -> io::Result<()> {
let mut at = inner.valid_len;
let end = inner.file.metadata()?.len();
inner.file.seek(SeekFrom::Start(at))?;
let mut reader = io::BufReader::new(inner.file.try_clone()?);
let mut head = [0u8; HEAD_LEN];
Expand All @@ -842,6 +843,11 @@ fn scan(inner: &mut Inner) -> io::Result<()> {
let path_len = u16::from_be_bytes([head[2], head[3]]) as usize;
let stored_len = u32::from_be_bytes([head[4], head[5], head[6], head[7]]) as usize;
let raw = u32::from_be_bytes([head[8], head[9], head[10], head[11]]);
// The lengths come from bytes a crash may have torn: the buffer is
// sized only for a record the file holds.
if at + (HEAD_LEN + path_len + stored_len + DIGEST_LEN) as u64 > end {
break;
}
record.clear();
record.extend_from_slice(&head);
record.resize(HEAD_LEN + path_len + stored_len, 0);
Expand Down
66 changes: 66 additions & 0 deletions crates/rfc9162-tiles/tests/pack_scan.rs
Original file line number Diff line number Diff line change
@@ -0,0 +1,66 @@
//! A torn record head costs the scan that opens a pack no more memory than
//! the bytes that are there. The test counts every allocation in this
//! binary, so it is the only test here.

use std::alloc::{GlobalAlloc, Layout, System};
use std::io::Write;
use std::sync::atomic::{AtomicUsize, Ordering};

use rfc9162_tiles::{PackStore, TileStore};

/// The system allocator, counting the bytes held and the most held at once.
struct Counting;

static HELD: AtomicUsize = AtomicUsize::new(0);
static PEAK: AtomicUsize = AtomicUsize::new(0);

unsafe impl GlobalAlloc for Counting {
unsafe fn alloc(&self, layout: Layout) -> *mut u8 {
let p = unsafe { System.alloc(layout) };
if !p.is_null() {
let held = HELD.fetch_add(layout.size(), Ordering::Relaxed) + layout.size();
PEAK.fetch_max(held, Ordering::Relaxed);
}
p
}

unsafe fn dealloc(&self, p: *mut u8, layout: Layout) {
unsafe { System.dealloc(p, layout) };
HELD.fetch_sub(layout.size(), Ordering::Relaxed);
}
}

#[global_allocator]
static COUNTING: Counting = Counting;

#[test]
fn a_torn_head_declaring_a_large_record_is_not_allocated() {
let dir = tempfile::tempdir().unwrap();
let pack = dir.path().join("pack");
PackStore::open(&pack)
.unwrap()
.put("kept", b"kept bytes")
.unwrap();
// A crash after a put's head: kind, codec, a 4-byte path and 256 MiB
// stored and raw, with none of them written.
let declared = 256u32 << 20;
let mut head = vec![1, 0, 0, 4];
head.extend_from_slice(&declared.to_be_bytes());
head.extend_from_slice(&declared.to_be_bytes());
let mut file = std::fs::OpenOptions::new()
.append(true)
.open(&pack)
.unwrap();
file.write_all(&head).unwrap();
drop(file);

let before = HELD.load(Ordering::Relaxed);
PEAK.store(before, Ordering::Relaxed);
let reader = PackStore::open_read(&pack).unwrap();
let peak = PEAK.load(Ordering::Relaxed) - before;
assert_eq!(
reader.get("kept").unwrap().as_deref(),
Some(&b"kept bytes"[..])
);
assert!(peak < 1 << 20, "opening the pack held {peak} bytes at once");
}
Loading