From 0d689269fee4eef8d20bb8a6f7e4246721a8d209 Mon Sep 17 00:00:00 2001 From: Rohit Goswami Date: Mon, 16 Nov 2026 02:33:49 +0000 Subject: [PATCH 1/2] docs(log): the store writes run format v3, not v2 The module doc said the writer writes v2 runs in the format RUN_MAGIC describes; Run::encode_blocks writes RUN_MAGIC_V3, front-coded blocks of BLOCK_PAIRS pairs behind an index of first keys, counts and lengths, which the book's runs page already gives. The doc now describes v3 and keeps v2 and v1 as formats readers accept. The code was right, so there is no failing test; a test reads the run a seal writes through the store and pins the v3 magic the doc now names. --- changelog.d/+log-run-format.doc.md | 2 ++ crates/rfc9162-log/src/store.rs | 18 +++++++++++------- crates/rfc9162-log/tests/store.rs | 19 +++++++++++++++++++ 3 files changed, 32 insertions(+), 7 deletions(-) create mode 100644 changelog.d/+log-run-format.doc.md diff --git a/changelog.d/+log-run-format.doc.md b/changelog.d/+log-run-format.doc.md new file mode 100644 index 0000000..db1d8e0 --- /dev/null +++ b/changelog.d/+log-run-format.doc.md @@ -0,0 +1,2 @@ +`rfc9162-log`: The `store` module doc describes the run format the writer writes, v3: front-coded blocks of `BLOCK_PAIRS` pairs behind a block index. +It said v2 and pointed at `RUN_MAGIC`, which only readers still accept. diff --git a/crates/rfc9162-log/src/store.rs b/crates/rfc9162-log/src/store.rs index f747aef..63a0283 100644 --- a/crates/rfc9162-log/src/store.rs +++ b/crates/rfc9162-log/src/store.rs @@ -17,13 +17,17 @@ //! //! - `runs/-`: one sealed run, the sorted `(key, index)` pairs of //! data entries `lo..=hi` and a Bloom filter over the keys, in the format -//! `RUN_MAGIC` describes. Its SHA-256 is the `run_hash` the logged manifest -//! carries, so a reader checks the file before trusting a seek. The writer -//! writes format v2: each key is stored as the length it shares with the -//! key before it and the bytes that follow (front coding), and each index -//! as a LEB128 varint offset from `lo`. Runs written as v1 (every key and -//! index in full) stay readable; the manifest binds the file's hash, not -//! its format. +//! [`RUN_MAGIC_V3`] describes. Its SHA-256 is the `run_hash` the logged +//! manifest carries, so a reader checks the file before trusting a seek. +//! The writer writes format v3: the pairs in blocks of [`BLOCK_PAIRS`], +//! behind an index of each block's first key, pair count and byte length, +//! so a lookup decodes only the blocks whose key range can hold the key. +//! Inside a block each key is stored as the length it shares with the key +//! before it and the bytes that follow (front coding, restarted at every +//! block), and each index as a LEB128 varint offset from `lo`. Runs +//! written as v2 (one front-coded sequence) or v1 (every key and index in +//! full) stay readable; the manifest binds the file's hash, not its +//! format. //! - `runs/live`: the manifest indices of the runs a lookup goes through, one //! per line, oldest first. A cache: when it is missing or stale the live set //! is rebuilt from the manifest entries in the log. diff --git a/crates/rfc9162-log/tests/store.rs b/crates/rfc9162-log/tests/store.rs index 6e4bd3a..a232d53 100644 --- a/crates/rfc9162-log/tests/store.rs +++ b/crates/rfc9162-log/tests/store.rs @@ -407,3 +407,22 @@ fn a_split_log_keeps_the_data_tree_as_the_entries_alone() { assert_eq!(reader.live_runs().len(), 1); assert_eq!(reader.find(b"module=m4").expect("find")[0].entry.index, 4); } + +#[test] +fn a_sealed_run_is_written_in_format_v3() { + let memory = MemoryStore::new(); + let mut store = Store::open(memory.clone(), fields()).expect("open"); + for i in 0..5 { + store.append(&entry(i)).expect("append"); + } + store.seal().expect("seal"); + let runs: Vec = memory + .list("runs") + .expect("list") + .into_iter() + .filter(|path| path != "runs/live") + .collect(); + assert_eq!(runs.len(), 1, "{runs:?}"); + let bytes = memory.get(&runs[0]).expect("get").expect("the run"); + assert!(bytes.starts_with(rfc9162_log::store::RUN_MAGIC_V3)); +} From bec807b5d554884841062fbf7e9a8a5eb0432405 Mon Sep 17 00:00:00 2001 From: Rohit Goswami Date: Mon, 16 Nov 2026 02:34:10 +0000 Subject: [PATCH 2/2] docs(log): Recovery.quarantined counts store paths, not entries The field's doc called it the entries found beyond the checkpoint that could not be replayed, but recover reports the length of the data writer's quarantined list: the tile and bundle paths beyond the tree state that Writer::open moved to .state/torn-, as the book's runs page says. The doc now says so. The code was right, so there is no failing test; a test leaves a bundle and a tile past the tree state and pins the count at two paths. --- changelog.d/+recovery-quarantined.doc.md | 2 ++ crates/rfc9162-log/src/lib.rs | 6 ++++-- crates/rfc9162-log/tests/store.rs | 26 ++++++++++++++++++++++++ 3 files changed, 32 insertions(+), 2 deletions(-) create mode 100644 changelog.d/+recovery-quarantined.doc.md diff --git a/changelog.d/+recovery-quarantined.doc.md b/changelog.d/+recovery-quarantined.doc.md new file mode 100644 index 0000000..6898e80 --- /dev/null +++ b/changelog.d/+recovery-quarantined.doc.md @@ -0,0 +1,2 @@ +`rfc9162-log`: The `Recovery::quarantined` doc says what it counts: store paths of the data log beyond its tree state that opening the store moved to `.state/torn-`. +It said entries. diff --git a/crates/rfc9162-log/src/lib.rs b/crates/rfc9162-log/src/lib.rs index 891142c..eec4ffb 100644 --- a/crates/rfc9162-log/src/lib.rs +++ b/crates/rfc9162-log/src/lib.rs @@ -123,8 +123,10 @@ pub struct Recovery { pub checkpoint_size: u64, /// Entries found beyond it and replayed into the tree. pub replayed: u64, - /// Entries found beyond it that could not be replayed (torn, or past a - /// gap) and were quarantined. + /// Store paths of the data log that its tree state did not cover (tiles + /// and bundles a crash left beyond the tree), which opening the store + /// moved to `.state/torn-`. A count of paths, not of entries; no + /// entry in them is in the recovered tree. pub quarantined: u64, /// The consistency path from the checkpoint's tree to the recovered one. pub consistency: Vec<[u8; 32]>, diff --git a/crates/rfc9162-log/tests/store.rs b/crates/rfc9162-log/tests/store.rs index a232d53..5251e39 100644 --- a/crates/rfc9162-log/tests/store.rs +++ b/crates/rfc9162-log/tests/store.rs @@ -426,3 +426,29 @@ fn a_sealed_run_is_written_in_format_v3() { let bytes = memory.get(&runs[0]).expect("get").expect("the run"); assert!(bytes.starts_with(rfc9162_log::store::RUN_MAGIC_V3)); } + +#[test] +fn recovery_counts_the_paths_moved_aside() { + let memory = MemoryStore::new(); + let (size, root) = { + let mut store = Store::open(memory.clone(), fields()).expect("open"); + for i in 0..3 { + store.append(&entry(i)).expect("append"); + } + (store.size(), store.root()) + }; + // A crash while six more entries went in: their bundle and tile were + // written, the tree state was not. + memory.put("tile/entries/000.p/9", b"torn").expect("put"); + memory.put("tile/0/000.p/9", b"torn").expect("put"); + let mut store = Store::open_trusting(memory.clone(), fields(), size, root).expect("open"); + let done = store.recover().expect("recover"); + assert_eq!((done.checkpoint_size, done.replayed), (3, 0)); + assert_eq!(done.quarantined, 2); + let moved = memory.list(".state").expect("list"); + assert_eq!( + moved.iter().filter(|path| path.contains("/torn-")).count(), + 2, + "{moved:?}" + ); +}