diff --git a/changelog.d/+log-run-format.doc.md b/changelog.d/+log-run-format.doc.md new file mode 100644 index 0000000..db1d8e0 --- /dev/null +++ b/changelog.d/+log-run-format.doc.md @@ -0,0 +1,2 @@ +`rfc9162-log`: The `store` module doc describes the run format the writer writes, v3: front-coded blocks of `BLOCK_PAIRS` pairs behind a block index. +It said v2 and pointed at `RUN_MAGIC`, which only readers still accept. diff --git a/changelog.d/+recovery-quarantined.doc.md b/changelog.d/+recovery-quarantined.doc.md new file mode 100644 index 0000000..6898e80 --- /dev/null +++ b/changelog.d/+recovery-quarantined.doc.md @@ -0,0 +1,2 @@ +`rfc9162-log`: The `Recovery::quarantined` doc says what it counts: store paths of the data log beyond its tree state that opening the store moved to `.state/torn-`. +It said entries. diff --git a/crates/rfc9162-log/src/lib.rs b/crates/rfc9162-log/src/lib.rs index 891142c..eec4ffb 100644 --- a/crates/rfc9162-log/src/lib.rs +++ b/crates/rfc9162-log/src/lib.rs @@ -123,8 +123,10 @@ pub struct Recovery { pub checkpoint_size: u64, /// Entries found beyond it and replayed into the tree. pub replayed: u64, - /// Entries found beyond it that could not be replayed (torn, or past a - /// gap) and were quarantined. + /// Store paths of the data log that its tree state did not cover (tiles + /// and bundles a crash left beyond the tree), which opening the store + /// moved to `.state/torn-`. A count of paths, not of entries; no + /// entry in them is in the recovered tree. pub quarantined: u64, /// The consistency path from the checkpoint's tree to the recovered one. pub consistency: Vec<[u8; 32]>, diff --git a/crates/rfc9162-log/src/store.rs b/crates/rfc9162-log/src/store.rs index f747aef..63a0283 100644 --- a/crates/rfc9162-log/src/store.rs +++ b/crates/rfc9162-log/src/store.rs @@ -17,13 +17,17 @@ //! //! - `runs/-`: one sealed run, the sorted `(key, index)` pairs of //! data entries `lo..=hi` and a Bloom filter over the keys, in the format -//! `RUN_MAGIC` describes. Its SHA-256 is the `run_hash` the logged manifest -//! carries, so a reader checks the file before trusting a seek. The writer -//! writes format v2: each key is stored as the length it shares with the -//! key before it and the bytes that follow (front coding), and each index -//! as a LEB128 varint offset from `lo`. Runs written as v1 (every key and -//! index in full) stay readable; the manifest binds the file's hash, not -//! its format. +//! [`RUN_MAGIC_V3`] describes. Its SHA-256 is the `run_hash` the logged +//! manifest carries, so a reader checks the file before trusting a seek. +//! The writer writes format v3: the pairs in blocks of [`BLOCK_PAIRS`], +//! behind an index of each block's first key, pair count and byte length, +//! so a lookup decodes only the blocks whose key range can hold the key. +//! Inside a block each key is stored as the length it shares with the key +//! before it and the bytes that follow (front coding, restarted at every +//! block), and each index as a LEB128 varint offset from `lo`. Runs +//! written as v2 (one front-coded sequence) or v1 (every key and index in +//! full) stay readable; the manifest binds the file's hash, not its +//! format. //! - `runs/live`: the manifest indices of the runs a lookup goes through, one //! per line, oldest first. A cache: when it is missing or stale the live set //! is rebuilt from the manifest entries in the log. diff --git a/crates/rfc9162-log/tests/store.rs b/crates/rfc9162-log/tests/store.rs index 6e4bd3a..5251e39 100644 --- a/crates/rfc9162-log/tests/store.rs +++ b/crates/rfc9162-log/tests/store.rs @@ -407,3 +407,48 @@ fn a_split_log_keeps_the_data_tree_as_the_entries_alone() { assert_eq!(reader.live_runs().len(), 1); assert_eq!(reader.find(b"module=m4").expect("find")[0].entry.index, 4); } + +#[test] +fn a_sealed_run_is_written_in_format_v3() { + let memory = MemoryStore::new(); + let mut store = Store::open(memory.clone(), fields()).expect("open"); + for i in 0..5 { + store.append(&entry(i)).expect("append"); + } + store.seal().expect("seal"); + let runs: Vec = memory + .list("runs") + .expect("list") + .into_iter() + .filter(|path| path != "runs/live") + .collect(); + assert_eq!(runs.len(), 1, "{runs:?}"); + let bytes = memory.get(&runs[0]).expect("get").expect("the run"); + assert!(bytes.starts_with(rfc9162_log::store::RUN_MAGIC_V3)); +} + +#[test] +fn recovery_counts_the_paths_moved_aside() { + let memory = MemoryStore::new(); + let (size, root) = { + let mut store = Store::open(memory.clone(), fields()).expect("open"); + for i in 0..3 { + store.append(&entry(i)).expect("append"); + } + (store.size(), store.root()) + }; + // A crash while six more entries went in: their bundle and tile were + // written, the tree state was not. + memory.put("tile/entries/000.p/9", b"torn").expect("put"); + memory.put("tile/0/000.p/9", b"torn").expect("put"); + let mut store = Store::open_trusting(memory.clone(), fields(), size, root).expect("open"); + let done = store.recover().expect("recover"); + assert_eq!((done.checkpoint_size, done.replayed), (3, 0)); + assert_eq!(done.quarantined, 2); + let moved = memory.list(".state").expect("list"); + assert_eq!( + moved.iter().filter(|path| path.contains("/torn-")).count(), + 2, + "{moved:?}" + ); +}