Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 2 additions & 0 deletions changelog.d/+log-run-format.doc.md
Original file line number Diff line number Diff line change
@@ -0,0 +1,2 @@
`rfc9162-log`: The `store` module doc describes the run format the writer writes, v3: front-coded blocks of `BLOCK_PAIRS` pairs behind a block index.
It said v2 and pointed at `RUN_MAGIC`, which only readers still accept.
2 changes: 2 additions & 0 deletions changelog.d/+recovery-quarantined.doc.md
Original file line number Diff line number Diff line change
@@ -0,0 +1,2 @@
`rfc9162-log`: The `Recovery::quarantined` doc says what it counts: store paths of the data log beyond its tree state that opening the store moved to `.state/torn-<utc>`.
It said entries.
6 changes: 4 additions & 2 deletions crates/rfc9162-log/src/lib.rs
Original file line number Diff line number Diff line change
Expand Up @@ -123,8 +123,10 @@ pub struct Recovery {
pub checkpoint_size: u64,
/// Entries found beyond it and replayed into the tree.
pub replayed: u64,
/// Entries found beyond it that could not be replayed (torn, or past a
/// gap) and were quarantined.
/// Store paths of the data log that its tree state did not cover (tiles
/// and bundles a crash left beyond the tree), which opening the store
/// moved to `.state/torn-<utc>`. A count of paths, not of entries; no
/// entry in them is in the recovered tree.
pub quarantined: u64,
/// The consistency path from the checkpoint's tree to the recovered one.
pub consistency: Vec<[u8; 32]>,
Expand Down
18 changes: 11 additions & 7 deletions crates/rfc9162-log/src/store.rs
Original file line number Diff line number Diff line change
Expand Up @@ -17,13 +17,17 @@
//!
//! - `runs/<lo>-<hi>`: one sealed run, the sorted `(key, index)` pairs of
//! data entries `lo..=hi` and a Bloom filter over the keys, in the format
//! `RUN_MAGIC` describes. Its SHA-256 is the `run_hash` the logged manifest
//! carries, so a reader checks the file before trusting a seek. The writer
//! writes format v2: each key is stored as the length it shares with the
//! key before it and the bytes that follow (front coding), and each index
//! as a LEB128 varint offset from `lo`. Runs written as v1 (every key and
//! index in full) stay readable; the manifest binds the file's hash, not
//! its format.
//! [`RUN_MAGIC_V3`] describes. Its SHA-256 is the `run_hash` the logged
//! manifest carries, so a reader checks the file before trusting a seek.
//! The writer writes format v3: the pairs in blocks of [`BLOCK_PAIRS`],
//! behind an index of each block's first key, pair count and byte length,
//! so a lookup decodes only the blocks whose key range can hold the key.
//! Inside a block each key is stored as the length it shares with the key
//! before it and the bytes that follow (front coding, restarted at every
//! block), and each index as a LEB128 varint offset from `lo`. Runs
//! written as v2 (one front-coded sequence) or v1 (every key and index in
//! full) stay readable; the manifest binds the file's hash, not its
//! format.
//! - `runs/live`: the manifest indices of the runs a lookup goes through, one
//! per line, oldest first. A cache: when it is missing or stale the live set
//! is rebuilt from the manifest entries in the log.
Expand Down
45 changes: 45 additions & 0 deletions crates/rfc9162-log/tests/store.rs
Original file line number Diff line number Diff line change
Expand Up @@ -407,3 +407,48 @@ fn a_split_log_keeps_the_data_tree_as_the_entries_alone() {
assert_eq!(reader.live_runs().len(), 1);
assert_eq!(reader.find(b"module=m4").expect("find")[0].entry.index, 4);
}

#[test]
fn a_sealed_run_is_written_in_format_v3() {
let memory = MemoryStore::new();
let mut store = Store::open(memory.clone(), fields()).expect("open");
for i in 0..5 {
store.append(&entry(i)).expect("append");
}
store.seal().expect("seal");
let runs: Vec<String> = memory
.list("runs")
.expect("list")
.into_iter()
.filter(|path| path != "runs/live")
.collect();
assert_eq!(runs.len(), 1, "{runs:?}");
let bytes = memory.get(&runs[0]).expect("get").expect("the run");
assert!(bytes.starts_with(rfc9162_log::store::RUN_MAGIC_V3));
}

#[test]
fn recovery_counts_the_paths_moved_aside() {
let memory = MemoryStore::new();
let (size, root) = {
let mut store = Store::open(memory.clone(), fields()).expect("open");
for i in 0..3 {
store.append(&entry(i)).expect("append");
}
(store.size(), store.root())
};
// A crash while six more entries went in: their bundle and tile were
// written, the tree state was not.
memory.put("tile/entries/000.p/9", b"torn").expect("put");
memory.put("tile/0/000.p/9", b"torn").expect("put");
let mut store = Store::open_trusting(memory.clone(), fields(), size, root).expect("open");
let done = store.recover().expect("recover");
assert_eq!((done.checkpoint_size, done.replayed), (3, 0));
assert_eq!(done.quarantined, 2);
let moved = memory.list(".state").expect("list");
assert_eq!(
moved.iter().filter(|path| path.contains("/torn-")).count(),
2,
"{moved:?}"
);
}
Loading