diff --git a/.config/nextest.toml b/.config/nextest.toml index a24eff7de..9ff3f9904 100644 --- a/.config/nextest.toml +++ b/.config/nextest.toml @@ -157,7 +157,7 @@ threads-required = 'num-test-threads' # Every test in the `wire` target spawns its own real `nodedb` server process. # At full fan-out the box runs one server per core, and each server carries a # Data Plane plus Raft groups — enough CPU starvation that a metadata propose -# misses its 5s deadline and the statement fails. Capping concurrent servers +# misses its statement deadline and the statement fails. Capping concurrent servers # fixes the starvation; widening that deadline would only hide it, and the # deadline is real. # diff --git a/Cargo.lock b/Cargo.lock index 1fa939bb9..53ab4b8cd 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -4679,6 +4679,7 @@ dependencies = [ "smallvec", "sonic-rs", "tempfile", + "thiserror 2.0.20", "tokio", "tokio-postgres", "tokio-tungstenite 0.30.0", diff --git a/nodedb-array/src/tile/layout.rs b/nodedb-array/src/tile/layout.rs index cb61577d3..d6c22f102 100644 --- a/nodedb-array/src/tile/layout.rs +++ b/nodedb-array/src/tile/layout.rs @@ -83,19 +83,21 @@ pub fn tile_id_for_cell( /// The space-filling-curve key of the tile at `tile_indices`. /// /// The memtable, the segment index, the transaction overlay and retention -/// group cells by this key, and a cluster routes a tile by its top bits. -/// Every cell inside a group stays addressed by its full coordinate: a -/// lookup matches the coordinate within the group +/// group cells by this key. A cluster routes a tile by the top bits of +/// `fmix64(key)` (`nodedb_types::array_route::prefix_bucket`), not by the +/// key's own top bits. Every cell inside a group stays addressed by its +/// full coordinate: a lookup matches the coordinate within the group /// (`TileBuffer::get_cell_bytes`, `extract_cell_bytes`), and a tile's MBR /// is built from the cells it holds. Two tiles that share a key therefore -/// share one group and lose no cell. +/// share one group and one route, and lose no cell. /// /// A dimension whose tile count fits its share of the key bits scales each /// index over that count to the full bit width. Its tiles keep distinct, -/// ordered keys that spread over the top bits. A dimension with more tiles -/// than its bits can number, such as one with the default full-range -/// domain, keeps the low bits of the index instead. Nearby tiles then keep -/// distinct keys, and tiles `2^bits` apart share one. +/// ordered keys. A dimension with more tiles than its bits can number, such +/// as one with the default full-range domain, keeps the low bits of the +/// index instead. Nearby tiles then keep distinct keys, and tiles `2^bits` +/// apart share one. Such keys can share their top bits, and the routing mix +/// still spreads them over buckets. pub fn tile_prefix(schema: &ArraySchema, tile_indices: &[u64]) -> ArrayResult { if tile_indices.len() != schema.arity() { return Err(ArrayError::CoordArityMismatch { @@ -178,6 +180,7 @@ mod tests { use crate::schema::attr_spec::{AttrSpec, AttrType}; use crate::schema::dim_spec::DimSpec; use crate::types::domain::Domain; + use nodedb_types::array_route::prefix_bucket; fn schema() -> ArraySchema { ArraySchemaBuilder::new("g") @@ -221,20 +224,63 @@ mod tests { } #[test] - fn tile_keys_spread_over_the_top_bits() { + fn tile_keys_spread_over_routing_buckets() { let s = schema(); - let tops: std::collections::BTreeSet = (0..10) + let buckets: std::collections::BTreeSet = (0..10) .flat_map(|x| (0..10).map(move |y| (x * 10, y * 10))) .map(|(x, y)| { - tile_id_for_cell(&s, &[CoordValue::Int64(x), CoordValue::Int64(y)], 0) + let key = tile_id_for_cell(&s, &[CoordValue::Int64(x), CoordValue::Int64(y)], 0) .unwrap() - .hilbert_prefix - >> 56 + .hilbert_prefix; + prefix_bucket(key, 8) + }) + .collect(); + // 100 tiles over 256 buckets: well over half of them distinct. + assert!( + buckets.len() >= 60, + "the tiles of one array route to {} buckets", + buckets.len() + ); + } + + fn full_range_2d() -> ArraySchema { + ArraySchemaBuilder::new("open") + .dim(DimSpec::new( + "x", + DimType::Int64, + Domain::new(DomainBound::Int64(i64::MIN), DomainBound::Int64(i64::MAX)), + )) + .dim(DimSpec::new( + "y", + DimType::Int64, + Domain::new(DomainBound::Int64(i64::MIN), DomainBound::Int64(i64::MAX)), + )) + .attr(AttrSpec::new("v", AttrType::Int64, false)) + .tile_extents(vec![1, 1]) + .build() + .unwrap() + } + + /// The tiles of coordinates 0..64 x 0..64 in a full-range 2-D array + /// share their key's top bits, yet spread over the routing buckets. + #[test] + fn full_range_tiles_near_zero_spread_over_routing_buckets() { + let s = full_range_2d(); + let keys: Vec = (0..64) + .flat_map(|x| (0..64).map(move |y| (x, y))) + .map(|(x, y)| { + cell_tile_prefix(&s, &[CoordValue::Int64(x), CoordValue::Int64(y)]).unwrap() }) .collect(); + let distinct: std::collections::BTreeSet = keys.iter().copied().collect(); + assert_eq!(distinct.len(), 4096, "every tile keeps a distinct key"); + let buckets: std::collections::BTreeSet = + keys.iter().map(|&k| prefix_bucket(k, 8)).collect(); + // 4096 distinct keys over 256 buckets reach at least 7/8 of them. assert!( - tops.len() > 1, - "the tiles of one array route to more than one bucket: {tops:?}" + buckets.len() >= 224, + "4096 tiles route to only {} of 256 buckets", + buckets.len() ); } diff --git a/nodedb-cluster-tests/tests/common_suite/cases/calvin_backup_cut_snapshot_install.rs b/nodedb-cluster-tests/tests/common_suite/cases/calvin_backup_cut_snapshot_install.rs new file mode 100644 index 000000000..eee85303e --- /dev/null +++ b/nodedb-cluster-tests/tests/common_suite/cases/calvin_backup_cut_snapshot_install.rs @@ -0,0 +1,164 @@ +// SPDX-License-Identifier: BUSL-1.1 + +//! A replica that catches up by `InstallSnapshot` past a backup cut's +//! barrier takes that cut at once. +//! +//! 1. A 3-node cluster with a low log-compaction threshold holds one +//! collection in data group `G`. The replica `F` neither leads `G` by +//! preference nor leads the metadata group. +//! 2. `F` stops. A live node takes a backup cut at watermark `W`: every data +//! group applies the cut's barrier on the live nodes. +//! 3. Writes compact `G`'s log past the barrier. `F` restarts and can catch +//! up only by a snapshot install, so it never applies the barrier entry. +//! 4. `F` takes the same cut at `W`, as a remote source node of a backup +//! does. The snapshot carried the barrier, so the cut finishes at once. +//! Without it, `F` waits for a barrier no leader places again and the cut +//! fails with "Retry the backup". + +use std::time::{Duration, Instant}; + +use crate::common::cluster_harness::{TestCluster, TestClusterNode, wait_for}; + +/// Low enough that a few dozen single-row inserts compact `G`'s log. +const COMPACTION_THRESHOLD: u64 = 4; + +const COLLECTION: &str = "cut_install_docs"; + +/// Most rows the test writes to compact `G` past the barrier. +const MAX_ROWS: usize = 200; + +const STEP_DEADLINE: Duration = Duration::from_secs(30); +const STEP_POLL: Duration = Duration::from_millis(50); + +/// Well inside the cut's 30 s statement deadline: a cut that waits for a +/// barrier no leader places runs to that deadline. +const CUT_BUDGET: Duration = Duration::from_secs(20); + +fn node_by_id(cluster: &TestCluster, node_id: u64) -> &TestClusterNode { + cluster + .nodes + .iter() + .find(|node| node.node_id == node_id) + .unwrap_or_else(|| panic!("node {node_id} is live")) +} + +/// The preferred leader of `group_id`, from `node`'s routing table. +fn preferred_leader(node: &TestClusterNode, group_id: u64) -> u64 { + let routing = node + .shared + .cluster_routing + .as_ref() + .expect("cluster routing") + .read() + .unwrap_or_else(|p| p.into_inner()); + nodedb_cluster::rebalancer::preferred_leaders(&routing) + .get(&group_id) + .copied() + .unwrap_or_else(|| panic!("group {group_id} has a preferred leader")) +} + +#[tokio::test(flavor = "multi_thread", worker_threads = 6)] +async fn a_snapshot_installed_replica_takes_a_cut_whose_barrier_it_skipped() { + let mut cluster = TestCluster::spawn_three_with_compaction_threshold(COMPACTION_THRESHOLD) + .await + .expect("3-node cluster with a low compaction threshold"); + cluster + .exec_ddl_on_any_leader(&format!( + "CREATE COLLECTION {COLLECTION} (id TEXT PRIMARY KEY, payload TEXT) \ + WITH (engine='document_strict')" + )) + .await + .expect("create the collection"); + cluster.wait_for_preferred_leaders(STEP_DEADLINE).await; + cluster.wait_for_full_apply_convergence(STEP_DEADLINE).await; + + let probe = &cluster.nodes[0]; + let group_id = probe + .group_id_for_collection(COLLECTION) + .expect("the collection maps to a data group"); + let group_leader = preferred_leader(probe, group_id); + let metadata_leader = probe.metadata_group_leader(); + let lagging_index = cluster + .nodes + .iter() + .position(|node| node.node_id != group_leader && node.node_id != metadata_leader) + .expect("a third node leads neither group"); + let lagging_id = cluster.nodes[lagging_index].node_id; + assert!( + cluster.nodes[lagging_index].hosts_data_group(group_id), + "node {lagging_id} replicates group {group_id}" + ); + + let stopped = cluster + .stop_member(lagging_index) + .await + .expect("stop the lagging replica"); + + // A live node takes the cut. Every group applies its barrier there. + let cutter = node_by_id(&cluster, group_leader); + let watermark = cutter.shared.hlc_clock.now().wall_ns; + nodedb::control::backup::cut::cut_at(&cutter.shared, watermark) + .await + .unwrap_or_else(|e| panic!("the cut on node {group_leader}: {e}")); + let barrier_floor = cutter.shared.applied_index_watcher(group_id).current(); + + // Compact the group's log past the barrier. + let mut written = 0; + while cutter.group_snapshot_index(group_id) <= barrier_floor { + assert!( + written < MAX_ROWS, + "group {group_id} did not compact past index {barrier_floor} after {written} \ + writes: {}", + cutter.group_status_line(group_id) + ); + cutter + .client + .simple_query(&format!( + "INSERT INTO {COLLECTION} (id, payload) VALUES ('r{written}', 'v')" + )) + .await + .unwrap_or_else(|e| panic!("insert r{written}: {e}")); + written += 1; + if written % 10 == 0 { + tokio::time::sleep(STEP_POLL).await; + } + } + + cluster + .restart_member(stopped) + .await + .expect("restart the lagging replica"); + let lagging = node_by_id(&cluster, lagging_id); + wait_for( + "the lagging replica installs a snapshot past the barrier", + STEP_DEADLINE, + STEP_POLL, + || lagging.local_snapshot_index_for_group(group_id) > barrier_floor, + ) + .await; + cluster.wait_for_preferred_leaders(STEP_DEADLINE).await; + assert!( + !lagging.leads_group(group_id), + "node {lagging_id} leads group {group_id}: its own driver could place the barrier" + ); + + let started = Instant::now(); + let cut = tokio::time::timeout( + CUT_BUDGET, + nodedb::control::backup::cut::cut_at(&lagging.shared, watermark), + ) + .await + .unwrap_or_else(|_| { + panic!( + "node {lagging_id}: the cut at {watermark} still waits after {CUT_BUDGET:?}: {}", + lagging.group_status_line(group_id) + ) + }); + cut.unwrap_or_else(|e| panic!("node {lagging_id}: the cut at {watermark}: {e}")); + assert!( + !lagging.leads_group(group_id), + "node {lagging_id} led group {group_id} during the cut after {:?}", + started.elapsed() + ); + cluster.shutdown().await; +} diff --git a/nodedb-cluster-tests/tests/common_suite/cases/calvin_multi_shard_redo_restart.rs b/nodedb-cluster-tests/tests/common_suite/cases/calvin_multi_shard_redo_restart.rs index c0a212ac2..21c3a69fc 100644 --- a/nodedb-cluster-tests/tests/common_suite/cases/calvin_multi_shard_redo_restart.rs +++ b/nodedb-cluster-tests/tests/common_suite/cases/calvin_multi_shard_redo_restart.rs @@ -44,6 +44,18 @@ fn sequencer_leader(node: &TestClusterNode) -> u64 { .unwrap_or(0) } +/// Whether `vshard_id`'s applied ledger on `node` holds an applied position. +fn ledger_holds_a_position(node: &TestClusterNode, vshard_id: u32) -> bool { + node.shared + .calvin + .applied + .get(vshard_id) + .is_some_and(|ledger| { + ledger.max_applied_epoch() + != nodedb::control::cluster::calvin::scheduler::NOT_YET_APPLIED_EPOCH + }) +} + /// Count of transactions the single-node sequencer has admitted to an epoch, /// or `0` if the sequencer metrics handle is not installed yet. fn admitted_total(node: &TestClusterNode) -> u64 { @@ -206,6 +218,19 @@ async fn calvin_multi_shard_write_in_explicit_block_commits_and_survives_restart tokio::time::sleep(Duration::from_millis(25)).await; } + // The txn applied on the KV collection's vShard, so this node holds + // stored Calvin state for it. + let kv_vshard = nodedb_types::CollectionKey::from_bare(nodedb::types::DatabaseId::DEFAULT, &kv) + .vshard() + .as_u32(); + wait_for( + "the KV vShard's applied ledger holds the txn", + Duration::from_secs(10), + Duration::from_millis(25), + || ledger_holds_a_position(&node, kv_vshard), + ) + .await; + // WAL-only restart: shut down cleanly (flushing the WAL, releasing every // redb handle) and reopen against the SAME directory — no vector // checkpoint, so the HNSW must be rebuilt purely from the committed @@ -264,5 +289,34 @@ async fn calvin_multi_shard_write_in_explicit_block_commits_and_survives_restart tokio::time::sleep(Duration::from_millis(25)).await; } + // Boot queued the stored Calvin state of every vShard for a forget + // check. Once the data groups replayed, the reconcile pass keeps the + // state of each vShard this node still replicates. + wait_for( + "boot recovery replays the data groups", + Duration::from_secs(30), + Duration::from_millis(50), + || node.shared.data_groups_replayed.load(Ordering::Acquire), + ) + .await; + wait_for( + "the reconcile pass settles every vShard queued at boot", + Duration::from_secs(10), + Duration::from_millis(50), + || { + node.shared + .calvin + .forget_pending + .lock() + .unwrap_or_else(|p| p.into_inner()) + .is_empty() + }, + ) + .await; + assert!( + ledger_holds_a_position(&node, kv_vshard), + "a vShard this node replicates keeps its applied ledger across a restart" + ); + node.shutdown().await; } diff --git a/nodedb-cluster-tests/tests/common_suite/cases/cluster_pitr_restore_point.rs b/nodedb-cluster-tests/tests/common_suite/cases/cluster_pitr_restore_point.rs index 5a9f7fbe5..aebdb954f 100644 --- a/nodedb-cluster-tests/tests/common_suite/cases/cluster_pitr_restore_point.rs +++ b/nodedb-cluster-tests/tests/common_suite/cases/cluster_pitr_restore_point.rs @@ -44,14 +44,27 @@ async fn insert_rows(cluster: &TestCluster, prefix: &str) { for collection in COLLECTIONS { for i in 0..3 { let sql = format!("INSERT INTO {collection} (id, v) VALUES ('{prefix}-{i}', {i})"); - cluster.nodes[0] - .exec(&sql) - .await - .unwrap_or_else(|e| panic!("{sql}: {e}")); + write_on(cluster, 0, collection, &sql).await; } } } +/// Run the write `sql` into `collection` on node `index`. On error, panic +/// with the collection's group and every node's group positions, so the +/// failing hop shows. +async fn write_on(cluster: &TestCluster, index: usize, collection: &str, sql: &str) { + let node = &cluster.nodes[index]; + if let Err(error) = node.exec(sql).await { + panic!( + "{sql} on node {}: {error}\n{collection} is in group {:?}\n\ + groups (leader, commit, applied per node):\n{}", + node.node_id, + node.group_id_for_collection(collection), + cluster.group_positions_report() + ); + } +} + fn ids(prefixes: &[&str]) -> Vec { let mut ids: Vec = prefixes .iter() @@ -104,10 +117,12 @@ async fn create_collection(cluster: &TestCluster, collection: &str) { "CREATE COLLECTION {collection} (id STRING PRIMARY KEY, v INT) \ WITH (engine='document_strict')" ); - cluster - .exec_ddl_on_any_leader(&sql) - .await - .unwrap_or_else(|e| panic!("{sql}: {e}")); + if let Err(error) = cluster.exec_ddl_on_any_leader(&sql).await { + panic!( + "{sql}: {error}\ngroups (leader, commit, applied per node):\n{}", + cluster.group_positions_report() + ); + } } /// Wait until every node archived `point`, stop every node, restore each to @@ -185,10 +200,13 @@ async fn a_cluster_restores_every_group_to_a_restore_point() { ); } - cluster.nodes[1] - .exec("INSERT INTO pitr_a (id, v) VALUES ('after-restore', 9)") - .await - .unwrap_or_else(|e| panic!("a write after the restore: {e}")); + write_on( + &cluster, + 1, + "pitr_a", + "INSERT INTO pitr_a (id, v) VALUES ('after-restore', 9)", + ) + .await; cluster .wait_for_full_apply_convergence(Duration::from_secs(30)) .await; @@ -263,10 +281,7 @@ async fn a_ddl_after_the_watermark_is_absent_though_it_precedes_the_point() { create_collection(&cluster, "pitr_mid").await; for i in 0..3 { let sql = format!("INSERT INTO pitr_mid (id, v) VALUES ('mid-{i}', {i})"); - cluster.nodes[0] - .exec(&sql) - .await - .unwrap_or_else(|e| panic!("{sql}: {e}")); + write_on(&cluster, 0, "pitr_mid", &sql).await; } assert!( !creator.is_finished(), diff --git a/nodedb-cluster-tests/tests/common_suite/cases/gateway_hung_node_budget.rs b/nodedb-cluster-tests/tests/common_suite/cases/gateway_hung_node_budget.rs new file mode 100644 index 000000000..23fbd6343 --- /dev/null +++ b/nodedb-cluster-tests/tests/common_suite/cases/gateway_hung_node_budget.rs @@ -0,0 +1,315 @@ +// SPDX-License-Identifier: BUSL-1.1 + +//! A live slow node gets a statement's whole budget. A dead node costs it +//! about one RPC timeout. +//! +//! A follower forwards its statements on the collection to the leader of the +//! collection's group. The leader's execute gate `exec::before_execute` +//! slows or parks the requests it receives: +//! +//! - A slow leader keeps its connection alive, so a read waits for it and +//! returns its row with no retry. +//! - A parked leader never answers. A read and a write each wait their whole +//! budget and end `57014`, with no retry. +//! - A leader cut off by `transport::blackhole` goes silent. Its connection +//! hits the QUIC idle timeout, the read retries on the new leader, and it +//! returns its row within two RPC timeouts. +//! +//! Requires `--features failpoints`. + +#![cfg(feature = "failpoints")] + +use std::time::{Duration, Instant}; + +use nodedb_cluster::rpc_codec::REPLY_MARGIN; +use nodedb_cluster::transport::endpoint::BLACKHOLE_FAIL_POINT; +use nodedb_test_support::fail_point::{FailAction, FailGuard}; +use nodedb_types::id::DatabaseId; + +use crate::common::cluster_harness::shared_steps::db_detail; +use crate::common::cluster_harness::{GroupLeader, TestCluster, TestClusterNode, wait_for}; + +const COLLECTION: &str = "hung_exec"; +const GATE: &str = "exec::before_execute"; +/// Every node's RPC timeout, and so its QUIC idle timeout. +const RPC_TIMEOUT: Duration = Duration::from_secs(3); +/// How long a slow leader holds each request it receives. +const SLOW_EXECUTE: Duration = Duration::from_secs(8); +/// The statement timeout of a read sent to a slow leader. +const SLOW_READ_TIMEOUT: Duration = Duration::from_secs(12); +/// The statement timeout of a request sent to a parked leader. +const PARKED_TIMEOUT: Duration = Duration::from_secs(4); +/// The statement timeout of a read sent to a silent leader. +const SILENT_READ_TIMEOUT: Duration = Duration::from_secs(20); +/// Scheduling slack on the upper bounds. +const SLACK: Duration = Duration::from_secs(2); + +/// A 3-node cluster with one seeded row, the collection's group leader, and +/// a follower that forwards to it. +struct Setup { + cluster: TestCluster, + leader_id: u64, + follower: usize, +} + +async fn setup() -> Setup { + let cluster = TestCluster::spawn_three_with_rpc_timeout(RPC_TIMEOUT.as_secs()) + .await + .expect("3-node cluster"); + cluster + .exec_ddl_on_any_leader(&format!( + "CREATE COLLECTION {COLLECTION} (id TEXT PRIMARY KEY, val INT) \ + WITH (engine='document_strict')" + )) + .await + .unwrap_or_else(|e| panic!("CREATE COLLECTION {COLLECTION}: {e}")); + wait_for( + "all 3 nodes see the collection", + Duration::from_secs(10), + Duration::from_millis(50), + || { + cluster + .nodes + .iter() + .all(|n| n.cached_collection_count() >= 1) + }, + ) + .await; + + let vshard = nodedb_cluster::routing::vshard_for_collection( + nodedb_types::CollectionKey::from_bare(DatabaseId::DEFAULT, COLLECTION), + ); + let group = cluster.nodes[0] + .shared + .cluster_routing + .as_ref() + .expect("cluster_routing") + .read() + .unwrap_or_else(|p| p.into_inner()) + .group_for_vshard(vshard) + .expect("the collection's group"); + let leader_id = cluster + .wait_for_group_leader(group, GroupLeader::Any, Duration::from_secs(20)) + .await; + let leader = cluster + .nodes + .iter() + .position(|n| n.node_id == leader_id) + .expect("the group's leader is a cluster node"); + + cluster.nodes[leader] + .client + .simple_query(&format!( + "INSERT INTO {COLLECTION} (id, val) VALUES ('k1', 1)" + )) + .await + .unwrap_or_else(|e| panic!("seed row: {}", db_detail(&e))); + cluster + .wait_for_full_apply_convergence(Duration::from_secs(20)) + .await; + Setup { + cluster, + leader_id, + follower: (leader + 1) % 3, + } +} + +async fn set_statement_timeout(node: &TestClusterNode, timeout: Duration) { + node.client + .simple_query(&format!( + "SET statement_timeout = '{}ms'", + timeout.as_millis() + )) + .await + .expect("SET statement_timeout"); +} + +fn read_sql() -> String { + format!("SELECT id, val FROM {COLLECTION} WHERE id = 'k1'") +} + +fn row_count(messages: Vec) -> usize { + messages + .into_iter() + .filter(|m| matches!(m, tokio_postgres::SimpleQueryMessage::Row(_))) + .count() +} + +fn sqlstate(err: &tokio_postgres::Error) -> Option<&str> { + err.as_db_error().map(|db| db.code().code()) +} + +#[tokio::test(flavor = "multi_thread", worker_threads = 6)] +async fn a_slow_leader_gets_a_read_its_whole_budget() { + let Setup { + cluster, + leader_id, + follower, + } = setup().await; + let follower = &cluster.nodes[follower]; + set_statement_timeout(follower, SLOW_READ_TIMEOUT).await; + + let gate = FailGuard::for_node(leader_id, GATE, FailAction::Sleep(SLOW_EXECUTE)); + let retries_before = follower.not_leader_retry_count(); + let started = Instant::now(); + let read = follower.client.simple_query(&read_sql()).await; + let elapsed = started.elapsed(); + drop(gate); + + let rows = read.unwrap_or_else(|e| panic!("the slow leader answers: {}", db_detail(&e))); + assert_eq!(row_count(rows), 1, "the read returns the seeded row"); + assert_eq!( + follower.not_leader_retry_count(), + retries_before, + "a live slow leader is never retried" + ); + assert!( + elapsed >= SLOW_EXECUTE, + "the read returned in {elapsed:?}, before the leader ran it" + ); + assert!( + elapsed < SLOW_READ_TIMEOUT, + "the read finished in {elapsed:?}, past its {SLOW_READ_TIMEOUT:?} budget" + ); + + cluster.shutdown().await; +} + +#[tokio::test(flavor = "multi_thread", worker_threads = 6)] +async fn a_parked_leader_costs_a_read_its_budget() { + let Setup { + cluster, + leader_id, + follower, + } = setup().await; + let follower = &cluster.nodes[follower]; + set_statement_timeout(follower, PARKED_TIMEOUT).await; + + // A read: the live leader holds it, so it waits its whole budget. + let read_dir = tempfile::tempdir().expect("read gate directory"); + let read_release = read_dir.path().join("release"); + let read_gate = FailGuard::for_node( + leader_id, + GATE, + FailAction::WaitForFile(read_release.clone()), + ); + let retries_before = follower.not_leader_retry_count(); + let started = Instant::now(); + let read = follower.client.simple_query(&read_sql()).await; + let read_elapsed = started.elapsed(); + std::fs::write(&read_release, b"release").expect("release the execute gate"); + drop(read_gate); + + let err = read.expect_err("a read the leader never answers runs out of budget"); + assert_eq!( + sqlstate(&err), + Some("57014"), + "the read reports its deadline: {}", + db_detail(&err) + ); + assert!( + read_dir.path().join("release.parked").exists(), + "the leader took the read and held it" + ); + assert_eq!( + follower.not_leader_retry_count(), + retries_before, + "a live leader that holds a read is never retried" + ); + assert!( + read_elapsed >= PARKED_TIMEOUT, + "the read waited {read_elapsed:?}, less than its {PARKED_TIMEOUT:?} budget" + ); + assert!( + read_elapsed < PARKED_TIMEOUT + REPLY_MARGIN + SLACK, + "the read waited {read_elapsed:?}, past its budget and reply margin" + ); + + // A write: it waits its whole budget and is never sent again. + let write_dir = tempfile::tempdir().expect("write gate directory"); + let write_release = write_dir.path().join("release"); + let write_gate = FailGuard::for_node( + leader_id, + GATE, + FailAction::WaitForFile(write_release.clone()), + ); + let retries_before = follower.not_leader_retry_count(); + let started = Instant::now(); + let write = follower + .client + .simple_query(&format!( + "INSERT INTO {COLLECTION} (id, val) VALUES ('k2', 2)" + )) + .await; + let write_elapsed = started.elapsed(); + std::fs::write(&write_release, b"release").expect("release the execute gate"); + drop(write_gate); + + let err = write.expect_err("a write whose answer never arrives is an unknown outcome"); + assert_eq!( + sqlstate(&err), + Some("57014"), + "the write reports its deadline: {}", + db_detail(&err) + ); + assert!( + write_dir.path().join("release.parked").exists(), + "the leader took the write and held it" + ); + assert_eq!( + follower.not_leader_retry_count(), + retries_before, + "a write with a lost answer is never sent again" + ); + assert!( + write_elapsed >= PARKED_TIMEOUT, + "the write waited {write_elapsed:?}, less than its {PARKED_TIMEOUT:?} budget" + ); + assert!( + write_elapsed < PARKED_TIMEOUT + REPLY_MARGIN + SLACK, + "the write waited {write_elapsed:?}, past its budget and reply margin" + ); + + cluster.shutdown().await; +} + +#[tokio::test(flavor = "multi_thread", worker_threads = 6)] +async fn a_silent_leader_costs_a_read_one_idle_timeout() { + let Setup { + cluster, + leader_id, + follower, + } = setup().await; + let follower = &cluster.nodes[follower]; + set_statement_timeout(follower, SILENT_READ_TIMEOUT).await; + + // The leader drops every datagram, both ways, while its process runs. + let blackhole = FailGuard::for_node( + leader_id, + BLACKHOLE_FAIL_POINT, + FailAction::Fail("cut off".into()), + ); + let retries_before = follower.not_leader_retry_count(); + let started = Instant::now(); + let read = follower.client.simple_query(&read_sql()).await; + let elapsed = started.elapsed(); + drop(blackhole); + + let rows = read + .unwrap_or_else(|e| panic!("the read retries past the silent leader: {}", db_detail(&e))); + assert_eq!( + row_count(rows), + 1, + "the retried read returns the seeded row" + ); + assert!( + follower.not_leader_retry_count() > retries_before, + "the read retried after the silent leader's connection timed out" + ); + assert!( + elapsed < RPC_TIMEOUT * 2, + "the read took {elapsed:?}, more than two {RPC_TIMEOUT:?} RPC timeouts" + ); + + cluster.shutdown().await; +} diff --git a/nodedb-cluster-tests/tests/common_suite/cases/listeners_typed_not_leader.rs b/nodedb-cluster-tests/tests/common_suite/cases/listeners_typed_not_leader.rs index 21819ef65..b75c3878f 100644 --- a/nodedb-cluster-tests/tests/common_suite/cases/listeners_typed_not_leader.rs +++ b/nodedb-cluster-tests/tests/common_suite/cases/listeners_typed_not_leader.rs @@ -29,7 +29,7 @@ //! NotLeader") when the QUIC connection itself fails (e.g. sending to a node that //! doesn't exist). In that case the hinted leader in the error is the bad node_id //! itself, so the retry loop would update the routing table to the same bad node -//! and exhaust all 3 attempts — the client sees `NotLeader`, not success. +//! and retry until the statement deadline — the client sees `NotLeader`, not success. //! //! The retry-on-success path exists for a FUTURE scenario where Raft-aware //! execution on follower nodes explicitly returns `TypedClusterError::NotLeader` @@ -492,7 +492,7 @@ async fn native_not_leader_gateway_error_mapping() { #[tokio::test(flavor = "multi_thread", worker_threads = 4)] async fn not_leader_counter_increments_per_retry_attempt() { - use nodedb::control::gateway::retry::retry_not_leader; + use nodedb::control::gateway::retry::{RetryPacing, retry_not_leader}; use std::sync::atomic::{AtomicU64, AtomicUsize}; let counter = Arc::new(AtomicU64::new(0)); @@ -501,12 +501,13 @@ async fn not_leader_counter_increments_per_retry_attempt() { let counter_inner = Arc::clone(&counter); let call_count_inner = Arc::clone(&call_count); - let result = retry_not_leader(None, move |attempt| { + let deadline = tokio::time::Instant::now() + std::time::Duration::from_secs(30); + let result = retry_not_leader(None, deadline, RetryPacing::default(), move |attempt| { let c = Arc::clone(&call_count_inner); let rc = Arc::clone(&counter_inner); async move { let n = c.fetch_add(1, AtomicOrdering::SeqCst); - if attempt > 0 { + if attempt.index > 0 { rc.fetch_add(1, AtomicOrdering::Relaxed); } if n < 2 { diff --git a/nodedb-cluster-tests/tests/common_suite/cases/mod.rs b/nodedb-cluster-tests/tests/common_suite/cases/mod.rs index 1e7a13aa4..be7dd1fbf 100644 --- a/nodedb-cluster-tests/tests/common_suite/cases/mod.rs +++ b/nodedb-cluster-tests/tests/common_suite/cases/mod.rs @@ -12,6 +12,7 @@ mod assign_surrogate_cross_node; mod backup_schedule_failover; mod bitemporal_array_cluster; mod calvin_backup_cut_order; +mod calvin_backup_cut_snapshot_install; mod calvin_cdc_net_kinds; mod calvin_cluster_array_read_set; mod calvin_cluster_pgwire_e2e; @@ -89,6 +90,7 @@ mod descriptor_lease_planner_integration; mod descriptor_replay_incarnation; mod descriptor_versioning_cross_node; mod gateway_execute; +mod gateway_hung_node_budget; mod gather_join_in_txn_occ; mod graph_index_edges_replicate_to_followers; mod graph_match_ryow_cross_node; @@ -127,6 +129,7 @@ mod pgwire_gateway_migration; mod planner_local_only; mod prepared_cache_invalidation; mod proposal_committed_twice_applies_once; +mod redo_stream_held_replica; mod resp_gateway_migration; mod retention_policy_cross_node; mod routing_leader_hint_follows_election; diff --git a/nodedb-cluster-tests/tests/common_suite/cases/redo_stream_held_replica.rs b/nodedb-cluster-tests/tests/common_suite/cases/redo_stream_held_replica.rs new file mode 100644 index 000000000..91747b792 --- /dev/null +++ b/nodedb-cluster-tests/tests/common_suite/cases/redo_stream_held_replica.rs @@ -0,0 +1,347 @@ +// SPDX-License-Identifier: BUSL-1.1 + +//! A replica that dropped a chunked redo stream holds its group at the +//! entry it cannot apply, never leads the group, and converges by snapshot. +//! +//! A 3-node cluster caps a redo entry at 64 KiB, so one large commit +//! travels as chunk entries and a final entry. Follower `C` of the +//! collection's group: +//! - applies the stream's first chunk and parks the second chunk on its +//! gate `redo_chunk::before_apply::1`; +//! - receives the commit and one later write in its log while the other +//! replicas apply them; +//! - unmounts the group, which drops the group's streams on `C`, and mounts +//! it again from its log. The group then owes a snapshot on `C`; +//! - releases the chunk. Its stream is gone, so the entry holds the group's +//! apply on `C`. Every snapshot build of the group parks on +//! `snapshot::before_build::`, so the hold lasts. +//! +//! While it holds, `C` stores no row of the collection, its applied index +//! stays below the held entry, a leadership transfer to it never makes it +//! lead, and a read at `C` returns the full count or waits. Once the build +//! gate opens, a snapshot installs and `C`'s content equals its peers'. +//! +//! Requires `--features failpoints`. + +#![cfg(feature = "failpoints")] + +use std::time::{Duration, Instant}; + +use nodedb_raft::ReplicaState; +use nodedb_test_support::fail_point::{FailAction, FailGuard}; +use tokio_postgres::SimpleQueryMessage; + +use super::calvin_replica_content::{local_content, vshard_of, wait_replicas_agree}; +use crate::common::cluster_harness::shared_steps::group_members; +use crate::common::cluster_harness::{ + GroupLeader, TestCluster, TestClusterNode, is_no_serving_leader, wait_for, +}; + +/// The redo entry limit every node runs: the smallest the config takes. +const ENTRY_LIMIT: usize = 64 * 1024; + +/// Rows one statement of the large transaction inserts. +const ROWS_PER_STATEMENT: usize = 100; + +/// Statements the large transaction runs. With 400-byte values its redo +/// holds several times `ENTRY_LIMIT`. +const STATEMENTS: usize = 6; + +/// Rows the collection holds once every write applied: the large +/// transaction's rows and one later write. +const EXPECTED_ROWS: usize = STATEMENTS * ROWS_PER_STATEMENT + 1; + +/// How long each step gets. +const STEP_DEADLINE: Duration = Duration::from_secs(30); + +/// Pause between two polls of a step. +const STEP_POLL: Duration = Duration::from_millis(50); + +/// How long the hold is watched before a snapshot may build. +const HOLD_WINDOW: Duration = Duration::from_secs(4); + +/// How long a read at the held replica gets. +const READ_DEADLINE: Duration = Duration::from_secs(5); + +const COLL: &str = "held_redo_stream"; + +/// The index in `cluster.nodes` of node `node_id`. +fn index_of(cluster: &TestCluster, node_id: u64) -> usize { + cluster + .nodes + .iter() + .position(|node| node.node_id == node_id) + .unwrap_or_else(|| panic!("node {node_id} is live")) +} + +/// A fresh session on `node`. +async fn session(node: &TestClusterNode) -> tokio_postgres::Client { + let (client, connection) = tokio_postgres::connect( + &format!( + "host=127.0.0.1 port={} user=nodedb dbname=default", + node.pg_addr.port() + ), + tokio_postgres::NoTls, + ) + .await + .expect("connect a session"); + tokio::spawn(async move { + let _ = connection.await; + }); + client +} + +/// Commit one transaction whose redo passes `ENTRY_LIMIT` several times. +async fn commit_large_transaction(client: &tokio_postgres::Client) { + let filler = "x".repeat(400); + client.simple_query("BEGIN").await.expect("BEGIN"); + for statement in 0..STATEMENTS { + let values: Vec = (0..ROWS_PER_STATEMENT) + .map(|row| { + let id = statement * ROWS_PER_STATEMENT + row; + format!("('r{id}', '{filler}')") + }) + .collect(); + client + .simple_query(&format!( + "INSERT INTO {COLL} (id, value) VALUES {}", + values.join(", ") + )) + .await + .expect("INSERT inside the transaction"); + } + client.simple_query("COMMIT").await.expect("COMMIT"); +} + +/// The commit index of `group_id` in node `node`'s Raft. +fn raft_committed(node: &TestClusterNode, group_id: u64) -> u64 { + node.shared.multi_raft.get().map_or(0, |multi_raft| { + multi_raft + .lock() + .unwrap_or_else(|p| p.into_inner()) + .commit_index_for(group_id) + .unwrap_or(0) + }) +} + +/// Entries of `group_id` node `node`'s Raft handed to its apply loop. +fn raft_delivered(node: &TestClusterNode, group_id: u64) -> u64 { + node.shared.multi_raft.get().map_or(0, |multi_raft| { + multi_raft + .lock() + .unwrap_or_else(|p| p.into_inner()) + .last_applied(group_id) + .unwrap_or(0) + }) +} + +/// What `node`'s replica of `group_id` needs before it takes log entries. +fn replica_state(node: &TestClusterNode, group_id: u64) -> Option { + node.shared.multi_raft.get().and_then(|multi_raft| { + multi_raft + .lock() + .unwrap_or_else(|p| p.into_inner()) + .replica_state(group_id) + }) +} + +/// Documents of `COLL` `node` stores, read from its own cores. +async fn stored_rows(node: &TestClusterNode) -> usize { + local_content(node, &[COLL]) + .await + .get(&vshard_of(COLL)) + .map_or(0, |content| { + content.keys().filter(|key| key.starts_with("doc:")).count() + }) +} + +/// Read `COUNT(*)` at `node`. A count must be the full count. A read that +/// waits past `READ_DEADLINE`, or finds no serving leader, returns no count. +async fn assert_read_is_whole_or_waits(node: &TestClusterNode) { + let client = session(node).await; + let count_sql = format!("SELECT COUNT(*) FROM {COLL}"); + let read = client.simple_query(&count_sql); + match tokio::time::timeout(READ_DEADLINE, read).await { + Err(_) => {} + Ok(Err(error)) if is_no_serving_leader(&error) => {} + Ok(Err(error)) => panic!("node {}: COUNT(*) failed: {error}", node.node_id), + Ok(Ok(messages)) => { + let count: Vec = messages + .iter() + .filter_map(|message| match message { + SimpleQueryMessage::Row(row) => row.get(0).map(str::to_owned), + _ => None, + }) + .collect(); + assert_eq!( + count, + vec![EXPECTED_ROWS.to_string()], + "node {}: a read served while the node's apply holds must count every \ + committed row", + node.node_id + ); + } + } +} + +#[tokio::test(flavor = "multi_thread", worker_threads = 6)] +async fn a_replica_that_lost_a_redo_stream_holds_its_group_until_a_snapshot() { + let cluster = TestCluster::spawn_three_with_redo_entry_limit(ENTRY_LIMIT) + .await + .expect("3-node cluster"); + cluster + .exec_ddl_on_any_leader(&format!( + "CREATE COLLECTION {COLL} (id TEXT PRIMARY KEY, value TEXT) \ + WITH (engine='document_strict')" + )) + .await + .unwrap_or_else(|e| panic!("create {COLL}: {e}")); + wait_for( + "every node replicates the collection's group", + STEP_DEADLINE, + STEP_POLL, + || { + cluster.nodes.iter().all(|node| { + node.group_id_for_collection(COLL) + .is_some_and(|group| node.replicates_data_group(group)) + }) + }, + ) + .await; + cluster.wait_for_preferred_leaders(STEP_DEADLINE).await; + + let group = cluster.nodes[0] + .group_id_for_collection(COLL) + .expect("the collection's data group"); + let leader = cluster + .wait_for_group_leader(group, GroupLeader::Any, STEP_DEADLINE) + .await; + let held_id = group_members(&cluster.nodes[0], group) + .into_iter() + .find(|member| *member != leader) + .expect("the group has a follower"); + let held = &cluster.nodes[index_of(&cluster, held_id)]; + let writer = &cluster.nodes[index_of(&cluster, leader)]; + + let gate_dir = tempfile::tempdir().expect("gate directory"); + let chunk_release = gate_dir.path().join("chunk"); + let chunk_parked = gate_dir.path().join("chunk.parked"); + let build_release = gate_dir.path().join("build"); + let chunk_gate = FailGuard::for_node( + held_id, + "redo_chunk::before_apply::1", + FailAction::WaitForFile(chunk_release.clone()), + ); + let build_gate = FailGuard::install( + &format!("snapshot::before_build::{group}"), + FailAction::WaitForFile(build_release.clone()), + ); + + let client = session(writer).await; + commit_large_transaction(&client).await; + client + .simple_query(&format!( + "INSERT INTO {COLL} (id, value) VALUES ('after', 'later write')" + )) + .await + .expect("a write after the large transaction"); + let leader_applied = raft_committed(writer, group); + wait_for( + "the leader applies every committed entry", + STEP_DEADLINE, + STEP_POLL, + || writer.shared.applied_index_watcher(group).current() >= leader_applied, + ) + .await; + + wait_for( + "the held node parks the stream's second chunk and its log holds every write", + STEP_DEADLINE, + STEP_POLL, + || chunk_parked.exists() && raft_delivered(held, group) >= leader_applied, + ) + .await; + let tracker = held.shared.propose_tracker.get().expect("propose tracker"); + let held_index = tracker + .applying(group) + .expect("the held node's group waits behind the parked chunk") + .log_index; + assert!( + held_index < leader_applied, + "the parked chunk at {held_index} comes before the later write at {leader_applied}" + ); + + held.remount_data_group(group) + .await + .unwrap_or_else(|e| panic!("{e}")); + assert!( + held.shared.redo_chunks.owes_snapshot(group), + "node {held_id} dropped the group's stream: it owes a snapshot" + ); + assert_eq!( + replica_state(held, group), + Some(ReplicaState::AwaitingSnapshot), + "node {held_id}'s data diverged: it must not lead" + ); + + std::fs::write(&chunk_release, b"release").expect("release the parked chunk"); + drop(chunk_gate); + + // A transfer to the held node starts its campaign. A refused transfer + // starts nothing. Either way it must never lead. + let _ = writer.transfer_group_leadership(group, held_id); + let watch_until = Instant::now() + HOLD_WINDOW; + while Instant::now() < watch_until { + let applied = held.shared.applied_index_watcher(group).current(); + assert!( + applied < held_index, + "node {held_id} applied through {applied}, past the held entry at {held_index}" + ); + assert_eq!( + tracker.applying(group).map(|entry| entry.log_index), + Some(held_index), + "node {held_id}'s group waits behind the held entry" + ); + assert!( + !held.leads_group(group), + "node {held_id} leads group {group} while it awaits a snapshot" + ); + assert_eq!( + replica_state(held, group), + Some(ReplicaState::AwaitingSnapshot) + ); + tokio::time::sleep(STEP_POLL).await; + } + assert_eq!( + stored_rows(held).await, + 0, + "node {held_id} stores rows the held stream's final entry or the later write carry" + ); + assert_read_is_whole_or_waits(held).await; + + std::fs::write(&build_release, b"release").expect("release the snapshot build"); + drop(build_gate); + + wait_for( + "the held node installs a snapshot and settles its debt", + STEP_DEADLINE, + STEP_POLL, + || { + !held.shared.redo_chunks.owes_snapshot(group) + && replica_state(held, group) == Some(ReplicaState::Whole) + && held.shared.applied_index_watcher(group).current() >= leader_applied + }, + ) + .await; + let content = wait_replicas_agree(&cluster, &[COLL]).await; + let rows = content.get(&vshard_of(COLL)).map_or(0, |stored| { + stored.keys().filter(|key| key.starts_with("doc:")).count() + }); + assert_eq!( + rows, EXPECTED_ROWS, + "every replica stores every committed row" + ); + assert_eq!(stored_rows(held).await, EXPECTED_ROWS); + + cluster.shutdown().await; +} diff --git a/nodedb-cluster-tests/tests/common_suite/cases/routing_redirect_hint_unhosted_group.rs b/nodedb-cluster-tests/tests/common_suite/cases/routing_redirect_hint_unhosted_group.rs index 6fe3b9af1..5fc1e4c79 100644 --- a/nodedb-cluster-tests/tests/common_suite/cases/routing_redirect_hint_unhosted_group.rs +++ b/nodedb-cluster-tests/tests/common_suite/cases/routing_redirect_hint_unhosted_group.rs @@ -16,7 +16,7 @@ use common::cluster_harness::{TestCluster, TestClusterNode, wait_for}; use std::time::Duration; -use nodedb::control::gateway::retry::retry_not_leader; +use nodedb::control::gateway::retry::{RetryPacing, retry_not_leader}; use nodedb::types::VShardId; const CONVERGE: Duration = Duration::from_secs(30); @@ -197,18 +197,24 @@ async fn an_outsider_follows_a_redirect_to_the_new_leader_and_ignores_a_stale_on // A stale redirect naming the old leader at the old term, through the // gateway's retry path, leaves the hint on the new leader. - let _ = retry_not_leader(Some(&**routing), |attempt| async move { - if attempt == 0 { - Err(nodedb::Error::NotLeader { - vshard_id: VShardId::new(vshard_id), - leader_node: old_leader, - leader_addr: String::new(), - leader_term: old_term, - }) - } else { - Ok::<(), nodedb::Error>(()) - } - }) + let deadline = tokio::time::Instant::now() + std::time::Duration::from_secs(30); + let _ = retry_not_leader( + Some(&**routing), + deadline, + RetryPacing::default(), + |attempt| async move { + if attempt.index == 0 { + Err(nodedb::Error::NotLeader { + vshard_id: VShardId::new(vshard_id), + leader_node: old_leader, + leader_addr: String::new(), + leader_term: old_term, + }) + } else { + Ok::<(), nodedb::Error>(()) + } + }, + ) .await; assert_eq!( hint(outsider, group_id), diff --git a/nodedb-cluster-tests/tests/common_suite/cases/sync_constraint_version_fence.rs b/nodedb-cluster-tests/tests/common_suite/cases/sync_constraint_version_fence.rs index b1d3c7df6..7ace48563 100644 --- a/nodedb-cluster-tests/tests/common_suite/cases/sync_constraint_version_fence.rs +++ b/nodedb-cluster-tests/tests/common_suite/cases/sync_constraint_version_fence.rs @@ -220,14 +220,11 @@ async fn peer_delta_fenced_until_constraint_installed() { "fenced delta A must not be imported, but crdt_state returned: {landed:?}" ); - // Install the constraint deterministically: drive one reconcile pass on - // every node (only the metadata leader actually proposes; the rest - // no-op), then poll until the UNIQUE(email) constraint is observed - // installed in node 0's validator. - for node in &cluster.nodes { - node.run_constraint_reconcile_once().await; - } + // Install the constraint on demand: a reconcile pass proposes only on the + // metadata leader, and leadership can move under load. Re-drive it on + // every node each round until node 0's validator holds UNIQUE(email). let deadline = Instant::now() + Duration::from_secs(15); + let mut accepted = 0usize; loop { let constraints = cluster.nodes[0] .crdt_constraints(TenantId::new(1), COLL) @@ -240,10 +237,15 @@ async fn peer_delta_fenced_until_constraint_installed() { } if Instant::now() >= deadline { panic!( - "UNIQUE(email) constraint on '{COLL}' did not install within 15s of driving \ - reconcile on demand; observed: {constraints:?}" + "UNIQUE(email) constraint on '{COLL}' did not install within 15s of re-driving \ + reconcile on every node ({accepted} proposals accepted); observed: \ + {constraints:?}; groups:\n{}", + cluster.group_positions_report() ); } + for node in &cluster.nodes { + accepted += node.run_constraint_reconcile_once().await; + } tokio::time::sleep(Duration::from_millis(200)).await; } diff --git a/nodedb-cluster-tests/tests/common_suite/cases/timeseries_ring_drop_catch_up.rs b/nodedb-cluster-tests/tests/common_suite/cases/timeseries_ring_drop_catch_up.rs index 0942bec47..c6f5a88f1 100644 --- a/nodedb-cluster-tests/tests/common_suite/cases/timeseries_ring_drop_catch_up.rs +++ b/nodedb-cluster-tests/tests/common_suite/cases/timeseries_ring_drop_catch_up.rs @@ -13,8 +13,8 @@ //! `TimeseriesIngest` opcode: a fresh column comes only from a raw ILP line //! (see `ts_native_ingest`). A parked apply holds every later entry of its //! group, so only A reaches the gates. B goes out once every replica's gate -//! holds A, and C once every replica committed B, so the log orders them as -//! sent: +//! holds A, and C once every replica's log holds B's ingest committed, so the +//! log orders them as sent: //! //! - A gives the new column `extra` a float. //! - B gives a new column `y` a float and names no `extra`. @@ -40,12 +40,13 @@ use common::cluster_harness::{TestCluster, TestClusterNode, wait_for}; use std::path::{Path, PathBuf}; use std::time::{Duration, Instant}; +use nodedb::control::wal_replication::{ReplicatedEntry, ReplicatedWrite}; use nodedb::event::cdc::consume::{ConsumeError, ConsumeParams, consume_local}; use nodedb_test_support::fail_point::{FailAction, FailGuard}; use nodedb_types::DatabaseId; use super::ts_native_ingest::{ - ingest_native, ingest_until_accepted, native_session, rejection_warnings, + ingest_native_within, ingest_until_accepted, native_session, rejection_warnings, }; const COLLECTION: &str = "ts_ring_drop"; @@ -118,6 +119,32 @@ fn group_commit_index(node: &TestClusterNode, group_id: u64) -> u64 { .map_or(0, |group| group.commit_index) } +/// Whether `node`'s log for `group_id` holds a committed ingest into the +/// collection above index `after`. +/// +/// One ingest is in flight at a time, and every earlier one sits at or below +/// `after`, so such an entry is the ingest just sent. +fn committed_ingest_above(node: &TestClusterNode, group_id: u64, after: u64) -> bool { + let Some(multi_raft) = node.shared.multi_raft.get() else { + return false; + }; + let mut multi_raft = multi_raft.lock().unwrap_or_else(|p| p.into_inner()); + let Some(raft) = multi_raft.groups_mut().get(&group_id) else { + return false; + }; + let collection = nodedb_types::QualifiedCollection::new(DatabaseId::DEFAULT, COLLECTION); + (after + 1..=raft.commit_index()) + .filter_map(|index| raft.log_entry_at(index)) + .filter_map(|entry| ReplicatedEntry::from_bytes(&entry.data)) + .any(|entry| { + matches!( + entry.write, + ReplicatedWrite::TimeseriesIngest { collection: ref written, .. } + if written == collection.as_str() + ) + }) +} + /// The new values of the change events `node` serves. fn event_values(node: &TestClusterNode) -> Vec { let params = ConsumeParams { @@ -199,8 +226,9 @@ async fn catch_up_after_an_apply_time_rejection_rebuilds_only_landed_rows() { format!("{COLLECTION} value=3.0,extra=\"text\" 4000000000"), ]; // A parked apply holds every later entry of its group, so only A reaches - // the gates. B and C each go out once every replica committed the line - // before it. + // the gates. B and C each go out once every replica's log holds the line + // before it committed. Each statement waits without a bound while parked: + // its reply comes only after the release. let group_id = cluster.nodes[0] .group_id_for_collection(COLLECTION) .expect("the collection maps to a data group"); @@ -219,7 +247,7 @@ async fn catch_up_after_an_apply_time_rejection_rebuilds_only_landed_rows() { .collect(); let mut session = native_session(node).await; sent.push(tokio::spawn(async move { - ingest_native(&mut session, 1, COLLECTION, &line).await + ingest_native_within(&mut session, 1, COLLECTION, &line, None).await })); if label == "A" { wait_for("every replica holds A's apply", CONVERGE, GATE_STEP, || { @@ -236,7 +264,7 @@ async fn catch_up_after_an_apply_time_rejection_rebuilds_only_landed_rows() { .nodes .iter() .zip(&committed_before) - .all(|(node, before)| group_commit_index(node, group_id) > *before) + .all(|(node, before)| committed_ingest_above(node, group_id, *before)) }, ) .await; @@ -248,9 +276,20 @@ async fn catch_up_after_an_apply_time_rejection_rebuilds_only_landed_rows() { for gate in &gates { gate.release(); } + // Every reply is due once the applies run, so the bound starts here. + let replied_by = tokio::time::Instant::now() + CONVERGE; let mut replies = Vec::new(); - for statement in sent { - replies.push(statement.await.expect("statement task")); + for (label, statement) in ["A", "B", "C"].into_iter().zip(sent) { + let reply = tokio::time::timeout_at(replied_by, statement) + .await + .unwrap_or_else(|_| { + panic!( + "{label} got no reply within {CONVERGE:?} of the release; groups:\n{}", + cluster.group_positions_report() + ) + }) + .expect("statement task"); + replies.push(reply); } let deadline = Instant::now() + CONVERGE; diff --git a/nodedb-cluster-tests/tests/common_suite/cases/ts_native_ingest.rs b/nodedb-cluster-tests/tests/common_suite/cases/ts_native_ingest.rs index 41ff198bb..5baf88ce6 100644 --- a/nodedb-cluster-tests/tests/common_suite/cases/ts_native_ingest.rs +++ b/nodedb-cluster-tests/tests/common_suite/cases/ts_native_ingest.rs @@ -12,7 +12,7 @@ use std::time::Duration; -use nodedb_test_support::native_harness::{do_handshake, send_request}; +use nodedb_test_support::native_harness::{RESPONSE_BOUND, do_handshake, send_request_within}; use nodedb_types::protocol::opcodes::ResponseStatus; use nodedb_types::protocol::text_fields::TextFields; use nodedb_types::protocol::{HelloFrame, NativeResponse, OpCode}; @@ -33,14 +33,28 @@ pub(super) async fn native_session(node: &TestClusterNode) -> TcpStream { } /// Ingest the raw ILP `lines` into `collection` through the native -/// `TimeseriesIngest` opcode. +/// `TimeseriesIngest` opcode, waiting at most [`RESPONSE_BOUND`] for the +/// response. pub(super) async fn ingest_native( stream: &mut TcpStream, seq: u64, collection: &str, lines: &str, ) -> NativeResponse { - send_request( + ingest_native_within(stream, seq, collection, lines, Some(RESPONSE_BOUND)).await +} + +/// [`ingest_native`] waiting at most `bound` for the response, or without +/// limit when `bound` is `None`. A test that parks the apply on purpose +/// bounds the wait itself. +pub(super) async fn ingest_native_within( + stream: &mut TcpStream, + seq: u64, + collection: &str, + lines: &str, + bound: Option, +) -> NativeResponse { + send_request_within( stream, seq, OpCode::TimeseriesIngest, @@ -50,6 +64,7 @@ pub(super) async fn ingest_native( format: Some("ilp".to_string()), ..Default::default() }, + bound, ) .await } diff --git a/nodedb-cluster-tests/tests/misc_suite/cases/transport_security.rs b/nodedb-cluster-tests/tests/misc_suite/cases/transport_security.rs index 8ae10a2e1..4b4edac70 100644 --- a/nodedb-cluster-tests/tests/misc_suite/cases/transport_security.rs +++ b/nodedb-cluster-tests/tests/misc_suite/cases/transport_security.rs @@ -65,7 +65,7 @@ impl RaftRpcHandler for EchoHandler { success: true, last_log_index: req.prev_log_index + req.entries.len() as u64, round: req.round, - needs_snapshot: false, + replica: nodedb_raft::ReplicaState::Whole, })) } RaftRpc::RequestVoteRequest(req) => { diff --git a/nodedb-cluster-tests/tests/misc_suite/cases/wire_version_handshake.rs b/nodedb-cluster-tests/tests/misc_suite/cases/wire_version_handshake.rs index ed7e01bf5..b1ef14b75 100644 --- a/nodedb-cluster-tests/tests/misc_suite/cases/wire_version_handshake.rs +++ b/nodedb-cluster-tests/tests/misc_suite/cases/wire_version_handshake.rs @@ -37,7 +37,7 @@ impl RaftRpcHandler for EchoHandler { success: true, last_log_index: req.prev_log_index, round: req.round, - needs_snapshot: false, + replica: nodedb_raft::ReplicaState::Whole, })) } other => Err(ClusterError::Transport { diff --git a/nodedb-cluster/Cargo.toml b/nodedb-cluster/Cargo.toml index eb0b0403a..c8c04a79b 100644 --- a/nodedb-cluster/Cargo.toml +++ b/nodedb-cluster/Cargo.toml @@ -19,8 +19,9 @@ default = [] # `faultbox::init`; a no-op on wasm32 regardless. Off by default so the crate # stays dependency-light for embedders that report failures their own way. diagnostics = ["dep:faultbox"] -# Arms the Raft fail gates cluster tests use. Off by default. -failpoints = ["nodedb-raft/failpoints"] +# Arms the Raft fail gates and the transport blackhole cluster tests use. +# Off by default. +failpoints = ["nodedb-raft/failpoints", "nodedb-types/failpoints"] [dependencies] # Workspace crates diff --git a/nodedb-cluster/src/applied_watcher/mod.rs b/nodedb-cluster/src/applied_watcher/mod.rs index 681ff9289..64232ec58 100644 --- a/nodedb-cluster/src/applied_watcher/mod.rs +++ b/nodedb-cluster/src/applied_watcher/mod.rs @@ -9,18 +9,15 @@ //! checks) can wait on the apply watermark of *their* group rather //! than only the metadata group. //! -//! Bump points (single source of truth for "applied on this node"): +//! Bump points: //! -//! 1. [`crate::raft_loop::tick`] after the per-group apply phase -//! — covers regular committed-entry application for every locally -//! mounted group, including the metadata group. -//! 2. [`crate::raft_loop::handle_rpc`] after `handle_install_snapshot` -//! — covers the snapshot-install path where `last_applied` jumps -//! to `last_included_index` without going through the apply loop. -//! -//! Both bump points read the canonical applied index from the Raft -//! node itself rather than mirroring it from a side channel, so the -//! watcher cannot drift from Raft's own state. +//! 1. The metadata lane, once a metadata entry applied. +//! 2. The tick loop, for the Calvin sequencer group, which the host applies +//! inline. +//! 3. The host apply loop, for data groups, once an entry's apply concluded +//! and every earlier entry of the group settled. +//! 4. A snapshot install, for every group whose state it restored. +//! 5. A group mount, seeded with the durable applied index. pub mod registry; pub mod watcher; diff --git a/nodedb-cluster/src/applied_watcher/registry.rs b/nodedb-cluster/src/applied_watcher/registry.rs index 666c875fd..d571458ad 100644 --- a/nodedb-cluster/src/applied_watcher/registry.rs +++ b/nodedb-cluster/src/applied_watcher/registry.rs @@ -20,7 +20,7 @@ use std::collections::HashMap; use std::sync::{Arc, RwLock}; -use std::time::Duration; +use std::time::Instant; use super::watcher::{AppliedIndexWatcher, WaitOutcome}; @@ -77,12 +77,18 @@ impl GroupAppliedWatchers { } } - /// Block until `group_id`'s applied watermark reaches `target`, - /// the timeout elapses, or the group is removed. Lazily creates - /// the watcher if it does not yet exist (waiters proposed before - /// the first apply on this node would otherwise miss the wake-up). - pub fn wait_for(&self, group_id: u64, target: u64, timeout: Duration) -> WaitOutcome { - self.get_or_create(group_id).wait_for(target, timeout) + /// Wait until `group_id`'s applied watermark reaches `target`, + /// `deadline` passes, or the group is removed. Creates the watcher + /// when absent, so a wait that starts before the first apply here + /// still wakes. + pub async fn wait_for_async( + &self, + group_id: u64, + target: u64, + deadline: Instant, + ) -> WaitOutcome { + let watcher = self.get_or_create(group_id); + watcher.wait_for_async(target, deadline).await } /// Number of groups currently registered. Useful for metrics. @@ -117,7 +123,11 @@ impl GroupAppliedWatchers { #[cfg(test)] mod tests { use super::*; - use std::thread; + use std::time::Duration; + + fn after(duration: Duration) -> Instant { + Instant::now() + duration + } #[test] fn separate_groups_have_independent_watermarks() { @@ -128,32 +138,40 @@ mod tests { assert_eq!(r.get(2).unwrap().current(), 10); } - #[test] - fn wait_returns_reached_after_bump() { + #[tokio::test] + async fn wait_returns_reached_after_bump() { let r = Arc::new(GroupAppliedWatchers::new()); - let r2 = r.clone(); - let h = thread::spawn(move || r2.wait_for(7, 5, Duration::from_secs(2))); - thread::sleep(Duration::from_millis(20)); + let waiter = { + let r = Arc::clone(&r); + tokio::spawn(async move { r.wait_for_async(7, 5, after(Duration::from_secs(2))).await }) + }; + tokio::time::sleep(Duration::from_millis(20)).await; r.bump(7, 5); - assert_eq!(h.join().unwrap(), WaitOutcome::Reached); + assert_eq!(waiter.await.unwrap(), WaitOutcome::Reached); } - #[test] - fn remove_wakes_waiter_with_group_gone() { + #[tokio::test] + async fn remove_wakes_waiter_with_group_gone() { let r = Arc::new(GroupAppliedWatchers::new()); - let r2 = r.clone(); // Pre-create so the waiter watches the same Arc that remove() closes. r.get_or_create(3); - let h = thread::spawn(move || r2.wait_for(3, 99, Duration::from_secs(2))); - thread::sleep(Duration::from_millis(20)); + let waiter = { + let r = Arc::clone(&r); + tokio::spawn( + async move { r.wait_for_async(3, 99, after(Duration::from_secs(2))).await }, + ) + }; + tokio::time::sleep(Duration::from_millis(20)).await; r.remove(3); - assert_eq!(h.join().unwrap(), WaitOutcome::GroupGone); + assert_eq!(waiter.await.unwrap(), WaitOutcome::GroupGone); } - #[test] - fn missing_group_creates_watcher_lazily() { + #[tokio::test] + async fn missing_group_creates_watcher_lazily() { let r = GroupAppliedWatchers::new(); - let outcome = r.wait_for(42, 1, Duration::from_millis(20)); + let outcome = r + .wait_for_async(42, 1, after(Duration::from_millis(20))) + .await; assert_eq!(outcome, WaitOutcome::TimedOut); assert!(r.get(42).is_some()); } diff --git a/nodedb-cluster/src/applied_watcher/watcher.rs b/nodedb-cluster/src/applied_watcher/watcher.rs index 080c769fa..62845a0d0 100644 --- a/nodedb-cluster/src/applied_watcher/watcher.rs +++ b/nodedb-cluster/src/applied_watcher/watcher.rs @@ -1,19 +1,18 @@ // SPDX-License-Identifier: BUSL-1.1 -//! Sync wait primitive used to block on a Raft group's local apply -//! watermark. +//! Async wait on a Raft group's local apply watermark. //! -//! Uses `std::sync::Condvar` so it is safe to call from synchronous -//! pgwire handler code without entering a tokio reactor. A `closed` -//! flag distinguishes "still waiting for apply" from "the group is -//! no longer hosted on this node" — a follower that gets removed via -//! conf-change must wake outstanding waiters with a terminal status -//! rather than time them out. +//! A waiter parks as a task on a [`Notify`], so it holds no thread while it +//! waits. A `closed` flag tells "still waiting for apply" apart from "the +//! group is no longer hosted on this node": a removed group wakes its +//! waiters with a terminal outcome instead of timing them out. -use std::sync::{Condvar, Mutex}; -use std::time::{Duration, Instant}; +use std::sync::Mutex; +use std::time::Instant; -/// Outcome of [`AppliedIndexWatcher::wait_for`]. +use tokio::sync::Notify; + +/// Outcome of [`AppliedIndexWatcher::wait_for_async`]. #[derive(Debug, Clone, Copy, PartialEq, Eq)] pub enum WaitOutcome { /// The applied watermark reached the requested target. @@ -35,7 +34,8 @@ impl WaitOutcome { #[derive(Debug, Default)] pub struct AppliedIndexWatcher { state: Mutex, - cv: Condvar, + /// Woken on every watermark advance and on close. + changed: Notify, } #[derive(Debug, Default)] @@ -53,29 +53,36 @@ impl AppliedIndexWatcher { Self::default() } + fn state(&self) -> std::sync::MutexGuard<'_, State> { + self.state.lock().unwrap_or_else(|p| p.into_inner()) + } + /// Advance the watermark. Idempotent: smaller indices are ignored. /// Bumps on a closed watcher are also ignored (the group is gone). pub fn bump(&self, applied_index: u64) { - let mut guard = self.state.lock().unwrap_or_else(|p| p.into_inner()); - if guard.closed { - return; - } - if applied_index > guard.applied { - guard.applied = applied_index; - self.cv.notify_all(); + let advanced = { + let mut guard = self.state(); + let advanced = !guard.closed && applied_index > guard.applied; + if advanced { + guard.applied = applied_index; + } + advanced + }; + if advanced { + self.changed.notify_waiters(); } } /// Read the current watermark without blocking. pub fn current(&self) -> u64 { - self.state.lock().unwrap_or_else(|p| p.into_inner()).applied + self.state().applied } /// Declare that the applier starts a batch ending at `last_index`. The /// applier calls it before the batch mutates any state a reader can see. /// Idempotent: smaller indices are ignored. pub fn begin_batch(&self, last_index: u64) { - let mut guard = self.state.lock().unwrap_or_else(|p| p.into_inner()); + let mut guard = self.state(); guard.applying_through = guard.applying_through.max(last_index); } @@ -84,51 +91,60 @@ impl AppliedIndexWatcher { /// /// A reader that sees an effect of the batch in progress gets a floor at /// or above the index of the entry that made it visible. A waiter for - /// that floor waits on [`Self::wait_for`], which the finished batch + /// that floor waits on [`Self::wait_for_async`], which the finished batch /// satisfies. pub fn floor(&self) -> u64 { - let guard = self.state.lock().unwrap_or_else(|p| p.into_inner()); + let guard = self.state(); guard.applied.max(guard.applying_through) } /// True once [`Self::close`] has been called. pub fn is_closed(&self) -> bool { - self.state.lock().unwrap_or_else(|p| p.into_inner()).closed + self.state().closed } /// Mark this watcher closed and wake every waiter with /// [`WaitOutcome::GroupGone`]. Idempotent. pub fn close(&self) { - let mut guard = self.state.lock().unwrap_or_else(|p| p.into_inner()); - if !guard.closed { + let closed_now = { + let mut guard = self.state(); + let closed_now = !guard.closed; guard.closed = true; - self.cv.notify_all(); + closed_now + }; + if closed_now { + self.changed.notify_waiters(); } } - /// Block until the watermark reaches `target`, the timeout elapses, - /// or the watcher is closed. - pub fn wait_for(&self, target: u64, timeout: Duration) -> WaitOutcome { - let deadline = Instant::now() + timeout; - let mut guard = self.state.lock().unwrap_or_else(|p| p.into_inner()); + /// The outcome of a wait for `target` as the state stands, `None` while + /// it is still pending. + fn settled(&self, target: u64) -> Option { + let guard = self.state(); + if guard.applied >= target { + Some(WaitOutcome::Reached) + } else if guard.closed { + Some(WaitOutcome::GroupGone) + } else { + None + } + } + + /// Wait until the watermark reaches `target`, `deadline` passes, or the + /// watcher is closed. The waiting task holds no thread. + pub async fn wait_for_async(&self, target: u64, deadline: Instant) -> WaitOutcome { + let deadline = tokio::time::Instant::from_std(deadline); loop { - if guard.applied >= target { - return WaitOutcome::Reached; + let notified = self.changed.notified(); + tokio::pin!(notified); + // Registered before the state is read, so a bump between the + // read and the await still wakes this waiter. + notified.as_mut().enable(); + if let Some(outcome) = self.settled(target) { + return outcome; } - if guard.closed { - return WaitOutcome::GroupGone; - } - let remaining = match deadline.checked_duration_since(Instant::now()) { - Some(r) if !r.is_zero() => r, - _ => return WaitOutcome::TimedOut, - }; - let wait = self - .cv - .wait_timeout(guard, remaining) - .unwrap_or_else(|p| p.into_inner()); - guard = wait.0; - if wait.1.timed_out() && guard.applied < target && !guard.closed { - return WaitOutcome::TimedOut; + if tokio::time::timeout_at(deadline, notified).await.is_err() { + return self.settled(target).unwrap_or(WaitOutcome::TimedOut); } } } @@ -138,47 +154,83 @@ impl AppliedIndexWatcher { mod tests { use super::*; use std::sync::Arc; - use std::thread; + use std::time::Duration; - #[test] - fn bump_notifies_waiter() { + fn after(duration: Duration) -> Instant { + Instant::now() + duration + } + + #[tokio::test] + async fn a_bump_wakes_the_waiter() { let w = Arc::new(AppliedIndexWatcher::new()); - let w2 = w.clone(); - let h = thread::spawn(move || w2.wait_for(5, Duration::from_secs(2))); - thread::sleep(Duration::from_millis(20)); + let waiter = { + let w = Arc::clone(&w); + tokio::spawn(async move { w.wait_for_async(5, after(Duration::from_secs(5))).await }) + }; + tokio::time::sleep(Duration::from_millis(20)).await; w.bump(3); - thread::sleep(Duration::from_millis(20)); + tokio::time::sleep(Duration::from_millis(20)).await; w.bump(5); - assert_eq!(h.join().unwrap(), WaitOutcome::Reached); + let started = Instant::now(); + assert_eq!(waiter.await.expect("join"), WaitOutcome::Reached); + assert!(started.elapsed() < Duration::from_secs(2)); } - #[test] - fn times_out_if_never_bumped() { + #[tokio::test] + async fn the_wait_times_out_at_its_deadline() { let w = AppliedIndexWatcher::new(); - assert_eq!( - w.wait_for(1, Duration::from_millis(30)), - WaitOutcome::TimedOut - ); + let started = Instant::now(); + let outcome = w.wait_for_async(1, after(Duration::from_millis(200))).await; + assert_eq!(outcome, WaitOutcome::TimedOut); + let waited = started.elapsed(); + assert!(waited >= Duration::from_millis(200), "{waited:?}"); + assert!(waited < Duration::from_secs(2), "{waited:?}"); } - #[test] - fn already_past_target_returns_immediately() { + #[tokio::test] + async fn a_passed_deadline_still_reports_a_reached_target() { let w = AppliedIndexWatcher::new(); w.bump(10); - assert_eq!( - w.wait_for(5, Duration::from_millis(10)), - WaitOutcome::Reached - ); + let outcome = w.wait_for_async(5, Instant::now()).await; + assert_eq!(outcome, WaitOutcome::Reached); } - #[test] - fn close_wakes_waiter_with_group_gone() { + #[tokio::test] + async fn close_wakes_the_waiter_with_group_gone() { let w = Arc::new(AppliedIndexWatcher::new()); - let w2 = w.clone(); - let h = thread::spawn(move || w2.wait_for(99, Duration::from_secs(2))); - thread::sleep(Duration::from_millis(20)); + let waiter = { + let w = Arc::clone(&w); + tokio::spawn(async move { w.wait_for_async(99, after(Duration::from_secs(5))).await }) + }; + tokio::time::sleep(Duration::from_millis(20)).await; w.close(); - assert_eq!(h.join().unwrap(), WaitOutcome::GroupGone); + assert_eq!(waiter.await.expect("join"), WaitOutcome::GroupGone); + } + + /// A bump that races the start of a wait is never lost: every waiter + /// ends `Reached` well before its deadline. + #[tokio::test(flavor = "multi_thread", worker_threads = 4)] + async fn a_bump_racing_the_wait_start_is_not_lost() { + for round in 1..=200u64 { + let w = Arc::new(AppliedIndexWatcher::new()); + let waiter = { + let w = Arc::clone(&w); + tokio::spawn(async move { + w.wait_for_async(round, after(Duration::from_secs(10))) + .await + }) + }; + let bumper = { + let w = Arc::clone(&w); + tokio::spawn(async move { w.bump(round) }) + }; + bumper.await.expect("join bumper"); + let outcome = tokio::time::timeout(Duration::from_secs(5), waiter) + .await + .expect("the waiter woke on the bump") + .expect("join waiter"); + assert_eq!(outcome, WaitOutcome::Reached, "round {round}"); + } } #[test] diff --git a/nodedb-cluster/src/array_routing.rs b/nodedb-cluster/src/array_routing.rs index 0085835e2..5610aea9a 100644 --- a/nodedb-cluster/src/array_routing.rs +++ b/nodedb-cluster/src/array_routing.rs @@ -16,10 +16,8 @@ //! //! # Returned type //! -//! Functions here return a raw `u16` vShard ID (0..1023). Callers in the -//! `nodedb` crate wrap this in `VShardId::new(…)`. `VShardId` lives in -//! `nodedb::types` which `nodedb-cluster` cannot depend on without creating -//! a circular dependency. +//! Functions here return a raw `u32` vShard ID below `VShardId::COUNT`. +//! Callers wrap it in `VShardId::new(…)`. //! //! # Fallbacks //! @@ -27,8 +25,8 @@ //! - Zero tile extent in any dimension: same fallback (no division by zero). //! - `coord.len() != tile_extents.len()`: fallback. -/// Number of virtual shards — must match `VShardId::COUNT` in `nodedb::types`. -pub const VSHARD_COUNT: u32 = 1024; +/// Number of virtual shards. +pub const VSHARD_COUNT: u32 = nodedb_types::id::VShardId::COUNT; // ─── Collection-level fallback hash ────────────────────────────────────────── @@ -120,7 +118,7 @@ pub fn tile_id_of_coord(coord: &[u64], tile_extents: &[u64]) -> Option { /// - `coord` / `tile_extents` are empty or mismatched. /// - Any `tile_extents[i]` is zero. /// -/// Returns a raw `u16` in `0..1023`. Callers wrap it in `VShardId::new(…)`. +/// Returns a raw `u32` below `VSHARD_COUNT`. Callers wrap it in `VShardId::new(…)`. pub fn vshard_for_array_coord(array_name: &str, coord: &[u64], tile_extents: &[u64]) -> u32 { match tile_id_of_coord(coord, tile_extents) { Some(tile_id) => vshard_for_array_tile(array_name, tile_id), @@ -133,7 +131,7 @@ pub fn vshard_for_array_coord(array_name: &str, coord: &[u64], tile_extents: &[u /// Use this when the tile_id is already known (e.g. from the op-log key or /// snapshot metadata) and coord re-derivation is unnecessary. /// -/// Returns a raw `u16` in `0..1023`. Callers wrap it in `VShardId::new(…)`. +/// Returns a raw `u32` below `VSHARD_COUNT`. Callers wrap it in `VShardId::new(…)`. pub fn vshard_for_array_tile(array_name: &str, tile_id: u64) -> u32 { let mut buf = Vec::with_capacity(array_name.len() + 8); buf.extend_from_slice(array_name.as_bytes()); diff --git a/nodedb-cluster/src/calvin/sequencer/validator.rs b/nodedb-cluster/src/calvin/sequencer/validator/batch.rs similarity index 66% rename from nodedb-cluster/src/calvin/sequencer/validator.rs rename to nodedb-cluster/src/calvin/sequencer/validator/batch.rs index d2e45f60f..b273323db 100644 --- a/nodedb-cluster/src/calvin/sequencer/validator.rs +++ b/nodedb-cluster/src/calvin/sequencer/validator/batch.rs @@ -20,7 +20,7 @@ //! # Conflict handling (read-aware) //! //! The scheduler locks a txn's write keys exclusive and its read keys shared, -//! so a reader and a writer of one key serialize by `position`. Per-key last-write-LSN +//! so a reader and a writer of conflicting keys serialize by `position`. Per-key last-write-LSN //! validation means a reader `R` of key `K` aborts at commit only if a //! *lower-position* writer `W` of `K` committed first (its write LSN exceeds //! `R`'s read version). Giving `R` a lower position than `W` converts that @@ -29,9 +29,14 @@ //! So the validator: //! //! - Builds, per key `K`, the set of transactions that read `K` and the set -//! that write `K`. -//! - For every `(reader r, writer w)` pair on the same key with `r != w`, adds -//! a directed edge `r -> w` ("`r` must get a lower position than `w`"). +//! that write `K`, and per collection its collection-scope and row keys. +//! - For every `(reader r, writer w)` pair with `r != w` whose keys conflict, +//! adds a directed edge `r -> w` ("`r` must get a lower position than +//! `w`"). Two keys conflict when they are the same key, or when they share +//! a collection and either covers the whole collection (see +//! [`super::keys::conflicts`]). The Data Plane validates a collection read +//! against every row write and a row read against every whole-collection +//! write, so these pairs order like same-key pairs. //! - Emits positions in a deterministic topological order of those edges, //! which places every reader ahead of the conflicting writers it can — the //! read-after-write → write-after-read reorder that avoids the abort. @@ -46,146 +51,17 @@ //! mirroring "the later transaction loses". Positions on the admitted set are //! dense `0..(n - num_rejected)` in emission order. -use std::collections::{BTreeMap, BTreeSet}; +use std::collections::BTreeSet; use xxhash_rust::xxh3::xxh3_64; use crate::calvin::sequencer::error::SequencerError; use crate::calvin::sequencer::inbox::{AdmittedTx, RejectedTx}; -use crate::calvin::sequencer::metrics::ConflictKey; -use crate::calvin::types::{EngineKeySet, ReadWriteSet, SequencedTxn}; - -/// Canonical, order-comparable identity of a single key across engines: -/// `(engine_discriminant, collection, key_bytes)`. Used as the `BTreeMap` key -/// for the reader/writer maps so all iteration feeding output is deterministic. -type KeyId = (u8, String, Vec); - -/// A single flattened key drawn from one transaction's read or write set. -#[derive(Debug)] -struct FlatKey { - /// Discriminant tag for the key-set variant (Document=0, Vector=1, Kv=2, - /// Edge=3, Array=4, Collection=5, Unique=6). - discriminant: u8, - /// Static engine name used when building conflict-context keys. - engine_name: &'static str, - /// Collection name. - collection: String, - /// Serialized key bytes. - key_bytes: Vec, -} - -impl FlatKey { - /// The order-comparable identity of this key. - fn key_id(&self) -> KeyId { - ( - self.discriminant, - self.collection.clone(), - self.key_bytes.clone(), - ) - } -} +use crate::calvin::types::SequencedTxn; -/// Flatten one read-or-write set into its constituent per-key entries. -fn flatten_set(set: &ReadWriteSet) -> Vec { - let mut out = Vec::new(); - for key_set in &set.0 { - match key_set { - EngineKeySet::Document { - collection, - surrogates, - } => { - for &s in surrogates.as_slice() { - out.push(FlatKey { - discriminant: 0, - engine_name: "document", - collection: collection.clone(), - key_bytes: s.to_le_bytes().to_vec(), - }); - } - } - EngineKeySet::Vector { - collection, - surrogates, - } => { - for &s in surrogates.as_slice() { - out.push(FlatKey { - discriminant: 1, - engine_name: "vector", - collection: collection.clone(), - key_bytes: s.to_le_bytes().to_vec(), - }); - } - } - EngineKeySet::Kv { collection, keys } => { - for k in keys.as_slice() { - out.push(FlatKey { - discriminant: 2, - engine_name: "kv", - collection: collection.clone(), - key_bytes: k.clone(), - }); - } - } - EngineKeySet::Edge { - collection, edges, .. - } => { - for &(src, dst) in edges.as_slice() { - let mut key_bytes = src.to_le_bytes().to_vec(); - key_bytes.extend_from_slice(&dst.to_le_bytes()); - out.push(FlatKey { - discriminant: 3, - engine_name: "edge", - collection: collection.clone(), - key_bytes, - }); - } - } - // The whole array on each vShard it names. - EngineKeySet::Array { - collection, - vshards, - } => { - for &vshard in vshards.as_slice() { - out.push(FlatKey { - discriminant: 4, - engine_name: "array", - collection: collection.clone(), - key_bytes: vshard.to_le_bytes().to_vec(), - }); - } - } - // The whole collection: one key with no row bytes. - EngineKeySet::Collection { collection, .. } => { - out.push(FlatKey { - discriminant: 5, - engine_name: "collection", - collection: collection.clone(), - key_bytes: Vec::new(), - }); - } - // One key per claimed value: the index name, a zero byte, the value. - EngineKeySet::Unique { - collection, - index, - values, - } => { - for value in values.as_slice() { - let mut key_bytes = Vec::with_capacity(index.len() + 1 + value.len()); - key_bytes.extend_from_slice(index.as_bytes()); - key_bytes.push(0); - key_bytes.extend_from_slice(value); - out.push(FlatKey { - discriminant: 6, - engine_name: "unique", - collection: collection.clone(), - key_bytes, - }); - } - } - } - } - out -} +use super::graph::ConflictIndex; +use super::keys::flatten_set; +use super::reject::reject_context; /// Sort key for admitted transactions: `(inbox_seq, tenant_id, hash(plans))`. fn admitted_sort_key(tx: &AdmittedTx) -> (u64, u64, u64) { @@ -218,49 +94,19 @@ pub fn validate_batch_with_assignments( candidates.sort_by_key(admitted_sort_key); let n = candidates.len(); - // Flatten both sets of every transaction and build the reader/writer maps - // keyed by canonical key identity. BTreeMap + ascending index vectors keep - // every iteration that feeds output deterministic. - let mut readers_of: BTreeMap> = BTreeMap::new(); - let mut writers_of: BTreeMap> = BTreeMap::new(); - let mut tx_reads: Vec> = Vec::with_capacity(n); - let mut tx_writes: Vec> = Vec::with_capacity(n); - + // Flatten both sets of every transaction into the conflict index, in + // sorted-index order. + let mut index = ConflictIndex::default(); for (i, tx) in candidates.iter().enumerate() { - let reads = flatten_set(&tx.tx_class.read_set); - let writes = flatten_set(&tx.tx_class.write_set); - for fk in &reads { - let entry = readers_of.entry(fk.key_id()).or_default(); - if entry.last() != Some(&i) { - entry.push(i); - } - } - for fk in &writes { - let entry = writers_of.entry(fk.key_id()).or_default(); - if entry.last() != Some(&i) { - entry.push(i); - } - } - tx_reads.push(reads); - tx_writes.push(writes); + index.add( + i, + flatten_set(&tx.tx_class.read_set), + flatten_set(&tx.tx_class.write_set), + ); } - // Build RAW edges `reader -> writer` (reader gets the lower position). A - // BTreeSet of successors deduplicates edges arising from multiple shared - // keys and keeps successor iteration deterministic. - let mut out: Vec> = vec![BTreeSet::new(); n]; - let mut in_degree: Vec = vec![0; n]; - for (key, readers) in &readers_of { - if let Some(writers) = writers_of.get(key) { - for &r in readers { - for &w in writers { - if r != w && out[r].insert(w) { - in_degree[w] += 1; - } - } - } - } - } + // RAW edges `reader -> writer`: the reader gets the lower position. + let (out, mut in_degree) = index.edges(n); // Deterministic topological emission with cycle-breaking. `ready` is a // BTreeSet so `pop_first` always yields the smallest index — i.e. the @@ -334,15 +180,8 @@ pub fn validate_batch_with_assignments( } None => { let tenant = tx.tx_class.tenant_id.as_u64(); - let (position_admitted, conflict_context) = reject_context( - idx, - tenant, - &tx_reads, - &tx_writes, - &readers_of, - &writers_of, - &emitted_position, - ); + let (position_admitted, conflict_context) = + reject_context(idx, tenant, &index, &emitted_position); rejected_out.push(RejectedTx { admitted: tx, reason: SequencerError::Conflict { position_admitted }, @@ -363,67 +202,6 @@ pub fn validate_batch( (admitted.into_iter().map(|(_, txn)| txn).collect(), rejected) } -/// Build the `(position_admitted, conflict_context)` for a cycle-rejected txn. -/// -/// A rejected txn conflicts with any admitted txn that writes a key it reads or -/// reads a key it writes. We select the conflict with the smallest admitted -/// position (ties broken by key identity) so both the reported position and the -/// [`ConflictKey`] are deterministic. If no conflicting txn was admitted, the -/// position defaults to `0` and the context to `None`. -fn reject_context( - idx: usize, - tenant: u64, - tx_reads: &[Vec], - tx_writes: &[Vec], - readers_of: &BTreeMap>, - writers_of: &BTreeMap>, - emitted_position: &[Option], -) -> (u32, Option) { - // Best conflict so far, ordered by (admitted position, key identity). - let mut best: Option<(u32, KeyId, &'static str)> = None; - let mut consider = |fk: &FlatKey, others: Option<&Vec>| { - let Some(others) = others else { return }; - for &other in others { - if other == idx { - continue; - } - let Some(pos) = emitted_position[other] else { - continue; - }; - let key = fk.key_id(); - let candidate = (pos, key, fk.engine_name); - let better = match &best { - None => true, - Some((bpos, bkey, _)) => (pos, &candidate.1) < (*bpos, bkey), - }; - if better { - best = Some(candidate); - } - } - }; - - // A writer of a key this txn reads. - for fk in &tx_reads[idx] { - consider(fk, writers_of.get(&fk.key_id())); - } - // A reader of a key this txn writes. - for fk in &tx_writes[idx] { - consider(fk, readers_of.get(&fk.key_id())); - } - - match best { - Some((pos, key, engine)) => ( - pos, - Some(ConflictKey { - tenant, - engine, - collection: key.1, - }), - ), - None => (0, None), - } -} - #[cfg(test)] mod tests { use super::*; @@ -694,4 +472,150 @@ mod tests { assert_eq!(a.position, b.position); } } + + /// A transaction that reads `read` and writes `write`, both whole key + /// sets. Single-vShard: the validator works on set contents only. + fn make_set_tx( + inbox_seq: u64, + read: Vec, + write: Vec, + ) -> AdmittedTx { + let tx_class = TxClass::new_single_vshard( + ReadWriteSet::new(read), + ReadWriteSet::new(write), + vec![inbox_seq as u8], + TenantId::new(1), + None, + crate::calvin::types::VersionedReadSet::default(), + ) + .expect("valid TxClass"); + AdmittedTx { + inbox_seq, + tx_class, + } + } + + fn rows(collection: &str, surrogates: Vec) -> EngineKeySet { + EngineKeySet::Document { + collection: collection.to_owned(), + surrogates: SortedVec::new(surrogates), + } + } + + fn whole(collection: &str) -> EngineKeySet { + EngineKeySet::Collection { + collection: collection.to_owned(), + vshards: SortedVec::new(vec![]), + } + } + + fn position_of(admitted: &[(u64, SequencedTxn)], inbox_seq: u64) -> u32 { + admitted + .iter() + .find(|(seq, _)| *seq == inbox_seq) + .map(|(_, t)| t.position) + .expect("txn admitted") + } + + #[test] + fn a_collection_reader_precedes_a_row_writer_of_its_collection() { + let (col_k, col_other) = find_two_distinct_collections(); + // tx0 writes one row of K. tx1 scans K and writes elsewhere. + let writer = make_set_tx(0, vec![], vec![rows(&col_k, vec![7])]); + let reader = make_set_tx(1, vec![whole(&col_k)], vec![rows(&col_other, vec![9])]); + let (admitted, rejected) = validate_batch_with_assignments(11, vec![writer, reader]); + assert!(rejected.is_empty()); + assert!( + position_of(&admitted, 1) < position_of(&admitted, 0), + "the collection reader must precede the row writer" + ); + } + + #[test] + fn a_row_reader_precedes_a_collection_writer() { + let (col_k, col_other) = find_two_distinct_collections(); + // tx0 truncates K. tx1 reads one row of K and writes elsewhere. + let truncate = make_set_tx(0, vec![], vec![whole(&col_k)]); + let reader = make_set_tx( + 1, + vec![rows(&col_k, vec![7])], + vec![rows(&col_other, vec![9])], + ); + let (admitted, rejected) = validate_batch_with_assignments(12, vec![truncate, reader]); + assert!(rejected.is_empty()); + assert!( + position_of(&admitted, 1) < position_of(&admitted, 0), + "the row reader must precede the truncate" + ); + + // An array write covers its collection on its vShard as well. + let array_write = make_set_tx( + 0, + vec![], + vec![EngineKeySet::Array { + collection: col_k.clone(), + vshards: SortedVec::new(vec![3]), + }], + ); + let reader = make_set_tx( + 1, + vec![rows(&col_k, vec![7])], + vec![rows(&col_other, vec![9])], + ); + let (admitted, rejected) = validate_batch_with_assignments(13, vec![array_write, reader]); + assert!(rejected.is_empty()); + assert!(position_of(&admitted, 1) < position_of(&admitted, 0)); + } + + #[test] + fn a_collection_scope_cycle_rejects_the_later_txn() { + let (col_a, col_b) = find_two_distinct_collections(); + // tx0 scans A and writes a row of B. tx1 reads a row of B and + // truncates A. Each must precede the other: no serial order exists. + let tx0 = make_set_tx(0, vec![whole(&col_a)], vec![rows(&col_b, vec![1])]); + let tx1 = make_set_tx(1, vec![rows(&col_b, vec![1])], vec![whole(&col_a)]); + let (admitted, rejected) = validate_batch_with_assignments(14, vec![tx0, tx1]); + assert_eq!(admitted.len(), 1); + assert_eq!(rejected.len(), 1); + assert_eq!(rejected[0].admitted.inbox_seq, 1, "the later txn loses"); + assert_eq!( + rejected[0].reason, + SequencerError::Conflict { + position_admitted: 0 + } + ); + let ctx = rejected[0] + .conflict_context + .as_ref() + .expect("a cycle rejection names its conflict"); + assert!( + ctx.collection == col_a || ctx.collection == col_b, + "collection {} is one of the cycle's", + ctx.collection + ); + + // A row-only variant of the same shapes is no cycle: tx1 reads a row + // of B that tx0 does not write. + let tx0 = make_set_tx(0, vec![rows(&col_a, vec![5])], vec![rows(&col_b, vec![1])]); + let tx1 = make_set_tx(1, vec![rows(&col_b, vec![2])], vec![rows(&col_a, vec![6])]); + let (admitted, rejected) = validate_batch_with_assignments(15, vec![tx0, tx1]); + assert_eq!(admitted.len(), 2); + assert!(rejected.is_empty()); + } + + #[test] + fn array_writes_on_distinct_vshards_add_no_edge() { + let (col_k, col_other) = find_two_distinct_collections(); + let array = |vshard: u32| EngineKeySet::Array { + collection: col_k.clone(), + vshards: SortedVec::new(vec![vshard]), + }; + // tx0 reads K on vShard 1 and writes a row of O. tx1 reads that row + // and writes K on vShard 2. Only tx1 -> tx0 holds: no cycle. + let tx0 = make_set_tx(0, vec![array(1)], vec![rows(&col_other, vec![1])]); + let tx1 = make_set_tx(1, vec![rows(&col_other, vec![1])], vec![array(2)]); + let (admitted, rejected) = validate_batch_with_assignments(16, vec![tx0, tx1]); + assert!(rejected.is_empty()); + assert!(position_of(&admitted, 1) < position_of(&admitted, 0)); + } } diff --git a/nodedb-cluster/src/calvin/sequencer/validator/graph.rs b/nodedb-cluster/src/calvin/sequencer/validator/graph.rs new file mode 100644 index 000000000..b458a2da1 --- /dev/null +++ b/nodedb-cluster/src/calvin/sequencer/validator/graph.rs @@ -0,0 +1,139 @@ +// SPDX-License-Identifier: BUSL-1.1 + +//! The reader/writer index of a candidate batch and the reorder edges it +//! implies. +//! +//! Every map is a `BTreeMap` and every list holds ascending transaction +//! indices, so every iteration that feeds output is deterministic. + +use std::collections::{BTreeMap, BTreeSet}; + +use super::keys::{FlatKey, KeyId, KeyScope, conflicts}; + +/// One key of one transaction: `(transaction index, key index)`. The key +/// index points into that transaction's flattened read or write keys. +pub(super) type KeyRef = (usize, usize); + +/// One collection's keys, split by scope. +#[derive(Debug, Default)] +pub(super) struct CollectionKeys { + /// Every collection-scope read key. + pub(super) scoped_reads: Vec, + /// Every collection-scope write key. + pub(super) scoped_writes: Vec, + /// One row read key per transaction that reads rows of the collection. + /// It stands for all of them: a row key conflicts with a collection-scope + /// key of its collection whatever its bytes. + pub(super) row_reads: Vec, + /// One row write key per transaction that writes rows of the collection. + pub(super) row_writes: Vec, +} + +/// The flattened keys of every candidate and the maps that find conflicting +/// pairs without comparing every key against every other. +#[derive(Debug, Default)] +pub(super) struct ConflictIndex { + /// Flattened read keys, per transaction. + pub(super) reads: Vec>, + /// Flattened write keys, per transaction. + pub(super) writes: Vec>, + /// Transactions that read each exact key. + pub(super) readers_of: BTreeMap>, + /// Transactions that write each exact key. + pub(super) writers_of: BTreeMap>, + /// Keys per collection, for the pairs a collection-scope key makes. + pub(super) by_collection: BTreeMap, +} + +impl ConflictIndex { + /// Add transaction `i`'s keys. Transactions are added in index order. + pub(super) fn add(&mut self, i: usize, reads: Vec, writes: Vec) { + for (k, fk) in reads.iter().enumerate() { + push_txn(self.readers_of.entry(fk.key_id()).or_default(), i); + let coll = self.by_collection.entry(fk.collection.clone()).or_default(); + match fk.scope { + KeyScope::Row => push_key(&mut coll.row_reads, (i, k)), + KeyScope::Collection => coll.scoped_reads.push((i, k)), + } + } + for (k, fk) in writes.iter().enumerate() { + push_txn(self.writers_of.entry(fk.key_id()).or_default(), i); + let coll = self.by_collection.entry(fk.collection.clone()).or_default(); + match fk.scope { + KeyScope::Row => push_key(&mut coll.row_writes, (i, k)), + KeyScope::Collection => coll.scoped_writes.push((i, k)), + } + } + self.reads.push(reads); + self.writes.push(writes); + } + + /// The read key `r` names. + pub(super) fn read(&self, (txn, key): KeyRef) -> &FlatKey { + &self.reads[txn][key] + } + + /// The write key `w` names. + pub(super) fn write(&self, (txn, key): KeyRef) -> &FlatKey { + &self.writes[txn][key] + } + + /// The reorder edges `reader -> writer` of every conflicting pair, as a + /// successor set per transaction, and each transaction's in-degree. + /// + /// A `BTreeSet` of successors deduplicates edges that several keys imply + /// and keeps successor iteration deterministic. + pub(super) fn edges(&self, n: usize) -> (Vec>, Vec) { + let mut out: Vec> = vec![BTreeSet::new(); n]; + let mut in_degree: Vec = vec![0; n]; + let mut add = |r: usize, w: usize| { + if r != w && out[r].insert(w) { + in_degree[w] += 1; + } + }; + + // A reader and a writer of one exact key. + for (key, readers) in &self.readers_of { + if let Some(writers) = self.writers_of.get(key) { + for &r in readers { + for &w in writers { + add(r, w); + } + } + } + } + + // A collection-scope key against the keys of its collection. + for coll in self.by_collection.values() { + for &r in &coll.scoped_reads { + for &w in coll.scoped_writes.iter().chain(&coll.row_writes) { + if conflicts(self.read(r), self.write(w)) { + add(r.0, w.0); + } + } + } + for &r in &coll.row_reads { + for &w in &coll.scoped_writes { + if conflicts(self.read(r), self.write(w)) { + add(r.0, w.0); + } + } + } + } + (out, in_degree) + } +} + +/// Append transaction `i` unless it is already the last entry. +fn push_txn(list: &mut Vec, i: usize) { + if list.last() != Some(&i) { + list.push(i); + } +} + +/// Append key `key` unless its transaction already has an entry. +fn push_key(list: &mut Vec, key: KeyRef) { + if list.last().map(|(txn, _)| *txn) != Some(key.0) { + list.push(key); + } +} diff --git a/nodedb-cluster/src/calvin/sequencer/validator/keys.rs b/nodedb-cluster/src/calvin/sequencer/validator/keys.rs new file mode 100644 index 000000000..cc98e86bd --- /dev/null +++ b/nodedb-cluster/src/calvin/sequencer/validator/keys.rs @@ -0,0 +1,246 @@ +// SPDX-License-Identifier: BUSL-1.1 + +//! Flattened per-key view of a transaction's read and write sets, and the +//! one predicate that decides whether a read key and a write key conflict. + +use crate::calvin::types::{EngineKeySet, ReadWriteSet}; + +/// Canonical, order-comparable identity of a single key across engines: +/// `(engine_discriminant, collection, key_bytes)`. Used as the `BTreeMap` key +/// for the reader/writer maps so all iteration feeding output is deterministic. +pub(super) type KeyId = (u8, String, Vec); + +/// Discriminant of an array key: one array on one vShard. +const ARRAY: u8 = 4; + +/// How much of its collection one key covers. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub(super) enum KeyScope { + /// One row: a surrogate, a KV key, an edge pair or a UNIQUE value. + Row, + /// Every row: a whole-collection key set, or an array on one vShard. + Collection, +} + +/// A single flattened key drawn from one transaction's read or write set. +#[derive(Debug)] +pub(super) struct FlatKey { + /// Discriminant tag for the key-set variant (Document=0, Vector=1, Kv=2, + /// Edge=3, Array=4, Collection=5, Unique=6). + pub(super) discriminant: u8, + /// Static engine name used when building conflict-context keys. + pub(super) engine_name: &'static str, + /// Collection name. + pub(super) collection: String, + /// Serialized key bytes. + pub(super) key_bytes: Vec, + /// How much of the collection the key covers. + pub(super) scope: KeyScope, +} + +impl FlatKey { + /// The order-comparable identity of this key. + pub(super) fn key_id(&self) -> KeyId { + ( + self.discriminant, + self.collection.clone(), + self.key_bytes.clone(), + ) + } +} + +/// Whether a reader of `read` and a writer of `write` conflict, so the +/// reader must take the lower position. +/// +/// Mirrors the scheduler's lock compatibility: +/// - One key conflicts with itself. +/// - A collection-scope key conflicts with every key of its collection. A row +/// key announces itself on its collection, and a whole-collection key +/// excludes every row announcement. +/// - Two array keys of one collection on distinct vShards are disjoint: each +/// vShard's lock table holds its own part of the array. +/// - Two distinct row keys never conflict. +pub(super) fn conflicts(read: &FlatKey, write: &FlatKey) -> bool { + if read.collection != write.collection { + return false; + } + if read.discriminant == write.discriminant && read.key_bytes == write.key_bytes { + return true; + } + match (read.scope, write.scope) { + (KeyScope::Row, KeyScope::Row) => false, + (KeyScope::Collection, KeyScope::Collection) + if read.discriminant == ARRAY && write.discriminant == ARRAY => + { + false + } + (KeyScope::Collection, KeyScope::Row | KeyScope::Collection) + | (KeyScope::Row, KeyScope::Collection) => true, + } +} + +/// Flatten one read-or-write set into its constituent per-key entries. +pub(super) fn flatten_set(set: &ReadWriteSet) -> Vec { + let mut out = Vec::new(); + for key_set in &set.0 { + match key_set { + EngineKeySet::Document { + collection, + surrogates, + } => { + for &s in surrogates.as_slice() { + out.push(FlatKey { + discriminant: 0, + engine_name: "document", + collection: collection.clone(), + key_bytes: s.to_le_bytes().to_vec(), + scope: KeyScope::Row, + }); + } + } + EngineKeySet::Vector { + collection, + surrogates, + } => { + for &s in surrogates.as_slice() { + out.push(FlatKey { + discriminant: 1, + engine_name: "vector", + collection: collection.clone(), + key_bytes: s.to_le_bytes().to_vec(), + scope: KeyScope::Row, + }); + } + } + EngineKeySet::Kv { collection, keys } => { + for k in keys.as_slice() { + out.push(FlatKey { + discriminant: 2, + engine_name: "kv", + collection: collection.clone(), + key_bytes: k.clone(), + scope: KeyScope::Row, + }); + } + } + EngineKeySet::Edge { + collection, edges, .. + } => { + for &(src, dst) in edges.as_slice() { + let mut key_bytes = src.to_le_bytes().to_vec(); + key_bytes.extend_from_slice(&dst.to_le_bytes()); + out.push(FlatKey { + discriminant: 3, + engine_name: "edge", + collection: collection.clone(), + key_bytes, + scope: KeyScope::Row, + }); + } + } + // The whole array on each vShard it names. + EngineKeySet::Array { + collection, + vshards, + } => { + for &vshard in vshards.as_slice() { + out.push(FlatKey { + discriminant: ARRAY, + engine_name: "array", + collection: collection.clone(), + key_bytes: vshard.to_le_bytes().to_vec(), + scope: KeyScope::Collection, + }); + } + } + // The whole collection: one key with no row bytes. + EngineKeySet::Collection { collection, .. } => { + out.push(FlatKey { + discriminant: 5, + engine_name: "collection", + collection: collection.clone(), + key_bytes: Vec::new(), + scope: KeyScope::Collection, + }); + } + // One key per claimed value: the index name, a zero byte, the value. + EngineKeySet::Unique { + collection, + index, + values, + } => { + for value in values.as_slice() { + let mut key_bytes = Vec::with_capacity(index.len() + 1 + value.len()); + key_bytes.extend_from_slice(index.as_bytes()); + key_bytes.push(0); + key_bytes.extend_from_slice(value); + out.push(FlatKey { + discriminant: 6, + engine_name: "unique", + collection: collection.clone(), + key_bytes, + scope: KeyScope::Row, + }); + } + } + } + } + out +} + +#[cfg(test)] +mod tests { + use super::*; + + fn key(discriminant: u8, collection: &str, bytes: &[u8], scope: KeyScope) -> FlatKey { + FlatKey { + discriminant, + engine_name: "test", + collection: collection.to_owned(), + key_bytes: bytes.to_vec(), + scope, + } + } + + fn row(collection: &str, surrogate: u32) -> FlatKey { + key(0, collection, &surrogate.to_le_bytes(), KeyScope::Row) + } + + fn whole(collection: &str) -> FlatKey { + key(5, collection, &[], KeyScope::Collection) + } + + fn array(collection: &str, vshard: u32) -> FlatKey { + key( + ARRAY, + collection, + &vshard.to_le_bytes(), + KeyScope::Collection, + ) + } + + #[test] + fn a_key_conflicts_with_itself_only_among_rows() { + assert!(conflicts(&row("c", 1), &row("c", 1))); + assert!(!conflicts(&row("c", 1), &row("c", 2))); + assert!(!conflicts(&row("c", 1), &row("d", 1))); + } + + #[test] + fn a_collection_scope_key_conflicts_with_every_key_of_its_collection() { + assert!(conflicts(&whole("c"), &row("c", 1))); + assert!(conflicts(&row("c", 1), &whole("c"))); + assert!(conflicts(&whole("c"), &whole("c"))); + assert!(conflicts(&array("c", 3), &row("c", 1))); + assert!(conflicts(&row("c", 1), &array("c", 3))); + assert!(conflicts(&whole("c"), &array("c", 3))); + assert!(!conflicts(&whole("c"), &row("d", 1))); + assert!(!conflicts(&row("d", 1), &whole("c"))); + } + + #[test] + fn array_keys_on_distinct_vshards_are_disjoint() { + assert!(conflicts(&array("c", 3), &array("c", 3))); + assert!(!conflicts(&array("c", 3), &array("c", 4))); + } +} diff --git a/nodedb-cluster/src/calvin/sequencer/validator/mod.rs b/nodedb-cluster/src/calvin/sequencer/validator/mod.rs new file mode 100644 index 000000000..f948cb9f5 --- /dev/null +++ b/nodedb-cluster/src/calvin/sequencer/validator/mod.rs @@ -0,0 +1,8 @@ +// SPDX-License-Identifier: BUSL-1.1 + +pub mod batch; +mod graph; +mod keys; +mod reject; + +pub use batch::{validate_batch, validate_batch_with_assignments}; diff --git a/nodedb-cluster/src/calvin/sequencer/validator/reject.rs b/nodedb-cluster/src/calvin/sequencer/validator/reject.rs new file mode 100644 index 000000000..8a960f89c --- /dev/null +++ b/nodedb-cluster/src/calvin/sequencer/validator/reject.rs @@ -0,0 +1,92 @@ +// SPDX-License-Identifier: BUSL-1.1 + +//! The reported position and conflict context of a cycle-rejected +//! transaction. + +use crate::calvin::sequencer::metrics::ConflictKey; + +use super::graph::{ConflictIndex, KeyRef}; +use super::keys::{FlatKey, KeyId, KeyScope, conflicts}; + +/// Build the `(position_admitted, conflict_context)` for a cycle-rejected txn. +/// +/// A rejected txn conflicts with any admitted txn that writes a key it reads +/// or reads a key it writes, by [`conflicts`]. We select the conflict with +/// the smallest admitted position (ties broken by key identity) so both the +/// reported position and the [`ConflictKey`] are deterministic. If no +/// conflicting txn was admitted, the position defaults to `0` and the context +/// to `None`. +pub(super) fn reject_context( + idx: usize, + tenant: u64, + index: &ConflictIndex, + emitted_position: &[Option], +) -> (u32, Option) { + // Best conflict so far, ordered by (admitted position, key identity). + let mut best: Option<(u32, KeyId, &'static str)> = None; + let mut consider = |fk: &FlatKey, other: usize| { + if other == idx { + return; + } + let Some(pos) = emitted_position[other] else { + return; + }; + let key = fk.key_id(); + let better = match &best { + None => true, + Some((bpos, bkey, _)) => (pos, &key) < (*bpos, bkey), + }; + if better { + best = Some((pos, key, fk.engine_name)); + } + }; + + for fk in &index.reads[idx] { + // A writer of a key this txn reads. + for &other in index.writers_of.get(&fk.key_id()).into_iter().flatten() { + consider(fk, other); + } + // A writer whose key's scope covers this read, or the reverse. + if let Some(coll) = index.by_collection.get(&fk.collection) { + let row_writes: &[KeyRef] = match fk.scope { + KeyScope::Row => &[], + KeyScope::Collection => &coll.row_writes, + }; + for &w in coll.scoped_writes.iter().chain(row_writes) { + if conflicts(fk, index.write(w)) { + consider(fk, w.0); + } + } + } + } + for fk in &index.writes[idx] { + // A reader of a key this txn writes. + for &other in index.readers_of.get(&fk.key_id()).into_iter().flatten() { + consider(fk, other); + } + // A reader whose key's scope covers this write, or the reverse. + if let Some(coll) = index.by_collection.get(&fk.collection) { + let row_reads: &[KeyRef] = match fk.scope { + KeyScope::Row => &[], + KeyScope::Collection => &coll.row_reads, + }; + for &r in coll.scoped_reads.iter().chain(row_reads) { + if conflicts(index.read(r), fk) { + consider(fk, r.0); + } + } + } + } + + match best { + Some((pos, key, engine)) => ( + pos, + Some(ConflictKey { + tenant, + engine, + collection: key.1, + }), + ), + None => (0, None), + } +} diff --git a/nodedb-cluster/src/cluster_info.rs b/nodedb-cluster/src/cluster_info.rs index 9d466d8d8..56fe44334 100644 --- a/nodedb-cluster/src/cluster_info.rs +++ b/nodedb-cluster/src/cluster_info.rs @@ -267,6 +267,7 @@ mod tests { last_applied: 5, last_log_index: 5, snapshot_index: 0, + replica_state: nodedb_raft::ReplicaState::Whole, member_count: 3, learner_count: 0, vshard_count: 512, diff --git a/nodedb-cluster/src/distributed_array/coordinator/read.rs b/nodedb-cluster/src/distributed_array/coordinator/read.rs index 760c0b184..09ea665fe 100644 --- a/nodedb-cluster/src/distributed_array/coordinator/read.rs +++ b/nodedb-cluster/src/distributed_array/coordinator/read.rs @@ -3,8 +3,8 @@ //! Read-path coordinator for distributed array operations. //! //! [`ArrayCoordinator`] drives fan-out reads (`coord_slice`, `coord_agg`, -//! `coord_surrogate_bitmap_scan`) to the set of vShards whose Hilbert range -//! overlaps the slice predicate. +//! `coord_surrogate_bitmap_scan`) to the vShards that own a tile the slice +//! predicate covers. use std::sync::Arc; @@ -15,8 +15,11 @@ use super::super::merge::{ ArrayAggPartial, any_truncated_before_horizon_agg, merge_read_versions, merge_slice_rows, merge_slice_rows_sorted, reduce_agg_partials, }; +use super::super::routing::shard_bucket_for_vshard; use super::super::rpc::ShardRpcDispatch; -use super::super::scatter::{FanOutParams, FanOutPartitionedParams, fan_out, fan_out_partitioned}; +use super::super::scatter::{ + ArrayCoordError, FanOutParams, FanOutPartitionedParams, fan_out, fan_out_partitioned, +}; use super::super::wire::{ ArrayShardAggReq, ArrayShardAggResp, ArrayShardDeleteReq, ArrayShardDeleteResp, ArrayShardSliceReq, ArrayShardSliceResp, ArrayShardSurrogateBitmapReq, @@ -30,14 +33,13 @@ pub struct ArrayCoordParams { pub shard_ids: Vec, /// Per-shard RPC timeout in milliseconds. pub timeout_ms: u64, - /// Hilbert routing granularity (1–16). 0 means no shard-side routing - /// validation (e.g. when the coordinator was constructed without slice - /// range information, as in tests or unbounded scans). + /// Routing granularity (1–16). 0 skips shard-side routing validation + /// and the per-shard bucket filter. pub prefix_bits: u8, - /// Inclusive Hilbert-prefix ranges `(lo, hi)` that this read covers. - /// Forwarded to each shard so it can verify it still owns the range. - /// Empty means unbounded — the shard skips routing validation. - pub slice_hilbert_ranges: Vec<(u64, u64)>, + /// Tile keys this read covers, at least one per bucket. Forwarded to + /// each shard so it checks that it still owns one. Empty means + /// unbounded, and the shard skips routing validation. + pub slice_tile_keys: Vec, } /// Result of a coordinated slice fan-out. @@ -69,24 +71,13 @@ pub struct CoordAggResult { pub read_versions: Vec, } -/// Compute the inclusive Hilbert-prefix range `[lo, hi]` that vShard `shard_id` -/// owns given the array's routing granularity `prefix_bits`. -/// -/// Each bucket `b = shard_id / stride` covers the Hilbert range -/// `[b << (64 - prefix_bits), ((b + 1) << (64 - prefix_bits)) - 1]`. -/// The stride is `VSHARD_COUNT >> prefix_bits` (floored at 1). -pub(super) fn shard_hilbert_range_for_vshard(shard_id: u32, prefix_bits: u8) -> (u64, u64) { - use crate::routing::VSHARD_COUNT; - let stride = (VSHARD_COUNT >> (prefix_bits as u32)).max(1); - let bucket = shard_id / stride; - let shift = 64u8.saturating_sub(prefix_bits); - let lo = (bucket as u64) << shift; - let hi = if shift == 0 { - u64::MAX - } else { - lo.saturating_add((1u64 << shift).saturating_sub(1)) - }; - (lo, hi) +/// The bucket filter for a read sent to vShard `shard_id`. `None` when +/// `prefix_bits` is 0, which reads every tile. +fn shard_bucket(shard_id: u32, prefix_bits: u8) -> Result> { + if prefix_bits == 0 { + return Ok(None); + } + shard_bucket_for_vshard(shard_id, prefix_bits).map(Some) } /// Coordinator for distributed array read operations. @@ -109,24 +100,24 @@ impl ArrayCoordinator { } } - /// Construct an `ArrayCoordinator` whose target shards are computed from - /// the Hilbert-prefix ranges that overlap a slice predicate. + /// Construct an `ArrayCoordinator` whose target shards own the tiles a + /// slice predicate covers. /// - /// `slice_hilbert_ranges` — `(lo, hi)` pairs computed by the planner from - /// the `Slice` predicate. Pass an empty slice for an unbounded scan. + /// `slice_tile_keys` — tile keys computed by the planner from the + /// `Slice` predicate. Pass an empty slice for an unbounded scan. /// `prefix_bits` — the array's routing granularity from the catalog entry. /// `total_shards` — the number of active vShards in the cluster. pub fn for_slice( source_node: u64, timeout_ms: u64, - slice_hilbert_ranges: &[(u64, u64)], + slice_tile_keys: &[u64], prefix_bits: u8, total_shards: u32, dispatch: Arc, circuit_breaker: Arc, ) -> crate::error::Result { let shard_ids = super::super::routing::array_vshards_for_slice( - slice_hilbert_ranges, + slice_tile_keys, prefix_bits, total_shards, )?; @@ -136,7 +127,7 @@ impl ArrayCoordinator { shard_ids, timeout_ms, prefix_bits, - slice_hilbert_ranges: slice_hilbert_ranges.to_vec(), + slice_tile_keys: slice_tile_keys.to_vec(), }, dispatch, circuit_breaker, @@ -147,10 +138,9 @@ impl ArrayCoordinator { /// /// Each shard receives the full slice request with the caller-supplied /// `limit` pushed down so shards can stop scanning early. The coordinator - /// stamps a per-shard `shard_hilbert_range` so each shard only returns - /// cells whose Hilbert prefix falls within its owned range, preventing - /// duplicate rows in single-node harnesses where all vShards share one - /// Data Plane. The coordinator applies the same `limit` as a final + /// stamps a per-shard `shard_bucket`, so each shard returns only cells + /// whose tile key routes to it. vShards that share one Data Plane then + /// return no duplicate rows. The coordinator applies the same `limit` as a final /// cut-off on the merged result. /// /// When `system_time` is `AllVersions`, the coordinator merge-sorts rows @@ -167,22 +157,17 @@ impl ArrayCoordinator { req: ArrayShardSliceReq, coordinator_limit: u32, system_time: nodedb_types::SystemTimeScope, - ) -> Result { + ) -> std::result::Result { let prefix_bits = self.params.prefix_bits; let per_shard: Vec<(u32, Vec)> = self .params .shard_ids .iter() .map(|&shard_id| { - let shard_hilbert_range = if prefix_bits > 0 { - Some(shard_hilbert_range_for_vshard(shard_id, prefix_bits)) - } else { - None - }; let per_shard_req = ArrayShardSliceReq { prefix_bits, - slice_hilbert_ranges: self.params.slice_hilbert_ranges.clone(), - shard_hilbert_range, + slice_tile_keys: self.params.slice_tile_keys.clone(), + shard_bucket: shard_bucket(shard_id, prefix_bits)?, ..req.clone() }; let bytes = @@ -224,24 +209,21 @@ impl ArrayCoordinator { /// Fan out an aggregate request and reduce partial aggregates from all shards. /// - /// Each shard receives its own `shard_hilbert_range` so it can apply a - /// Hilbert-prefix pre-filter and only count cells in its partition. This - /// prevents double-counting in configurations where multiple vShards share - /// a single Data Plane executor (e.g. single-node harnesses). - pub async fn coord_agg(&self, req: ArrayShardAggReq) -> Result { + /// Each shard receives its own `shard_bucket` and counts only cells whose + /// tile key routes to it. vShards that share one Data Plane executor then + /// count no cell twice. + pub async fn coord_agg( + &self, + req: ArrayShardAggReq, + ) -> std::result::Result { let prefix_bits = self.params.prefix_bits; let per_shard: Vec<(u32, Vec)> = self .params .shard_ids .iter() .map(|&shard_id| { - let hilbert_range = if prefix_bits > 0 { - Some(shard_hilbert_range_for_vshard(shard_id, prefix_bits)) - } else { - None - }; let per_shard_req = ArrayShardAggReq { - shard_hilbert_range: hilbert_range, + shard_bucket: shard_bucket(shard_id, prefix_bits)?, ..req.clone() }; let bytes = @@ -276,7 +258,7 @@ impl ArrayCoordinator { pub async fn coord_delete( &self, req: ArrayShardDeleteReq, - ) -> Result> { + ) -> std::result::Result, ArrayCoordError> { let req_bytes = zerompk::to_msgpack_vec(&req).map_err(|e| ClusterError::Codec { detail: format!("ArrayShardDeleteReq serialise: {e}"), })?; @@ -288,7 +270,7 @@ impl ArrayCoordinator { &self.circuit_breaker, ) .await?; - decode_resps::(&raw) + Ok(decode_resps::(&raw)?) } /// Fan out a surrogate bitmap scan, collect per-shard bitmap bytes, and @@ -298,7 +280,7 @@ impl ArrayCoordinator { pub async fn coord_surrogate_bitmap_scan( &self, req: ArrayShardSurrogateBitmapReq, - ) -> Result> { + ) -> std::result::Result, ArrayCoordError> { let req_bytes = zerompk::to_msgpack_vec(&req).map_err(|e| ClusterError::Codec { detail: format!("ArrayShardSurrogateBitmapReq serialise: {e}"), })?; @@ -310,7 +292,7 @@ impl ArrayCoordinator { &self.circuit_breaker, ) .await?; - decode_resps::(&raw) + Ok(decode_resps::(&raw)?) } pub(super) fn fan_out_params(&self) -> FanOutParams { @@ -414,11 +396,10 @@ mod tests { source_node: 1, shard_ids, timeout_ms: 1000, - // Tests use prefix_bits=0 so shard-side routing validation - // is skipped — mock executors don't need to match Hilbert - // ownership. + // prefix_bits=0 skips shard-side routing validation and the + // bucket filter, so mock executors ignore ownership. prefix_bits: 0, - slice_hilbert_ranges: vec![], + slice_tile_keys: vec![], }, dispatch, Arc::new(CircuitBreaker::new(CircuitBreakerConfig::default())), @@ -440,8 +421,8 @@ mod tests { limit: 100, cell_filter_msgpack: vec![], prefix_bits: 0, - slice_hilbert_ranges: vec![], - shard_hilbert_range: None, + slice_tile_keys: vec![], + shard_bucket: None, system_time: nodedb_types::SystemTimeScope::Current, valid_at_ms: None, txn_id: None, @@ -481,8 +462,8 @@ mod tests { limit: 3, cell_filter_msgpack: vec![], prefix_bits: 0, - slice_hilbert_ranges: vec![], - shard_hilbert_range: None, + slice_tile_keys: vec![], + shard_bucket: None, system_time: nodedb_types::SystemTimeScope::Current, valid_at_ms: None, txn_id: None, @@ -503,7 +484,7 @@ mod tests { reducer_msgpack: vec![0x00], group_by_dim: -1, cell_filter_msgpack: vec![], - shard_hilbert_range: None, + shard_bucket: None, system_as_of: None, valid_at_ms: None, txn_id: None, @@ -662,8 +643,8 @@ mod tests { limit: 0, cell_filter_msgpack: vec![], prefix_bits: 0, - slice_hilbert_ranges: vec![], - shard_hilbert_range: None, + slice_tile_keys: vec![], + shard_bucket: None, system_time: nodedb_types::SystemTimeScope::Current, valid_at_ms: None, txn_id: None, @@ -676,4 +657,85 @@ mod tests { .expect("coord_slice unlimited should succeed"); assert_eq!(result.rows.len(), 20); } + + /// Serves a fixed set of tile keys, each as one row, keeping only the + /// keys the request's `shard_bucket` admits. + struct BucketFilterDispatch { + tile_keys: Vec, + } + + #[async_trait] + impl ShardRpcDispatch for BucketFilterDispatch { + async fn call(&self, req: VShardEnvelope, _timeout_ms: u64) -> Result { + let slice: ArrayShardSliceReq = zerompk::from_msgpack(&req.payload).unwrap(); + let rows = self + .tile_keys + .iter() + .filter(|&&k| nodedb_types::array_route::shard_bucket_admits(slice.shard_bucket, k)) + .map(|k| zerompk::to_msgpack_vec(k).unwrap()) + .collect(); + let resp = ArrayShardSliceResp { + shard_id: req.vshard_id, + rows_msgpack: rows, + truncated: false, + truncated_before_horizon: false, + read_versions: Vec::new(), + }; + let payload = zerompk::to_msgpack_vec(&resp).unwrap(); + Ok(VShardEnvelope::new( + VShardMessageType::ArrayShardSliceResp, + req.target_node, + req.source_node, + req.vshard_id, + payload, + )) + } + } + + /// Over every primary vShard, the bucket filters return each tile once. + #[tokio::test] + async fn bucket_filters_partition_every_tile_across_shards() { + let tile_keys: Vec = (0u64..4096).chain([1 << 40, u64::MAX]).collect(); + for prefix_bits in [1u8, 8, 10, 16] { + let dispatch: Arc = Arc::new(BucketFilterDispatch { + tile_keys: tile_keys.clone(), + }); + let coord = ArrayCoordinator::for_slice( + 1, + 1000, + &[], + prefix_bits, + crate::routing::VSHARD_COUNT, + dispatch, + Arc::new(CircuitBreaker::new(CircuitBreakerConfig::default())), + ) + .expect("for_slice"); + let req = ArrayShardSliceReq { + array_id_msgpack: vec![], + slice_msgpack: vec![], + attr_projection: vec![], + limit: 0, + cell_filter_msgpack: vec![], + prefix_bits: 0, + slice_tile_keys: vec![], + shard_bucket: None, + system_time: nodedb_types::SystemTimeScope::Current, + valid_at_ms: None, + txn_id: None, + }; + let result = coord + .coord_slice(req, 0, nodedb_types::SystemTimeScope::Current) + .await + .expect("coord_slice"); + let mut got: Vec = result + .rows + .iter() + .map(|r| zerompk::from_msgpack::(r).unwrap()) + .collect(); + got.sort_unstable(); + let mut want = tile_keys.clone(); + want.sort_unstable(); + assert_eq!(got, want, "prefix_bits={prefix_bits}"); + } + } } diff --git a/nodedb-cluster/src/distributed_array/coordinator/write.rs b/nodedb-cluster/src/distributed_array/coordinator/write.rs index 80c61d54f..17c284ff3 100644 --- a/nodedb-cluster/src/distributed_array/coordinator/write.rs +++ b/nodedb-cluster/src/distributed_array/coordinator/write.rs @@ -3,7 +3,7 @@ //! Write-path coordinator for distributed array operations. //! //! Provides [`coord_put`], [`coord_put_partitioned`], and [`coord_delete`] -//! — functions that partition a flat cell/coord list by Hilbert tile and +//! — functions that partition a flat cell/coord list by routing bucket and //! fan writes to the owning vShards. use std::sync::Arc; @@ -13,7 +13,7 @@ use crate::error::{ClusterError, Result}; use super::super::partition::{partition_delete_coords, partition_put_cells}; use super::super::rpc::ShardRpcDispatch; -use super::super::scatter::{FanOutPartitionedParams, fan_out_partitioned}; +use super::super::scatter::{ArrayCoordError, FanOutPartitionedParams, fan_out_partitioned}; use super::super::wire::{ ArrayShardDeleteReq, ArrayShardDeleteResp, ArrayShardPutReq, ArrayShardPutResp, }; @@ -27,7 +27,7 @@ pub struct ArrayWriteCoordParams { /// Forward pre-partitioned cell writes to the owning shards. /// -/// The caller groups cells by Hilbert prefix bucket using +/// The caller groups cells by routing bucket using /// `array_vshard_for_tile` and produces one `ArrayShardPutReq` per /// target shard. This function dispatches each batch to its shard via /// `fan_out_partitioned` and collects acknowledgements. @@ -39,7 +39,7 @@ pub async fn coord_put_partitioned( per_shard: Vec<(u32, ArrayShardPutReq)>, dispatch: &Arc, circuit_breaker: &Arc, -) -> Result> { +) -> std::result::Result, ArrayCoordError> { if per_shard.is_empty() { return Ok(Vec::new()); } @@ -69,15 +69,15 @@ pub async fn coord_put_partitioned( ) .await?; - decode_resps::(&raw) + Ok(decode_resps::(&raw)?) } -/// Partition a flat cell list by Hilbert tile and fan out to owning shards. +/// Partition a flat cell list by routing bucket and fan out to owning shards. /// -/// `cells` — each element is `(hilbert_prefix, zerompk-encoded single-cell bytes)`. -/// The Hilbert prefix is computed by the caller (the Control Plane planner) from -/// the cell's coord tuple and the array schema; this function does not decode -/// cell bytes. +/// `cells` — each element is `(tile_key, zerompk-encoded single-cell bytes)`. +/// The caller (the Control Plane planner) computes the tile key from the +/// cell's coord tuple and the array schema. This function decodes no cell +/// bytes. /// /// `prefix_bits` — routing granularity (1–16) from the array catalog entry. /// `wal_lsn` — WAL sequence number allocated by the Control Plane for this batch. @@ -93,7 +93,7 @@ pub async fn coord_put( cells: &[(u64, Vec)], dispatch: &Arc, circuit_breaker: &Arc, -) -> Result> { +) -> std::result::Result, ArrayCoordError> { if cells.is_empty() { return Ok(Vec::new()); } @@ -107,7 +107,7 @@ pub async fn coord_put( array_id_msgpack: array_id_msgpack.clone(), cells_msgpack: b.cells_msgpack, wal_lsn, - representative_hilbert_prefix: b.representative_hilbert_prefix, + representative_tile_key: b.representative_tile_key, prefix_bits, }; (b.vshard_id, req) @@ -117,9 +117,9 @@ pub async fn coord_put( coord_put_partitioned(params, per_shard, dispatch, circuit_breaker).await } -/// Partition a flat coord list by Hilbert tile and fan delete requests to owning shards. +/// Partition a flat coord list by routing bucket and fan delete requests to owning shards. /// -/// `coords` — each element is `(hilbert_prefix, zerompk-encoded single-coord bytes)`. +/// `coords` — each element is `(tile_key, zerompk-encoded single-coord bytes)`. /// `prefix_bits` — routing granularity (1–16). /// `wal_lsn` — WAL sequence number allocated by the Control Plane. /// @@ -132,7 +132,7 @@ pub async fn coord_delete( coords: &[(u64, Vec)], dispatch: &Arc, circuit_breaker: &Arc, -) -> Result> { +) -> std::result::Result, ArrayCoordError> { if coords.is_empty() { return Ok(Vec::new()); } @@ -151,7 +151,7 @@ pub async fn coord_delete( array_id_msgpack: array_id_msgpack.clone(), coords_msgpack: b.coords_msgpack, wal_lsn, - representative_hilbert_prefix: b.representative_hilbert_prefix, + representative_tile_key: b.representative_tile_key, prefix_bits, }; zerompk::to_msgpack_vec(&req) @@ -171,7 +171,7 @@ pub async fn coord_delete( ) .await?; - decode_resps::(&raw) + Ok(decode_resps::(&raw)?) } #[cfg(test)] @@ -179,6 +179,7 @@ mod tests { use async_trait::async_trait; use crate::circuit_breaker::{CircuitBreaker, CircuitBreakerConfig}; + use crate::distributed_array::scatter::ShardFailure; use crate::wire::{VShardEnvelope, VShardMessageType}; use super::*; @@ -263,10 +264,8 @@ mod tests { #[tokio::test] async fn coord_put_partitions_cells_by_tile() { - // prefix_bits=10, stride=1 → vshard == top-10-bit bucket. - // p0 → bucket 0 → vshard 0 - // p1 → bucket 1 → vshard 1 - // p2 → bucket 2 → vshard 2 + // prefix_bits=10, stride=1 → vShard == mixed bucket. + // The three keys mix to three distinct buckets. let p0 = 0x0000_0000_0000_0000u64; let p1 = 0x0040_0000_0000_0000u64; let p2 = 0x0080_0000_0000_0000u64; @@ -285,10 +284,16 @@ mod tests { .expect("coord_put should succeed"); resps.sort_by_key(|r| r.shard_id); + let mut expected: Vec = [p0, p1, p2] + .iter() + .map(|&k| crate::distributed_array::array_vshard_for_tile(k, 10).unwrap()) + .collect(); + expected.sort_unstable(); assert_eq!(resps.len(), 3, "should fan-out to 3 shards"); - assert_eq!(resps[0].shard_id, 0); - assert_eq!(resps[1].shard_id, 1); - assert_eq!(resps[2].shard_id, 2); + assert_eq!( + resps.iter().map(|r| r.shard_id).collect::>(), + expected + ); // Each shard echoes back wal_lsn=42. for r in &resps { assert_eq!(r.applied_lsn, 42); @@ -325,8 +330,14 @@ mod tests { .await .expect_err("coord_put with failing shard should return error"); assert!( - matches!(err, ClusterError::Codec { .. }), - "expected Codec error, got {err:?}" + matches!( + err, + ArrayCoordError::Shard(ShardFailure { + error: ClusterError::Codec { .. }, + .. + }) + ), + "expected a shard's Codec error, got {err:?}" ); } @@ -343,9 +354,16 @@ mod tests { .expect("coord_delete should succeed"); resps.sort_by_key(|r| r.shard_id); + let mut expected: Vec = [p0, p1] + .iter() + .map(|&k| crate::distributed_array::array_vshard_for_tile(k, 10).unwrap()) + .collect(); + expected.sort_unstable(); assert_eq!(resps.len(), 2, "should fan-out to 2 shards"); - assert_eq!(resps[0].shard_id, 0); - assert_eq!(resps[1].shard_id, 1); + assert_eq!( + resps.iter().map(|r| r.shard_id).collect::>(), + expected + ); for r in &resps { assert_eq!(r.applied_lsn, 55); } diff --git a/nodedb-cluster/src/distributed_array/handler.rs b/nodedb-cluster/src/distributed_array/handler.rs index 01cafb3e5..9ac775379 100644 --- a/nodedb-cluster/src/distributed_array/handler.rs +++ b/nodedb-cluster/src/distributed_array/handler.rs @@ -174,15 +174,15 @@ async fn handle_surrogate_bitmap( /// Validate that a PUT request is routed to the correct shard. /// -/// Returns `Ok(())` when routing is correct or when `prefix_bits == 0` -/// (which means the request predates Hilbert routing and skips validation). -/// Returns `Err(ClusterError::WrongOwner)` when the coordinator used a stale -/// routing table; the coordinator should refresh and retry. +/// Returns `Ok(())` when routing is correct or when `prefix_bits == 0`, +/// which skips validation. Returns `Err(ClusterError::WrongOwner)` when the +/// coordinator used a stale routing table. The coordinator then refreshes +/// and retries. fn validate_put_routing(req: &ArrayShardPutReq, local_vshard_id: u32) -> Result<()> { if req.prefix_bits == 0 { return Ok(()); } - let expected = array_vshard_for_tile(req.representative_hilbert_prefix, req.prefix_bits)?; + let expected = array_vshard_for_tile(req.representative_tile_key, req.prefix_bits)?; if expected != local_vshard_id { return Err(ClusterError::WrongOwner { vshard_id: local_vshard_id, @@ -194,18 +194,16 @@ fn validate_put_routing(req: &ArrayShardPutReq, local_vshard_id: u32) -> Result< /// Validate that a SLICE request is routed to the correct shard. /// -/// Returns `Ok(())` when `prefix_bits == 0` (no validation) or when -/// `local_vshard_id` falls within the Hilbert ranges carried by the request. -/// Returns `Err(ClusterError::WrongOwner)` when this shard no longer covers -/// any of the requested ranges, indicating a stale routing table. +/// Returns `Ok(())` when `prefix_bits == 0` or `slice_tile_keys` is empty, +/// which skips validation, or when some tile key routes to +/// `local_vshard_id`. Returns `Err(ClusterError::WrongOwner)` when no key +/// routes here, which means the coordinator used a stale routing table. fn validate_slice_routing(req: &ArrayShardSliceReq, local_vshard_id: u32) -> Result<()> { - if req.prefix_bits == 0 || req.slice_hilbert_ranges.is_empty() { + if req.prefix_bits == 0 || req.slice_tile_keys.is_empty() { return Ok(()); } - // Compute which vShards overlap the slice ranges. If local_vshard_id is not - // among them, this node no longer owns the required Hilbert range. let covered = array_vshards_for_slice( - &req.slice_hilbert_ranges, + &req.slice_tile_keys, req.prefix_bits, crate::routing::VSHARD_COUNT, )?; @@ -226,7 +224,7 @@ fn validate_delete_routing(req: &ArrayShardDeleteReq, local_vshard_id: u32) -> R if req.prefix_bits == 0 { return Ok(()); } - let expected = array_vshard_for_tile(req.representative_hilbert_prefix, req.prefix_bits)?; + let expected = array_vshard_for_tile(req.representative_tile_key, req.prefix_bits)?; if expected != local_vshard_id { return Err(ClusterError::WrongOwner { vshard_id: local_vshard_id, @@ -351,8 +349,8 @@ mod tests { cell_filter_msgpack: vec![], // prefix_bits=0 skips routing validation in the handler. prefix_bits: 0, - slice_hilbert_ranges: vec![], - shard_hilbert_range: None, + slice_tile_keys: vec![], + shard_bucket: None, system_time: nodedb_types::SystemTimeScope::Current, valid_at_ms: None, txn_id: None, @@ -372,7 +370,7 @@ mod tests { reducer_msgpack: reducer_bytes, group_by_dim: -1, cell_filter_msgpack: vec![], - shard_hilbert_range: None, + shard_bucket: None, system_as_of: None, valid_at_ms: None, txn_id: None, @@ -528,7 +526,7 @@ mod tests { assert!((resp.partials[0].sum - 42.0).abs() < f64::EPSILON); } - fn make_put_req_bytes(vshard_id: u32, prefix_bits: u8) -> Vec { + fn make_put_req_bytes(prefix_bits: u8) -> Vec { // Build a minimal ArrayShardPutReq with routing metadata set so // validate_put_routing can confirm the request is destined for // the right shard. Use prefix_bits=0 to skip routing validation @@ -537,17 +535,10 @@ mod tests { array_id_msgpack: vec![], cells_msgpack: vec![], wal_lsn: 77, - representative_hilbert_prefix: if prefix_bits == 0 { - 0 - } else { - // Route to bucket 0 (top bits all zero) → vshard 0 with any prefix_bits. - 0u64 - }, + // Key 0 mixes to 0, so it routes to vShard 0 at any prefix_bits. + representative_tile_key: 0, prefix_bits, }; - // Adjust routing so the request would target vshard_id: set bits so - // bucket = vshard_id when stride = 1 (prefix_bits=10, VSHARD_COUNT=1024). - let _ = vshard_id; // routing byte chosen above; test uses prefix_bits=0. zerompk::to_msgpack_vec(&req).unwrap() } @@ -557,7 +548,7 @@ mod tests { coords_msgpack: vec![], wal_lsn: 88, // prefix_bits=0 skips routing validation in the handler. - representative_hilbert_prefix: 0, + representative_tile_key: 0, prefix_bits: 0, }; zerompk::to_msgpack_vec(&req).unwrap() @@ -573,7 +564,7 @@ mod tests { affected: 3, }); // prefix_bits=0 disables routing validation. - let payload = make_put_req_bytes(0, 0); + let payload = make_put_req_bytes(0); let resp_bytes = handle_array_shard_rpc(ARRAY_SHARD_PUT_REQ, 0, &payload, &executor) .await .expect("put handler should succeed"); @@ -623,14 +614,13 @@ mod tests { truncated_before_horizon: false, affected: 0, }); - // prefix_bits=10, stride=1 → bucket = top 10 bits of hilbert_prefix. - // hilbert_prefix = 0 → bucket 0 → expected vshard 0. - // We tell the handler we're vshard 7 → mismatch → error. + // Key 0 mixes to 0 → bucket 0 → expected vShard 0. + // The handler runs as vShard 7 → mismatch → error. let req = super::super::wire::ArrayShardPutReq { array_id_msgpack: vec![], cells_msgpack: vec![], wal_lsn: 1, - representative_hilbert_prefix: 0, + representative_tile_key: 0, prefix_bits: 10, }; let payload = zerompk::to_msgpack_vec(&req).unwrap(); @@ -646,53 +636,44 @@ mod tests { ); } - #[test] - fn validate_slice_routing_rejects_disjoint_range() { - // prefix_bits=10, stride=1 → vshard == bucket == top 10 bits. - // Hilbert range [0x0040_0000_0000_0000, 0x0040_0000_0000_0000] - // covers bucket 1 → vshard 1. - // Local shard is vshard 5 → disjoint → WrongOwner. - let req = super::super::wire::ArrayShardSliceReq { + fn slice_req_with_keys(slice_tile_keys: Vec) -> super::super::wire::ArrayShardSliceReq { + super::super::wire::ArrayShardSliceReq { array_id_msgpack: vec![], slice_msgpack: vec![], attr_projection: vec![], limit: 10, cell_filter_msgpack: vec![], prefix_bits: 10, - slice_hilbert_ranges: vec![(0x0040_0000_0000_0000, 0x0040_0000_0000_0000)], - shard_hilbert_range: None, + slice_tile_keys, + shard_bucket: None, system_time: nodedb_types::SystemTimeScope::Current, valid_at_ms: None, txn_id: None, - }; - let err = super::validate_slice_routing(&req, 5) - .expect_err("disjoint Hilbert range should reject"); + } + } + + #[test] + fn validate_slice_routing_rejects_a_shard_no_key_routes_to() { + let key = 0x0040_0000_0000_0000u64; + let owner = crate::distributed_array::array_vshard_for_tile(key, 10).unwrap(); + let other = (owner + 1) % crate::routing::VSHARD_COUNT; + let err = super::validate_slice_routing(&slice_req_with_keys(vec![key]), other) + .expect_err("a shard no key routes to should reject"); assert!( matches!( err, - crate::error::ClusterError::WrongOwner { vshard_id: 5, .. } + crate::error::ClusterError::WrongOwner { vshard_id, .. } if vshard_id == other ), - "expected WrongOwner for vshard 5, got {err:?}" + "expected WrongOwner for vshard {other}, got {err:?}" ); } #[test] - fn validate_slice_routing_accepts_overlapping_range() { - // Same range as above → vshard 1. Local shard IS vshard 1 → accept. - let req = super::super::wire::ArrayShardSliceReq { - array_id_msgpack: vec![], - slice_msgpack: vec![], - attr_projection: vec![], - limit: 10, - cell_filter_msgpack: vec![], - prefix_bits: 10, - slice_hilbert_ranges: vec![(0x0040_0000_0000_0000, 0x0040_0000_0000_0000)], - shard_hilbert_range: None, - system_time: nodedb_types::SystemTimeScope::Current, - valid_at_ms: None, - txn_id: None, - }; - super::validate_slice_routing(&req, 1).expect("overlapping range should accept"); + fn validate_slice_routing_accepts_the_owner_of_a_key() { + let key = 0x0040_0000_0000_0000u64; + let owner = crate::distributed_array::array_vshard_for_tile(key, 10).unwrap(); + super::validate_slice_routing(&slice_req_with_keys(vec![key]), owner) + .expect("the owning shard should accept"); } #[tokio::test] diff --git a/nodedb-cluster/src/distributed_array/local_executor.rs b/nodedb-cluster/src/distributed_array/local_executor.rs index 68fcf31ff..971737412 100644 --- a/nodedb-cluster/src/distributed_array/local_executor.rs +++ b/nodedb-cluster/src/distributed_array/local_executor.rs @@ -73,10 +73,9 @@ pub trait ArrayLocalExecutor: Send + Sync + 'static { /// `limit` — maximum rows to return per shard (0 = unlimited). /// `cell_filter_msgpack` — zerompk encoding of `SurrogateBitmap`; empty /// means no filter. - /// `shard_hilbert_range` — optional `[lo, hi]` Hilbert-prefix range; when - /// set only tiles whose prefix falls in this range are returned, preventing - /// duplicate rows in single-node harnesses where all vShards share one - /// Data Plane. `None` = no Hilbert filter. + /// `shard_bucket` — optional bucket filter. When set, only tiles whose + /// key routes to this bucket are returned, so vShards that share one + /// Data Plane return no duplicate rows. `None` = no filter. /// `txn_id` — the reading transaction's id; the executor stamps it on the /// Data Plane request so the shard folds that transaction's staged /// cells into the result. `None` = autocommit read. @@ -116,8 +115,8 @@ pub trait ArrayLocalExecutor: Send + Sync + 'static { /// /// `req.cells_msgpack` is a zerompk encoding of /// `Vec`. All cells belong to - /// the same Hilbert-prefix tile. The shard handler has already validated - /// that this shard owns the tile; the executor dispatches directly to the + /// the same routing bucket. The shard handler has already validated + /// that this shard owns the bucket; the executor dispatches directly to the /// Data Plane without further routing checks. /// /// Returns the applied LSN plus the real cell count the Data Plane diff --git a/nodedb-cluster/src/distributed_array/partition.rs b/nodedb-cluster/src/distributed_array/partition.rs index f6aea11ab..89ce0b32f 100644 --- a/nodedb-cluster/src/distributed_array/partition.rs +++ b/nodedb-cluster/src/distributed_array/partition.rs @@ -3,12 +3,11 @@ //! Cell-to-tile partitioning for distributed array writes. //! //! Both `coord_put` and `coord_delete` receive a flat list of cells (or -//! coord tuples) paired with their pre-computed Hilbert prefixes. This -//! module groups them by tile bucket (the top `prefix_bits` of the Hilbert -//! prefix) and maps each bucket to its owning vShard via -//! `array_vshard_for_tile`. +//! coord tuples) paired with their pre-computed tile keys. This module +//! groups them by owning vShard, the vShard of the key's routing bucket +//! from `array_vshard_for_tile`. //! -//! Callers pre-attach Hilbert prefixes because the query planner already +//! Callers pre-attach tile keys because the query planner already //! holds the schema and can compute them at planning time; doing the same //! computation again inside the coordinator would require an additional //! schema lookup that `nodedb-cluster` intentionally avoids (it carries no @@ -16,13 +15,14 @@ //! //! ## Output shape //! -//! `partition_put_cells` returns a `Vec<(vshard_id, representative_hilbert_prefix, -//! cells_msgpack)>` where `cells_msgpack` is a zerompk-encoded +//! `partition_put_cells` returns one `PutBucket` per vShard: its +//! `representative_tile_key` and `cells_msgpack`, a zerompk-encoded //! `Vec` (the same bytes the shard handler passes to //! `exec_put`). //! -//! `partition_delete_coords` returns a `Vec<(vshard_id, coords_msgpack)>` where -//! `coords_msgpack` is a zerompk-encoded `Vec`. +//! `partition_delete_coords` returns one `DeleteBucket` per vShard: its +//! `representative_tile_key` and `coords_msgpack`, a zerompk-encoded +//! `Vec`. use std::collections::HashMap; @@ -33,9 +33,9 @@ use super::routing::array_vshard_for_tile; /// One bucket produced by `partition_put_cells`. pub struct PutBucket { pub vshard_id: u32, - /// Hilbert prefix of the first cell added to this bucket. Used by the - /// shard handler for routing validation (`validate_put_routing`). - pub representative_hilbert_prefix: u64, + /// Tile key of the first cell added to this bucket. The shard handler + /// routes it to check ownership (`validate_put_routing`). + pub representative_tile_key: u64, /// zerompk-encoded `Vec` for all cells in this bucket. pub cells_msgpack: Vec, } @@ -43,16 +43,16 @@ pub struct PutBucket { /// One bucket produced by `partition_delete_coords`. pub struct DeleteBucket { pub vshard_id: u32, - /// Hilbert prefix of the first coord added to this bucket. Used by the - /// shard handler for routing validation (`validate_delete_routing`). - pub representative_hilbert_prefix: u64, + /// Tile key of the first coord added to this bucket. The shard handler + /// routes it to check ownership (`validate_delete_routing`). + pub representative_tile_key: u64, /// zerompk-encoded `Vec` for all coords in this bucket. pub coords_msgpack: Vec, } -/// Group `(hilbert_prefix, cell_msgpack)` pairs by tile bucket. +/// Group `(tile_key, cell_msgpack)` pairs by owning vShard. /// -/// `cells` — each element is `(hilbert_prefix, zerompk-encoded single cell)`. +/// `cells` — each element is `(tile_key, zerompk-encoded single cell)`. /// `prefix_bits` — routing granularity (1–16). /// /// Returns one `PutBucket` per unique owning vShard. The order of buckets @@ -62,14 +62,12 @@ pub fn partition_put_cells(cells: &[(u64, Vec)], prefix_bits: u8) -> Result< return Ok(Vec::new()); } - // vshard_id → (representative_hilbert_prefix, per-cell byte blobs) + // vshard_id → (representative tile key, per-cell byte blobs) let mut buckets: HashMap>)> = HashMap::new(); - for (hilbert_prefix, cell_bytes) in cells { - let vshard_id = array_vshard_for_tile(*hilbert_prefix, prefix_bits)?; - let entry = buckets - .entry(vshard_id) - .or_insert((*hilbert_prefix, Vec::new())); + for (tile_key, cell_bytes) in cells { + let vshard_id = array_vshard_for_tile(*tile_key, prefix_bits)?; + let entry = buckets.entry(vshard_id).or_insert((*tile_key, Vec::new())); entry.1.push(cell_bytes.clone()); } @@ -79,16 +77,16 @@ pub fn partition_put_cells(cells: &[(u64, Vec)], prefix_bits: u8) -> Result< let cells_msgpack = encode_blob_vec(&cell_blobs)?; Ok(PutBucket { vshard_id, - representative_hilbert_prefix: representative, + representative_tile_key: representative, cells_msgpack, }) }) .collect() } -/// Group `(hilbert_prefix, coord_msgpack)` pairs by tile bucket. +/// Group `(tile_key, coord_msgpack)` pairs by owning vShard. /// -/// `coords` — each element is `(hilbert_prefix, zerompk-encoded single coord)`. +/// `coords` — each element is `(tile_key, zerompk-encoded single coord)`. /// `prefix_bits` — routing granularity (1–16). /// /// Returns one `DeleteBucket` per unique owning vShard. @@ -100,14 +98,12 @@ pub fn partition_delete_coords( return Ok(Vec::new()); } - // vshard_id → (representative_hilbert_prefix, coord byte blobs) + // vshard_id → (representative tile key, coord byte blobs) let mut buckets: HashMap>)> = HashMap::new(); - for (hilbert_prefix, coord_bytes) in coords { - let vshard_id = array_vshard_for_tile(*hilbert_prefix, prefix_bits)?; - let entry = buckets - .entry(vshard_id) - .or_insert((*hilbert_prefix, Vec::new())); + for (tile_key, coord_bytes) in coords { + let vshard_id = array_vshard_for_tile(*tile_key, prefix_bits)?; + let entry = buckets.entry(vshard_id).or_insert((*tile_key, Vec::new())); entry.1.push(coord_bytes.clone()); } @@ -117,7 +113,7 @@ pub fn partition_delete_coords( let coords_msgpack = encode_blob_vec(&coord_blobs)?; Ok(DeleteBucket { vshard_id, - representative_hilbert_prefix: representative, + representative_tile_key: representative, coords_msgpack, }) }) @@ -142,7 +138,7 @@ fn encode_blob_vec(blobs: &[Vec]) -> Result> { mod tests { use super::*; - // Build a fake (hilbert_prefix, cell_bytes) pair. The cell_bytes are + // Build a fake (tile_key, cell_bytes) pair. The cell_bytes are // opaque for the partitioner — any non-empty Vec works. fn cell(prefix: u64, tag: u8) -> (u64, Vec) { (prefix, vec![tag]) @@ -153,7 +149,7 @@ mod tests { (prefix, vec![tag]) } - // Helper: compute expected vshard for a given hilbert prefix and prefix_bits. + // Helper: compute expected vshard for a given tile key and prefix_bits. fn expected_vshard(prefix: u64, prefix_bits: u8) -> u32 { array_vshard_for_tile(prefix, prefix_bits).unwrap() } @@ -172,23 +168,18 @@ mod tests { #[test] fn partition_put_single_cell_one_bucket() { - // prefix_bits=10, stride=1 → vshard == bucket == top 10 bits. - // All-zero hilbert prefix → bucket 0 → vshard 0. + // Key 0 mixes to 0 → bucket 0 → vShard 0. let cells = vec![cell(0, 0xAA)]; let buckets = partition_put_cells(&cells, 10).unwrap(); assert_eq!(buckets.len(), 1); assert_eq!(buckets[0].vshard_id, 0); - assert_eq!(buckets[0].representative_hilbert_prefix, 0); + assert_eq!(buckets[0].representative_tile_key, 0); } #[test] fn partition_put_cells_by_tile_three_buckets() { - // With prefix_bits=10 (stride=1), every distinct top-10-bit value - // is its own vshard. Use three well-separated prefixes. - // - // prefix 0x0000_0000_0000_0000 → bucket 0 → vshard 0 - // prefix 0x0040_0000_0000_0000 → top 10 bits = 1 → vshard 1 - // prefix 0x0080_0000_0000_0000 → top 10 bits = 2 → vshard 2 + // With prefix_bits=10 (stride=1), each mixed bucket is its own + // vShard. The three keys mix to three distinct buckets. let p0 = 0x0000_0000_0000_0000u64; let p1 = 0x0040_0000_0000_0000u64; let p2 = 0x0080_0000_0000_0000u64; @@ -201,22 +192,19 @@ mod tests { cell(p1, 0x05), // same bucket as second ]; - let mut buckets = partition_put_cells(&cells, 10).unwrap(); - // Sort by vshard_id for deterministic assertion. - buckets.sort_by_key(|b| b.vshard_id); + let buckets = partition_put_cells(&cells, 10).unwrap(); assert_eq!(buckets.len(), 3); - assert_eq!(buckets[0].vshard_id, expected_vshard(p0, 10)); - assert_eq!(buckets[1].vshard_id, expected_vshard(p1, 10)); - assert_eq!(buckets[2].vshard_id, expected_vshard(p2, 10)); - - // Bucket 0 and 1 each have 2 cells, bucket 2 has 1. - let blobs0: Vec> = zerompk::from_msgpack(&buckets[0].cells_msgpack).unwrap(); - let blobs1: Vec> = zerompk::from_msgpack(&buckets[1].cells_msgpack).unwrap(); - let blobs2: Vec> = zerompk::from_msgpack(&buckets[2].cells_msgpack).unwrap(); - assert_eq!(blobs0.len(), 2); - assert_eq!(blobs1.len(), 2); - assert_eq!(blobs2.len(), 1); + // p0 and p1 each own 2 cells, p2 owns 1. + for (key, want) in [(p0, 2), (p1, 2), (p2, 1)] { + let vshard = expected_vshard(key, 10); + let bucket = buckets + .iter() + .find(|b| b.vshard_id == vshard) + .expect("bucket for key"); + let blobs: Vec> = zerompk::from_msgpack(&bucket.cells_msgpack).unwrap(); + assert_eq!(blobs.len(), want, "key={key:#x}"); + } } #[test] @@ -226,16 +214,17 @@ mod tests { let coords = vec![coord(p0, 0xAA), coord(p1, 0xBB), coord(p0, 0xCC)]; - let mut buckets = partition_delete_coords(&coords, 10).unwrap(); - buckets.sort_by_key(|b| b.vshard_id); + let buckets = partition_delete_coords(&coords, 10).unwrap(); assert_eq!(buckets.len(), 2); - assert_eq!(buckets[0].vshard_id, expected_vshard(p0, 10)); - assert_eq!(buckets[1].vshard_id, expected_vshard(p1, 10)); - - let blobs0: Vec> = zerompk::from_msgpack(&buckets[0].coords_msgpack).unwrap(); - let blobs1: Vec> = zerompk::from_msgpack(&buckets[1].coords_msgpack).unwrap(); - assert_eq!(blobs0.len(), 2); - assert_eq!(blobs1.len(), 1); + for (key, want) in [(p0, 2), (p1, 1)] { + let vshard = expected_vshard(key, 10); + let bucket = buckets + .iter() + .find(|b| b.vshard_id == vshard) + .expect("bucket for key"); + let blobs: Vec> = zerompk::from_msgpack(&bucket.coords_msgpack).unwrap(); + assert_eq!(blobs.len(), want, "key={key:#x}"); + } } #[test] diff --git a/nodedb-cluster/src/distributed_array/routing.rs b/nodedb-cluster/src/distributed_array/routing.rs index d8a5791f6..29125d5aa 100644 --- a/nodedb-cluster/src/distributed_array/routing.rs +++ b/nodedb-cluster/src/distributed_array/routing.rs @@ -2,264 +2,129 @@ //! vShard routing for array tiles. //! -//! Arrays are sharded by Hilbert-curve prefix: the top `prefix_bits` -//! of a tile's `hilbert_prefix` determine which vShard bucket owns it. +//! A tile routes by the top bits of `fmix64(tile_key)`. The tile-to-vShard +//! functions live in `nodedb_types::array_route`, shared with the Data +//! Plane. This module adds the read-side bucket filter and slice fan-out. //! -//! With `prefix_bits = P` there are `2^P` distinct buckets numbered -//! `0 .. 2^P - 1`. Each bucket maps to a contiguous range of vShards -//! within the cluster's `VSHARD_COUNT`-wide space. The stride is -//! `VSHARD_COUNT / 2^P` (integer division). Bucket `b` owns vShards -//! `[b * stride .. (b + 1) * stride)`. The primary vShard for a bucket -//! is the first one in that range (`b * stride`). - -use crate::error::{ClusterError, Result}; -use crate::routing::VSHARD_COUNT; - -// Array Hilbert-range ownership is derived entirely from vShard ownership: -// bucket→vshard is a deterministic function of `prefix_bits`, and -// vshard→node is maintained by `RoutingTable` (updated by committed -// `RoutingChange::ReassignVShard` entries). No separate array-specific -// routing variant is needed in the metadata log — array migration is "free" -// because a vShard rebalance already atomically re-points all array tile -// ownership through the same `RoutingTable` cut-over path. - -/// Determine the primary owning vShard for a single tile identified by -/// its raw Hilbert-prefix value. -/// -/// `hilbert_prefix` — the tile's u64 Hilbert key. -/// `prefix_bits` — number of high-order bits used for routing (1–16). -/// -/// Returns the vShard ID in `[0, VSHARD_COUNT)`. -pub fn array_vshard_for_tile(hilbert_prefix: u64, prefix_bits: u8) -> Result { - validate_prefix_bits(prefix_bits)?; - let bucket = prefix_bucket(hilbert_prefix, prefix_bits); - Ok(bucket_to_vshard(bucket, prefix_bits)) +//! Bucket-to-vShard is a fixed function of `prefix_bits`, and vShard-to-node +//! lives in `RoutingTable`. A vShard rebalance therefore moves array tiles +//! with no array-specific routing entry in the metadata log. + +pub use nodedb_types::array_route::array_vshard_for_tile; +use nodedb_types::array_route::{ShardBucket, bucket_to_vshard, prefix_bucket, routing_bits}; +use nodedb_types::id::VShardId; + +use crate::error::Result; + +/// The bucket filter a read sent to primary vShard `vshard_id` applies, so +/// the shard returns only the tiles it owns. +pub fn shard_bucket_for_vshard(vshard_id: u32, prefix_bits: u8) -> Result { + let bits = routing_bits(prefix_bits)?; + Ok(ShardBucket { + bucket: vshard_id >> u32::from(VShardId::BITS - bits), + prefix_bits: bits, + }) } -/// Determine all vShards that intersect a set of Hilbert-prefix ranges. +/// The primary vShards that own the tiles in `slice_tile_keys`. /// -/// `slice_hilbert_ranges` — list of `(lo, hi)` inclusive Hilbert-prefix -/// ranges covering the slice's MBR. An empty slice means "unbounded" -/// (returns all vShards up to `total_shards`). +/// `slice_tile_keys` — tile keys covering the slice, at least one per +/// bucket the slice touches. An empty list means an unbounded slice, which +/// returns the primary vShard of every bucket. /// /// `prefix_bits` — routing granularity (1–16). -/// `total_shards` — number of active vShards; returned IDs are -/// clamped to `[0, total_shards)`. +/// `total_shards` — number of active vShards. IDs at or above it are dropped. /// /// Returns a sorted, deduplicated list of vShard IDs. pub fn array_vshards_for_slice( - slice_hilbert_ranges: &[(u64, u64)], + slice_tile_keys: &[u64], prefix_bits: u8, total_shards: u32, ) -> Result> { if total_shards == 0 { return Ok(Vec::new()); } - validate_prefix_bits(prefix_bits)?; - - let stride = vshard_stride(prefix_bits); - let mut out: Vec = Vec::new(); - - // Unbounded slice: return one canonical primary vShard per bucket. - // Within a bucket all vShards serve the same Hilbert range, so fanning - // out to every member would multiply the result by `stride`. - if slice_hilbert_ranges.is_empty() { - let mut vshard_id: u32 = 0; - while vshard_id < total_shards && vshard_id < VSHARD_COUNT { - out.push(vshard_id); - vshard_id = vshard_id.saturating_add(stride); - if stride == 0 { - break; - } - } - return Ok(out); - } - - for &(lo, hi) in slice_hilbert_ranges { - let lo_bucket = prefix_bucket(lo, prefix_bits); - // `hi` is inclusive, so we take the bucket that contains `hi`. - let hi_bucket = prefix_bucket(hi, prefix_bits); - - for bucket in lo_bucket..=hi_bucket { - // One canonical primary vShard per bucket. All other vShards in - // `[vshard_start, vshard_start + stride)` serve identical Hilbert - // ranges; fanning out to all of them double-counts on aggregate. - let vshard_start = bucket_to_vshard(bucket, prefix_bits); - if vshard_start < total_shards { - out.push(vshard_start); - } - } - } - + let bits = routing_bits(prefix_bits)?; + + // One primary vShard per bucket. The other vShards of a bucket serve the + // same tiles, so a fan-out to them double-counts aggregates. + let mut out: Vec = if slice_tile_keys.is_empty() { + (0..(1u32 << bits)) + .map(|bucket| bucket_to_vshard(bucket, bits)) + .collect() + } else { + slice_tile_keys + .iter() + .map(|&key| bucket_to_vshard(prefix_bucket(key, bits), bits)) + .collect() + }; + out.retain(|&vshard| vshard < total_shards); out.sort_unstable(); out.dedup(); Ok(out) } -// ── Internal helpers ────────────────────────────────────────────────────── - -fn validate_prefix_bits(prefix_bits: u8) -> Result<()> { - if prefix_bits == 0 || prefix_bits > 16 { - return Err(ClusterError::Codec { - detail: format!("array routing: prefix_bits must be 1-16, got {prefix_bits}"), - }); - } - Ok(()) -} - -/// Extract the top `prefix_bits` bits of a Hilbert prefix as a bucket index. -fn prefix_bucket(hilbert_prefix: u64, prefix_bits: u8) -> u32 { - // Shift right so the top `prefix_bits` bits are in the low-order position. - let shift = 64u8.saturating_sub(prefix_bits); - (hilbert_prefix >> shift) as u32 -} - -/// Map a bucket index to the primary vShard in that bucket's range. -fn bucket_to_vshard(bucket: u32, prefix_bits: u8) -> u32 { - let stride = vshard_stride(prefix_bits); - bucket.saturating_mul(stride) -} - -/// Number of vShards per Hilbert bucket. -/// -/// With `P` prefix bits there are `2^P` buckets and `VSHARD_COUNT` -/// total vShards. Stride = `VSHARD_COUNT >> P`. When `P >= log2(VSHARD_COUNT)` -/// the stride is 1 (one vShard per bucket or less). -fn vshard_stride(prefix_bits: u8) -> u32 { - // VSHARD_COUNT is a power of two (1024 = 2^10). - // Right-shifting by prefix_bits gives the stride, floored at 1. - let shifted = VSHARD_COUNT >> (prefix_bits as u32); - shifted.max(1) -} - #[cfg(test)] mod tests { use super::*; use crate::routing::VSHARD_COUNT; #[test] - fn tile_routing_invalid_prefix_bits() { - assert!(array_vshard_for_tile(0, 0).is_err()); - assert!(array_vshard_for_tile(0, 17).is_err()); - } - - #[test] - fn tile_routing_top_bit_prefix1() { - // With prefix_bits=1: stride=512. Bucket 0 → vshard 0, bucket 1 → vshard 512. - let shard_low = array_vshard_for_tile(0x0000_0000_0000_0000, 1).unwrap(); - let shard_high = array_vshard_for_tile(0x8000_0000_0000_0000, 1).unwrap(); - assert_eq!(shard_low, 0); - assert_eq!(shard_high, 512); - } - - #[test] - fn tile_routing_prefix10_is_direct() { - // With prefix_bits=10: stride=1, bucket == vshard_id. - // Top 10 bits of 0x0100_0000_0000_0000 = 0b0000_0000_01 = 1. - let vshard = array_vshard_for_tile(0x0040_0000_0000_0000, 10).unwrap(); - // 0x0040... >> (64-10) = 0x0040... >> 54 = 1 - assert_eq!(vshard, 1); - } - - #[test] - fn tile_routing_prefix8_stride4() { - // With prefix_bits=8: stride=4. Bucket 0 → vshard 0, bucket 1 → vshard 4. - let shard0 = array_vshard_for_tile(0x0000_0000_0000_0000, 8).unwrap(); - // Top 8 bits of 0x0100... = 1. - let shard1 = array_vshard_for_tile(0x0100_0000_0000_0000, 8).unwrap(); - assert_eq!(shard0, 0); - assert_eq!(shard1, 4); + fn invalid_prefix_bits_are_refused() { + assert!(shard_bucket_for_vshard(0, 0).is_err()); + assert!(shard_bucket_for_vshard(0, 17).is_err()); + assert!(array_vshards_for_slice(&[], 0, VSHARD_COUNT).is_err()); } #[test] - fn tile_routing_output_in_vshard_range() { - // Every vshard returned must be < VSHARD_COUNT. - for bits in 1u8..=10 { - for &prefix in &[0u64, u64::MAX / 3, u64::MAX / 2, u64::MAX] { - let vshard = array_vshard_for_tile(prefix, bits).unwrap(); - assert!( - vshard < VSHARD_COUNT, - "bits={bits} prefix={prefix:#x} → vshard={vshard} >= {VSHARD_COUNT}" - ); + fn shard_bucket_admits_exactly_the_tiles_routed_to_its_vshard() { + for bits in [1u8, 4, 8, 10, 12, 16] { + let primaries = array_vshards_for_slice(&[], bits, VSHARD_COUNT).unwrap(); + for key in 0u64..512 { + let owner = array_vshard_for_tile(key, bits).unwrap(); + let admitting: Vec = primaries + .iter() + .copied() + .filter(|&v| shard_bucket_for_vshard(v, bits).unwrap().contains(key)) + .collect(); + assert_eq!(admitting, vec![owner], "bits={bits} key={key}"); } } } #[test] fn slice_routing_zero_shards() { - let shards = array_vshards_for_slice(&[(0, u64::MAX)], 8, 0).unwrap(); + let shards = array_vshards_for_slice(&[0, u64::MAX], 8, 0).unwrap(); assert!(shards.is_empty()); } #[test] - fn slice_routing_empty_ranges_returns_all() { - // Empty range list = unbounded slice → one canonical primary per bucket. - // With prefix_bits=8 (stride=4) and total_shards=4, only primary 0 fits. - let shards = array_vshards_for_slice(&[], 8, 4).unwrap(); - assert_eq!(shards, vec![0]); - } - - #[test] - fn slice_routing_full_range_returns_all_active() { - // Full range covers all 256 buckets → one primary per bucket. - // total_shards=4 only fits primary 0. - let shards = array_vshards_for_slice(&[(0, u64::MAX)], 8, 4).unwrap(); - assert_eq!(shards, vec![0]); - } - - #[test] - fn slice_routing_single_bucket_prefix8() { - // Hilbert range [0, 0] covers only bucket 0 → primary vShard 0. - let shards = array_vshards_for_slice(&[(0, 0)], 8, 16).unwrap(); - assert_eq!(shards, vec![0]); - } - - #[test] - fn slice_routing_two_adjacent_buckets() { - // prefix_bits=8, stride=4. One canonical primary per bucket. - // Range 1 → bucket 0 → primary vShard 0. - // Range 2 → bucket 1 → primary vShard 4. - let shards = array_vshards_for_slice( - &[ - (0x0000_0000_0000_0000, 0x00FF_FFFF_FFFF_FFFF), - (0x0100_0000_0000_0000, 0x01FF_FFFF_FFFF_FFFF), - ], - 8, - 1024, - ) - .unwrap(); - assert_eq!(shards, vec![0, 4]); - } - - #[test] - fn slice_routing_clamped_to_total_shards() { - // With total_shards=2, only the primary for bucket 0 (= vShard 0) fits. - let shards = array_vshards_for_slice(&[(0, u64::MAX)], 8, 2).unwrap(); - assert_eq!(shards, vec![0]); + fn slice_routing_empty_keys_returns_every_primary() { + // prefix_bits=8 (stride=4): primaries 0, 4, …, 1020. + let shards = array_vshards_for_slice(&[], 8, VSHARD_COUNT).unwrap(); + assert_eq!(shards.len(), 256); + assert!(shards.iter().all(|v| v % 4 == 0)); + // total_shards=4 keeps only primary 0. + assert_eq!(array_vshards_for_slice(&[], 8, 4).unwrap(), vec![0]); } #[test] - fn slice_routing_dedup_overlapping_ranges() { - // Two ranges that map to the same bucket → one canonical primary vShard. - let shards = array_vshards_for_slice( - &[ - (0x0000_0000_0000_0000, 0x0000_0000_0000_0001), - (0x0000_0000_0000_0002, 0x0000_0000_0000_0003), - ], - 8, - 16, - ) - .unwrap(); - // Both ranges are in bucket 0 → primary vShard 0. - assert_eq!(shards, vec![0]); + fn slice_routing_maps_each_key_to_its_owner() { + let keys = [5u64, 6, 7, 1 << 40]; + let shards = array_vshards_for_slice(&keys, 8, VSHARD_COUNT).unwrap(); + let mut expected: Vec = keys + .iter() + .map(|&k| array_vshard_for_tile(k, 8).unwrap()) + .collect(); + expected.sort_unstable(); + expected.dedup(); + assert_eq!(shards, expected); } #[test] - fn vshard_stride_values() { - // VSHARD_COUNT=1024. - assert_eq!(vshard_stride(1), 512); - assert_eq!(vshard_stride(8), 4); - assert_eq!(vshard_stride(10), 1); - assert_eq!(vshard_stride(16), 1); // floored at 1 + fn slice_routing_dedups_keys_of_one_bucket() { + let key = 42u64; + let shards = array_vshards_for_slice(&[key, key], 8, VSHARD_COUNT).unwrap(); + assert_eq!(shards, vec![array_vshard_for_tile(key, 8).unwrap()]); } } diff --git a/nodedb-cluster/src/distributed_array/scatter.rs b/nodedb-cluster/src/distributed_array/scatter.rs index 4c58f172d..9dd0fc845 100644 --- a/nodedb-cluster/src/distributed_array/scatter.rs +++ b/nodedb-cluster/src/distributed_array/scatter.rs @@ -6,12 +6,12 @@ //! - `fan_out` — broadcast one request to all listed shards (slice, agg, //! delete, surrogate scan). //! - `fan_out_partitioned` — send a different payload to each shard -//! (used by `coord_put` where cells are routed by Hilbert prefix). +//! (used by `coord_put` where cells are routed by tile key). //! //! Both functions dispatch concurrently via `FuturesUnordered`, apply a //! per-shard `tokio::time::timeout`, and wrap each call in the cluster //! `CircuitBreaker`. Any shard failure is propagated as `Err` — partial -//! results are not silently dropped. +//! results are not silently dropped. The error names the failing shard. use std::sync::Arc; @@ -74,6 +74,28 @@ async fn call_with_wrong_owner_retry( use super::rpc::ShardRpcDispatch; +/// A fan-out failure and the shard it names. +#[derive(Debug, thiserror::Error)] +#[error("array shard {vshard_id}: {error}")] +pub struct ShardFailure { + /// The shard whose call failed. + pub vshard_id: u32, + /// Why it failed. + #[source] + pub error: ClusterError, +} + +/// The error of an array coordinator call. +#[derive(Debug, thiserror::Error)] +pub enum ArrayCoordError { + /// A shard call failed. The failure names the shard. + #[error(transparent)] + Shard(#[from] ShardFailure), + /// The coordinator failed outside any shard call, such as an encode. + #[error(transparent)] + Coordinator(#[from] ClusterError), +} + /// Classify whether a shard error should count against the peer circuit breaker. /// /// `WrongOwner` is a transient routing/ownership-churn signal, not a peer-health @@ -92,8 +114,9 @@ fn counts_against_breaker(err: &ClusterError) -> bool { ClusterError::DataPlane { .. } | ClusterError::ShardExecution { .. } | ClusterError::StreamTerminal { .. } => false, - // An unresponsive peer is exactly what the breaker exists to shed - // load from, so a shard timeout counts like any other liveness failure. + // The breaker counts peer liveness, not the send fate of + // `is_lost_answer`. An unresponsive peer is what the breaker sheds + // load from, so a lost answer counts like any other liveness failure. ClusterError::ShardTimeout { .. } | ClusterError::Unanswered { .. } => true, ClusterError::Raft(_) | ClusterError::VShardNotMapped { .. } @@ -166,7 +189,7 @@ pub async fn fan_out( req_bytes: &[u8], dispatch: &Arc, circuit_breaker: &CircuitBreaker, -) -> Result)>> { +) -> std::result::Result)>, ShardFailure> { if params.shard_ids.is_empty() { return Ok(Vec::new()); } @@ -176,11 +199,15 @@ pub async fn fan_out( let mut futs = futures::stream::FuturesUnordered::new(); for &shard_id in ¶ms.shard_ids { + let failed = |error| ShardFailure { + vshard_id: shard_id, + error, + }; // Circuit-breaker gate: treat shard_id as the peer identifier. - let admission = circuit_breaker.check(shard_id as u64)?; + let admission = circuit_breaker.check(shard_id as u64).map_err(failed)?; let env = VShardEnvelope::new( - msg_type_from_opcode(opcode)?, + msg_type_from_opcode(opcode).map_err(failed)?, params.source_node, 0, // target_node resolved by the dispatch impl shard_id, @@ -209,7 +236,10 @@ pub async fn fan_out( if counts_against_breaker(&e) { circuit_breaker.record_failure(shard_id as u64, admission); } - return Err(e); + return Err(ShardFailure { + vshard_id: shard_id, + error: e, + }); } } } @@ -227,7 +257,7 @@ pub async fn fan_out_partitioned( per_shard: &[(u32, Vec)], dispatch: &Arc, circuit_breaker: &CircuitBreaker, -) -> Result)>> { +) -> std::result::Result)>, ShardFailure> { if per_shard.is_empty() { return Ok(Vec::new()); } @@ -235,10 +265,14 @@ pub async fn fan_out_partitioned( let mut futs = futures::stream::FuturesUnordered::new(); for (shard_id, payload) in per_shard { - let admission = circuit_breaker.check(*shard_id as u64)?; + let failed = |error| ShardFailure { + vshard_id: *shard_id, + error, + }; + let admission = circuit_breaker.check(*shard_id as u64).map_err(failed)?; let env = VShardEnvelope::new( - msg_type_from_opcode(opcode)?, + msg_type_from_opcode(opcode).map_err(failed)?, params.source_node, 0, *shard_id, @@ -267,7 +301,10 @@ pub async fn fan_out_partitioned( if counts_against_breaker(&e) { circuit_breaker.record_failure(shard_id as u64, admission); } - return Err(e); + return Err(ShardFailure { + vshard_id: shard_id, + error: e, + }); } } } @@ -425,11 +462,49 @@ mod tests { .await .expect_err("fan_out should propagate shard failure"); assert!( - matches!(err, ClusterError::Transport { .. }), + matches!(err.error, ClusterError::Transport { .. }), "expected Transport error, got {err:?}" ); } + /// A failing shard's id rides with its error, so the caller can name + /// the vShard that refused. + #[tokio::test] + async fn a_failure_names_the_failing_shard() { + let dispatch: Arc = Arc::new(FailDispatch); + let params = FanOutParams { + shard_ids: vec![17], + timeout_ms: 1000, + source_node: 1, + }; + let err = fan_out( + ¶ms, + super::super::opcodes::ARRAY_SHARD_SLICE_REQ, + b"", + &dispatch, + &cb(), + ) + .await + .expect_err("a failing shard fails the fan-out"); + assert_eq!(err.vshard_id, 17); + assert!(err.error.is_unsent(), "got {err:?}"); + + let per_shard = vec![(23u32, b"cells".to_vec())]; + let err = fan_out_partitioned( + &FanOutPartitionedParams { + timeout_ms: 1000, + source_node: 1, + }, + super::super::opcodes::ARRAY_SHARD_PUT_REQ, + &per_shard, + &dispatch, + &cb(), + ) + .await + .expect_err("a failing shard fails the partitioned fan-out"); + assert_eq!(err.vshard_id, 23); + } + #[tokio::test] async fn fan_out_partitioned_dispatches_distinct_payloads() { let dispatch: Arc = Arc::new(EchoDispatch); @@ -486,7 +561,7 @@ mod tests { .await .expect_err("open circuit should block fan_out"); assert!( - matches!(err, ClusterError::CircuitOpen { .. }), + matches!(err.error, ClusterError::CircuitOpen { .. }), "expected CircuitOpen, got {err:?}" ); } @@ -579,7 +654,7 @@ mod tests { .expect_err("fan_out should propagate WrongOwner when both attempts fail"); assert!( - matches!(err, ClusterError::WrongOwner { .. }), + matches!(err.error, ClusterError::WrongOwner { .. }), "expected WrongOwner, got {err:?}" ); assert_eq!( @@ -617,7 +692,7 @@ mod tests { .expect_err("fan_out should propagate WrongOwner"); assert!( - matches!(err, ClusterError::WrongOwner { .. }), + matches!(err.error, ClusterError::WrongOwner { .. }), "expected WrongOwner, got {err:?}" ); // WrongOwner is a routing signal, not a health failure: the breaker @@ -653,7 +728,7 @@ mod tests { .await .expect_err("fan_out should surface a timeout once the shard call never resolves"); - match err { + match err.error { ClusterError::ShardTimeout { vshard_id, elapsed_ms, diff --git a/nodedb-cluster/src/distributed_array/wire.rs b/nodedb-cluster/src/distributed_array/wire.rs index 2183236bb..683f796d5 100644 --- a/nodedb-cluster/src/distributed_array/wire.rs +++ b/nodedb-cluster/src/distributed_array/wire.rs @@ -22,25 +22,20 @@ pub struct ArrayShardSliceReq { /// Optional surrogate pre-filter bitmap bytes (zerompk encoding of `SurrogateBitmap`). /// Empty slice means no filter. pub cell_filter_msgpack: Vec, - /// Prefix bits used for Hilbert routing (1–16). 0 means no routing validation. + /// The array's routing granularity (1–16). 0 skips routing validation. /// - /// When non-zero the shard verifies that its `local_vshard_id` is covered - /// by the Hilbert ranges in `slice_hilbert_ranges` before executing the - /// scan. A mismatch returns `ClusterError::WrongOwner` so the coordinator - /// can retry against a refreshed routing table. + /// When non-zero the shard checks that some key in `slice_tile_keys` + /// routes to its `local_vshard_id`. A mismatch returns + /// `ClusterError::WrongOwner`, so the coordinator retries against a + /// refreshed routing table. pub prefix_bits: u8, - /// Inclusive Hilbert-prefix ranges `(lo, hi)` that this slice covers, - /// pre-computed by the coordinator from the slice predicate. Each entry - /// is encoded as two consecutive `u64` values (little-endian). Empty - /// means unbounded (no routing validation is performed). - pub slice_hilbert_ranges: Vec<(u64, u64)>, - /// Hilbert-prefix range `[lo, hi]` that this shard owns. The shard - /// applies this as a pre-filter so it only returns cells whose Hilbert - /// prefix falls within the range. `None` means unbounded (return all - /// matching cells). Used by the distributed shard handler to prevent - /// duplicate rows in single-node harnesses where all vShards share one - /// Data Plane. - pub shard_hilbert_range: Option<(u64, u64)>, + /// Tile keys covering the slice, at least one per bucket it touches, + /// computed by the planner. Empty means unbounded and skips validation. + pub slice_tile_keys: Vec, + /// The bucket this shard owns. The shard returns only cells whose tile + /// key falls in it, so vShards that share one Data Plane return no + /// duplicate rows. `None` returns every matching cell. + pub shard_bucket: Option, /// Bitemporal system-time scope forwarded from `ArrayOp::Slice::system_time`. /// /// - `Current` = live read. @@ -91,10 +86,9 @@ pub struct ArrayShardAggReq { pub group_by_dim: i32, /// Optional surrogate pre-filter; empty = all cells. pub cell_filter_msgpack: Vec, - /// Hilbert-prefix range `[lo, hi]` that this shard owns. The shard - /// applies this as a pre-filter so it only counts cells whose Hilbert - /// prefix falls within the range. `None` means unbounded (scan all). - pub shard_hilbert_range: Option<(u64, u64)>, + /// The bucket this shard owns. The shard counts only cells whose tile + /// key falls in it. `None` counts every cell. + pub shard_bucket: Option, /// Bitemporal system-time cutoff forwarded from `ArrayOp::Aggregate::system_as_of`. /// `None` = live read. pub system_as_of: Option, @@ -131,16 +125,15 @@ pub struct ArrayShardAggResp { /// /// `cells_msgpack` is a zerompk encoding of /// `Vec`. -/// All cells in this batch belong to the same Hilbert-prefix bucket. +/// All cells in this batch route to the same bucket (`prefix_bucket` of the tile key). #[derive(Debug, Clone, zerompk::ToMessagePack, zerompk::FromMessagePack)] pub struct ArrayShardPutReq { pub array_id_msgpack: Vec, pub cells_msgpack: Vec, pub wal_lsn: u64, - /// Hilbert prefix of any representative cell in this batch (used by - /// the shard handler to validate routing). Set to 0 when routing - /// metadata is unavailable (pre-routing clients). - pub representative_hilbert_prefix: u64, + /// Tile key of any cell in this batch. The shard handler routes it to + /// check that it owns the batch. + pub representative_tile_key: u64, /// Prefix bits used for routing (1–16). 0 means no validation. pub prefix_bits: u8, } @@ -165,11 +158,10 @@ pub struct ArrayShardDeleteReq { pub array_id_msgpack: Vec, pub coords_msgpack: Vec, pub wal_lsn: u64, - /// Hilbert prefix of any representative coord in this batch. Used by the - /// shard handler to validate that it still owns the target Hilbert bucket. - /// Set to 0 when routing metadata is unavailable. - pub representative_hilbert_prefix: u64, - /// Prefix bits used for Hilbert routing (1–16). 0 means no validation. + /// Tile key of any coord in this batch. The shard handler routes it to + /// check that it still owns the batch's bucket. + pub representative_tile_key: u64, + /// Prefix bits used for routing (1–16). 0 means no validation. pub prefix_bits: u8, } diff --git a/nodedb-cluster/src/error.rs b/nodedb-cluster/src/error.rs index 26e716a99..ee107278e 100644 --- a/nodedb-cluster/src/error.rs +++ b/nodedb-cluster/src/error.rs @@ -271,7 +271,78 @@ pub enum ClusterError { }, } +/// What a failed send says about the request it carried. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +enum SendFate { + /// The request never reached the peer whole. A resend cannot run it twice. + Unsent, + /// The request reached the peer and no answer came back. The peer can + /// have run it, so its outcome is unknown. + LostAnswer, + /// An answer, or a refusal the caller handles itself. + Answered, +} + impl ClusterError { + /// Whether the request never reached the peer whole. + /// + /// The peer ran nothing, so any request can be sent again. + pub fn is_unsent(&self) -> bool { + self.send_fate() == SendFate::Unsent + } + + /// Whether the request reached the peer and its answer was lost. + /// + /// The peer can have run it. Only a resend-safe request goes again. Any + /// other request reports an unknown outcome. + pub fn is_lost_answer(&self) -> bool { + self.send_fate() == SendFate::LostAnswer + } + + fn send_fate(&self) -> SendFate { + match self { + Self::Transport { .. } => SendFate::Unsent, + Self::ShardTimeout { .. } | Self::Unanswered { .. } => SendFate::LostAnswer, + Self::Raft(_) + | Self::VShardNotMapped { .. } + | Self::GroupNotFound { .. } + | Self::LearnerNotCaughtUp { .. } + | Self::MigrationInProgress { .. } + | Self::MigrationPauseBudgetExceeded { .. } + | Self::NodeUnreachable { .. } + | Self::GhostNotFound { .. } + | Self::StreamTerminal { .. } + | Self::Storage { .. } + | Self::DataPlane { .. } + | Self::Codec { .. } + | Self::UnsupportedWireVersion { .. } + | Self::CircuitOpen { .. } + | Self::JoinGroupDisappeared { .. } + | Self::JoinCommitTimeout { .. } + | Self::ReadIndexNotLeader { .. } + | Self::ReadIndexTimeout { .. } + | Self::Config { .. } + | Self::MigrationCheckpoint(_) + | Self::MigrationRecovery(_) + | Self::WrongOwner { .. } + | Self::Calvin(_) + | Self::SnapshotCrcMismatch { .. } + | Self::SnapshotOffsetRegression { .. } + | Self::PartialSnapshotCorrupt { .. } + | Self::PartialSnapshotCleanupFailed { .. } + | Self::SnapshotApplyFailed { .. } + | Self::Mirror(_) + | Self::BspBarrier(_) + | Self::VectorGather(_) + | Self::SpatialGather(_) + | Self::Bm25Gather(_) + | Self::TsGather(_) + | Self::ShufflePush(_) + | Self::RemoteUntyped { .. } + | Self::ShardExecution { .. } => SendFate::Answered, + } + } + /// Whether the error means the link to the peer failed. /// /// A link failure counts against the peer's circuit breaker and ends a @@ -288,6 +359,16 @@ impl ClusterError { } } +/// An array routing granularity out of range is a codec error: it arrives +/// on a wire request or a catalog entry. +impl From for ClusterError { + fn from(e: nodedb_types::array_route::PrefixBitsOutOfRange) -> Self { + ClusterError::Codec { + detail: e.to_string(), + } + } +} + #[cfg(test)] mod tests { use super::*; @@ -316,4 +397,54 @@ mod tests { .is_link_failure() ); } + + #[test] + fn only_a_transport_error_is_unsent() { + let unsent = ClusterError::Transport { + detail: "connect refused".into(), + }; + assert!(unsent.is_unsent()); + assert!(!unsent.is_lost_answer()); + } + + #[test] + fn a_timeout_or_a_missing_answer_is_a_lost_answer() { + let lost = [ + ClusterError::ShardTimeout { + vshard_id: 3, + elapsed_ms: 50, + }, + ClusterError::Unanswered { + node_id: 2, + detail: "stream reset".into(), + }, + ]; + for error in &lost { + assert!(error.is_lost_answer(), "{error}"); + assert!(!error.is_unsent(), "{error}"); + } + } + + #[test] + fn an_answer_is_neither_unsent_nor_lost() { + let answered = [ + ClusterError::Raft(nodedb_raft::RaftError::NotLeader { + leader_hint: Some(2), + term: 3, + }), + ClusterError::CircuitOpen { + node_id: 1, + failures: 5, + }, + ClusterError::NodeUnreachable { node_id: 1 }, + ClusterError::RemoteUntyped { + detail: "refused".into(), + }, + ClusterError::Calvin(CalvinError::AdmissionTimedOut), + ]; + for error in &answered { + assert!(!error.is_unsent(), "{error}"); + assert!(!error.is_lost_answer(), "{error}"); + } + } } diff --git a/nodedb-cluster/src/multi_raft/conf_change.rs b/nodedb-cluster/src/multi_raft/conf_change.rs index cca910be2..4ac4dd356 100644 --- a/nodedb-cluster/src/multi_raft/conf_change.rs +++ b/nodedb-cluster/src/multi_raft/conf_change.rs @@ -62,34 +62,16 @@ impl MultiRaft { group_id: u64, change: &ConfChange, ) -> Result<(u64, u64)> { + // Promoting a learner that has not caught up grows the voter set + // while it cannot ack, shrinking the effective quorum. Removing the + // last whole voter leaves no replica to lead and send a snapshot. + self.admit_conf_change(group_id, change)?; let (log_index, committed_immediately) = { let node = self .groups .get_mut(&group_id) .ok_or(ClusterError::GroupNotFound { group_id })?; - // PromoteLearner safety gate (propose side): a learner must have - // caught up to the group's commit index BEFORE the promote entry - // enters the log. Promoting an un-caught-up learner grows the - // voter set while it cannot yet ack anything, shrinking the - // effective quorum (e.g. 3 voters + 1 stale learner -> no - // majority) and risking cluster unavailability. The check cannot - // live in `apply_conf_change` — at apply time the commit index - // has already advanced past the promote entry itself, so it - // would spuriously reject promotions that were valid when - // proposed. - if change.change_type == ConfChangeType::PromoteLearner - && node.role() == nodedb_raft::NodeRole::Leader - && node.match_index_for(change.node_id).unwrap_or(0) < node.commit_index() - { - return Err(ClusterError::LearnerNotCaughtUp { - group_id, - node_id: change.node_id, - match_index: node.match_index_for(change.node_id).unwrap_or(0), - commit_index: node.commit_index(), - }); - } - let data = change.to_entry_data()?; let log_index = node.propose(data)?; // A single-voter group commits once its own disk holds the @@ -376,7 +358,7 @@ mod tests { success: true, last_log_index: last, round: nodedb_raft::node::leader_lease::UNTRACKED_ROUND, - needs_snapshot: false, + replica: nodedb_raft::ReplicaState::Whole, }; mr.handle_append_entries_response(0, 2, &resp).unwrap(); assert_eq!(mr.match_index_for(0, 2), Some(last)); diff --git a/nodedb-cluster/src/multi_raft/conf_change_gate.rs b/nodedb-cluster/src/multi_raft/conf_change_gate.rs new file mode 100644 index 000000000..b226a064e --- /dev/null +++ b/nodedb-cluster/src/multi_raft/conf_change_gate.rs @@ -0,0 +1,265 @@ +// SPDX-License-Identifier: BUSL-1.1 + +//! The leader's check on a configuration change before it enters the log. +//! +//! A replica awaiting a snapshot holds no state to lead from, and only a +//! leader sends it one. So a group must keep a whole voter: +//! - A learner is promoted only once it caught up and awaits no snapshot. +//! - A voter is removed only when a voter that may lead stays. The leader +//! may lead: a replica whose data diverged steps down and never +//! campaigns. A peer counts by the state its latest reply this term +//! reported: one that only wants a snapshot may lead, one whose data +//! diverged may not. A peer not heard this term does not count: its +//! state is unknown. +//! +//! Every proposal path runs the check: a change proposed as a conf change, +//! and conf-change bytes proposed as a plain entry. + +use nodedb_raft::{NodeRole, RaftError, ReplicaState}; + +use crate::conf_change::{ConfChange, ConfChangeType}; +use crate::error::{ClusterError, Result}; + +use super::core::MultiRaft; + +impl MultiRaft { + /// Refuse `change` to `group_id` when it would promote a learner that is + /// not caught up, or remove the last whole voter. A node that does not + /// lead the group admits it: its propose refuses with `NotLeader`. + pub(super) fn admit_conf_change(&self, group_id: u64, change: &ConfChange) -> Result<()> { + let node = self + .groups + .get(&group_id) + .ok_or(ClusterError::GroupNotFound { group_id })?; + if node.role() != NodeRole::Leader { + return Ok(()); + } + match change.change_type { + // At apply time the commit index has already passed the promote + // entry, so the check runs here, before the entry enters the log. + ConfChangeType::PromoteLearner => { + if self.peer_caught_up(group_id, change.node_id) { + return Ok(()); + } + Err(ClusterError::LearnerNotCaughtUp { + group_id, + node_id: change.node_id, + match_index: node.match_index_for(change.node_id).unwrap_or(0), + commit_index: node.commit_index(), + }) + } + ConfChangeType::RemoveNode => { + let whole_voter_stays = std::iter::once(self.node_id) + .chain(node.voters().iter().copied()) + .filter(|&voter| voter != change.node_id) + .any(|voter| { + if voter == self.node_id { + node.replica_state().may_lead() + } else { + node.peer_replica_state(voter) + .is_some_and(ReplicaState::may_lead) + } + }); + if whole_voter_stays { + return Ok(()); + } + Err(ClusterError::Raft(RaftError::ProposalRejected { + reason: format!( + "removing voter {} from group {group_id} leaves no voter that may \ + lead; every other voter is unheard this term or its data diverged; \ + retry once another voter answers that it may lead", + change.node_id + ), + })) + } + ConfChangeType::AddNode + | ConfChangeType::AddLearner + | ConfChangeType::RemoveLearner => Ok(()), + } + } +} + +#[cfg(test)] +mod tests { + use std::time::{Duration, Instant}; + + use nodedb_raft::message::AppendEntriesResponse; + use nodedb_raft::node::leader_lease::UNTRACKED_ROUND; + + use super::*; + use crate::routing::RoutingTable; + + /// Node 1 leads group 0 alone, with its election no-op committed. + fn leader_mr() -> (MultiRaft, tempfile::TempDir) { + let dir = tempfile::tempdir().expect("tempdir"); + let rt = RoutingTable::uniform(1, &[1], 1); + let mut mr = MultiRaft::new(1, rt, dir.path().to_path_buf()); + mr.add_group(0, vec![]).expect("add group"); + for node in mr.groups.values_mut() { + node.election_deadline_override(Instant::now() - Duration::from_millis(1)); + } + mr.tick().expect("tick"); + mr.wait_all_durable_blocking(); + mr.tick().expect("tick"); + assert!(mr.group_role_is_leader(0)); + (mr, dir) + } + + fn reply(mr: &MultiRaft, last_log_index: u64, replica: ReplicaState) -> AppendEntriesResponse { + AppendEntriesResponse { + term: mr.groups.get(&0).map_or(0, |node| node.current_term()), + success: !replica.wants_snapshot(), + last_log_index, + round: UNTRACKED_ROUND, + replica, + } + } + + fn change(change_type: ConfChangeType, node_id: u64) -> ConfChange { + ConfChange { + change_type, + node_id, + } + } + + /// The leader cannot remove itself while every other voter awaits a + /// snapshot: no replica would be left to lead and send it one. + #[test] + fn removing_the_last_whole_voter_is_refused() { + let (mut mr, _dir) = leader_mr(); + mr.apply_conf_change(0, &change(ConfChangeType::AddNode, 2)) + .expect("add voter 2"); + let awaiting = reply(&mr, 0, ReplicaState::AwaitingSnapshot); + mr.handle_append_entries_response(0, 2, &awaiting) + .expect("reply"); + + let refused = mr.propose_conf_change(0, &change(ConfChangeType::RemoveNode, 1)); + assert!( + matches!( + refused, + Err(ClusterError::Raft(RaftError::ProposalRejected { .. })) + ), + "got {refused:?}" + ); + let bytes = change(ConfChangeType::RemoveNode, 1) + .to_entry_data() + .expect("encode"); + assert!( + mr.propose_to_group(0, bytes).is_err(), + "the plain proposal path runs the same check" + ); + + assert!( + mr.admit_conf_change(0, &change(ConfChangeType::RemoveNode, 2)) + .is_ok(), + "removing the awaiting voter keeps the whole leader" + ); + } + + /// A fresh leader has heard no other voter this term: their state is + /// unknown, so it cannot remove itself. + #[test] + fn a_leader_that_heard_no_voter_cannot_remove_itself() { + let (mut mr, _dir) = leader_mr(); + for voter in [2, 3] { + mr.apply_conf_change(0, &change(ConfChangeType::AddNode, voter)) + .expect("add voter"); + } + let refused = mr.admit_conf_change(0, &change(ConfChangeType::RemoveNode, 1)); + assert!( + matches!( + refused, + Err(ClusterError::Raft(RaftError::ProposalRejected { .. })) + ), + "got {refused:?}" + ); + + let answer = reply(&mr, 0, ReplicaState::WantsSnapshot); + mr.handle_append_entries_response(0, 3, &answer) + .expect("reply"); + assert!( + mr.admit_conf_change(0, &change(ConfChangeType::RemoveNode, 1)) + .is_ok(), + "voter 3 answered and may lead" + ); + } + + /// Node 1 leads with voters 2 and 3, which both answered with `state`. + fn leader_with_two_voters_in(state: ReplicaState) -> (MultiRaft, tempfile::TempDir) { + let (mut mr, dir) = leader_mr(); + for voter in [2, 3] { + mr.apply_conf_change(0, &change(ConfChangeType::AddNode, voter)) + .expect("add voter"); + } + let answer = reply(&mr, 0, state); + for voter in [2, 3] { + mr.handle_append_entries_response(0, voter, &answer) + .expect("reply"); + } + (mr, dir) + } + + /// Voters that only want a snapshot for state beside their log may + /// lead: the leader can remove itself. Voters whose data diverged may + /// not: the removal is refused. + #[test] + fn a_voter_that_wants_a_snapshot_may_lead_and_a_diverged_one_may_not() { + let (mr, _dir) = leader_with_two_voters_in(ReplicaState::WantsSnapshot); + assert!( + mr.admit_conf_change(0, &change(ConfChangeType::RemoveNode, 1)) + .is_ok() + ); + assert!( + !mr.peer_caught_up(0, 2), + "a peer that wants a snapshot is behind" + ); + + let (mr, _dir) = leader_with_two_voters_in(ReplicaState::AwaitingSnapshot); + assert!(matches!( + mr.admit_conf_change(0, &change(ConfChangeType::RemoveNode, 1)), + Err(ClusterError::Raft(RaftError::ProposalRejected { .. })) + )); + assert!(!mr.peer_caught_up(0, 2)); + } + + #[test] + fn removing_a_voter_while_a_whole_voter_stays_is_admitted() { + let (mut mr, _dir) = leader_mr(); + mr.apply_conf_change(0, &change(ConfChangeType::AddNode, 2)) + .expect("add voter 2"); + let last = mr.groups.get(&0).map_or(0, |node| node.last_log_index()); + let whole = reply(&mr, last, ReplicaState::Whole); + mr.handle_append_entries_response(0, 2, &whole) + .expect("reply"); + assert!( + mr.admit_conf_change(0, &change(ConfChangeType::RemoveNode, 1)) + .is_ok() + ); + } + + /// A remounted learner whose log matches still awaits its snapshot: it + /// is promoted only once the snapshot installs. + #[test] + fn a_learner_awaiting_a_snapshot_is_not_promoted() { + let (mut mr, _dir) = leader_mr(); + mr.apply_conf_change(0, &change(ConfChangeType::AddLearner, 2)) + .expect("add learner 2"); + let last = mr.groups.get(&0).map_or(0, |node| node.last_log_index()); + let matched = reply(&mr, last, ReplicaState::Whole); + mr.handle_append_entries_response(0, 2, &matched) + .expect("reply"); + let awaiting = reply(&mr, last, ReplicaState::AwaitingSnapshot); + mr.handle_append_entries_response(0, 2, &awaiting) + .expect("reply"); + + assert!(mr.ready_learners(0).is_empty()); + assert!(matches!( + mr.admit_conf_change(0, &change(ConfChangeType::PromoteLearner, 2)), + Err(ClusterError::LearnerNotCaughtUp { .. }) + )); + + mr.handle_append_entries_response(0, 2, &matched) + .expect("reply"); + assert_eq!(mr.ready_learners(0), vec![2]); + } +} diff --git a/nodedb-cluster/src/multi_raft/core.rs b/nodedb-cluster/src/multi_raft/core.rs index beaefed7d..9665f9a8e 100644 --- a/nodedb-cluster/src/multi_raft/core.rs +++ b/nodedb-cluster/src/multi_raft/core.rs @@ -74,11 +74,12 @@ pub struct MultiRaft { pub(super) snapshot_requirement: Option, } -/// Whether this node's replica of a data group must take a snapshot before -/// it takes log entries. Called with the group id and the first index the +/// What this node's replica of a data group needs before it takes log +/// entries: nothing, a snapshot it may lead without, or a snapshot it must +/// not lead without. Called with the group id and the first index the /// Calvin sequencer log still holds here, under the `MultiRaft` lock: it /// must not take that lock. -pub type SnapshotRequirement = Arc bool + Send + Sync>; +pub type SnapshotRequirement = Arc nodedb_raft::ReplicaState + Send + Sync>; /// Aggregated output from all Raft groups after a tick. #[derive(Debug, Default)] @@ -148,7 +149,7 @@ impl MultiRaft { let sequencer_first = self.sequencer_first_available(); for (&group_id, node) in &mut self.groups { if is_data_group(group_id) { - node.set_snapshot_required(requirement(group_id, sequencer_first)); + node.set_replica_state(requirement(group_id, sequencer_first)); } } self.snapshot_requirement = Some(requirement); diff --git a/nodedb-cluster/src/multi_raft/membership.rs b/nodedb-cluster/src/multi_raft/membership.rs index 59c83fa5e..118fa7252 100644 --- a/nodedb-cluster/src/multi_raft/membership.rs +++ b/nodedb-cluster/src/multi_raft/membership.rs @@ -9,8 +9,8 @@ //! - `ready_learners(group)`: used by the tick loop's "promote //! caught-up learners" phase — returns every learner in the group //! whose `match_index` on this (leader) node is at least the current -//! `commit_index`, i.e. learners that have replicated enough log to be -//! safely promoted. +//! `commit_index` and that answered whole this term, i.e. learners that hold +//! the group's state and enough log to be safely promoted. //! - `group_leader(group)`: leader id observed by this node's local //! RaftNode state, used by the join flow to decide redirect vs admit. //! - `group_role_is_leader(group)`: cheap leader-check helper. @@ -36,8 +36,10 @@ impl MultiRaft { self.groups.get(&group_id).map(|n| n.commit_index()) } - /// Learners in `group_id` whose `match_index` on this leader has - /// caught up to the current `commit_index` — safe to promote. + /// Learners in `group_id` that caught up to this leader's commit index + /// and answered whole this term — safe to promote. A remounted learner whose + /// log matches still lacks the group's state until its snapshot + /// installs. /// /// Returns an empty vec if this node is not the leader of the group /// or the group is not hosted here. @@ -45,14 +47,10 @@ impl MultiRaft { let Some(node) = self.groups.get(&group_id) else { return Vec::new(); }; - if node.role() != NodeRole::Leader { - return Vec::new(); - } - let commit = node.commit_index(); node.learners() .iter() .copied() - .filter(|&learner| node.match_index_for(learner).unwrap_or(0) >= commit) + .filter(|&learner| self.peer_caught_up(group_id, learner)) .collect() } @@ -89,15 +87,16 @@ impl MultiRaft { } /// Whether `peer` holds every entry this leader has committed in - /// `group_id`, and did not ask for a snapshot last. `false` when this - /// node does not lead the group. + /// `group_id`, and its latest reply this term reported it whole. `false` + /// when this node does not lead the group, or has not heard `peer` this + /// term. pub fn peer_caught_up(&self, group_id: u64, peer: u64) -> bool { let Some(node) = self.groups.get(&group_id) else { return false; }; node.role() == NodeRole::Leader && node.match_index_for(peer).unwrap_or(0) >= node.commit_index() - && !node.peer_awaits_snapshot(peer) + && node.peer_answered_whole(peer) } /// Initiate a leadership transfer for `group_id` to `target`. @@ -114,20 +113,12 @@ impl MultiRaft { node.transfer_leadership(target).map_err(ClusterError::Raft) } - /// Hand this node's leadership of `group_id` to another voter that holds - /// every committed entry and did not ask for a snapshot last. Returns the - /// target. `None` when this node does not lead the group, no voter - /// qualifies, or the transfer did not start. + /// Hand this node's leadership of `group_id` to the voter + /// `RaftNode::hand_off_target` names. Returns the target. `None` when + /// this node does not lead the group, no voter qualifies, or the transfer + /// did not start. pub fn hand_off_leadership(&mut self, group_id: u64) -> Option { - let membership = self.group_membership(group_id)?; - if membership.leader_id != self.node_id { - return None; - } - let target = membership - .voters - .iter() - .copied() - .find(|&voter| voter != self.node_id && self.peer_caught_up(group_id, voter))?; + let target = self.groups.get(&group_id)?.hand_off_target()?; self.transfer_leadership(group_id, target).ok()?; Some(target) } diff --git a/nodedb-cluster/src/multi_raft/mod.rs b/nodedb-cluster/src/multi_raft/mod.rs index e285def53..493d92a59 100644 --- a/nodedb-cluster/src/multi_raft/mod.rs +++ b/nodedb-cluster/src/multi_raft/mod.rs @@ -14,6 +14,8 @@ //! corresponding response handlers. //! - [`conf_change`]: `propose_conf_change` / `apply_conf_change` with //! proper voter / learner / promotion semantics. +//! - [`conf_change_gate`]: the leader's check that a change keeps a voter +//! that holds the group's state. //! - [`membership`]: learner catch-up / promotion helpers //! (`commit_index_for`, `ready_learners`, etc.) driven by the tick loop. //! - [`membership_sync`]: set a group's voters and learners from routing @@ -21,6 +23,7 @@ //! - [`read_index`]: leadership confirmation for linearizable reads. pub mod conf_change; +pub mod conf_change_gate; pub mod core; pub mod membership; pub mod membership_sync; diff --git a/nodedb-cluster/src/multi_raft/mount.rs b/nodedb-cluster/src/multi_raft/mount.rs index 3f277a650..c149c3909 100644 --- a/nodedb-cluster/src/multi_raft/mount.rs +++ b/nodedb-cluster/src/multi_raft/mount.rs @@ -121,7 +121,7 @@ impl MultiRaft { if super::core::is_data_group(group_id) && let Some(requirement) = self.snapshot_requirement.as_ref() { - node.set_snapshot_required(requirement(group_id, self.sequencer_first_available())); + node.set_replica_state(requirement(group_id, self.sequencer_first_available())); } self.groups.insert(group_id, node); self.apply_gates.mount(group_id); diff --git a/nodedb-cluster/src/multi_raft/peer_contact.rs b/nodedb-cluster/src/multi_raft/peer_contact.rs index 6adfbae23..cb10491fd 100644 --- a/nodedb-cluster/src/multi_raft/peer_contact.rs +++ b/nodedb-cluster/src/multi_raft/peer_contact.rs @@ -4,7 +4,7 @@ use std::time::Duration; -use nodedb_raft::{NodeRole, StalenessVerdict}; +use nodedb_raft::NodeRole; use crate::multi_raft::core::MultiRaft; @@ -43,7 +43,7 @@ impl MultiRaft { pub fn leader_contact_within(&self, group_id: u64, window: Duration) -> bool { self.groups .get(&group_id) - .is_some_and(|node| node.staleness_verdict(window) != StalenessVerdict::NoRecentContact) + .is_some_and(|node| node.leader_contact_within(window)) } /// Response counts from every peer of `group_id`. `None` when the group diff --git a/nodedb-cluster/src/multi_raft/proposals.rs b/nodedb-cluster/src/multi_raft/proposals.rs index 5d8033ee7..31f8f0082 100644 --- a/nodedb-cluster/src/multi_raft/proposals.rs +++ b/nodedb-cluster/src/multi_raft/proposals.rs @@ -4,6 +4,7 @@ use nodedb_raft::RaftNode; +use crate::conf_change::ConfChange; use crate::error::{ClusterError, Result}; use crate::group_disk::StagedLogStorage; use crate::multi_raft::core::MultiRaft; @@ -53,6 +54,9 @@ impl MultiRaft { .read() .unwrap_or_else(|p| p.into_inner()) .group_for_vshard(vshard_id)?; + if let Some(change) = ConfChange::from_entry_data(&data) { + self.admit_conf_change(group_id, &change)?; + } let node = self .groups .get_mut(&group_id) @@ -78,6 +82,9 @@ impl MultiRaft { /// /// Returns the committed log index on success. pub fn propose_to_group(&mut self, group_id: u64, data: Vec) -> Result { + if let Some(change) = ConfChange::from_entry_data(&data) { + self.admit_conf_change(group_id, &change)?; + } let node = self .groups .get_mut(&group_id) @@ -123,13 +130,19 @@ impl MultiRaft { .map(|n| n.first_available_index()) } - /// Make this node's replica of `group_id` refuse log entries until a - /// snapshot installs, or lift that. Its leader answers the refusal with - /// a snapshot. Returns whether this node hosts the group. - pub fn set_snapshot_required(&mut self, group_id: u64, required: bool) -> bool { + /// Raise what this node's replica of `group_id` needs to `state`, never + /// lowering it: it refuses log entries until a snapshot installs, and + /// its leader answers the refusal with a snapshot. A replica whose data + /// diverged also steps down from leadership and never campaigns. Only an + /// install lifts it (see [`Self::refresh_snapshot_requirement`]). + /// Returns whether this node hosts the group. + pub fn require_snapshot(&mut self, group_id: u64, state: nodedb_raft::ReplicaState) -> bool { match self.groups.get_mut(&group_id) { Some(node) => { - node.set_snapshot_required(required); + let raised = node.replica_state().raised_to(state); + if raised != node.replica_state() { + node.set_replica_state(raised); + } true } None => false, @@ -159,7 +172,7 @@ impl MultiRaft { let sequencer_first = self.sequencer_first_available(); for group_id in groups { if let Some(node) = self.groups.get_mut(&group_id) { - node.set_snapshot_required(requirement(group_id, sequencer_first)); + node.set_replica_state(requirement(group_id, sequencer_first)); } } } @@ -182,6 +195,12 @@ impl MultiRaft { .is_some_and(|node| node.snapshot_required()) } + /// What this node's replica of `group_id` needs before it takes log + /// entries. `None` when the group is not mounted here. + pub fn replica_state(&self, group_id: u64) -> Option { + self.groups.get(&group_id).map(|node| node.replica_state()) + } + /// Auto-compact a group's log if its configured threshold has been /// reached, given the DATA-PLANE applied watermark `applied_index`. /// diff --git a/nodedb-cluster/src/multi_raft/read_index.rs b/nodedb-cluster/src/multi_raft/read_index.rs index 92edcdc36..92601186d 100644 --- a/nodedb-cluster/src/multi_raft/read_index.rs +++ b/nodedb-cluster/src/multi_raft/read_index.rs @@ -51,11 +51,20 @@ impl MultiRaft { } /// Whether this node's replica of `group_id` is within `max_staleness` - /// of the leader. False for a group not hosted here. - pub fn within_staleness_bound(&self, group_id: u64, max_staleness: Duration) -> bool { - self.groups - .get(&group_id) - .is_some_and(|node| node.within_staleness_bound(max_staleness)) + /// of the leader. `applied` is the highest index whose state reads see + /// here, the host's applied watermark, never Raft's hand-off index. + /// False for a group not hosted here, and for a replica that awaits a + /// snapshot install: its data diverged from the log. + pub fn within_staleness_bound( + &self, + group_id: u64, + max_staleness: Duration, + applied: u64, + ) -> bool { + self.groups.get(&group_id).is_some_and(|node| { + node.replica_state() != nodedb_raft::ReplicaState::AwaitingSnapshot + && node.within_staleness_bound(max_staleness, applied) + }) } } diff --git a/nodedb-cluster/src/multi_raft/status.rs b/nodedb-cluster/src/multi_raft/status.rs index 2a019f7bc..863053cc1 100644 --- a/nodedb-cluster/src/multi_raft/status.rs +++ b/nodedb-cluster/src/multi_raft/status.rs @@ -22,6 +22,9 @@ pub struct GroupStatus { /// lagging peer below this index can only be caught up via /// `InstallSnapshot`, never `AppendEntries`. pub snapshot_index: u64, + /// What this replica needs before it takes log entries. A replica that + /// awaits a snapshot applies nothing until one installs. + pub replica_state: nodedb_raft::ReplicaState, pub member_count: usize, pub learner_count: usize, pub vshard_count: usize, @@ -100,6 +103,7 @@ impl MultiRaft { last_applied: node.last_applied(), last_log_index: node.last_log_index(), snapshot_index: node.log_snapshot_index(), + replica_state: node.replica_state(), member_count: node.voters().len() + usize::from(self_is_voter), learner_count: node.learners().len() + usize::from(node.role() == nodedb_raft::NodeRole::Learner), diff --git a/nodedb-cluster/src/raft_loop/tick/snapshot_dispatch.rs b/nodedb-cluster/src/raft_loop/tick/snapshot_dispatch.rs index dfd6c0be1..ceaae2a9f 100644 --- a/nodedb-cluster/src/raft_loop/tick/snapshot_dispatch.rs +++ b/nodedb-cluster/src/raft_loop/tick/snapshot_dispatch.rs @@ -326,7 +326,7 @@ async fn send_snapshot( success: false, last_log_index: 0, round: nodedb_raft::node::leader_lease::UNTRACKED_ROUND, - needs_snapshot: false, + replica: nodedb_raft::ReplicaState::Whole, }, ) { tracing::error!(group_id, peer, resp_term, error = %e, "apply higher term from install_snapshot response"); diff --git a/nodedb-cluster/src/routing.rs b/nodedb-cluster/src/routing.rs index 87c97784d..2030d85bd 100644 --- a/nodedb-cluster/src/routing.rs +++ b/nodedb-cluster/src/routing.rs @@ -321,9 +321,11 @@ impl RoutingTable { self.group_members.len() } - /// All group IDs. + /// All group IDs, in ascending order. pub fn group_ids(&self) -> Vec { - self.group_members.keys().copied().collect() + let mut ids: Vec = self.group_members.keys().copied().collect(); + ids.sort_unstable(); + ids } /// Update the voting members of a Raft group (for membership changes). diff --git a/nodedb-cluster/src/rpc_codec/data_propose.rs b/nodedb-cluster/src/rpc_codec/data_propose.rs index 6bd70dae7..9d188e155 100644 --- a/nodedb-cluster/src/rpc_codec/data_propose.rs +++ b/nodedb-cluster/src/rpc_codec/data_propose.rs @@ -291,6 +291,7 @@ mod tests { std::time::Duration::from_millis(u64::MAX) + REPLY_MARGIN ); } + use crate::rpc_codec::{decode, encode}; fn roundtrip(target: ProposeTarget) -> DataProposeRequest { diff --git a/nodedb-cluster/src/rpc_codec/raft_msgs.rs b/nodedb-cluster/src/rpc_codec/raft_msgs.rs index 4518f7c98..8d57a2c1f 100644 --- a/nodedb-cluster/src/rpc_codec/raft_msgs.rs +++ b/nodedb-cluster/src/rpc_codec/raft_msgs.rs @@ -214,21 +214,27 @@ mod tests { #[test] fn roundtrip_append_entries_response() { - let resp = AppendEntriesResponse { - term: 5, - success: true, - last_log_index: 100, - round: 11, - needs_snapshot: true, - }; - match roundtrip(RaftRpc::AppendEntriesResponse(resp)) { - RaftRpc::AppendEntriesResponse(d) => { - assert_eq!(d.term, 5); - assert_eq!(d.round, 11); - assert!(d.success); - assert!(d.needs_snapshot); + for replica in [ + nodedb_raft::ReplicaState::Whole, + nodedb_raft::ReplicaState::WantsSnapshot, + nodedb_raft::ReplicaState::AwaitingSnapshot, + ] { + let resp = AppendEntriesResponse { + term: 5, + success: true, + last_log_index: 100, + round: 11, + replica, + }; + match roundtrip(RaftRpc::AppendEntriesResponse(resp)) { + RaftRpc::AppendEntriesResponse(d) => { + assert_eq!(d.term, 5); + assert_eq!(d.round, 11); + assert!(d.success); + assert_eq!(d.replica, replica, "the follower's state crosses the wire"); + } + other => panic!("expected AppendEntriesResponse, got {other:?}"), } - other => panic!("expected AppendEntriesResponse, got {other:?}"), } } diff --git a/nodedb-cluster/src/rpc_codec/raft_rpc.rs b/nodedb-cluster/src/rpc_codec/raft_rpc.rs index 5e6ebb452..414e9c598 100644 --- a/nodedb-cluster/src/rpc_codec/raft_rpc.rs +++ b/nodedb-cluster/src/rpc_codec/raft_rpc.rs @@ -472,7 +472,7 @@ mod tests { success: true, last_log_index: 5, round: 3, - needs_snapshot: false, + replica: nodedb_raft::ReplicaState::Whole, }); let encoded = encode(&rpc, &crate::cluster_epoch::ClusterEpochState::default()).unwrap(); let header: [u8; HEADER_SIZE] = encoded[..HEADER_SIZE].try_into().unwrap(); diff --git a/nodedb-cluster/src/transport/client/attempt.rs b/nodedb-cluster/src/transport/client/attempt.rs index 3ad90fd5c..e12c22649 100644 --- a/nodedb-cluster/src/transport/client/attempt.rs +++ b/nodedb-cluster/src/transport/client/attempt.rs @@ -24,6 +24,11 @@ //! A live peer acknowledges the request within its ACK delay. A read that //! times out on a connection that received datagrams meanwhile is a slow //! handler, and the connection stays pooled. +//! +//! A peer silent for the RPC timeout is dead or cut off: its connection hit +//! the QUIC idle timeout, or a dial got no answer. The call ends at once with +//! no resend to that peer, so the caller can try another node. A written +//! request then ends as `ClusterError::Unanswered`. use std::time::Duration; @@ -59,15 +64,18 @@ struct LinkFailure { /// Whether the whole request was written before the failure. The peer /// can have run a written request. written: bool, + /// The peer sent nothing for the RPC timeout. No resend goes to it. + silent: bool, } impl LinkFailure { - /// A failure before the request went out that leaves the pool as it is. - fn keep_connection(error: ClusterError) -> Self { + /// A dial the peer never answered. + fn from_dial(failed: super::pool::DialError) -> Self { Self { - error: Box::new(error), + error: Box::new(failed.error), condemned: None, written: false, + silent: failed.silent, } } @@ -85,10 +93,16 @@ impl LinkFailure { /// `Ok` holds the error kept while the next attempt goes out. `Err` ends the /// call. A written request that is not [`RaftRpc::resend_safe`] can have run /// on the peer. It is never resent and ends as `ClusterError::Unanswered`. +/// +/// A silent peer gets no resend: a written request ends as `Unanswered`, an +/// unsent one with its own error. fn resend_verdict(failure: LinkFailure, target: u64, resend_safe: bool) -> Result { - if failure.written && !resend_safe { + if failure.written && (failure.silent || !resend_safe) { return Err(failure.into_unanswered(target)); } + if failure.silent { + return Err(*failure.error); + } if RetryPolicy::is_retryable(&failure.error) { Ok(*failure.error) } else { @@ -234,15 +248,16 @@ impl NexarTransport { read_timeout: Duration, ) -> std::result::Result, LinkFailure> { let conn = self - .get_or_connect(target) + .connect_peer(target) .await - .map_err(LinkFailure::keep_connection)?; + .map_err(LinkFailure::from_dial)?; let stable_id = conn.stable_id(); if let Err(error) = self.verify_connection_target(&conn, target) { return Err(LinkFailure { error: Box::new(error), condemned: Some(stable_id), written: false, + silent: false, }); } let received_before = received_datagrams(&conn); @@ -255,11 +270,13 @@ impl NexarTransport { StreamFailure::Unanswered(error) => (error, false, true), }; let heard = received_datagrams(&conn) > received_before; - let broken = condemns_connection(timed_out, conn.close_reason().is_some(), heard); + let close_reason = conn.close_reason(); + let broken = condemns_connection(timed_out, close_reason.is_some(), heard); LinkFailure { error: Box::new(error), condemned: broken.then_some(stable_id), written, + silent: matches!(close_reason, Some(quinn::ConnectionError::TimedOut)), } }) } @@ -349,7 +366,13 @@ fn received_datagrams(conn: &quinn::Connection) -> u64 { mod tests { use nodedb_raft::message::RequestVoteRequest; - use super::super::transport::tests::{STALL, STALLED_GROUP, serve_echo}; + use std::net::SocketAddr; + use std::sync::Arc; + use std::sync::atomic::{AtomicBool, Ordering}; + + use super::super::transport::tests::{ + SLOW, SLOW_GROUP, STALL, STALLED_GROUP, serve_echo, serve_echo_with_rpc_timeout, + }; use super::*; use crate::rpc_codec::{FrameRefusal, PongResponse, RequestRefusal}; @@ -399,6 +422,106 @@ mod tests { assert_eq!(client.circuit_breaker().failure_count(1), 1); } + /// RPC timeout, and so QUIC idle timeout, of the liveness tests. + const LIVENESS_RPC_TIMEOUT: Duration = Duration::from_millis(300); + + fn vote(group_id: u64) -> RaftRpc { + RaftRpc::RequestVoteRequest(RequestVoteRequest { + term: 1, + candidate_id: 2, + last_log_index: 0, + last_log_term: 0, + group_id, + transfer: false, + }) + } + + /// A handler slower than the idle timeout on a live peer gets the whole + /// read timeout: keep-alives hold the connection open. + #[tokio::test] + async fn a_stalled_handler_on_a_live_peer_answers_past_the_idle_timeout() { + let (_server, client, _shutdown) = serve_echo_with_rpc_timeout(LIVENESS_RPC_TIMEOUT).await; + client.warm_peer(1).await.expect("warm"); + let pooled = client.peer_connection_stable_id(1).expect("pooled"); + assert!(SLOW > LIVENESS_RPC_TIMEOUT * 3); + + let reply = client + .send_rpc_with_read_timeout(1, vote(SLOW_GROUP), Duration::from_secs(2)) + .await + .expect("a live peer answers"); + + assert!( + matches!(reply, RaftRpc::RequestVoteResponse(_)), + "{reply:?}" + ); + assert_eq!(client.peer_connection_stable_id(1), Some(pooled)); + assert_eq!(client.circuit_breaker().failure_count(1), 0); + } + + /// A UDP relay between a client and a server that drops every datagram + /// once `cut` is set. + async fn relay(server: SocketAddr, cut: Arc) -> SocketAddr { + let socket = Arc::new( + tokio::net::UdpSocket::bind("127.0.0.1:0") + .await + .expect("bind relay"), + ); + let addr = socket.local_addr().expect("relay address"); + tokio::spawn(async move { + let mut client: Option = None; + let mut buf = vec![0u8; 65_536]; + while let Ok((len, from)) = socket.recv_from(&mut buf).await { + if cut.load(Ordering::SeqCst) { + continue; + } + let to = if from == server { + match client { + Some(client) => client, + None => continue, + } + } else { + client = Some(from); + server + }; + let _ = socket.send_to(&buf[..len], to).await; + } + }); + addr + } + + /// A peer cut off mid-read fails a long read at the idle timeout. The + /// read is not resent to it, and its connection leaves the pool. + #[tokio::test] + async fn a_silent_peer_fails_a_long_read_within_the_idle_timeout() { + let (server, client, _shutdown) = serve_echo_with_rpc_timeout(LIVENESS_RPC_TIMEOUT).await; + let cut = Arc::new(AtomicBool::new(false)); + client.register_peer(1, relay(server.local_addr(), Arc::clone(&cut)).await); + client.warm_peer(1).await.expect("warm through the relay"); + assert!(client.peer_connection_stable_id(1).is_some()); + + cut.store(true, Ordering::SeqCst); + let started = tokio::time::Instant::now(); + let err = client + .send_rpc_with_read_timeout(1, vote(STALLED_GROUP), Duration::from_secs(5)) + .await + .unwrap_err(); + let elapsed = started.elapsed(); + + assert!( + matches!(err, ClusterError::Unanswered { node_id: 1, .. }), + "{err:?}" + ); + let slack = Duration::from_millis(700); + assert!( + elapsed < LIVENESS_RPC_TIMEOUT + slack, + "failed after {elapsed:?}, idle timeout {LIVENESS_RPC_TIMEOUT:?}" + ); + assert!( + !client.peers.read().unwrap().contains_key(&1), + "the dead connection is evicted" + ); + } + fn lost(written: bool) -> LinkFailure { LinkFailure { error: Box::new(ClusterError::Transport { @@ -406,6 +529,7 @@ mod tests { }), condemned: None, written, + silent: false, } } @@ -439,6 +563,34 @@ mod tests { )); } + fn silent(written: bool) -> LinkFailure { + LinkFailure { + silent: true, + ..lost(written) + } + } + + /// A silent peer is dead or cut off. A written read is not resent to it + /// and ends unanswered, so the caller can try another node. + #[test] + fn a_written_request_to_a_silent_peer_ends_unanswered() { + for resend_safe in [false, true] { + match resend_verdict(silent(true), 2, resend_safe) { + Err(ClusterError::Unanswered { node_id: 2, .. }) => {} + other => panic!("expected Unanswered, got {other:?}"), + } + } + } + + /// A request a silent peer never got ends with its own error, unsent. + #[test] + fn an_unsent_request_to_a_silent_peer_is_not_resent() { + assert!(matches!( + resend_verdict(silent(false), 2, true), + Err(ClusterError::Transport { .. }) + )); + } + #[test] fn a_typed_refusal_is_a_final_answer() { let refusal = RaftRpc::RequestRefused(RequestRefusal::from(ClusterError::GroupNotFound { @@ -463,10 +615,10 @@ mod tests { #[test] fn only_the_outer_error_is_a_link_failure() { - let link = LinkFailure::keep_connection(ClusterError::Transport { - detail: "connection lost".into(), - }); - assert!(matches!(classify(1, Err(link)), Attempt::LinkFailed(_))); + assert!(matches!( + classify(1, Err(lost(false))), + Attempt::LinkFailed(_) + )); let unreadable = ClusterError::Codec { detail: "bad crc".into(), diff --git a/nodedb-cluster/src/transport/client/pool.rs b/nodedb-cluster/src/transport/client/pool.rs index bbb97c8c9..ee83c8e3e 100644 --- a/nodedb-cluster/src/transport/client/pool.rs +++ b/nodedb-cluster/src/transport/client/pool.rs @@ -71,15 +71,25 @@ impl NexarTransport { /// a dial that ended while it waited. Only a caller with neither dials, /// so a burst of sends to one peer opens one connection. pub(super) async fn get_or_connect(&self, target: u64) -> Result { + self.connect_peer(target) + .await + .map_err(|failed| failed.error) + } + + /// [`get_or_connect`](Self::get_or_connect), with whether the peer stayed + /// silent through the dial. + pub(super) async fn connect_peer( + &self, + target: u64, + ) -> std::result::Result { if let Some(conn) = self.pooled_connection(target) { return Ok(conn); } let addr = { let addrs = self.peer_addrs.read().unwrap_or_else(|p| p.into_inner()); - addrs - .get(&target) - .copied() - .ok_or(ClusterError::NodeUnreachable { node_id: target })? + addrs.get(&target).copied().ok_or(DialError::answered( + ClusterError::NodeUnreachable { node_id: target }, + ))? }; let waiting_since = Instant::now(); @@ -90,10 +100,14 @@ impl NexarTransport { return Ok(conn); } if let Some(failure) = last_dial.failed_since(waiting_since) { - return Err(ClusterError::Transport { - detail: format!( - "dial to node {target} at {addr} failed while this send waited: {failure}" - ), + return Err(DialError { + error: ClusterError::Transport { + detail: format!( + "dial to node {target} at {addr} failed while this send waited: {}", + failure.detail + ), + }, + silent: failure.silent, }); } self.drop_closed_connection(target); @@ -104,37 +118,51 @@ impl NexarTransport { peers.insert(target, conn.clone()); Ok(conn) } - Err(error) => { + Err(failed) => { last_dial.failure = Some(DialFailure { at: Instant::now(), - detail: error.to_string(), + detail: failed.error.to_string(), + silent: failed.silent, }); - Err(error) + Err(failed) } } } /// Connect to `target` at `addr` and negotiate the wire version. - async fn dial(&self, target: u64, addr: SocketAddr) -> Result { - // Connect — bounded by rpc_timeout so a hung QUIC handshake - // (peer not yet serving) doesn't block for the full 30s idle timeout. + /// + /// A peer that answers nothing within the RPC timeout fails the dial as + /// silent: it is dead or cut off, and a redial in the same call is waste. + async fn dial( + &self, + target: u64, + addr: SocketAddr, + ) -> std::result::Result { + let silent = || { + DialError::silent(ClusterError::Transport { + detail: format!( + "handshake timeout ({}ms) with node {target} at {addr}", + self.rpc_timeout.as_millis() + ), + }) + }; let connecting = self .listener .endpoint() .connect_with(self.client_config.clone(), addr, SNI_HOSTNAME) - .map_err(|e| ClusterError::Transport { - detail: format!("connect to node {target} at {addr}: {e}"), + .map_err(|e| { + DialError::answered(ClusterError::Transport { + detail: format!("connect to node {target} at {addr}: {e}"), + }) })?; let conn = tokio::time::timeout(self.rpc_timeout, connecting) .await - .map_err(|_| ClusterError::Transport { - detail: format!( - "handshake timeout ({}ms) with node {target} at {addr}", - self.rpc_timeout.as_millis() - ), - })? - .map_err(|e| ClusterError::Transport { - detail: format!("handshake with node {target} at {addr}: {e}"), + .map_err(|_| silent())? + .map_err(|e| match e { + quinn::ConnectionError::TimedOut => silent(), + other => DialError::answered(ClusterError::Transport { + detail: format!("handshake with node {target} at {addr}: {other}"), + }), })?; debug!(target, %addr, "connected to peer"); @@ -142,21 +170,18 @@ impl NexarTransport { // Open a dedicated bidi stream for the wire-version handshake. // This must complete before any RPC stream is opened on this connection. let agreed = { - let (mut hs_send, mut hs_recv) = - conn.open_bi().await.map_err(|e| ClusterError::Transport { + let (mut hs_send, mut hs_recv) = conn.open_bi().await.map_err(|e| { + DialError::answered(ClusterError::Transport { detail: format!("open handshake stream to node {target} at {addr}: {e}"), - })?; + }) + })?; let version = tokio::time::timeout( self.rpc_timeout, perform_version_handshake_client(&mut hs_send, &mut hs_recv), ) .await - .map_err(|_| ClusterError::Transport { - detail: format!( - "handshake timeout ({}ms) with node {target} at {addr}", - self.rpc_timeout.as_millis() - ), - })??; + .map_err(|_| silent())? + .map_err(DialError::answered)?; // Finish the handshake send stream — the server reads it until FIN. let _ = hs_send.finish(); version @@ -208,12 +233,40 @@ impl NexarTransport { } } +/// A failed dial, and whether the peer stayed silent through it. +#[derive(Debug)] +pub(super) struct DialError { + pub(super) error: ClusterError, + /// The peer sent nothing within the RPC timeout: it is dead or cut off. + pub(super) silent: bool, +} + +impl DialError { + /// A dial the peer refused or broke off: the peer is up. + fn answered(error: ClusterError) -> Self { + Self { + error, + silent: false, + } + } + + /// A dial the peer never answered within the RPC timeout. + fn silent(error: ClusterError) -> Self { + Self { + error, + silent: true, + } + } +} + /// The last failed dial to one peer. #[derive(Debug)] struct DialFailure { /// When the dial ended. at: Instant, detail: String, + /// The peer stayed silent through the dial. + silent: bool, } /// Outcome of the last dial to one peer, behind the peer's dial gate. @@ -224,11 +277,8 @@ struct LastDial { impl LastDial { /// The failure of a dial that ended at or after `since`. - fn failed_since(&self, since: Instant) -> Option<&str> { - self.failure - .as_ref() - .filter(|failure| failure.at >= since) - .map(|failure| failure.detail.as_str()) + fn failed_since(&self, since: Instant) -> Option<&DialFailure> { + self.failure.as_ref().filter(|failure| failure.at >= since) } } diff --git a/nodedb-cluster/src/transport/client/send.rs b/nodedb-cluster/src/transport/client/send.rs index fc6942836..d51ad9023 100644 --- a/nodedb-cluster/src/transport/client/send.rs +++ b/nodedb-cluster/src/transport/client/send.rs @@ -178,6 +178,9 @@ impl NexarTransport { /// There is NO retry wrapper on the stream body — retry only applies to the /// eager pre-first-frame phase, which the caller (coordinator) owns. Once /// the first frame has been observed, any error is terminal. + /// + /// The request is written before the stream returns, so a link failure + /// while reading it is `ClusterError::Unanswered`: the peer can have run it. pub async fn send_rpc_stream( &self, target: u64, @@ -211,7 +214,9 @@ impl NexarTransport { // server-side `accept_bi` stream is not reset early. let _send = send; loop { - let envelope = read_envelope(&mut recv).await?; + let envelope = read_envelope(&mut recv) + .await + .map_err(|e| written_stream_failure(target, e))?; let (fields, inner_frame) = auth_envelope::parse_envelope(&envelope, &auth.mac_key)?; if fields.from_node_id != target { @@ -361,6 +366,20 @@ impl NexarTransport { } } +/// The error of a failed read on a stream whose request is written. +/// +/// A link failure leaves the outcome unknown, so it is `Unanswered`. A +/// malformed frame keeps its own error. +fn written_stream_failure(target: u64, error: ClusterError) -> ClusterError { + if let ClusterError::Transport { detail } = error { + return ClusterError::Unanswered { + node_id: target, + detail, + }; + } + error +} + /// Map a terminal [`TypedClusterError`] carried by `ExecuteStreamEnd` into a /// [`ClusterError`] for the stream's `Err` item. /// diff --git a/nodedb-cluster/src/transport/client/transport.rs b/nodedb-cluster/src/transport/client/transport.rs index 74c569bf7..0e4608cb0 100644 --- a/nodedb-cluster/src/transport/client/transport.rs +++ b/nodedb-cluster/src/transport/client/transport.rs @@ -15,6 +15,7 @@ use crate::error::{ClusterError, Result}; use crate::transport::auth_context::AuthContext; use crate::transport::config; use crate::transport::credentials::{self, TransportCredentials}; +use crate::transport::endpoint::RaftListener; use crate::transport::peer_identity_store::{NoopIdentityStore, PeerIdentityStore}; use crate::transport::topology_identity_store::TopologyIdentityStore; @@ -33,7 +34,7 @@ use crate::transport::topology_identity_store::TopologyIdentityStore; /// [`serve`]: Self::serve pub struct NexarTransport { pub(super) node_id: u64, - pub(super) listener: nexar::TransportListener, + pub(super) listener: RaftListener, pub(super) client_config: quinn::ClientConfig, /// Cached connections to peers. Stale connections are replaced on next use. pub(super) peers: RwLock>, @@ -114,8 +115,8 @@ impl NexarTransport { /// Create a new transport using values from `ClusterTransportTuning`. /// - /// All QUIC parameters (streams, windows, keep-alive, idle timeout) and - /// the RPC timeout are read from `tuning`. `creds` selects channel-level + /// All QUIC parameters (streams, windows) and the RPC timeout are read + /// from `tuning`. The idle timeout and keep-alive follow the RPC timeout. `creds` selects channel-level /// authentication. Use this in production so that operators can override /// defaults via the `[tuning.cluster_transport]` section of `config.toml`. /// @@ -185,14 +186,24 @@ impl NexarTransport { let (server_config, client_config) = match &creds { TransportCredentials::Mtls(tls) => ( - config::make_raft_server_config_mtls(tls, tuning, Arc::clone(&identity_store))?, - config::make_raft_client_config_mtls(tls, tuning, Arc::clone(&identity_store))?, + config::make_raft_server_config_mtls( + tls, + tuning, + rpc_timeout, + Arc::clone(&identity_store), + )?, + config::make_raft_client_config_mtls( + tls, + tuning, + rpc_timeout, + Arc::clone(&identity_store), + )?, ), TransportCredentials::Insecure => { credentials::announce_insecure_transport(node_id); ( - config::make_raft_server_config(tuning)?, - config::make_raft_client_config(tuning)?, + config::make_raft_server_config(tuning, rpc_timeout)?, + config::make_raft_client_config(tuning, rpc_timeout)?, ) } }; @@ -204,10 +215,7 @@ impl NexarTransport { let auth = Arc::new(AuthContext::from_credentials(node_id, &creds)); - let listener = nexar::TransportListener::bind_with_config(listen_addr, server_config) - .map_err(|e| ClusterError::Transport { - detail: format!("bind {listen_addr}: {e}"), - })?; + let listener = RaftListener::bind(node_id, listen_addr, server_config)?; info!( node_id, @@ -248,6 +256,12 @@ impl NexarTransport { &self.circuit_breaker } + /// The fixed timeout of one RPC on this transport, from + /// `ClusterTransportTuning::rpc_timeout_secs`. + pub fn rpc_timeout(&self) -> Duration { + self.rpc_timeout + } + /// The local address this transport is listening on. pub fn local_addr(&self) -> SocketAddr { self.listener.local_addr() @@ -449,6 +463,12 @@ pub(super) mod tests { /// How long `EchoHandler` holds a vote request of [`STALLED_GROUP`]. pub(crate) const STALL: Duration = Duration::from_secs(2); + /// The group whose vote requests `EchoHandler` answers after [`SLOW`]. + pub(crate) const SLOW_GROUP: u64 = 504; + + /// How long `EchoHandler` holds a vote request of [`SLOW_GROUP`]. + pub(crate) const SLOW: Duration = Duration::from_secs(1); + /// AppendEntries requests `EchoHandler` refused for `UNHOSTED_GROUP`. static UNHOSTED_APPENDS: AtomicU32 = AtomicU32::new(0); @@ -471,7 +491,14 @@ pub(super) mod tests { success: true, last_log_index: req.prev_log_index + req.entries.len() as u64, round: req.round, - needs_snapshot: false, + replica: nodedb_raft::ReplicaState::Whole, + })) + } + RaftRpc::RequestVoteRequest(req) if req.group_id == SLOW_GROUP => { + tokio::time::sleep(SLOW).await; + Ok(RaftRpc::RequestVoteResponse(RequestVoteResponse { + term: req.term, + vote_granted: true, })) } RaftRpc::RequestVoteRequest(req) if req.group_id == STALLED_GROUP => { @@ -969,6 +996,36 @@ pub(super) mod tests { (server, client, shutdown_tx) } + /// [`serve_echo`] with `rpc_timeout` on both nodes, so the QUIC idle + /// timeout is `rpc_timeout` too. + pub(crate) async fn serve_echo_with_rpc_timeout( + rpc_timeout: Duration, + ) -> ( + Arc, + Arc, + tokio::sync::watch::Sender, + ) { + let bind = |node_id| { + NexarTransport::with_timeout( + node_id, + "127.0.0.1:0".parse().unwrap(), + rpc_timeout, + TransportCredentials::Insecure, + ) + .unwrap() + }; + let server = Arc::new(bind(1)); + let client = Arc::new(bind(2)); + client.register_peer(1, server.local_addr()); + let (shutdown_tx, shutdown_rx) = tokio::sync::watch::channel(false); + let srv = server.clone(); + tokio::spawn(async move { + srv.serve(Arc::new(EchoHandler), shutdown_rx).await.unwrap(); + }); + tokio::time::sleep(Duration::from_millis(20)).await; + (server, client, shutdown_tx) + } + #[tokio::test] async fn a_handler_refusal_is_answered_and_not_resent() { let (_server, client, _shutdown) = serve_echo().await; diff --git a/nodedb-cluster/src/transport/config.rs b/nodedb-cluster/src/transport/config.rs index 36ac0042d..e23e9b7fd 100644 --- a/nodedb-cluster/src/transport/config.rs +++ b/nodedb-cluster/src/transport/config.rs @@ -9,6 +9,7 @@ use std::time::Duration; use nodedb_types::config::tuning::ClusterTransportTuning; use crate::error::{ClusterError, Result}; +use crate::transport::liveness::raft_transport_config; use crate::transport::peer_identity_store::PeerIdentityStore; use crate::transport::pinned_verifier::{PinnedClientVerifier, PinnedServerVerifier}; @@ -42,30 +43,6 @@ pub const SNI_HOSTNAME: &str = "nodedb"; /// constructing `NexarTransport::with_tuning()` with a custom `ClusterTransportTuning`. pub const DEFAULT_RPC_TIMEOUT: Duration = Duration::from_secs(5); -/// Transport config tuned for Raft RPCs in a datacenter. -/// -/// All values are read from `tuning`. Pass `&ClusterTransportTuning::default()` -/// to get the same behaviour as the previous hardcoded defaults. -pub fn raft_transport_config(tuning: &ClusterTransportTuning) -> quinn::TransportConfig { - let mut config = quinn::TransportConfig::default(); - // Raft RPCs use bidi streams: one per request-response pair. - config.max_concurrent_bidi_streams(quinn::VarInt::from_u32(tuning.quic_max_bi_streams)); - // Also allow uni streams for future migration/snapshot streaming. - config.max_concurrent_uni_streams(quinn::VarInt::from_u32(tuning.quic_max_uni_streams)); - // Datacenter tuning: generous windows, low RTT estimate. - config.receive_window(quinn::VarInt::from_u32(tuning.quic_receive_window)); - config.send_window(u64::from(tuning.quic_send_window)); - config.stream_receive_window(quinn::VarInt::from_u32(tuning.quic_stream_receive_window)); - config.initial_rtt(Duration::from_micros(100)); - config.keep_alive_interval(Some(Duration::from_secs(tuning.quic_keep_alive_secs))); - config.max_idle_timeout(Some( - Duration::from_secs(tuning.quic_idle_timeout_secs) - .try_into() - .expect("idle timeout fits IdleTimeout"), - )); - config -} - /// Build a QUIC server config with self-signed TLS (unauthenticated). /// /// **Crate-private** — reachable only via @@ -75,6 +52,7 @@ pub fn raft_transport_config(tuning: &ClusterTransportTuning) -> quinn::Transpor /// Production clusters use the mTLS variant. pub(crate) fn make_raft_server_config( tuning: &ClusterTransportTuning, + rpc_timeout: Duration, ) -> Result { ensure_rustls_crypto_provider(); let (cert, key) = nexar::transport::tls::generate_self_signed_cert().map_err(|e| { @@ -103,7 +81,7 @@ pub(crate) fn make_raft_server_config( })?; let mut server_config = quinn::ServerConfig::with_crypto(Arc::new(quic_crypto)); - server_config.transport_config(Arc::new(raft_transport_config(tuning))); + server_config.transport_config(Arc::new(raft_transport_config(tuning, rpc_timeout)?)); Ok(server_config) } @@ -116,6 +94,7 @@ pub(crate) fn make_raft_server_config( /// Production clusters use the mTLS variant. pub(crate) fn make_raft_client_config( tuning: &ClusterTransportTuning, + rpc_timeout: Duration, ) -> Result { ensure_rustls_crypto_provider(); let provider = rustls::crypto::ring::default_provider(); @@ -136,7 +115,7 @@ pub(crate) fn make_raft_client_config( })?; let mut client_config = quinn::ClientConfig::new(Arc::new(quic_crypto)); - client_config.transport_config(Arc::new(raft_transport_config(tuning))); + client_config.transport_config(Arc::new(raft_transport_config(tuning, rpc_timeout)?)); Ok(client_config) } @@ -188,6 +167,7 @@ pub struct TlsCredentials { pub fn make_raft_server_config_mtls( creds: &TlsCredentials, tuning: &ClusterTransportTuning, + rpc_timeout: Duration, identity_store: Arc, ) -> Result { ensure_rustls_crypto_provider(); @@ -247,7 +227,7 @@ pub fn make_raft_server_config_mtls( })?; let mut server_config = quinn::ServerConfig::with_crypto(Arc::new(quic_crypto)); - server_config.transport_config(Arc::new(raft_transport_config(tuning))); + server_config.transport_config(Arc::new(raft_transport_config(tuning, rpc_timeout)?)); Ok(server_config) } @@ -261,6 +241,7 @@ pub fn make_raft_server_config_mtls( pub fn make_raft_client_config_mtls( creds: &TlsCredentials, tuning: &ClusterTransportTuning, + rpc_timeout: Duration, identity_store: Arc, ) -> Result { ensure_rustls_crypto_provider(); @@ -320,7 +301,7 @@ pub fn make_raft_client_config_mtls( })?; let mut client_config = quinn::ClientConfig::new(Arc::new(quic_crypto)); - client_config.transport_config(Arc::new(raft_transport_config(tuning))); + client_config.transport_config(Arc::new(raft_transport_config(tuning, rpc_timeout)?)); Ok(client_config) } diff --git a/nodedb-cluster/src/transport/endpoint.rs b/nodedb-cluster/src/transport/endpoint.rs new file mode 100644 index 000000000..7de0b48f6 --- /dev/null +++ b/nodedb-cluster/src/transport/endpoint.rs @@ -0,0 +1,169 @@ +// SPDX-License-Identifier: BUSL-1.1 + +//! The QUIC endpoint a node serves and dials Raft RPCs on. +//! +//! A build with the `failpoints` feature binds the endpoint over a UDP socket +//! that drops every datagram while the node's `transport::blackhole` fail +//! point is armed. The node then looks partitioned to its peers while its +//! process keeps running. Any other build binds a plain UDP socket. + +use std::net::SocketAddr; + +use crate::error::{ClusterError, Result}; + +/// Node-scoped fail point that cuts a node off the network while armed. +pub const BLACKHOLE_FAIL_POINT: &str = "transport::blackhole"; + +/// A bound QUIC endpoint that accepts Raft RPC connections. +#[derive(Debug)] +pub struct RaftListener { + endpoint: quinn::Endpoint, + local_addr: SocketAddr, +} + +impl RaftListener { + /// Bind the endpoint of node `node_id` on `addr`. + pub fn bind(node_id: u64, addr: SocketAddr, config: quinn::ServerConfig) -> Result { + let endpoint = + bind_endpoint(node_id, addr, config).map_err(|e| ClusterError::Transport { + detail: format!("bind {addr}: {e}"), + })?; + let local_addr = endpoint.local_addr().map_err(|e| ClusterError::Transport { + detail: format!("local address of {addr}: {e}"), + })?; + Ok(Self { + endpoint, + local_addr, + }) + } + + /// Accept the next incoming QUIC connection. + pub async fn accept(&self) -> Result { + let incoming = self + .endpoint + .accept() + .await + .ok_or_else(|| ClusterError::Transport { + detail: "endpoint closed".into(), + })?; + incoming.await.map_err(|e| ClusterError::Transport { + detail: format!("accept: {e}"), + }) + } + + /// The local address the endpoint is bound to. + pub fn local_addr(&self) -> SocketAddr { + self.local_addr + } + + /// The endpoint, which also dials outbound connections. + pub fn endpoint(&self) -> &quinn::Endpoint { + &self.endpoint + } +} + +#[cfg(not(feature = "failpoints"))] +fn bind_endpoint( + _node_id: u64, + addr: SocketAddr, + config: quinn::ServerConfig, +) -> std::io::Result { + quinn::Endpoint::server(config, addr) +} + +#[cfg(feature = "failpoints")] +fn bind_endpoint( + node_id: u64, + addr: SocketAddr, + config: quinn::ServerConfig, +) -> std::io::Result { + let socket = std::net::UdpSocket::bind(addr)?; + let runtime = + quinn::default_runtime().ok_or_else(|| std::io::Error::other("no async runtime found"))?; + let inner = runtime.wrap_udp_socket(socket)?; + let socket = std::sync::Arc::new(blackhole::BlackholeSocket::new(node_id, inner)); + quinn::Endpoint::new_with_abstract_socket( + quinn::EndpointConfig::default(), + Some(config), + socket, + runtime, + ) +} + +#[cfg(feature = "failpoints")] +mod blackhole { + use std::io::{self, IoSliceMut}; + use std::net::SocketAddr; + use std::pin::Pin; + use std::sync::Arc; + use std::task::{Context, Poll}; + + use nodedb_types::fail_point::{FailScope, lookup}; + use quinn::udp::{RecvMeta, Transmit}; + use quinn::{AsyncUdpSocket, UdpPoller}; + + use super::BLACKHOLE_FAIL_POINT; + + /// A UDP socket that drops every datagram, both ways, while its node's + /// blackhole fail point is armed. + #[derive(Debug)] + pub(super) struct BlackholeSocket { + node_id: u64, + inner: Arc, + } + + impl BlackholeSocket { + pub(super) fn new(node_id: u64, inner: Arc) -> Self { + Self { node_id, inner } + } + + fn cut_off(&self) -> bool { + lookup(FailScope::Node(self.node_id), BLACKHOLE_FAIL_POINT).is_some() + } + } + + impl AsyncUdpSocket for BlackholeSocket { + fn create_io_poller(self: Arc) -> Pin> { + Arc::clone(&self.inner).create_io_poller() + } + + fn try_send(&self, transmit: &Transmit) -> io::Result<()> { + if self.cut_off() { + return Ok(()); + } + self.inner.try_send(transmit) + } + + fn poll_recv( + &self, + cx: &mut Context, + bufs: &mut [IoSliceMut<'_>], + meta: &mut [RecvMeta], + ) -> Poll> { + // Received datagrams are drained and dropped until the inner + // socket has none left, so it registers the waker. + loop { + match self.inner.poll_recv(cx, bufs, meta) { + Poll::Ready(Ok(_)) if self.cut_off() => {} + other => return other, + } + } + } + + fn local_addr(&self) -> io::Result { + self.inner.local_addr() + } + + fn max_transmit_segments(&self) -> usize { + self.inner.max_transmit_segments() + } + + fn max_receive_segments(&self) -> usize { + self.inner.max_receive_segments() + } + + fn may_fragment(&self) -> bool { + self.inner.may_fragment() + } + } +} diff --git a/nodedb-cluster/src/transport/liveness.rs b/nodedb-cluster/src/transport/liveness.rs new file mode 100644 index 000000000..43557c6af --- /dev/null +++ b/nodedb-cluster/src/transport/liveness.rs @@ -0,0 +1,104 @@ +// SPDX-License-Identifier: BUSL-1.1 + +//! QUIC transport settings for Raft RPCs, and peer liveness. +//! +//! The QUIC idle timeout is the RPC timeout. A peer that sends nothing for +//! that long, keep-alives included, is dead: its connection closes and every +//! pending read on it fails. A live peer's keep-alives hold the connection +//! open while a slow handler runs, so a long read gets its whole budget. + +use std::time::Duration; + +use nodedb_types::config::tuning::ClusterTransportTuning; + +use crate::error::{ClusterError, Result}; + +/// Smallest RPC timeout a node starts with. +/// +/// The RPC timeout is also the QUIC idle timeout. A shorter one drops a live +/// peer whose keep-alives meet a short scheduling stall. +pub const MIN_RPC_TIMEOUT: Duration = Duration::from_millis(500); + +/// Shortest keep-alive interval, so a short RPC timeout cannot flood a peer. +const MIN_KEEP_ALIVE: Duration = Duration::from_millis(100); + +/// Keep-alives sent per idle timeout. Several must be lost in a row before +/// a live peer reads as silent. +const KEEP_ALIVES_PER_IDLE_TIMEOUT: u32 = 5; + +/// The keep-alive interval for `rpc_timeout`: a fifth of it, at least 100 ms. +pub fn keep_alive_interval(rpc_timeout: Duration) -> Duration { + (rpc_timeout / KEEP_ALIVES_PER_IDLE_TIMEOUT).max(MIN_KEEP_ALIVE) +} + +/// Transport config tuned for Raft RPCs in a datacenter. +/// +/// Peer liveness is the QUIC idle timeout, set to `rpc_timeout`. Keep-alives +/// hold a live connection open while a handler runs past it. A peer silent +/// for `rpc_timeout` closes the connection, and every pending read on it fails. +pub fn raft_transport_config( + tuning: &ClusterTransportTuning, + rpc_timeout: Duration, +) -> Result { + let idle_timeout = + quinn::IdleTimeout::try_from(rpc_timeout).map_err(|e| ClusterError::Config { + detail: format!( + "rpc timeout {}ms does not fit a QUIC idle timeout: {e}", + rpc_timeout.as_millis() + ), + })?; + let mut config = quinn::TransportConfig::default(); + // Raft RPCs use bidi streams: one per request-response pair. + config.max_concurrent_bidi_streams(quinn::VarInt::from_u32(tuning.quic_max_bi_streams)); + // Also allow uni streams for future migration/snapshot streaming. + config.max_concurrent_uni_streams(quinn::VarInt::from_u32(tuning.quic_max_uni_streams)); + // Datacenter tuning: generous windows, low RTT estimate. + config.receive_window(quinn::VarInt::from_u32(tuning.quic_receive_window)); + config.send_window(u64::from(tuning.quic_send_window)); + config.stream_receive_window(quinn::VarInt::from_u32(tuning.quic_stream_receive_window)); + config.initial_rtt(Duration::from_micros(100)); + config.keep_alive_interval(Some(keep_alive_interval(rpc_timeout))); + config.max_idle_timeout(Some(idle_timeout)); + Ok(config) +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn the_keep_alive_is_a_fifth_of_the_rpc_timeout() { + assert_eq!( + keep_alive_interval(Duration::from_secs(5)), + Duration::from_secs(1) + ); + assert_eq!( + keep_alive_interval(Duration::from_millis(500)), + Duration::from_millis(100) + ); + } + + #[test] + fn the_keep_alive_never_drops_below_its_floor() { + assert_eq!( + keep_alive_interval(Duration::from_millis(300)), + MIN_KEEP_ALIVE + ); + } + + #[test] + fn the_idle_timeout_follows_the_rpc_timeout() { + let tuning = ClusterTransportTuning::default(); + assert!(raft_transport_config(&tuning, Duration::from_millis(300)).is_ok()); + assert!(raft_transport_config(&tuning, Duration::from_secs(5)).is_ok()); + } + + #[test] + fn an_idle_timeout_past_the_quic_range_is_refused() { + let tuning = ClusterTransportTuning::default(); + assert!(matches!( + raft_transport_config(&tuning, Duration::MAX), + Err(ClusterError::Config { .. }) + )); + } +} diff --git a/nodedb-cluster/src/transport/mod.rs b/nodedb-cluster/src/transport/mod.rs index 5e0cecad5..ab1325967 100644 --- a/nodedb-cluster/src/transport/mod.rs +++ b/nodedb-cluster/src/transport/mod.rs @@ -4,8 +4,10 @@ pub mod auth_context; pub mod client; pub mod config; pub mod credentials; +pub mod endpoint; mod frame_io; mod identity_admission; +pub mod liveness; pub mod peer_identity_store; pub mod peer_identity_verifier; pub mod pinned_verifier; diff --git a/nodedb-cluster/src/transport/server.rs b/nodedb-cluster/src/transport/server.rs index a52417851..95f1e637c 100644 --- a/nodedb-cluster/src/transport/server.rs +++ b/nodedb-cluster/src/transport/server.rs @@ -343,9 +343,12 @@ async fn handle_stream( reply_and_finish(&mut send, &auth, &response, "response").await }; + // A dead connection can carry no answer, so its request stops using the + // executor. Handlers already tolerate the drop: shutdown drops `work` too. let outcome = tokio::select! { biased; _ = shutdown.changed() => Ok(false), + _ = conn.closed() => Ok(false), result = work => result.map(|()| true), }; if let Ok(true) = outcome { diff --git a/nodedb-physical/src/physical_plan/array.rs b/nodedb-physical/src/physical_plan/array.rs index b01e0b039..b0e3606dd 100644 --- a/nodedb-physical/src/physical_plan/array.rs +++ b/nodedb-physical/src/physical_plan/array.rs @@ -13,7 +13,7 @@ use nodedb_array::types::ArrayId; use nodedb_types::sync::wire::SyncProvenance; -use nodedb_types::{SurrogateBitmap, SystemTimeScope}; +use nodedb_types::{ShardBucket, SurrogateBitmap, SystemTimeScope}; /// Reducer for [`ArrayOp::Aggregate`]. Numeric `c_enum` keeps the /// wire encoding to a single byte. @@ -81,7 +81,7 @@ pub enum ArrayOp { array_id: ArrayId, schema_msgpack: Vec, schema_hash: u64, - /// Hilbert-prefix bits for vShard routing. Immutable post-create. + /// Routing bucket bits for vShard routing. Immutable post-create. prefix_bits: u8, /// Array audit retention selected by CREATE ARRAY. This stays on the /// authorized task so planning remains read-only. @@ -150,12 +150,10 @@ pub enum ArrayOp { /// Only cells whose surrogate is present in this bitmap are included. /// `None` = no restriction. cell_filter: Option, - /// Optional Hilbert-prefix range `[lo, hi]`. When set, only cells - /// whose Hilbert prefix falls within this range are included. - /// Used by distributed shard slice to prevent duplicate rows in - /// single-node harnesses where all vShards share one Data Plane. - /// `None` = no Hilbert filter (all cells included). - hilbert_range: Option<(u64, u64)>, + /// Optional shard bucket. When set, only cells whose tile key routes + /// to this bucket are included, so vShards that share one Data + /// Plane return no duplicate rows. `None` includes every cell. + shard_bucket: Option, /// Bitemporal system-time scope. /// /// - `Current`: live read (returns the latest committed state). @@ -193,12 +191,10 @@ pub enum ArrayOp { /// merge partials across shards before finalizing. /// Single-node SQL callers always pass `false`. return_partial: bool, - /// Optional Hilbert-prefix range `[lo, hi]`. When set, only cells - /// whose Hilbert prefix falls within this range are included. - /// Used by distributed shard agg to prevent double-counting in - /// single-node harnesses where all vShards share one Data Plane. - /// `None` = no Hilbert filter (all cells included). - hilbert_range: Option<(u64, u64)>, + /// Optional shard bucket. When set, only cells whose tile key routes + /// to this bucket contribute, so vShards that share one Data Plane + /// count no cell twice. `None` includes every cell. + shard_bucket: Option, /// Bitemporal system-time cutoff. When `Some(t)`, only tile versions /// with `system_from_ms <= t` contribute to the aggregate. `None` = /// live read. @@ -309,7 +305,7 @@ mod tests { group_by_dim: -1, cell_filter: None, return_partial: false, - hilbert_range: None, + shard_bucket: None, system_as_of: None, valid_at_ms: None, }; @@ -327,7 +323,7 @@ mod tests { attr_projection: vec![], limit: 0, cell_filter: None, - hilbert_range: None, + shard_bucket: None, system_time: SystemTimeScope::AllVersions, valid_at_ms: None, }; @@ -355,7 +351,7 @@ mod tests { attr_projection: vec![], limit: 10, cell_filter: None, - hilbert_range: None, + shard_bucket: None, system_time: SystemTimeScope::AsOf(1_700_000_000_000), valid_at_ms: Some(1_600_000_000_000), }; diff --git a/nodedb-physical/src/physical_plan/cluster_array.rs b/nodedb-physical/src/physical_plan/cluster_array.rs index add4e26d0..acf6487c2 100644 --- a/nodedb-physical/src/physical_plan/cluster_array.rs +++ b/nodedb-physical/src/physical_plan/cluster_array.rs @@ -30,9 +30,9 @@ use nodedb_types::SystemTimeScope; pub enum ClusterArrayOp { /// Fan-out a slice query across all target vShards and merge rows. /// - /// `slice_hilbert_ranges` — pre-computed `(lo, hi)` Hilbert-prefix - /// ranges that determine which shards to contact. An empty vec means - /// "scan all shards". + /// `slice_tile_keys` — pre-computed tile keys, at least one per routing + /// bucket the slice touches. They pick the shards to contact. An empty + /// vec scans every shard. /// /// `prefix_bits` — routing granularity from the array catalog entry. Slice { @@ -43,8 +43,8 @@ pub enum ClusterArrayOp { attr_projection: Vec, /// Maximum rows returned. 0 = unlimited. limit: u32, - /// Pre-computed Hilbert-prefix ranges for shard selection. - slice_hilbert_ranges: Vec<(u64, u64)>, + /// Pre-computed tile keys for shard selection. + slice_tile_keys: Vec, /// Routing granularity from the array catalog entry. prefix_bits: u8, /// Bitemporal system-time scope for this slice fan-out. @@ -68,8 +68,8 @@ pub enum ClusterArrayOp { reducer_msgpack: Vec, /// Dimension index to group by (-1 = no grouping). group_by_dim: i32, - /// Pre-computed Hilbert ranges for shard selection (empty = all). - slice_hilbert_ranges: Vec<(u64, u64)>, + /// Pre-computed tile keys for shard selection (empty = all). + slice_tile_keys: Vec, /// Routing granularity. prefix_bits: u8, /// Bitemporal system-time cutoff. `None` = live read. @@ -78,16 +78,16 @@ pub enum ClusterArrayOp { valid_at_ms: Option, }, - /// Fan-out a cell write batch, partitioned by Hilbert tile. + /// Fan-out a cell write batch, partitioned by routing bucket. /// - /// `cells` — list of `(hilbert_prefix, zerompk-encoded single-cell - /// bytes)`. The coordinator partitions these by prefix and sends each - /// shard's subset via `coord_put`. + /// `cells` — list of `(tile_key, zerompk-encoded single-cell bytes)`. + /// The coordinator partitions these by the tile key's vShard and sends + /// each shard's subset via `coord_put`. Put { array_id: ArrayId, /// zerompk-encoded `ArrayId` bytes for the wire request. array_id_msgpack: Vec, - /// Cells as `(hilbert_prefix, cell_msgpack)` pairs. + /// Cells as `(tile_key, cell_msgpack)` pairs. cells: Vec<(u64, Vec)>, /// WAL LSN allocated by the Control Plane for this batch. wal_lsn: u64, @@ -95,14 +95,14 @@ pub enum ClusterArrayOp { prefix_bits: u8, }, - /// Fan-out a coord delete batch, partitioned by Hilbert tile. + /// Fan-out a coord delete batch, partitioned by routing bucket. /// - /// `coords` — `(hilbert_prefix, zerompk-encoded single-coord bytes)`. + /// `coords` — `(tile_key, zerompk-encoded single-coord bytes)`. Delete { array_id: ArrayId, /// zerompk-encoded `ArrayId` bytes for the wire request. array_id_msgpack: Vec, - /// Coords as `(hilbert_prefix, coord_msgpack)` pairs. + /// Coords as `(tile_key, coord_msgpack)` pairs. coords: Vec<(u64, Vec)>, /// WAL LSN. wal_lsn: u64, diff --git a/nodedb-raft/src/lib.rs b/nodedb-raft/src/lib.rs index 203c93d80..d54544787 100644 --- a/nodedb-raft/src/lib.rs +++ b/nodedb-raft/src/lib.rs @@ -35,6 +35,6 @@ pub use snapshot_framing::{ SNAPSHOT_FORMAT_VERSION, SNAPSHOT_MAGIC, SnapshotEngineId, SnapshotFramingError, decode_snapshot_chunk, encode_snapshot_chunk, }; -pub use state::{HardState, NodeRole, PeerRole}; +pub use state::{HardState, NodeRole, PeerRole, ReplicaState}; pub use storage::LogStorage; pub use transport::RaftTransport; diff --git a/nodedb-raft/src/message.rs b/nodedb-raft/src/message.rs index 368a054b3..e83281f63 100644 --- a/nodedb-raft/src/message.rs +++ b/nodedb-raft/src/message.rs @@ -89,9 +89,10 @@ pub struct AppendEntriesResponse { pub last_log_index: u64, /// `round` of the request this answers. pub round: u64, - /// Set on a rejection from a follower that holds no state it can resume - /// the log from. The leader sends it a snapshot instead of entries. - pub needs_snapshot: bool, + /// What the follower needs before it takes entries. A rejection from a + /// follower that wants a snapshot gets one instead of entries. The + /// leader records it per peer: a peer whose data diverged cannot lead. + pub replica: crate::state::ReplicaState, } /// RequestVote RPC (Raft paper Figure 2). diff --git a/nodedb-raft/src/node/core.rs b/nodedb-raft/src/node/core.rs index d895ec21a..f2b319664 100644 --- a/nodedb-raft/src/node/core.rs +++ b/nodedb-raft/src/node/core.rs @@ -16,7 +16,7 @@ use crate::error::{RaftError, Result}; use crate::log::RaftLog; use crate::message::{AppendEntriesRequest, LogEntry, PreVoteRequest, TimeoutNowRequest}; use crate::state::{ - HardState, LeaderState, LeadershipTransfer, NodeRole, PreVoteRound, VolatileState, + HardState, LeaderState, LeadershipTransfer, NodeRole, PreVoteRound, ReplicaState, VolatileState, }; use crate::storage::LogStorage; @@ -134,10 +134,12 @@ pub struct RaftNode { /// An outside bound on compaction: the log never discards an entry above /// it. An archiver that must copy entries before they go raises it. pub(super) compaction_ceiling: Option>, - /// Whether this follower holds no state it can resume the log from. It - /// then refuses every `AppendEntries` with `needs_snapshot`, and its - /// leader sends a snapshot instead. The driver sets and clears it. - pub(super) snapshot_required: bool, + /// Whether this replica holds the state it can resume the log from. One + /// that wants a snapshot refuses every `AppendEntries`, reporting this + /// state, and its leader sends a snapshot instead. One whose + /// data diverged also never campaigns. The driver sets and clears it + /// (see [`super::snapshot_debt`]). + pub(super) replica: ReplicaState, /// The highest `replicated_floor` a leader sent this node, or this /// node's own as leader. See [`Self::replicated_floor`]. pub(super) replicated_floor: u64, @@ -189,23 +191,12 @@ impl RaftNode { lease: LeaseState::new(), boot_vote_fence: now + config.election_timeout_max, compaction_ceiling: None, - snapshot_required: false, + replica: ReplicaState::Whole, replicated_floor: 0, config, } } - /// Require a snapshot before this follower takes log entries again, or - /// lift the requirement. See [`Self::snapshot_required`]. - pub fn set_snapshot_required(&mut self, required: bool) { - self.snapshot_required = required; - } - - /// Whether this follower refuses log entries until a snapshot installs. - pub fn snapshot_required(&self) -> bool { - self.snapshot_required - } - /// Restore state from persistent storage. Must be called before ticking. /// /// Seeds `volatile.last_applied` from the durable applied index so diff --git a/nodedb-raft/src/node/election.rs b/nodedb-raft/src/node/election.rs new file mode 100644 index 000000000..2cff12512 --- /dev/null +++ b/nodedb-raft/src/node/election.rs @@ -0,0 +1,121 @@ +// SPDX-License-Identifier: BUSL-1.1 + +//! Election entry points: the pre-vote round and the vote round. A replica +//! whose data diverged starts neither. + +use std::collections::HashSet; + +use tracing::{debug, info}; + +use crate::message::{PreVoteRequest, RequestVoteRequest}; +use crate::state::{NodeRole, PreVoteRound}; +use crate::storage::LogStorage; + +use super::core::RaftNode; + +impl RaftNode { + /// Ask peers whether they WOULD vote for this node at `current_term + 1`. + /// + /// Touches neither `current_term` nor `voted_for` and persists nothing, so + /// a node cut off from the cluster can retry indefinitely without inflating + /// its term and deposing the healthy leader when the partition heals. + pub(super) fn start_pre_election(&mut self) { + match self.role { + NodeRole::Learner | NodeRole::Observer => return, + NodeRole::Follower | NodeRole::Candidate | NodeRole::Leader => {} + } + // A replica whose data diverged cannot lead. + if !self.replica.may_lead() { + return; + } + + // Single voter: there is nobody to probe, and the existing immediate + // win must be preserved. + if self.config.peers.is_empty() { + self.start_election(); + return; + } + + // A round that wins nothing retries on the usual randomized backoff. + self.reset_election_timeout(); + + let term = self.hard_state.current_term + 1; + self.pre_vote = Some(PreVoteRound { + term, + granted: HashSet::new(), + }); + + debug!( + node = self.config.node_id, + group = self.config.group_id, + term, + "starting pre-vote round" + ); + + for &peer in &self.config.peers { + self.ready.pre_vote_requests.push(( + peer, + PreVoteRequest { + term, + candidate_id: self.config.node_id, + last_log_index: self.log.last_index(), + last_log_term: self.log.last_term(), + group_id: self.config.group_id, + }, + )); + } + } + + pub(super) fn start_election(&mut self) { + // Learners and observers never stand for election — defensive check + // in case `tick()` is bypassed (e.g., tests forcing a deadline). + match self.role { + NodeRole::Learner | NodeRole::Observer => return, + NodeRole::Follower | NodeRole::Candidate | NodeRole::Leader => {} + } + // A replica whose data diverged cannot lead, so it never asks for + // votes, and a single voter never elects itself. + if !self.replica.may_lead() { + return; + } + + self.pre_vote = None; + self.hard_state.current_term += 1; + self.role = NodeRole::Candidate; + self.hard_state.voted_for = self.config.node_id; + self.votes_received.clear(); + self.leader_id = 0; + + self.persist_hard_state(); + self.reset_election_timeout(); + + info!( + node = self.config.node_id, + group = self.config.group_id, + term = self.hard_state.current_term, + "starting election" + ); + + // Single-voter cluster: win immediately. (Learners in the group do + // not count — single voter + N learners still elects the single + // voter as leader.) + if self.config.peers.is_empty() { + self.become_leader(); + return; + } + + for &peer in &self.config.peers { + self.ready.vote_requests.push(( + peer, + RequestVoteRequest { + term: self.hard_state.current_term, + candidate_id: self.config.node_id, + last_log_index: self.log.last_index(), + last_log_term: self.log.last_term(), + group_id: self.config.group_id, + transfer: false, + }, + )); + } + } +} diff --git a/nodedb-raft/src/node/internal.rs b/nodedb-raft/src/node/internal.rs index cc9eb796a..006b96104 100644 --- a/nodedb-raft/src/node/internal.rs +++ b/nodedb-raft/src/node/internal.rs @@ -2,116 +2,19 @@ //! Internal state transitions and replication logic. -use std::collections::HashSet; use std::time::{Duration, Instant}; use rand::RngExt; use tracing::{debug, info}; use crate::error::{RaftError, Result}; -use crate::message::{AppendEntriesRequest, LogEntry, PreVoteRequest, RequestVoteRequest}; -use crate::state::{LeaderState, NodeRole, PreVoteRound}; +use crate::message::{AppendEntriesRequest, LogEntry}; +use crate::state::{LeaderState, NodeRole}; use crate::storage::LogStorage; use super::core::RaftNode; impl RaftNode { - /// Ask peers whether they WOULD vote for this node at `current_term + 1`. - /// - /// Touches neither `current_term` nor `voted_for` and persists nothing, so - /// a node cut off from the cluster can retry indefinitely without inflating - /// its term and deposing the healthy leader when the partition heals. - pub(super) fn start_pre_election(&mut self) { - match self.role { - NodeRole::Learner | NodeRole::Observer => return, - NodeRole::Follower | NodeRole::Candidate | NodeRole::Leader => {} - } - - // Single voter: there is nobody to probe, and the existing immediate - // win must be preserved. - if self.config.peers.is_empty() { - self.start_election(); - return; - } - - // A round that wins nothing retries on the usual randomized backoff. - self.reset_election_timeout(); - - let term = self.hard_state.current_term + 1; - self.pre_vote = Some(PreVoteRound { - term, - granted: HashSet::new(), - }); - - debug!( - node = self.config.node_id, - group = self.config.group_id, - term, - "starting pre-vote round" - ); - - for &peer in &self.config.peers { - self.ready.pre_vote_requests.push(( - peer, - PreVoteRequest { - term, - candidate_id: self.config.node_id, - last_log_index: self.log.last_index(), - last_log_term: self.log.last_term(), - group_id: self.config.group_id, - }, - )); - } - } - - pub(super) fn start_election(&mut self) { - // Learners and observers never stand for election — defensive check - // in case `tick()` is bypassed (e.g., tests forcing a deadline). - match self.role { - NodeRole::Learner | NodeRole::Observer => return, - NodeRole::Follower | NodeRole::Candidate | NodeRole::Leader => {} - } - - self.pre_vote = None; - self.hard_state.current_term += 1; - self.role = NodeRole::Candidate; - self.hard_state.voted_for = self.config.node_id; - self.votes_received.clear(); - self.leader_id = 0; - - self.persist_hard_state(); - self.reset_election_timeout(); - - info!( - node = self.config.node_id, - group = self.config.group_id, - term = self.hard_state.current_term, - "starting election" - ); - - // Single-voter cluster: win immediately. (Learners in the group do - // not count — single voter + N learners still elects the single - // voter as leader.) - if self.config.peers.is_empty() { - self.become_leader(); - return; - } - - for &peer in &self.config.peers { - self.ready.vote_requests.push(( - peer, - RequestVoteRequest { - term: self.hard_state.current_term, - candidate_id: self.config.node_id, - last_log_index: self.log.last_index(), - last_log_term: self.log.last_term(), - group_id: self.config.group_id, - transfer: false, - }, - )); - } - } - /// Step down to follower (or keep learner/observer role if the node is one). /// /// Learners and observers that receive an `AppendEntries` with a higher diff --git a/nodedb-raft/src/node/leader_lease.rs b/nodedb-raft/src/node/leader_lease.rs index bfc7ca729..047303958 100644 --- a/nodedb-raft/src/node/leader_lease.rs +++ b/nodedb-raft/src/node/leader_lease.rs @@ -305,7 +305,7 @@ mod tests { success: true, last_log_index, round, - needs_snapshot: false, + replica: crate::state::ReplicaState::Whole, }, ); } @@ -422,7 +422,7 @@ mod tests { success: false, last_log_index: 0, round: stale, - needs_snapshot: false, + replica: crate::state::ReplicaState::Whole, }, ); assert_eq!(node.role(), NodeRole::Follower); diff --git a/nodedb-raft/src/node/mod.rs b/nodedb-raft/src/node/mod.rs index 75c0ac82b..ae6c6a2e4 100644 --- a/nodedb-raft/src/node/mod.rs +++ b/nodedb-raft/src/node/mod.rs @@ -7,7 +7,9 @@ //! - [`core`]: `RaftNode` struct, constructors, simple accessors, `tick`, //! `propose`, and the `Ready` output type. //! - [`durability`]: Applied-index durability floor and log compaction. -//! - [`internal`]: Internal state transitions (elections, replication, +//! - [`election`]: The pre-vote and vote rounds a replica starts, and the +//! guard that keeps a replica whose data diverged out of both. +//! - [`internal`]: Internal state transitions (role changes, replication, //! commit advancement) and timeout math. //! - [`leader_lease`]: Serving a linearizable read from the leader's commit //! index without a quorum round, and the vote refusal that makes it safe. @@ -20,12 +22,15 @@ //! - [`rpc`]: Incoming RPC handlers (`AppendEntries`, `PreVote`, //! `RequestVote`, `InstallSnapshot`, `TimeoutNow`, and their response //! handlers). +//! - [`snapshot_debt`]: A replica awaiting a snapshot refuses entries and +//! never leads. //! - [`staleness`]: How far behind the leader a replica is, for //! bounded-staleness reads. pub mod config; pub mod core; pub mod durability; +mod election; mod internal; pub mod leader_lease; pub mod membership; @@ -33,6 +38,7 @@ pub mod peer_contact; pub mod quorum_contact; pub mod read_index; pub mod rpc; +pub mod snapshot_debt; pub mod staleness; pub use self::config::RaftConfig; diff --git a/nodedb-raft/src/node/peer_contact.rs b/nodedb-raft/src/node/peer_contact.rs index c8dcfe5bf..eb54a6d0d 100644 --- a/nodedb-raft/src/node/peer_contact.rs +++ b/nodedb-raft/src/node/peer_contact.rs @@ -6,7 +6,7 @@ use std::time::Instant; use crate::node::core::RaftNode; -use crate::state::NodeRole; +use crate::state::{NodeRole, ReplicaState}; use crate::storage::LogStorage; impl RaftNode { @@ -57,12 +57,19 @@ impl RaftNode { .max(lowest_live_voter.min(self.volatile.commit_index)) } - /// Whether `peer`'s latest response to this leader asked for a snapshot. - /// Such a peer holds no state to lead from. `false` off the leader. - pub fn peer_awaits_snapshot(&self, peer: u64) -> bool { + /// Whether `peer`'s latest response this term reported it whole. `false` + /// for a peer not heard this term, and off the leader. + pub fn peer_answered_whole(&self, peer: u64) -> bool { + self.peer_replica_state(peer) == Some(ReplicaState::Whole) + } + + /// `peer`'s state as its latest response this term reported it. `None` + /// for a peer not heard this term, and off the leader: its state is + /// unknown. + pub fn peer_replica_state(&self, peer: u64) -> Option { self.leader_state .as_ref() - .is_some_and(|leader| leader.awaiting_snapshot.contains(&peer)) + .and_then(|leader| leader.peer_states.get(&peer).copied()) } /// The leader this node names at `now`, when it has proof the leader is @@ -178,7 +185,7 @@ mod tests { success: true, last_log_index: 1, round: 1, - needs_snapshot: false, + replica: crate::state::ReplicaState::Whole, }; node.handle_append_entries_response(2, &ack); assert_eq!(node.replicated_floor(), 0, "voter 3 holds nothing yet"); @@ -216,7 +223,7 @@ mod tests { success, last_log_index, round: node.lease.next_round - 1, - needs_snapshot: false, + replica: crate::state::ReplicaState::Whole, }; node.handle_append_entries_response(peer, &resp); } diff --git a/nodedb-raft/src/node/quorum_contact.rs b/nodedb-raft/src/node/quorum_contact.rs index 74f32f7a2..f57c43feb 100644 --- a/nodedb-raft/src/node/quorum_contact.rs +++ b/nodedb-raft/src/node/quorum_contact.rs @@ -206,7 +206,7 @@ mod tests { success: true, last_log_index: node.last_log_index(), round, - needs_snapshot: false, + replica: crate::state::ReplicaState::Whole, }, ); } @@ -226,7 +226,7 @@ mod tests { success: false, last_log_index: 0, round: latest_round(&node), - needs_snapshot: false, + replica: crate::state::ReplicaState::Whole, }, ); @@ -257,7 +257,7 @@ mod tests { success: true, last_log_index: 1, round: latest_round(&node), - needs_snapshot: false, + replica: crate::state::ReplicaState::Whole, }, ); @@ -296,7 +296,7 @@ mod tests { success: true, last_log_index: node.last_log_index(), round: latest_round(&node), - needs_snapshot: false, + replica: crate::state::ReplicaState::Whole, }, ); @@ -380,7 +380,7 @@ mod tests { success: true, last_log_index: node.last_log_index(), round: latest_round(&node), - needs_snapshot: false, + replica: crate::state::ReplicaState::Whole, }, ); node.tick(); @@ -468,7 +468,7 @@ mod tests { success: false, last_log_index: 0, round: stale, - needs_snapshot: false, + replica: crate::state::ReplicaState::Whole, }, ); assert_eq!(node.role(), NodeRole::Follower); @@ -520,7 +520,7 @@ mod tests { success: true, last_log_index: node.last_log_index(), round: latest_round(&node), - needs_snapshot: false, + replica: crate::state::ReplicaState::Whole, }, ); diff --git a/nodedb-raft/src/node/read_index.rs b/nodedb-raft/src/node/read_index.rs index ca3111af0..ab1b970e5 100644 --- a/nodedb-raft/src/node/read_index.rs +++ b/nodedb-raft/src/node/read_index.rs @@ -190,7 +190,7 @@ mod tests { success, last_log_index: node.last_log_index(), round: node.lease.next_round - 1, - needs_snapshot: false, + replica: crate::state::ReplicaState::Whole, }, ); } @@ -268,7 +268,7 @@ mod tests { success: false, last_log_index: 0, round: 0, - needs_snapshot: false, + replica: crate::state::ReplicaState::Whole, }, ); diff --git a/nodedb-raft/src/node/rpc/append_entries.rs b/nodedb-raft/src/node/rpc/append_entries.rs index 95637f3fa..1bb0ca819 100644 --- a/nodedb-raft/src/node/rpc/append_entries.rs +++ b/nodedb-raft/src/node/rpc/append_entries.rs @@ -18,7 +18,7 @@ impl RaftNode { success: false, last_log_index: self.log.last_index(), round: req.round, - needs_snapshot: false, + replica: self.replica, }; } @@ -49,13 +49,13 @@ impl RaftNode { // A follower with no state to resume from takes no entries. The // leader answers the refusal with a snapshot. - if self.snapshot_required { + if self.replica.wants_snapshot() { return AppendEntriesResponse { term: self.hard_state.current_term, success: false, last_log_index: self.log.last_index(), round: req.round, - needs_snapshot: true, + replica: self.replica, }; } @@ -68,7 +68,7 @@ impl RaftNode { success: false, last_log_index: self.volatile.last_applied, round: req.round, - needs_snapshot: false, + replica: self.replica, }; } @@ -82,7 +82,7 @@ impl RaftNode { success: false, last_log_index: self.log.last_index(), round: req.round, - needs_snapshot: false, + replica: self.replica, }; } } @@ -97,7 +97,7 @@ impl RaftNode { success: false, last_log_index: self.log.last_index(), round: req.round, - needs_snapshot: false, + replica: self.replica, }; } }; @@ -116,7 +116,7 @@ impl RaftNode { success: true, last_log_index: self.claimable_match(matched, wrote), round: req.round, - needs_snapshot: false, + replica: self.replica, } } @@ -149,6 +149,11 @@ impl RaftNode { self.become_follower(resp.term); return; } + // A reply to a request of an earlier term says nothing about this + // term's log, acks, or peer states. + if resp.term < self.hard_state.current_term { + return; + } if self.role != NodeRole::Leader { return; @@ -205,7 +210,7 @@ impl RaftNode { // the follower recorded this leader's contact before its log check. // The lease settles at the round's send time. Check-quorum dates the // answer at its arrival. - if peer_is_voter && resp.term == self.hard_state.current_term { + if peer_is_voter { let now = std::time::Instant::now(); self.record_lease_ack(peer, resp.round, now); self.settle_lease(); @@ -216,11 +221,7 @@ impl RaftNode { None => return, }; - if resp.needs_snapshot { - leader.awaiting_snapshot.insert(peer); - } else { - leader.awaiting_snapshot.remove(&peer); - } + leader.peer_states.insert(peer, resp.replica); if resp.success { let new_match = resp.last_log_index; if new_match > leader.match_index_for(peer) { @@ -231,7 +232,7 @@ impl RaftNode { self.try_advance_commit_index(); self.replicated_floor = self.replicated_floor(); } - } else if resp.needs_snapshot { + } else if resp.replica.wants_snapshot() { // The peer holds no state to resume from: entries from any index // leave it as it is, so it gets a snapshot. if !self.ready.snapshots_needed.contains(&peer) { @@ -365,12 +366,12 @@ mod tests { } /// A follower that requires a snapshot refuses entries with - /// `needs_snapshot`, and its leader flags it for a snapshot even though + /// the state it wants, and its leader flags it for a snapshot even though /// the leader's log holds every entry. #[test] fn a_follower_that_requires_a_snapshot_gets_one() { let mut follower = RaftNode::new(test_config(2, vec![1]), MemStorage::new()); - follower.set_snapshot_required(true); + follower.set_replica_state(crate::state::ReplicaState::WantsSnapshot); let req = AppendEntriesRequest { term: 1, leader_id: 1, @@ -388,7 +389,7 @@ mod tests { }; let refusal = follower.handle_append_entries(&req); assert!(!refusal.success); - assert!(refusal.needs_snapshot); + assert_eq!(refusal.replica, crate::state::ReplicaState::WantsSnapshot); assert_eq!(follower.last_log_index(), 0, "no entry was appended"); assert_eq!(follower.leader_id(), 1, "the leader's contact still counts"); @@ -406,7 +407,7 @@ mod tests { leader.handle_append_entries_response(2, &refusal); assert!(leader.take_ready().snapshots_needed.contains(&2)); - follower.set_snapshot_required(false); + follower.set_replica_state(crate::state::ReplicaState::Whole); assert!(follower.handle_append_entries(&req).success); } @@ -430,7 +431,7 @@ mod tests { success, last_log_index, round: 1, - needs_snapshot: false, + replica: crate::state::ReplicaState::Whole, }; node.handle_append_entries_response(2, &answer(true, 1)); node.handle_append_entries_response(2, &answer(false, 0)); @@ -463,6 +464,44 @@ mod tests { assert_eq!(node.hard_state.voted_for, 0, "a new term frees the vote"); } + /// A reply carrying an earlier term answers a request of that term. It + /// records no ack, match, or peer state in this term. + #[test] + fn a_reply_from_an_earlier_term_records_nothing() { + let mut node = RaftNode::new(test_config(1, vec![2, 3]), MemStorage::new()); + force_election(&mut node); + let _ = node.take_ready(); + node.handle_request_vote_response( + 2, + &RequestVoteResponse { + term: 1, + vote_granted: true, + }, + ); + assert_eq!(node.role(), NodeRole::Leader); + + let stale = AppendEntriesResponse { + term: 0, + success: true, + last_log_index: node.last_log_index(), + round: UNTRACKED_ROUND, + replica: crate::state::ReplicaState::AwaitingSnapshot, + }; + node.handle_append_entries_response(2, &stale); + assert_eq!(node.peer_replica_state(2), None); + assert_eq!(node.peer_ack_count(2), Some(0)); + assert_eq!(node.match_index_for(2), Some(0)); + assert!(node.take_ready().snapshots_needed.is_empty()); + + let current = AppendEntriesResponse { term: 1, ..stale }; + node.handle_append_entries_response(2, ¤t); + assert_eq!( + node.peer_replica_state(2), + Some(crate::state::ReplicaState::AwaitingSnapshot) + ); + assert_eq!(node.peer_ack_count(2), Some(1)); + } + #[test] fn leader_steps_down_on_higher_term() { let config = test_config(1, vec![2, 3]); @@ -564,7 +603,7 @@ mod tests { success: true, last_log_index: 2, round: UNTRACKED_ROUND, - needs_snapshot: false, + replica: crate::state::ReplicaState::Whole, }; node.handle_append_entries_response(4, &ae_ok); assert_eq!( @@ -669,7 +708,7 @@ mod tests { success: true, last_log_index: idx, round: UNTRACKED_ROUND, - needs_snapshot: false, + replica: crate::state::ReplicaState::Whole, }; leader.handle_append_entries_response(2, &ae_ok); assert_eq!(leader.commit_index(), idx); @@ -723,7 +762,7 @@ mod tests { success: true, last_log_index: idx, round: UNTRACKED_ROUND, - needs_snapshot: false, + replica: crate::state::ReplicaState::Whole, }; node1.handle_append_entries_response(5, &obs_ack); assert_eq!( @@ -749,7 +788,7 @@ mod tests { success: true, last_log_index: idx, round: UNTRACKED_ROUND, - needs_snapshot: false, + replica: crate::state::ReplicaState::Whole, }; leader.handle_append_entries_response(2, &voter_ack); assert_eq!( diff --git a/nodedb-raft/src/node/rpc/install_snapshot.rs b/nodedb-raft/src/node/rpc/install_snapshot.rs index 278c757b6..62d2c2207 100644 --- a/nodedb-raft/src/node/rpc/install_snapshot.rs +++ b/nodedb-raft/src/node/rpc/install_snapshot.rs @@ -261,7 +261,7 @@ mod tests { success: true, last_log_index: tip, round: UNTRACKED_ROUND, - needs_snapshot: false, + replica: crate::state::ReplicaState::Whole, }, ); assert_eq!(leader.commit_index(), tip); diff --git a/nodedb-raft/src/node/rpc/timeout_now.rs b/nodedb-raft/src/node/rpc/timeout_now.rs index 60fcdd492..a59248964 100644 --- a/nodedb-raft/src/node/rpc/timeout_now.rs +++ b/nodedb-raft/src/node/rpc/timeout_now.rs @@ -115,7 +115,7 @@ mod tests { success: true, last_log_index: idx, round: UNTRACKED_ROUND, - needs_snapshot: false, + replica: crate::state::ReplicaState::Whole, } } @@ -261,7 +261,7 @@ mod tests { success: false, last_log_index: 0, round: UNTRACKED_ROUND, - needs_snapshot: false, + replica: crate::state::ReplicaState::Whole, }, ); assert_eq!(leader.role(), NodeRole::Follower); diff --git a/nodedb-raft/src/node/snapshot_debt.rs b/nodedb-raft/src/node/snapshot_debt.rs new file mode 100644 index 000000000..52d786e61 --- /dev/null +++ b/nodedb-raft/src/node/snapshot_debt.rs @@ -0,0 +1,341 @@ +// SPDX-License-Identifier: BUSL-1.1 + +//! A replica that takes a snapshot before it takes log entries again. +//! +//! - [`ReplicaState::WantsSnapshot`]: its data matches its log, and state +//! its host keeps beside the log lags. It refuses entries so its leader +//! sends a snapshot. It still campaigns and leads. +//! - [`ReplicaState::AwaitingSnapshot`]: its data diverged. Only a leader +//! sends a snapshot, so a leader awaiting one would wait forever. Entering +//! this state steps a leader or candidate down, and the election paths +//! refuse to start while it holds. A leader hands its term to the voter +//! [`RaftNode::hand_off_target`] names, when one qualifies. + +use tracing::info; + +use crate::message::TimeoutNowRequest; +use crate::node::core::RaftNode; +use crate::state::{NodeRole, ReplicaState}; +use crate::storage::LogStorage; + +impl RaftNode { + /// Set what this replica needs before it takes log entries again. A + /// leader or candidate whose data diverged steps down. + pub fn set_replica_state(&mut self, state: ReplicaState) { + self.replica = state; + if !state.may_lead() { + self.stand_down_for_snapshot(); + } + } + + /// Whether this replica refuses log entries until a snapshot installs. + pub fn snapshot_required(&self) -> bool { + self.replica.wants_snapshot() + } + + /// Whether this replica holds the state it serves and leads from. + pub fn replica_state(&self) -> ReplicaState { + self.replica + } + + /// Leave a leadership or candidacy this replica can no longer hold. + fn stand_down_for_snapshot(&mut self) { + let term = self.hard_state.current_term; + match self.role { + NodeRole::Leader => { + let target = self.hand_off_target(); + if let Some(target) = target { + // The target campaigns at once. Voters grant it although + // they still hear this leader, so no lease holds past + // here. + self.lease.revoke(); + self.ready.timeout_now.push(( + target, + TimeoutNowRequest { + term, + leader_id: self.config.node_id, + group_id: self.config.group_id, + }, + )); + } + info!( + node = self.config.node_id, + group = self.config.group_id, + term, + ?target, + "replica awaits a snapshot; stepping down from leader" + ); + self.become_follower(term); + } + NodeRole::Candidate => self.become_follower(term), + NodeRole::Follower => self.pre_vote = None, + NodeRole::Learner | NodeRole::Observer => {} + } + } + + /// The voter peer this leader hands its leadership to. `None` off the + /// leader, or when no voter qualifies. + /// + /// A target answered whole this term and holds every committed entry. A + /// voter missing a committed entry cannot win an election. Of the + /// qualified voters, the highest match goes first. One at the log + /// frontier wins its campaign against every voter. + pub fn hand_off_target(&self) -> Option { + let commit = self.volatile.commit_index; + self.config + .peers + .iter() + .copied() + .filter(|&peer| self.peer_answered_whole(peer)) + .filter_map(|peer| Some((peer, self.match_index_for(peer)?))) + .filter(|&(_, matched)| matched >= commit) + .max_by_key(|&(_, matched)| matched) + .map(|(peer, _)| peer) + } +} + +#[cfg(test)] +mod tests { + use std::time::{Duration, Instant}; + + use crate::message::{AppendEntriesRequest, AppendEntriesResponse, RequestVoteResponse}; + use crate::node::core::RaftNode; + use crate::node::leader_lease::UNTRACKED_ROUND; + use crate::state::{NodeRole, ReplicaState}; + use crate::storage::MemStorage; + use crate::test_support::{force_election, test_config}; + + fn leader_3voter() -> RaftNode { + let mut node = RaftNode::new(test_config(1, vec![2, 3]), MemStorage::new()); + force_election(&mut node); + let _ = node.take_ready(); + node.handle_request_vote_response( + 2, + &RequestVoteResponse { + term: 1, + vote_granted: true, + }, + ); + assert_eq!(node.role(), NodeRole::Leader); + let _ = node.take_ready(); + node + } + + fn reply(last_log_index: u64, replica: ReplicaState) -> AppendEntriesResponse { + AppendEntriesResponse { + term: 1, + success: !replica.wants_snapshot(), + last_log_index, + round: UNTRACKED_ROUND, + replica, + } + } + + #[test] + fn a_replica_awaiting_a_snapshot_never_campaigns() { + let mut node = RaftNode::new(test_config(2, vec![1, 3]), MemStorage::new()); + node.set_replica_state(ReplicaState::AwaitingSnapshot); + assert_eq!(node.replica_state(), ReplicaState::AwaitingSnapshot); + + force_election(&mut node); + let ready = node.take_ready(); + assert!(ready.pre_vote_requests.is_empty(), "no pre-vote round"); + assert!(ready.vote_requests.is_empty(), "no vote request"); + assert_eq!(node.role(), NodeRole::Follower); + assert_eq!(node.current_term(), 0); + + node.set_replica_state(ReplicaState::Whole); + force_election(&mut node); + assert_eq!( + node.role(), + NodeRole::Candidate, + "a whole replica campaigns" + ); + } + + #[test] + fn a_sole_voter_awaiting_a_snapshot_never_elects_itself() { + let mut node = RaftNode::new(test_config(1, vec![]), MemStorage::new()); + node.set_replica_state(ReplicaState::AwaitingSnapshot); + node.election_deadline_override(Instant::now() - Duration::from_millis(1)); + node.tick(); + assert_eq!(node.role(), NodeRole::Follower); + assert_eq!(node.leader_id(), 0); + } + + #[test] + fn a_transfer_never_makes_an_awaiting_replica_campaign() { + let mut node = RaftNode::new(test_config(2, vec![1, 3]), MemStorage::new()); + node.handle_append_entries(&AppendEntriesRequest { + term: 1, + leader_id: 1, + prev_log_index: 0, + prev_log_term: 0, + entries: vec![], + leader_commit: 0, + group_id: 1, + round: 1, + replicated_floor: 0, + }); + node.set_replica_state(ReplicaState::AwaitingSnapshot); + node.handle_timeout_now(&crate::message::TimeoutNowRequest { + term: 1, + leader_id: 1, + group_id: 1, + }); + assert_eq!(node.role(), NodeRole::Follower); + assert_eq!(node.current_term(), 1); + assert!(node.take_ready().vote_requests.is_empty()); + } + + #[test] + fn a_leader_given_the_requirement_steps_down_and_hands_off() { + let mut leader = leader_3voter(); + let frontier = leader.last_log_index(); + leader.handle_append_entries_response(2, &reply(frontier, ReplicaState::Whole)); + let _ = leader.take_ready(); + + leader.set_replica_state(ReplicaState::AwaitingSnapshot); + assert_eq!(leader.role(), NodeRole::Follower); + assert_eq!(leader.leader_id(), 0); + let ready = leader.take_ready(); + assert_eq!(ready.timeout_now.len(), 1); + let (target, request) = ready.timeout_now[0].clone(); + assert_eq!(target, 2); + assert_eq!((request.term, request.leader_id), (1, 1)); + + leader.election_deadline_override(Instant::now() - Duration::from_millis(1)); + leader.tick(); + assert_eq!(leader.role(), NodeRole::Follower, "it never campaigns back"); + } + + #[test] + fn a_leader_never_hands_off_to_a_peer_awaiting_a_snapshot() { + let mut leader = leader_3voter(); + let frontier = leader.last_log_index(); + // Peer 2 matched the frontier, then asked for a snapshot. Peer 3 + // never answered. + leader.handle_append_entries_response(2, &reply(frontier, ReplicaState::Whole)); + leader.handle_append_entries_response(2, &reply(frontier, ReplicaState::AwaitingSnapshot)); + let _ = leader.take_ready(); + + leader.set_replica_state(ReplicaState::AwaitingSnapshot); + assert_eq!(leader.role(), NodeRole::Follower); + assert!(leader.take_ready().timeout_now.is_empty()); + } + + /// A replica that wants a snapshot only for state beside its log, such + /// as a lost Calvin base, still campaigns and leads, and a leader that + /// takes that state keeps leading. + #[test] + fn a_replica_that_wants_a_snapshot_still_campaigns_and_leads() { + let mut node = RaftNode::new(test_config(1, vec![2, 3]), MemStorage::new()); + node.set_replica_state(ReplicaState::WantsSnapshot); + assert!(node.snapshot_required()); + force_election(&mut node); + assert_eq!(node.role(), NodeRole::Candidate); + node.handle_request_vote_response( + 2, + &RequestVoteResponse { + term: 1, + vote_granted: true, + }, + ); + assert_eq!(node.role(), NodeRole::Leader); + + let mut leader = leader_3voter(); + leader.set_replica_state(ReplicaState::WantsSnapshot); + assert_eq!(leader.role(), NodeRole::Leader); + assert!(leader.take_ready().timeout_now.is_empty()); + + let mut sole = RaftNode::new(test_config(1, vec![]), MemStorage::new()); + sole.set_replica_state(ReplicaState::WantsSnapshot); + sole.election_deadline_override(Instant::now() - Duration::from_millis(1)); + sole.tick(); + assert_eq!(sole.role(), NodeRole::Leader); + } + + #[test] + fn a_stricter_state_wins() { + assert_eq!( + ReplicaState::AwaitingSnapshot.raised_to(ReplicaState::WantsSnapshot), + ReplicaState::AwaitingSnapshot + ); + assert_eq!( + ReplicaState::Whole.raised_to(ReplicaState::WantsSnapshot), + ReplicaState::WantsSnapshot + ); + } + + #[test] + fn a_candidate_given_the_requirement_stands_down() { + let mut node = RaftNode::new(test_config(1, vec![2, 3]), MemStorage::new()); + force_election(&mut node); + assert_eq!(node.role(), NodeRole::Candidate); + node.set_replica_state(ReplicaState::AwaitingSnapshot); + assert_eq!(node.role(), NodeRole::Follower); + node.handle_request_vote_response( + 2, + &RequestVoteResponse { + term: 1, + vote_granted: true, + }, + ); + assert_eq!( + node.role(), + NodeRole::Follower, + "a late grant elects nothing" + ); + } + + /// A voter not heard this term is never a hand-off target. + #[test] + fn a_voter_not_heard_this_term_is_no_hand_off_target() { + let mut leader = leader_3voter(); + assert_eq!(leader.hand_off_target(), None); + leader.handle_append_entries_response(3, &reply(0, ReplicaState::AwaitingSnapshot)); + assert_eq!(leader.hand_off_target(), None, "voter 2 is still unheard"); + } + + /// The target holds every committed entry, and the highest match goes + /// first. + #[test] + fn the_hand_off_target_holds_every_committed_entry() { + let mut node = RaftNode::new(test_config(1, vec![2, 3, 4, 5]), MemStorage::new()); + force_election(&mut node); + let _ = node.take_ready(); + for peer in [2, 3] { + node.handle_request_vote_response( + peer, + &RequestVoteResponse { + term: 1, + vote_granted: true, + }, + ); + } + assert_eq!(node.role(), NodeRole::Leader); + let frontier = node.last_log_index(); + for peer in [2, 3, 4] { + node.handle_append_entries_response(peer, &reply(frontier, ReplicaState::Whole)); + } + assert_eq!(node.commit_index(), frontier); + node.handle_append_entries_response(5, &reply(0, ReplicaState::Whole)); + for peer in [2, 3] { + node.handle_append_entries_response( + peer, + &reply(frontier, ReplicaState::AwaitingSnapshot), + ); + } + assert_eq!(node.hand_off_target(), Some(4)); + + node.handle_append_entries_response(4, &reply(frontier, ReplicaState::WantsSnapshot)); + assert_eq!( + node.hand_off_target(), + None, + "voter 5 answered whole but misses a committed entry" + ); + + node.handle_append_entries_response(5, &reply(frontier, ReplicaState::Whole)); + assert_eq!(node.hand_off_target(), Some(5)); + } +} diff --git a/nodedb-raft/src/node/staleness.rs b/nodedb-raft/src/node/staleness.rs index 98ba6be76..591093c8f 100644 --- a/nodedb-raft/src/node/staleness.rs +++ b/nodedb-raft/src/node/staleness.rs @@ -31,14 +31,19 @@ pub enum StalenessVerdict { } impl RaftNode { - /// Whether this replica is within `max_staleness` of the leader. + /// Whether this replica is within `max_staleness` of the leader, given + /// `applied`: the highest index whose state is visible to reads here. + /// + /// `applied` is the caller's, not Raft's `last_applied`. Raft advances + /// `last_applied` when it hands an entry to the state machine, before + /// the entry's state is visible. /// /// The leader is always fresh — it is the source of the bound, not a /// consumer of it. A follower is fresh when it has applied everything /// the leader had committed as of a contact no older than the bound: /// anything committed since is at most `max_staleness` old, which is /// precisely what was promised. - pub fn staleness_verdict(&self, max_staleness: Duration) -> StalenessVerdict { + pub fn staleness_verdict(&self, max_staleness: Duration, applied: u64) -> StalenessVerdict { if self.role == NodeRole::Leader { return StalenessVerdict::Fresh; } @@ -48,18 +53,30 @@ impl RaftNode { if contact.at.elapsed() > max_staleness { return StalenessVerdict::NoRecentContact; } - if self.volatile.last_applied < contact.leader_commit { + if applied < contact.leader_commit { return StalenessVerdict::Behind { leader_commit: contact.leader_commit, - applied: self.volatile.last_applied, + applied, }; } StalenessVerdict::Fresh } - /// Whether a read with `max_staleness` can be served from this replica. - pub fn within_staleness_bound(&self, max_staleness: Duration) -> bool { - self.staleness_verdict(max_staleness) == StalenessVerdict::Fresh + /// Whether a read with `max_staleness` can be served from this replica, + /// whose reads see state through `applied`. + pub fn within_staleness_bound(&self, max_staleness: Duration, applied: u64) -> bool { + self.staleness_verdict(max_staleness, applied) == StalenessVerdict::Fresh + } + + /// Whether this replica heard from its leader within `window`. The + /// leader itself always has. How far behind the replica applies does not + /// count. + pub fn leader_contact_within(&self, window: Duration) -> bool { + if self.role == NodeRole::Leader { + return true; + } + self.leader_contact + .is_some_and(|contact| contact.at.elapsed() <= window) } } @@ -115,16 +132,17 @@ mod tests { fn a_replica_that_never_heard_from_a_leader_is_not_fresh() { let node = follower(); assert_eq!( - node.staleness_verdict(BOUND), + node.staleness_verdict(BOUND, 0), StalenessVerdict::NoRecentContact ); + assert!(!node.leader_contact_within(BOUND)); } #[test] fn a_caught_up_replica_in_recent_contact_is_fresh() { let mut node = follower(); heartbeat(&mut node, 0); - assert_eq!(node.staleness_verdict(BOUND), StalenessVerdict::Fresh); + assert_eq!(node.staleness_verdict(BOUND, 0), StalenessVerdict::Fresh); } /// The failure the old measure could not see: applying steadily while @@ -134,9 +152,8 @@ mod tests { fn a_replica_behind_the_leader_is_not_fresh_however_recent_the_contact() { let mut node = follower(); heartbeat(&mut node, 500); - node.advance_applied(10); assert_eq!( - node.staleness_verdict(BOUND), + node.staleness_verdict(BOUND, 10), StalenessVerdict::Behind { leader_commit: 500, applied: 10, @@ -150,8 +167,31 @@ mod tests { fn an_idle_cluster_does_not_make_a_caught_up_replica_look_stale() { let mut node = follower(); heartbeat(&mut node, 7); - node.advance_applied(7); - assert_eq!(node.staleness_verdict(BOUND), StalenessVerdict::Fresh); + assert_eq!(node.staleness_verdict(BOUND, 7), StalenessVerdict::Fresh); + } + + /// Raft's own `last_applied` at the leader's commit proves nothing: the + /// entries can still be applying. The verdict reads the caller's + /// watermark, so a lagging one is stale and one at the commit is fresh. + #[test] + fn the_verdict_reads_the_callers_watermark_not_the_hand_off_index() { + let mut node = follower(); + heartbeat(&mut node, 9); + node.advance_applied(9); + assert_eq!(node.last_applied(), 9); + assert_eq!( + node.staleness_verdict(BOUND, 4), + StalenessVerdict::Behind { + leader_commit: 9, + applied: 4, + } + ); + assert!(!node.within_staleness_bound(BOUND, 4)); + assert!(node.within_staleness_bound(BOUND, 9)); + assert!( + node.leader_contact_within(BOUND), + "a lagging apply keeps contact" + ); } #[test] @@ -160,9 +200,10 @@ mod tests { heartbeat(&mut node, 0); node.leader_contact_at_override(Instant::now() - Duration::from_secs(30)); assert_eq!( - node.staleness_verdict(BOUND), + node.staleness_verdict(BOUND, 0), StalenessVerdict::NoRecentContact ); + assert!(!node.leader_contact_within(BOUND)); } /// The leader is the source of the bound, so it needs no contact of its @@ -173,6 +214,7 @@ mod tests { node.election_deadline_override(Instant::now() - Duration::from_millis(1)); node.tick(); assert_eq!(node.role(), NodeRole::Leader); - assert_eq!(node.staleness_verdict(BOUND), StalenessVerdict::Fresh); + assert_eq!(node.staleness_verdict(BOUND, 0), StalenessVerdict::Fresh); + assert!(node.leader_contact_within(BOUND)); } } diff --git a/nodedb-raft/src/state.rs b/nodedb-raft/src/state.rs index 18faf94b4..f7bd867c2 100644 --- a/nodedb-raft/src/state.rs +++ b/nodedb-raft/src/state.rs @@ -50,6 +50,63 @@ pub enum NodeRole { Observer, } +/// Whether a replica holds the state beside its log that it serves and +/// leads from. A follower reports it on every `AppendEntries` reply. +/// +/// Variants order from least to most strict. +#[repr(u8)] +#[derive( + Debug, + Clone, + Copy, + PartialEq, + Eq, + PartialOrd, + Ord, + serde::Serialize, + serde::Deserialize, + rkyv::Archive, + rkyv::Serialize, + rkyv::Deserialize, + zerompk::ToMessagePack, + zerompk::FromMessagePack, +)] +#[msgpack(c_enum)] +pub enum ReplicaState { + /// Its state matches the log it holds. + Whole = 0, + /// Its data matches its log, but state its host keeps beside the log + /// lags. It refuses log entries so its leader sends a snapshot. It + /// still campaigns and leads. + WantsSnapshot = 1, + /// Its data diverged from the log. It refuses log entries, never stands + /// for election, and leads nothing until a snapshot installs. + AwaitingSnapshot = 2, +} + +impl ReplicaState { + /// The stricter of `self` and `other`. + pub fn raised_to(self, other: Self) -> Self { + self.max(other) + } + + /// Whether the replica refuses log entries until a snapshot installs. + pub fn wants_snapshot(self) -> bool { + match self { + Self::Whole => false, + Self::WantsSnapshot | Self::AwaitingSnapshot => true, + } + } + + /// Whether the replica may campaign and lead. + pub fn may_lead(self) -> bool { + match self { + Self::Whole | Self::WantsSnapshot => true, + Self::AwaitingSnapshot => false, + } + } +} + /// Role of a remote peer as seen by the local leader. /// /// Used to classify tracked peers so the leader can send entries to both @@ -183,9 +240,9 @@ pub struct LeaderState { /// timed: leadership is confirmed by a quorum of these rising after a /// heartbeat round. pub ack_count: Vec<(u64, u64)>, - /// Peers whose latest `AppendEntries` response asked for a snapshot. - /// Such a peer holds no state to lead from. - pub awaiting_snapshot: std::collections::HashSet, + /// Each peer's state as its latest `AppendEntries` response this term + /// reported it. A peer not heard this term has no entry. + pub peer_states: std::collections::HashMap, } impl LeaderState { @@ -195,7 +252,7 @@ impl LeaderState { next_index: peers.iter().map(|&id| (id, last_log_index + 1)).collect(), match_index: peers.iter().map(|&id| (id, 0)).collect(), ack_count: peers.iter().map(|&id| (id, 0)).collect(), - awaiting_snapshot: std::collections::HashSet::new(), + peer_states: std::collections::HashMap::new(), observer_states: observers .iter() .map(|&id| { @@ -254,7 +311,7 @@ impl LeaderState { self.next_index.retain(|&(id, _)| id != peer); self.match_index.retain(|&(id, _)| id != peer); self.ack_count.retain(|&(id, _)| id != peer); - self.awaiting_snapshot.remove(&peer); + self.peer_states.remove(&peer); } /// Responses received from `peer` in this term. diff --git a/nodedb-raft/tests/membership.rs b/nodedb-raft/tests/membership.rs index 57ecd2089..2698ac7ad 100644 --- a/nodedb-raft/tests/membership.rs +++ b/nodedb-raft/tests/membership.rs @@ -192,7 +192,7 @@ fn learner_catchup_then_promotion_lifecycle() { success: true, last_log_index: idx, round: 0, - needs_snapshot: false, + replica: nodedb_raft::ReplicaState::Whole, }; node.handle_append_entries_response(2, &voter_ack); assert_eq!(node.commit_index(), idx); @@ -203,7 +203,7 @@ fn learner_catchup_then_promotion_lifecycle() { success: true, last_log_index: idx, round: 0, - needs_snapshot: false, + replica: nodedb_raft::ReplicaState::Whole, }; node.handle_append_entries_response(3, &learner_ack); assert_eq!(node.match_index_for(3), Some(idx)); diff --git a/nodedb-test-support/Cargo.toml b/nodedb-test-support/Cargo.toml index 385462da9..ba22e37a3 100644 --- a/nodedb-test-support/Cargo.toml +++ b/nodedb-test-support/Cargo.toml @@ -42,6 +42,7 @@ futures = { workspace = true } smallvec = { workspace = true } uuid = { workspace = true } tracing-subscriber = { workspace = true } +thiserror = { workspace = true } # JWT fixture: mint RS256 tokens against an in-process JWKS endpoint. rsa = { workspace = true } diff --git a/nodedb-test-support/src/cluster_harness/cluster/bringup.rs b/nodedb-test-support/src/cluster_harness/cluster/bringup.rs index d1bea0ea7..2d5c94dc5 100644 --- a/nodedb-test-support/src/cluster_harness/cluster/bringup.rs +++ b/nodedb-test-support/src/cluster_harness/cluster/bringup.rs @@ -36,6 +36,7 @@ impl TestCluster { backup_storage: None, pitr: None, node_timeseries_tuning: std::collections::HashMap::new(), + redo_chunk_limits: None, }; Self::spawn_three_with_config(config).await } @@ -60,6 +61,7 @@ impl TestCluster { backup_storage: None, pitr: None, node_timeseries_tuning: std::collections::HashMap::new(), + redo_chunk_limits: None, }) .await } @@ -83,6 +85,7 @@ impl TestCluster { backup_storage: None, pitr: None, node_timeseries_tuning: std::collections::HashMap::new(), + redo_chunk_limits: None, }) .await } @@ -106,6 +109,7 @@ impl TestCluster { backup_storage: None, pitr: None, node_timeseries_tuning: std::collections::HashMap::from([(node_id, timeseries_tuning)]), + redo_chunk_limits: None, }) .await } @@ -131,6 +135,7 @@ impl TestCluster { }), pitr: None, node_timeseries_tuning: std::collections::HashMap::new(), + redo_chunk_limits: None, }) .await } @@ -153,6 +158,37 @@ impl TestCluster { backup_storage: None, pitr: Some(pitr), node_timeseries_tuning: std::collections::HashMap::new(), + redo_chunk_limits: None, + }) + .await + } + + /// Spawn a 3-node cluster whose nodes cap a redo entry at + /// `max_entry_bytes`, so a commit larger than it travels as a chunked + /// redo stream. Uses the standard fast-election tuning and 1 core per + /// node. + pub async fn spawn_three_with_redo_entry_limit( + max_entry_bytes: usize, + ) -> Result> { + let calvin = nodedb_types::config::tuning::CalvinTuning::default(); + Self::spawn_three_with_config(ClusterSpawnConfig { + tuning: super::types::fast_cluster_tuning(), + graph_tuning: nodedb_types::config::tuning::GraphTuning::default(), + query_tuning: nodedb_types::config::tuning::QueryTuning::default(), + num_cores: 1, + log_compaction_threshold: None, + replication_factor: 3, + num_groups: DEFAULT_NUM_GROUPS, + single_node_calvin: false, + backup_storage: None, + pitr: None, + node_timeseries_tuning: std::collections::HashMap::new(), + redo_chunk_limits: Some( + nodedb::control::wal_replication::transaction_redo::chunks::RedoChunkLimits { + max_entry_bytes, + max_open_bytes: calvin.max_open_redo_bytes, + }, + ), }) .await } diff --git a/nodedb-test-support/src/cluster_harness/cluster/convergence.rs b/nodedb-test-support/src/cluster_harness/cluster/convergence.rs new file mode 100644 index 000000000..e6b7a82b3 --- /dev/null +++ b/nodedb-test-support/src/cluster_harness/cluster/convergence.rs @@ -0,0 +1,169 @@ +// SPDX-License-Identifier: BUSL-1.1 + +//! Apply-convergence barriers. +//! +//! A barrier waits until every node applied every entry its hosted groups +//! committed when the barrier looked. On timeout it panics with each lagging +//! group's commit target and every node's commit and applied index. +//! +//! The target of a group is the highest commit index any node reports for it. +//! A node's own applied index alone is no target: after a restart, every node +//! can hold the same applied index while committed entries above it are +//! unapplied everywhere. + +use std::collections::BTreeMap; +use std::fmt::Write as _; +use std::time::{Duration, Instant}; + +use super::TestCluster; + +/// One hosted group's positions on one node. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub struct GroupPosition { + pub node_id: u64, + pub group_id: u64, + /// The leader this node knows for the group, `0` when it knows none. + pub leader_id: u64, + pub commit: u64, + /// This node's apply watermark for the group. + pub applied: u64, +} + +impl TestCluster { + /// Every hosted group's positions on every node, keyed by group. A node + /// that runs no Raft yet reports nothing. + pub fn group_positions(&self) -> BTreeMap> { + let mut groups: BTreeMap> = BTreeMap::new(); + for node in &self.nodes { + let Some(status_fn) = node.shared.raft_status_fn.get() else { + continue; + }; + for status in status_fn() { + groups + .entry(status.group_id) + .or_default() + .push(GroupPosition { + node_id: node.node_id, + group_id: status.group_id, + leader_id: status.leader_id, + commit: status.commit_index, + applied: node.shared.applied_index_watcher(status.group_id).current(), + }); + } + } + groups + } + + /// [`Self::group_positions`] as text, one group per line, for a panic + /// message. + pub fn group_positions_report(&self) -> String { + render(&self.group_positions(), &BTreeMap::new()) + } + + /// Block until every node applied every entry its hosted groups + /// committed. Groups and commits are re-read on every poll, so a group + /// that appears late is covered. + /// + /// # Panics + /// + /// At `timeout`, naming each lagging group with its commit target and + /// every node's commit and applied index. + pub async fn wait_for_full_apply_convergence(&self, timeout: Duration) { + self.wait_for_groups_applied(timeout, None).await; + } + + /// [`Self::wait_for_full_apply_convergence`] for the metadata group only. + pub(crate) async fn wait_for_metadata_apply_convergence(&self, timeout: Duration) { + self.wait_for_groups_applied(timeout, Some(nodedb_cluster::METADATA_GROUP_ID)) + .await; + } + + async fn wait_for_groups_applied(&self, timeout: Duration, only: Option) { + let deadline = Instant::now() + timeout; + loop { + // Commit targets first, apply positions after: an applied index + // only grows, so a node at or past the target applied every + // entry committed when the targets were read. + let targets = commit_targets(&self.group_positions(), only); + let mut positions = self.group_positions(); + positions.retain(|group_id, _| targets.contains_key(group_id)); + let lagging = lagging_groups(&positions, &targets); + let silent = self.nodes_without_status(); + if lagging.is_empty() && silent.is_empty() { + return; + } + if Instant::now() >= deadline { + let lagging_positions: BTreeMap> = positions + .into_iter() + .filter(|(group_id, _)| lagging.contains(group_id)) + .collect(); + panic!( + "the cluster did not apply every committed entry within {timeout:?}.\n\ + nodes reporting no group status: {silent:?}\n\ + lagging groups:\n{}", + render(&lagging_positions, &targets) + ); + } + tokio::time::sleep(Duration::from_millis(5)).await; + } + } + + /// Nodes whose Raft status is not installed yet. + fn nodes_without_status(&self) -> Vec { + self.nodes + .iter() + .filter(|node| node.shared.raft_status_fn.get().is_none()) + .map(|node| node.node_id) + .collect() + } +} + +/// The highest commit index any node reports, per group. +fn commit_targets( + groups: &BTreeMap>, + only: Option, +) -> BTreeMap { + groups + .iter() + .filter(|(group_id, _)| only.is_none_or(|only| only == **group_id)) + .map(|(group_id, positions)| { + let target = positions.iter().map(|p| p.commit).max().unwrap_or(0); + (*group_id, target) + }) + .collect() +} + +/// Groups where some hosting node applied less than the group's target. +fn lagging_groups( + groups: &BTreeMap>, + targets: &BTreeMap, +) -> Vec { + groups + .iter() + .filter(|(group_id, positions)| { + let target = targets.get(group_id).copied().unwrap_or(0); + positions.iter().any(|p| p.applied < target) + }) + .map(|(group_id, _)| *group_id) + .collect() +} + +fn render(groups: &BTreeMap>, targets: &BTreeMap) -> String { + let mut out = String::new(); + for (group_id, positions) in groups { + let _ = write!(out, " group {group_id}"); + if let Some(target) = targets.get(group_id) { + let _ = write!(out, " (commit target {target})"); + } + let _ = write!(out, ":"); + for p in positions { + let _ = write!( + out, + " [node {} leader {} commit {} applied {}]", + p.node_id, p.leader_id, p.commit, p.applied + ); + } + out.push('\n'); + } + out +} diff --git a/nodedb-test-support/src/cluster_harness/cluster/membership.rs b/nodedb-test-support/src/cluster_harness/cluster/membership.rs index 7c5fb72fe..e51f786ca 100644 --- a/nodedb-test-support/src/cluster_harness/cluster/membership.rs +++ b/nodedb-test-support/src/cluster_harness/cluster/membership.rs @@ -96,17 +96,10 @@ impl TestCluster { /// the retry loop tries the next node on failure so the test /// doesn't have to discover the leader explicitly. /// - /// After the DDL is accepted, **blocks until every node's - /// metadata applier has caught up to the proposer's applied - /// index**. `propose_catalog_entry` already waits for the entry - /// to be applied on the proposing node before returning, but - /// followers apply asynchronously — without this barrier a - /// subsequent `wait_for("x visible on every node")` would race - /// the follower appliers and trip its timeout on the cold-start - /// attempt. Polling the watermark directly is O(applied_index) - /// and converges as soon as the followers drain their commit - /// queues, so it's both strictly more correct and strictly - /// faster than waiting on the visibility check itself. + /// After the DDL is accepted, blocks until every node applied the + /// metadata group through its commit index. Followers apply + /// asynchronously, so a later visibility check otherwise races their + /// appliers. Panics with every node's positions after 10 s. pub async fn exec_ddl_on_any_leader(&self, sql: &str) -> Result { let deadline = std::time::Instant::now() + Duration::from_secs(10); let mut last_err = String::new(); @@ -114,7 +107,8 @@ impl TestCluster { for (idx, node) in self.nodes.iter().enumerate() { match node.exec(sql).await { Ok(()) => { - self.wait_for_applied_index_convergence(idx).await; + self.wait_for_metadata_apply_convergence(Duration::from_secs(10)) + .await; return Ok(idx); } Err(e) => last_err = e, @@ -127,121 +121,6 @@ impl TestCluster { )) } - /// Block until every node's metadata applier has caught up to the - /// proposer's current applied index. Called after every successful - /// DDL by `exec_ddl_on_any_leader`. - async fn wait_for_applied_index_convergence(&self, proposer_idx: usize) { - let group_id = nodedb_cluster::METADATA_GROUP_ID; - let target = self.nodes[proposer_idx] - .shared - .applied_index_watcher(group_id) - .current(); - if target == 0 { - return; - } - let deadline = std::time::Instant::now() + Duration::from_secs(10); - loop { - let all_caught_up = self - .nodes - .iter() - .all(|n| n.shared.applied_index_watcher(group_id).current() >= target); - if all_caught_up { - return; - } - if std::time::Instant::now() >= deadline { - // Don't panic — the caller's own `wait_for` assertion - // will report the specific visibility failure with a - // better error than "convergence timed out". - return; - } - tokio::time::sleep(Duration::from_millis(5)).await; - } - } - - /// Block until every node's per-group applied watermark has - /// caught up to the maximum observed across the cluster for that - /// group. This is the deterministic barrier for "every Raft - /// group has fully propagated" — replaces the SQL-polling - /// pattern (`wait_for_async("rows visible from node N", ...)`) - /// that races the follower applier under load. - /// - /// For every group registered on *any* node, the target is - /// `max(applied_index across all nodes)`. Each node then waits - /// for that group's local watcher to reach the target. New - /// groups that show up partway through (e.g. a vshard the test - /// has not written to yet) are handled by re-snapshotting on - /// every iteration of the outer poll, so the helper is - /// idempotent against late-bound group registration. - /// - /// Returns once every (node, group) pair has converged or the - /// deadline expires. On expiry, falls through silently — the - /// caller's own assertion will surface the specific row-level - /// failure with a more useful error than "convergence timed - /// out". - pub async fn wait_for_full_apply_convergence(&self, timeout: Duration) { - let deadline = std::time::Instant::now() + timeout; - loop { - // Targets: for each group_id seen on any node, take the - // max applied_index. Asymmetric group membership is - // expected — replication factor may be < node count, so - // not every group is hosted on every node. - let mut targets: std::collections::HashMap = std::collections::HashMap::new(); - for node in &self.nodes { - for (gid, applied) in node.shared.group_watchers().snapshot() { - let entry = targets.entry(gid).or_insert(0); - if applied > *entry { - *entry = applied; - } - } - } - - // Group membership is read from the routing table (the - // authoritative source) rather than inferred from the - // watcher registry. Inferring from the registry has a - // cold-start race: a follower that hosts group X but - // hasn't yet applied its first entry has no registry - // entry, would be treated as "not hosted", and the - // helper would return prematurely. Routing knows the - // members + learners list as soon as the conf-change - // commits. - let all_caught_up = { - let routing = self.nodes[0] - .shared - .cluster_routing - .as_ref() - .expect("cluster_routing") - .read() - .unwrap_or_else(|p| p.into_inner()); - - self.nodes.iter().all(|node| { - let nid = node.shared.node_id; - let watcher = node.shared.group_watchers(); - targets.iter().all(|(&gid, &target)| { - let hosts = routing - .group_info(gid) - .map(|info| info.members.contains(&nid) || info.learners.contains(&nid)) - .unwrap_or(false); - if !hosts { - return true; - } - watcher.get_or_create(gid).current() >= target - }) - }) - }; - if all_caught_up { - return; - } - if std::time::Instant::now() >= deadline { - // Falls through silently — the caller's own - // assertion will surface the specific row-level - // failure with a more useful error than - // "convergence timed out". - return; - } - tokio::time::sleep(Duration::from_millis(5)).await; - } - } - /// Cooperatively shut down every node. Reverse order so peers /// observe their neighbours' drop without rejecting inbound /// traffic on an already-closed transport. diff --git a/nodedb-test-support/src/cluster_harness/cluster/mod.rs b/nodedb-test-support/src/cluster_harness/cluster/mod.rs index d0ed6e4ba..6ee9896c8 100644 --- a/nodedb-test-support/src/cluster_harness/cluster/mod.rs +++ b/nodedb-test-support/src/cluster_harness/cluster/mod.rs @@ -4,12 +4,14 @@ //! //! [`TestCluster`] + [`ClusterSpawnConfig`] type definitions live in //! [`types`]; spawn-variant convenience wrappers in [`spawn_variants`]; -//! the bringup body in [`bringup`]; the convergence barriers in [`ready`]; +//! the bringup body in [`bringup`]; the readiness barriers in [`ready`]; +//! the apply-convergence barriers in [`convergence`]; //! the in-place restart in [`restart`]; post-spawn membership + DDL helpers //! in [`membership`]; data-group leadership moves and waits in //! [`leadership`]. mod bringup; +mod convergence; mod leadership; mod membership; mod ready; @@ -17,6 +19,7 @@ mod restart; mod spawn_variants; mod types; +pub use convergence::GroupPosition; pub use leadership::GroupLeader; pub use restart::{StoppedCluster, StoppedMember, StoppedNodeInfo}; pub use types::TestCluster; diff --git a/nodedb-test-support/src/cluster_harness/cluster/spawn_variants.rs b/nodedb-test-support/src/cluster_harness/cluster/spawn_variants.rs index a1c57240d..b9f2e5855 100644 --- a/nodedb-test-support/src/cluster_harness/cluster/spawn_variants.rs +++ b/nodedb-test-support/src/cluster_harness/cluster/spawn_variants.rs @@ -69,6 +69,19 @@ impl TestCluster { .await } + /// [`Self::spawn_three`] with `rpc_timeout_secs` as every node's fixed + /// RPC timeout. A test sets it to size the time a read reserves for its + /// retries. + pub async fn spawn_three_with_rpc_timeout( + rpc_timeout_secs: u64, + ) -> Result> { + Self::spawn_three_with_tuning(ClusterTransportTuning { + rpc_timeout_secs, + ..fast_cluster_tuning() + }) + .await + } + /// Spawn a 3-node cluster with a custom `ClusterTransportTuning`. /// Used by the descriptor-lease renewal tests to drive the /// renewal loop on a much faster cadence than the production diff --git a/nodedb-test-support/src/cluster_harness/cluster/types.rs b/nodedb-test-support/src/cluster_harness/cluster/types.rs index 23983f3cd..e27a2b87b 100644 --- a/nodedb-test-support/src/cluster_harness/cluster/types.rs +++ b/nodedb-test-support/src/cluster_harness/cluster/types.rs @@ -49,6 +49,10 @@ pub(crate) struct ClusterSpawnConfig { /// runs `TimeseriesToning::default()`. pub(crate) node_timeseries_tuning: std::collections::HashMap, + /// Chunked-redo bounds every node runs. `None` keeps the default + /// `[tuning.calvin]` bounds. + pub(crate) redo_chunk_limits: + Option, } impl ClusterSpawnConfig { diff --git a/nodedb-test-support/src/cluster_harness/mod.rs b/nodedb-test-support/src/cluster_harness/mod.rs index eb4f9f0e3..4aa023190 100644 --- a/nodedb-test-support/src/cluster_harness/mod.rs +++ b/nodedb-test-support/src/cluster_harness/mod.rs @@ -22,7 +22,7 @@ pub mod retriable; pub mod shared_steps; pub mod wait; -pub use cluster::{GroupLeader, StoppedCluster, StoppedNodeInfo, TestCluster}; +pub use cluster::{GroupLeader, GroupPosition, StoppedCluster, StoppedNodeInfo, TestCluster}; pub use node::TestClusterNode; pub use pitr::PitrStorage; pub use retriable::{is_no_serving_leader, read_once_a_leader_exists}; diff --git a/nodedb-test-support/src/cluster_harness/node/lifecycle/spawn_full.rs b/nodedb-test-support/src/cluster_harness/node/lifecycle/spawn_full.rs index 004496689..b88c03706 100644 --- a/nodedb-test-support/src/cluster_harness/node/lifecycle/spawn_full.rs +++ b/nodedb-test-support/src/cluster_harness/node/lifecycle/spawn_full.rs @@ -200,6 +200,16 @@ impl TestClusterNode { // cloned. Before `EventPlane::spawn` below: the Event Plane starts the // cross-shard drain only when its sender is wired. if let Some(state) = Arc::get_mut(&mut shared) { + // No stream is open yet: the store with the test's bounds + // replaces the default one before any entry applies. + if let Some(limits) = config.redo_chunk_limits { + state.redo_chunks = Arc::new( + nodedb::control::wal_replication::transaction_redo::RedoChunkStore::new( + Arc::clone(&wal), + limits, + ), + ); + } // Consumer offsets, job history, MV state and the array-sync // stores live in the data directory in the production layout. A // restart reopens them there, since no metadata entry below the @@ -326,6 +336,12 @@ impl TestClusterNode { .unwrap_or_else(|p| p.into_inner()) .take(); + // The data-group recovery production boot runs, in the background: + // the harness starts nodes one at a time, and a group elects only + // once its peers run. It sets `data_groups_replayed`, which gates + // the forget of a departed vShard's Calvin state. + nodedb::bootstrap::data_group_recovery::spawn_data_group_recovery(&shared); + // CRDT constraint reconcile loop (leader-gated). The production server // spawns this from `spawn_background_loops`, which the harness does not // call; wire it directly so cluster tests exercise constraint delivery diff --git a/nodedb-test-support/src/cluster_harness/node/lifecycle/spawn_variants.rs b/nodedb-test-support/src/cluster_harness/node/lifecycle/spawn_variants.rs index 1f46d3508..3f816eae0 100644 --- a/nodedb-test-support/src/cluster_harness/node/lifecycle/spawn_variants.rs +++ b/nodedb-test-support/src/cluster_harness/node/lifecycle/spawn_variants.rs @@ -108,6 +108,7 @@ impl TestClusterNode { backup_storage: None, pitr: None, node_timeseries_tuning: std::collections::HashMap::new(), + redo_chunk_limits: None, }; Self::spawn_with_full_config(node_id, seed_nodes, &config).await } @@ -166,6 +167,7 @@ impl TestClusterNode { backup_storage: None, pitr: None, node_timeseries_tuning: std::collections::HashMap::new(), + redo_chunk_limits: None, }; Self::spawn_with_full_config_at(1, vec![], &config, data_dir_path, None).await } diff --git a/nodedb-test-support/src/cluster_harness/node/mod.rs b/nodedb-test-support/src/cluster_harness/node/mod.rs index 19ac65c78..5768ebde8 100644 --- a/nodedb-test-support/src/cluster_harness/node/mod.rs +++ b/nodedb-test-support/src/cluster_harness/node/mod.rs @@ -9,5 +9,6 @@ pub mod leadership; pub mod lifecycle; pub mod native_client; pub mod raw_pgwire; +pub mod remount; pub use lifecycle::TestClusterNode; diff --git a/nodedb-test-support/src/cluster_harness/node/remount.rs b/nodedb-test-support/src/cluster_harness/node/remount.rs new file mode 100644 index 000000000..51bab42e6 --- /dev/null +++ b/nodedb-test-support/src/cluster_harness/node/remount.rs @@ -0,0 +1,110 @@ +// SPDX-License-Identifier: BUSL-1.1 + +//! Unmount one data group on a [`TestClusterNode`] and mount it again from +//! its log on disk, as a node that left a group and rejoined it does. + +use std::sync::Arc; + +use nodedb::control::state::SharedState; +use nodedb_cluster::ClusterError; + +use crate::cluster_harness::node::lifecycle::TestClusterNode; + +/// Why a remount of a data group failed. Every case names the node and the +/// group. +#[derive(Debug, thiserror::Error)] +pub enum RemountError { + #[error("node {node_id}: the remount task of group {group_id} did not finish: {source}")] + Task { + node_id: u64, + group_id: u64, + source: tokio::task::JoinError, + }, + #[error("node {node_id}: its Raft groups are not started, so group {group_id} has none")] + NotStarted { node_id: u64, group_id: u64 }, + #[error("node {node_id}: group {group_id} is not mounted")] + NotMounted { node_id: u64, group_id: u64 }, + #[error("node {node_id}: dropping the redo streams of group {group_id}: {source}")] + DropStreams { + node_id: u64, + group_id: u64, + source: nodedb::Error, + }, + #[error("node {node_id}: reopening group {group_id}: {source}")] + Reopen { + node_id: u64, + group_id: u64, + source: ClusterError, + }, + #[error("node {node_id}: group {group_id} mounted again while its lock was held")] + MountedTwice { node_id: u64, group_id: u64 }, +} + +impl TestClusterNode { + /// Unmount `group_id` here, drop its redo streams as the membership pass + /// drops the streams of a group this node no longer mounts, and mount the + /// group again from its log on disk with the same voters and learners. + /// + /// The node's `MultiRaft` lock is held throughout, so no tick mounts the + /// group in between. + pub async fn remount_data_group(&self, group_id: u64) -> Result<(), RemountError> { + let shared = Arc::clone(&self.shared); + let node_id = self.node_id; + tokio::task::spawn_blocking(move || remount(&shared, node_id, group_id)) + .await + .map_err(|source| RemountError::Task { + node_id, + group_id, + source, + })? + } +} + +/// Run the remount on a blocking thread: closing and opening a group's log +/// blocks on disk. +fn remount(shared: &SharedState, node_id: u64, group_id: u64) -> Result<(), RemountError> { + let multi_raft = shared + .multi_raft + .get() + .ok_or(RemountError::NotStarted { node_id, group_id })?; + let mut mr = multi_raft.lock().unwrap_or_else(|p| p.into_inner()); + let membership = mr + .group_membership(group_id) + .ok_or(RemountError::NotMounted { node_id, group_id })?; + let starts_as_learner = membership.learners.contains(&node_id); + let peers: Vec = membership + .voters + .into_iter() + .filter(|&id| id != node_id) + .collect(); + let learners: Vec = membership + .learners + .into_iter() + .filter(|&id| id != node_id) + .collect(); + // Dropping the replica closes its log file, which the mount reopens. + drop(mr.unmount_group(group_id)); + shared + .redo_chunks + .drop_unhosted_groups( + |group| mr.contains_group(group), + shared.credentials.catalog(), + ) + .map_err(|source| RemountError::DropStreams { + node_id, + group_id, + source, + })?; + let opened = mr + .mount_spec(group_id, peers, learners, starts_as_learner) + .open() + .map_err(|source| RemountError::Reopen { + node_id, + group_id, + source, + })?; + if mr.insert_opened(opened).is_some() { + return Err(RemountError::MountedTwice { node_id, group_id }); + } + Ok(()) +} diff --git a/nodedb-test-support/src/native_harness/frames.rs b/nodedb-test-support/src/native_harness/frames.rs index f99c9f2b7..2e549ad01 100644 --- a/nodedb-test-support/src/native_harness/frames.rs +++ b/nodedb-test-support/src/native_harness/frames.rs @@ -132,11 +132,29 @@ pub async fn read_frame(stream: &mut TcpStream) -> Option> { /// must send one JSON frame before calling this. Shared by [`send_sql`] and /// any test driving a direct-op opcode (`PointGet`, `RangeScan`, /// `VectorSearch`, `KvBatchPut`, ...) directly rather than through SQL text. +/// +/// Waits at most [`RESPONSE_BOUND`] for the response. pub async fn send_request( stream: &mut TcpStream, seq: u64, op: OpCode, fields: TextFields, +) -> NativeResponse { + send_request_within(stream, seq, op, fields, Some(RESPONSE_BOUND)).await +} + +/// How long [`send_request`] waits for a response. +pub const RESPONSE_BOUND: Duration = Duration::from_secs(5); + +/// [`send_request`] waiting at most `bound` for the response, or without +/// limit when `bound` is `None`. A caller that parks the server on purpose +/// bounds the wait itself. +pub async fn send_request_within( + stream: &mut TcpStream, + seq: u64, + op: OpCode, + fields: TextFields, + bound: Option, ) -> NativeResponse { let req = NativeRequest { op, @@ -146,14 +164,18 @@ pub async fn send_request( let json_bytes = sonic_rs::to_vec(&req).expect("json encode"); write_frame(stream, &json_bytes).await; - let deadline = tokio::time::Instant::now() + Duration::from_secs(5); + let deadline = bound.map(|bound| tokio::time::Instant::now() + bound); let mut accumulated: Option = None; loop { - let remaining = deadline.saturating_duration_since(tokio::time::Instant::now()); - let response_payload = tokio::time::timeout(remaining, read_frame(stream)) - .await - .expect("timeout waiting for response") - .expect("response frame"); + let response_payload = match deadline { + Some(deadline) => tokio::time::timeout_at(deadline, read_frame(stream)) + .await + .unwrap_or_else(|_| { + panic!("no response to request seq {seq} ({op:?}) within {bound:?}") + }), + None => read_frame(stream).await, + } + .expect("response frame"); let response: NativeResponse = sonic_rs::from_slice(&response_payload).expect("json decode NativeResponse"); if response.seq == seq { diff --git a/nodedb-test-support/src/native_harness/mod.rs b/nodedb-test-support/src/native_harness/mod.rs index cb0df1c10..9bdf9d15f 100644 --- a/nodedb-test-support/src/native_harness/mod.rs +++ b/nodedb-test-support/src/native_harness/mod.rs @@ -10,7 +10,7 @@ mod frames; mod server; pub use frames::{ - do_handshake, do_handshake_from, open_trust_session, read_frame, send_api_key_auth, - send_request, send_sql, write_frame, + RESPONSE_BOUND, do_handshake, do_handshake_from, open_trust_session, read_frame, + send_api_key_auth, send_request, send_request_within, send_sql, write_frame, }; pub use server::NativeTestServer; diff --git a/nodedb-types/src/array_route.rs b/nodedb-types/src/array_route.rs new file mode 100644 index 000000000..506b939eb --- /dev/null +++ b/nodedb-types/src/array_route.rs @@ -0,0 +1,241 @@ +// SPDX-License-Identifier: Apache-2.0 + +//! Routing bucket and vShard of an array tile. +//! +//! A tile key orders tiles on a space-filling curve, so its top bits carry +//! little entropy when tile indices are small. Routing therefore takes the +//! top bits of a bijective mix of the key. Stored keys stay unchanged. +//! The cluster router, the Data Plane shard filter and the snapshot paths +//! use these functions, so a tile has one bucket and one vShard everywhere. +//! +//! The router uses at most [`VShardId::BITS`] bucket bits. Bucket `b` of `R` +//! routing bits owns the vShards `[b * stride, (b + 1) * stride)`, with +//! `stride = VShardId::COUNT >> R`. Its primary vShard is `b * stride`. + +use serde::{Deserialize, Serialize}; + +use crate::id::VShardId; + +/// The largest routing granularity an array declares. +pub const MAX_PREFIX_BITS: u8 = 16; + +/// An array routing granularity outside `1..=MAX_PREFIX_BITS`. +#[derive(Debug, Clone, Copy, PartialEq, Eq, thiserror::Error)] +#[error("array routing: prefix_bits must be 1-{}, got {}", MAX_PREFIX_BITS, .0)] +pub struct PrefixBitsOutOfRange(pub u8); + +/// MurmurHash3 64-bit finalizer. A bijection on `u64`: distinct keys keep +/// distinct values, and every input bit affects every output bit. +pub const fn fmix64(key: u64) -> u64 { + let mut k = key; + k ^= k >> 33; + k = k.wrapping_mul(0xff51_afd7_ed55_8ccd); + k ^= k >> 33; + k = k.wrapping_mul(0xc4ce_b9fe_1a85_ec53); + k ^= k >> 33; + k +} + +/// The routing bucket of tile key `tile_key`: the top `prefix_bits` bits of +/// `fmix64(tile_key)`. Returns a value in `[0, 2^prefix_bits)`. With +/// `prefix_bits == 0` every key maps to bucket 0. +pub const fn prefix_bucket(tile_key: u64, prefix_bits: u8) -> u32 { + let bits = if prefix_bits > 32 { 32 } else { prefix_bits }; + if bits == 0 { + return 0; + } + (fmix64(tile_key) >> (64 - bits as u32)) as u32 +} + +/// The bucket one shard serves. A shard read keeps only the tiles whose +/// key falls in this bucket, so shards that share one Data Plane never +/// return the same tile twice. +#[derive( + Debug, + Clone, + Copy, + PartialEq, + Eq, + Hash, + Serialize, + Deserialize, + zerompk::ToMessagePack, + zerompk::FromMessagePack, +)] +pub struct ShardBucket { + /// Bucket index in `[0, 2^prefix_bits)`. + pub bucket: u32, + /// Number of mixed key bits the bucket is taken from. + pub prefix_bits: u8, +} + +impl ShardBucket { + /// Whether tile key `tile_key` routes to this bucket. + pub const fn contains(&self, tile_key: u64) -> bool { + prefix_bucket(tile_key, self.prefix_bits) == self.bucket + } +} + +/// Whether `tile_key` passes the optional shard filter. `None` keeps every +/// tile. +pub fn shard_bucket_admits(filter: Option, tile_key: u64) -> bool { + filter.is_none_or(|b| b.contains(tile_key)) +} + +/// The bucket bits the router uses for `prefix_bits`: at most +/// [`VShardId::BITS`], since a finer bucket has no vShard of its own. +pub fn routing_bits(prefix_bits: u8) -> Result { + if prefix_bits == 0 || prefix_bits > MAX_PREFIX_BITS { + return Err(PrefixBitsOutOfRange(prefix_bits)); + } + Ok(prefix_bits.min(VShardId::BITS)) +} + +/// The primary vShard of `bucket` at `bits` routing bits. `bits` comes from +/// [`routing_bits`], so it is at most [`VShardId::BITS`]. +pub const fn bucket_to_vshard(bucket: u32, bits: u8) -> u32 { + bucket << (VShardId::BITS - bits) as u32 +} + +/// The primary vShard that owns the tile with key `tile_key`. +/// +/// `prefix_bits` is the array's routing granularity (1–16). Returns a vShard +/// ID in `[0, VShardId::COUNT)`. +pub fn array_vshard_for_tile(tile_key: u64, prefix_bits: u8) -> Result { + let bits = routing_bits(prefix_bits)?; + Ok(bucket_to_vshard(prefix_bucket(tile_key, bits), bits)) +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn fmix64_is_injective_on_a_sample() { + let mut seen = std::collections::HashSet::new(); + for k in 0u64..100_000 { + assert!(seen.insert(fmix64(k)), "collision at {k}"); + } + assert_eq!(fmix64(0), 0); + } + + #[test] + fn prefix_bucket_stays_in_range() { + for bits in 1u8..=16 { + for k in [0u64, 1, 2, 63, 1 << 40, u64::MAX / 3, u64::MAX] { + assert!(prefix_bucket(k, bits) < (1u32 << bits), "bits={bits} k={k}"); + } + } + assert_eq!(prefix_bucket(u64::MAX, 0), 0); + } + + #[test] + fn small_keys_spread_over_buckets() { + // Keys whose top bits are all zero land in most of the 256 buckets. + let buckets: std::collections::HashSet = + (0u64..4096).map(|k| prefix_bucket(k, 8)).collect(); + assert!( + buckets.len() >= 240, + "only {} of 256 buckets used", + buckets.len() + ); + } + + #[test] + fn shard_bucket_partitions_keys() { + let bits = 4u8; + for k in 0u64..2048 { + let owners = (0u32..(1 << bits)) + .filter(|&b| { + ShardBucket { + bucket: b, + prefix_bits: bits, + } + .contains(k) + }) + .count(); + assert_eq!(owners, 1, "key {k} has {owners} owners"); + } + assert!(shard_bucket_admits(None, 7)); + } + + #[test] + fn tile_routing_invalid_prefix_bits() { + assert_eq!(array_vshard_for_tile(0, 0), Err(PrefixBitsOutOfRange(0))); + assert_eq!(array_vshard_for_tile(0, 17), Err(PrefixBitsOutOfRange(17))); + } + + #[test] + fn tile_routing_uses_the_mixed_bucket() { + // prefix_bits=8: stride=4, so the vShard is the mixed bucket times 4. + for key in [0u64, 1, 2, 0x0100_0000_0000_0000, u64::MAX] { + let vshard = array_vshard_for_tile(key, 8).unwrap(); + assert_eq!(vshard, prefix_bucket(key, 8) * 4, "key={key:#x}"); + } + // prefix_bits=10: stride=1, so the vShard is the mixed bucket. + let key = 0x0040_0000_0000_0000; + assert_eq!( + array_vshard_for_tile(key, 10).unwrap(), + prefix_bucket(key, 10) + ); + } + + #[test] + fn tile_routing_output_in_vshard_range() { + // Every vShard returned is below VShardId::COUNT, including + // prefix_bits above VShardId::BITS. + for bits in 1u8..=MAX_PREFIX_BITS { + for &key in &[0u64, 1, u64::MAX / 3, u64::MAX / 2, u64::MAX] { + let vshard = array_vshard_for_tile(key, bits).unwrap(); + assert!( + vshard < VShardId::COUNT, + "bits={bits} key={key:#x} → vshard={vshard}" + ); + } + } + } + + #[test] + fn prefix_bits_above_vshard_bits_route_like_vshard_bits() { + for key in [3u64, 99, 1 << 50, u64::MAX - 7] { + let at_max = array_vshard_for_tile(key, VShardId::BITS).unwrap(); + for bits in VShardId::BITS..=MAX_PREFIX_BITS { + assert_eq!(array_vshard_for_tile(key, bits).unwrap(), at_max); + } + } + } + + #[test] + fn small_tile_keys_spread_over_many_vshards() { + // Keys whose top bits are all zero reach most of the 256 buckets. + let vshards: std::collections::BTreeSet = (0u64..4096) + .map(|key| array_vshard_for_tile(key, 8).unwrap()) + .collect(); + assert!( + vshards.len() >= 240, + "only {} of 256 buckets used", + vshards.len() + ); + } + + #[test] + fn bucket_to_vshard_strides() { + assert_eq!(VShardId::COUNT, 1 << VShardId::BITS); + assert_eq!(VShardId::BITS, 10); + assert_eq!(bucket_to_vshard(1, 1), 512); + assert_eq!(bucket_to_vshard(1, 8), 4); + assert_eq!(bucket_to_vshard(1, 10), 1); + assert_eq!(routing_bits(16), Ok(10)); + } + + #[test] + fn shard_bucket_roundtrips_through_msgpack() { + let b = ShardBucket { + bucket: 41, + prefix_bits: 8, + }; + let bytes = zerompk::to_msgpack_vec(&b).expect("encode"); + let back: ShardBucket = zerompk::from_msgpack(&bytes).expect("decode"); + assert_eq!(b, back); + } +} diff --git a/nodedb-types/src/config/tuning/network.rs b/nodedb-types/src/config/tuning/network.rs index 6f2ef37c6..f6b72d335 100644 --- a/nodedb-types/src/config/tuning/network.rs +++ b/nodedb-types/src/config/tuning/network.rs @@ -170,12 +170,11 @@ pub struct ClusterTransportTuning { /// [`Self::election_timeout_min_ms`]. #[serde(default)] pub election_timeout_max_ms: u64, + /// Timeout of one RPC round trip, and the QUIC idle timeout: a peer + /// silent this long is dead. Keep-alives go out at a fifth of it. + /// Startup refuses a value below 500 ms. #[serde(default = "default_rpc_timeout_secs")] pub rpc_timeout_secs: u64, - #[serde(default = "default_quic_keep_alive_secs")] - pub quic_keep_alive_secs: u64, - #[serde(default = "default_quic_idle_timeout_secs")] - pub quic_idle_timeout_secs: u64, #[serde(default = "default_quic_max_bi_streams")] pub quic_max_bi_streams: u32, #[serde(default = "default_quic_max_uni_streams")] @@ -262,8 +261,6 @@ impl Default for ClusterTransportTuning { election_timeout_min_ms: 0, election_timeout_max_ms: 0, rpc_timeout_secs: default_rpc_timeout_secs(), - quic_keep_alive_secs: default_quic_keep_alive_secs(), - quic_idle_timeout_secs: default_quic_idle_timeout_secs(), quic_max_bi_streams: default_quic_max_bi_streams(), quic_max_uni_streams: default_quic_max_uni_streams(), quic_receive_window: default_quic_receive_window(), @@ -309,12 +306,6 @@ fn default_election_timeout_max_secs() -> u64 { fn default_rpc_timeout_secs() -> u64 { 5 } -fn default_quic_keep_alive_secs() -> u64 { - 5 -} -fn default_quic_idle_timeout_secs() -> u64 { - 30 -} fn default_quic_max_bi_streams() -> u32 { 256 } diff --git a/nodedb-types/src/id/vshard.rs b/nodedb-types/src/id/vshard.rs index 0230f2a07..7732b78c6 100644 --- a/nodedb-types/src/id/vshard.rs +++ b/nodedb-types/src/id/vshard.rs @@ -26,10 +26,15 @@ use serde::{Deserialize, Serialize}; )] pub struct VShardId(pub(super) u32); +const _: () = assert!(VShardId::COUNT.is_power_of_two()); + impl VShardId { /// Total number of virtual shards in the system. pub const COUNT: u32 = 1024; + /// Bits that address every vShard once: `COUNT == 1 << BITS`. + pub const BITS: u8 = Self::COUNT.trailing_zeros() as u8; + pub const fn new(id: u32) -> Self { assert!(id < Self::COUNT, "vShard ID must be < 1024"); Self(id) diff --git a/nodedb-types/src/lib.rs b/nodedb-types/src/lib.rs index 13b965607..f8a7e9372 100644 --- a/nodedb-types/src/lib.rs +++ b/nodedb-types/src/lib.rs @@ -15,6 +15,7 @@ pub mod quota; pub mod approx; pub mod array_cell; +pub mod array_route; pub mod ascii; pub mod backup_envelope; pub mod bbox; @@ -78,6 +79,7 @@ pub mod write_version; pub use approx::{CountMinSketch, HyperLogLog, SpaceSaving, TDigest}; pub use array_cell::ArrayCell; +pub use array_route::ShardBucket; pub use ascii::{ find_ascii_case_insensitive, find_ascii_case_insensitive_from, rfind_ascii_case_insensitive, starts_with_ascii_case_insensitive, strip_prefix_ascii_case_insensitive, diff --git a/nodedb/src/bootstrap/cluster_ready.rs b/nodedb/src/bootstrap/cluster_ready.rs index f6b24d6e0..b3627e86b 100644 --- a/nodedb/src/bootstrap/cluster_ready.rs +++ b/nodedb/src/bootstrap/cluster_ready.rs @@ -44,8 +44,8 @@ pub async fn await_cluster_ready( // Boot-time readiness gate: wait until the metadata raft group has // applied its first entry on this node before opening any // client-facing listener. This eliminates the restart-window race - // where the first DDL observes `metadata propose: not leader` - // because the election has not yet completed. + // where the first DDL finds no metadata leader because the election + // has not yet completed. wait_for_raft_ready( shared, &mut raft_ready_rx, diff --git a/nodedb/src/bootstrap/constraint_reconcile.rs b/nodedb/src/bootstrap/constraint_reconcile.rs index 90e2b28b1..b1f3e671e 100644 --- a/nodedb/src/bootstrap/constraint_reconcile.rs +++ b/nodedb/src/bootstrap/constraint_reconcile.rs @@ -169,7 +169,9 @@ pub async fn reconcile_once( }, ); - let deadline = crate::control::wal_replication::statement_propose_deadline(shared); + let deadline = + crate::control::server::shared::session::StatementDeadline::of_statement(shared) + .tokio_instant(); match propose_replicated_entry(shared, &proposer, entry, deadline).await { Ok(_) => { // Record only on commit. A transient / NotLeader error diff --git a/nodedb/src/bootstrap/data_group_recovery.rs b/nodedb/src/bootstrap/data_group_recovery.rs index 950a0e87e..6dbc89986 100644 --- a/nodedb/src/bootstrap/data_group_recovery.rs +++ b/nodedb/src/bootstrap/data_group_recovery.rs @@ -7,23 +7,28 @@ //! (KV hash index, columnar/timeseries/spatial in-memory state, graph node //! labels, …) is rebuilt only by re-applying that log into the Data Plane. //! -//! Raft's `commit_index` / `last_applied` are volatile: after a restart every -//! locally hosted group must win its own randomized election before it commits -//! a no-op and carries its retained log forward to the applier. Those elections -//! run independently of the metadata group's, so waiting only on metadata +//! Raft's `commit_index` is volatile: after a restart every locally hosted +//! group must win its own randomized election before it commits a no-op and +//! carries its retained log forward to the applier. Those elections run +//! independently of the metadata group's, so waiting only on metadata //! readiness lets the gateway open while a data group has replayed nothing — an //! acknowledged write then reads back as if it never happened, with no error to //! explain it. //! //! This gate closes that window: startup is held until every locally hosted //! data group has elected a leader, committed through the end of its retained -//! log, and applied that log. It fails closed — a timeout aborts startup rather -//! than opening a gateway that would serve incomplete state. +//! log, and applied that log on this node. It fails closed — a timeout aborts +//! startup rather than opening a gateway that would serve incomplete state. +//! +//! A group whose replica here awaits a snapshot install does not hold startup. +//! Its data diverged from the log, so it applies nothing until a snapshot +//! installs, and its reads are refused until then. use std::sync::Arc; use std::time::{Duration, Instant}; -use tracing::{debug, info}; +use nodedb_raft::ReplicaState; +use tracing::{debug, info, warn}; use crate::control::state::SharedState; @@ -47,7 +52,7 @@ fn is_data_group(group_id: u64) -> bool { && group_id != nodedb_cluster::calvin::SEQUENCER_GROUP_ID } -/// Why a data group is not yet recovered, or `None` when it is. +/// Why a data group is not yet recovered. #[derive(Debug, Clone, Copy, PartialEq, Eq)] enum GroupWait { /// No leader yet — the post-restart election is still running, so the group @@ -59,21 +64,46 @@ enum GroupWait { commit_index: u64, last_log_index: u64, }, - /// The committed log is known, but has not been fully applied yet. + /// The committed log is known, but this node has not applied all of it. ReplayLagging { commit_index: u64, - last_applied: u64, + host_applied: u64, }, } -/// Recovery predicate for one group, from a single `GroupStatus` snapshot. +/// Where one data group stands against the gate. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +enum GroupRecovery { + /// Applied on this node through its committed log. + Replayed, + /// Its replica here awaits a snapshot install. It applies nothing until + /// one installs and serves no reads meanwhile, so it does not hold + /// startup. + AwaitingSnapshot, + /// Still holding startup. + Pending(GroupWait), +} + +/// One group's progress, from its Raft status and this node's watermark. +#[derive(Debug, Clone, Copy)] +struct GroupProgress { + leader_id: u64, + commit_index: u64, + last_log_index: u64, + /// The highest index whose apply concluded on this node: the watermark + /// reads wait on. + host_applied: u64, + replica_state: ReplicaState, +} + +/// Recovery verdict for one group. /// -/// `last_applied` is Raft's own applied watermark, and is the only usable -/// progress signal here. The Data-Plane-side apply watermark cannot be the -/// target: a leader's election no-op is committed but is never delivered to the -/// distributed applier, so a Data-Plane watermark stops one short of -/// `commit_index` and never converges — and every post-restart log ends in -/// exactly such a no-op. +/// The progress signal is the host watermark, not Raft's `last_applied`. Raft +/// advances `last_applied` when it hands an entry to the apply loop, before +/// the entry's apply concludes. Reads wait on the host watermark, so a gate on +/// `last_applied` opens while replay still runs. The election no-op reaches +/// the apply loop and concludes like any entry, so the host watermark reaches +/// `commit_index`. /// /// `commit_index` alone is not a sufficient target either. It is volatile and /// starts at 0 after a restart, and Raft publishes `leader_id` before the new @@ -87,32 +117,32 @@ enum GroupWait { /// Once `commit_index >= last_log_index` the target is static while the gateway /// is closed — no new entries can be proposed — so this converges rather than /// chasing a moving head. -fn group_wait( - leader_id: u64, - commit_index: u64, - last_log_index: u64, - last_applied: u64, -) -> Option { - if leader_id == 0 { - return Some(GroupWait::NoLeader); +fn group_recovery(progress: GroupProgress) -> GroupRecovery { + // Checked first: such a replica never leads, so a group it holds alone + // never elects. + if progress.replica_state == ReplicaState::AwaitingSnapshot { + return GroupRecovery::AwaitingSnapshot; + } + if progress.leader_id == 0 { + return GroupRecovery::Pending(GroupWait::NoLeader); } // Empty log: nothing was ever written to this group, so there is nothing to // replay once it has a leader. - if last_log_index == 0 { - return None; + if progress.last_log_index == 0 { + return GroupRecovery::Replayed; } - if commit_index < last_log_index { - return Some(GroupWait::LogUncommitted { - commit_index, - last_log_index, + if progress.commit_index < progress.last_log_index { + return GroupRecovery::Pending(GroupWait::LogUncommitted { + commit_index: progress.commit_index, + last_log_index: progress.last_log_index, }); } - if last_applied >= commit_index { - return None; + if progress.host_applied >= progress.commit_index { + return GroupRecovery::Replayed; } - Some(GroupWait::ReplayLagging { - commit_index, - last_applied, + GroupRecovery::Pending(GroupWait::ReplayLagging { + commit_index: progress.commit_index, + host_applied: progress.host_applied, }) } @@ -137,38 +167,53 @@ impl std::fmt::Display for PendingGroup { ), GroupWait::ReplayLagging { commit_index, - last_applied, + host_applied, } => write!( f, - "group {} applied {last_applied} of {commit_index} committed entries", + "group {} applied {host_applied} of {commit_index} committed entries on this node", self.group_id ), } } } -/// Collect the data groups that are not yet recovered. -fn pending_groups(statuses: Vec) -> Vec { - statuses - .into_iter() - .filter(|s| is_data_group(s.group_id)) - .filter_map(|s| { - group_wait( - s.leader_id, - s.commit_index, - s.last_log_index, - s.last_applied, - ) - .map(|wait| PendingGroup { - group_id: s.group_id, +/// The data groups still holding startup, and the groups awaiting a snapshot. +#[derive(Debug, Default)] +struct GateView { + pending: Vec, + awaiting_snapshot: Vec, +} + +/// Sort every data group in `statuses` against the gate. `host_applied` +/// reads this node's watermark of a group. +fn gate_view( + statuses: Vec, + host_applied: impl Fn(u64) -> u64, +) -> GateView { + let mut view = GateView::default(); + for status in statuses.into_iter().filter(|s| is_data_group(s.group_id)) { + let progress = GroupProgress { + leader_id: status.leader_id, + commit_index: status.commit_index, + last_log_index: status.last_log_index, + host_applied: host_applied(status.group_id), + replica_state: status.replica_state, + }; + match group_recovery(progress) { + GroupRecovery::Replayed => {} + GroupRecovery::AwaitingSnapshot => view.awaiting_snapshot.push(status.group_id), + GroupRecovery::Pending(wait) => view.pending.push(PendingGroup { + group_id: status.group_id, wait, - }) - }) - .collect() + }), + } + } + view } /// Hold startup until every locally hosted data group has replayed its retained -/// Raft log. +/// Raft log on this node. A group awaiting a snapshot install does not hold it. +/// On success it sets `SharedState::data_groups_replayed`. /// /// Refuses when `start_raft` installed no Raft status source. pub async fn await_data_group_recovery(shared: &Arc) -> anyhow::Result<()> { @@ -179,14 +224,27 @@ pub async fn await_data_group_recovery(shared: &Arc) -> anyhow::Res let deadline = Instant::now() + DATA_GROUP_RECOVERY_TIMEOUT; loop { - let pending = pending_groups(status_fn()); - if pending.is_empty() { + let view = gate_view(status_fn(), |group_id| { + shared.applied_index_watcher(group_id).current() + }); + if view.pending.is_empty() { + for group_id in &view.awaiting_snapshot { + warn!( + group_id, + "data raft group awaits a snapshot install on this node; it serves no reads \ + until one installs" + ); + } info!("all local data raft groups replayed — opening client gateway"); + shared + .data_groups_replayed + .store(true, std::sync::atomic::Ordering::Release); return Ok(()); } if Instant::now() >= deadline { - let detail = pending + let detail = view + .pending .iter() .map(|p| p.to_string()) .collect::>() @@ -197,17 +255,79 @@ pub async fn await_data_group_recovery(shared: &Arc) -> anyhow::Res } debug!( - pending = pending.len(), + pending = view.pending.len(), "waiting for data raft group replay" ); tokio::time::sleep(POLL_INTERVAL).await; } } +/// Run [`await_data_group_recovery`] in a loop on `shared`'s shutdown +/// watch until it succeeds, without holding the caller. A host that brings +/// nodes up one at a time uses it: a group here elects only once its peers +/// run. Each failed attempt is logged, and the next one starts at once. +pub fn spawn_data_group_recovery(shared: &Arc) { + let task_shared = Arc::clone(shared); + crate::control::shutdown::spawn_loop( + &shared.loop_registry, + &shared.shutdown, + "data_group_recovery", + crate::control::shutdown::ShutdownPhase::DrainingControlPlane, + move |mut shutdown| async move { + loop { + tokio::select! { + biased; + _ = shutdown.wait_cancelled() => break, + recovered = await_data_group_recovery(&task_shared) => match recovered { + Ok(()) => break, + Err(error) => warn!(%error, "data raft group recovery did not finish; retrying"), + }, + } + } + }, + ); +} + #[cfg(test)] mod tests { use super::*; + fn progress( + leader_id: u64, + commit_index: u64, + last_log_index: u64, + host_applied: u64, + ) -> GroupProgress { + GroupProgress { + leader_id, + commit_index, + last_log_index, + host_applied, + replica_state: ReplicaState::Whole, + } + } + + fn status( + group_id: u64, + raft_applied: u64, + replica_state: ReplicaState, + ) -> nodedb_cluster::GroupStatus { + nodedb_cluster::GroupStatus { + group_id, + role: "Leader".into(), + leader_id: 1, + term: 2, + commit_index: 12, + last_applied: raft_applied, + last_log_index: 12, + snapshot_index: 0, + replica_state, + member_count: 1, + learner_count: 0, + vshard_count: 1, + } + } + #[test] fn metadata_and_sequencer_groups_are_not_data_groups() { assert!(!is_data_group(nodedb_cluster::METADATA_GROUP_ID)); @@ -222,13 +342,22 @@ mod tests { #[test] fn leaderless_group_waits_for_election() { - assert_eq!(group_wait(0, 0, 0, 0), Some(GroupWait::NoLeader)); - assert_eq!(group_wait(0, 9, 9, 9), Some(GroupWait::NoLeader)); + assert_eq!( + group_recovery(progress(0, 0, 0, 0)), + GroupRecovery::Pending(GroupWait::NoLeader) + ); + assert_eq!( + group_recovery(progress(0, 9, 9, 9)), + GroupRecovery::Pending(GroupWait::NoLeader) + ); } #[test] fn empty_log_is_recovered_once_leader_exists() { - assert_eq!(group_wait(1, 0, 0, 0), None); + assert_eq!( + group_recovery(progress(1, 0, 0, 0)), + GroupRecovery::Replayed + ); } /// The window this gate exists to close: a restart resets the volatile @@ -238,8 +367,8 @@ mod tests { #[test] fn fresh_leader_with_uncommitted_retained_log_waits() { assert_eq!( - group_wait(1, 0, 12, 0), - Some(GroupWait::LogUncommitted { + group_recovery(progress(1, 0, 12, 0)), + GroupRecovery::Pending(GroupWait::LogUncommitted { commit_index: 0, last_log_index: 12, }) @@ -249,8 +378,8 @@ mod tests { #[test] fn partially_committed_retained_log_waits() { assert_eq!( - group_wait(1, 5, 12, 5), - Some(GroupWait::LogUncommitted { + group_recovery(progress(1, 5, 12, 5)), + GroupRecovery::Pending(GroupWait::LogUncommitted { commit_index: 5, last_log_index: 12, }) @@ -260,26 +389,84 @@ mod tests { #[test] fn committed_but_unapplied_log_waits() { assert_eq!( - group_wait(1, 12, 12, 4), - Some(GroupWait::ReplayLagging { + group_recovery(progress(1, 12, 12, 4)), + GroupRecovery::Pending(GroupWait::ReplayLagging { commit_index: 12, - last_applied: 4, + host_applied: 4, }) ); } - /// A freshly elected leader whose only committed entry is its own election - /// no-op is recovered once Raft has applied it. The no-op never reaches the - /// Data Plane, which is why Raft's `last_applied` — not a Data-Plane - /// watermark — is the progress signal here. + /// Raft hands every entry to the apply loop, so its `last_applied` reaches + /// `commit_index` while replay still runs. The host watermark below it + /// holds the gate. + #[test] + fn a_handed_off_log_waits_for_the_host_watermark() { + let view = gate_view(vec![status(4, 12, ReplicaState::Whole)], |_| 7); + assert!(view.awaiting_snapshot.is_empty()); + assert_eq!(view.pending.len(), 1); + assert_eq!( + view.pending[0].wait, + GroupWait::ReplayLagging { + commit_index: 12, + host_applied: 7, + } + ); + } + + /// The host watermark at `commit_index` opens the gate, the election no-op + /// included: it concludes on the apply loop like any entry. #[test] - fn election_noop_only_log_converges() { - assert_eq!(group_wait(1, 1, 1, 1), None); + fn a_host_watermark_at_the_commit_index_is_recovered() { + let view = gate_view(vec![status(4, 12, ReplicaState::Whole)], |_| 12); + assert!(view.pending.is_empty()); + assert!(view.awaiting_snapshot.is_empty()); + assert_eq!( + group_recovery(progress(1, 1, 1, 1)), + GroupRecovery::Replayed + ); + assert_eq!( + group_recovery(progress(1, 12, 12, 13)), + GroupRecovery::Replayed + ); } + /// A replica that awaits a snapshot install applies nothing until one + /// installs. It does not hold startup, even with no leader, and is + /// reported apart. #[test] - fn caught_up_group_is_recovered() { - assert_eq!(group_wait(1, 12, 12, 12), None); - assert_eq!(group_wait(1, 12, 12, 13), None); + fn a_group_awaiting_a_snapshot_does_not_hold_startup() { + let awaiting = GroupProgress { + replica_state: ReplicaState::AwaitingSnapshot, + ..progress(0, 0, 12, 3) + }; + assert_eq!(group_recovery(awaiting), GroupRecovery::AwaitingSnapshot); + + let view = gate_view( + vec![ + status(4, 12, ReplicaState::AwaitingSnapshot), + status(5, 12, ReplicaState::Whole), + ], + |group_id| if group_id == 4 { 3 } else { 12 }, + ); + assert!(view.pending.is_empty()); + assert_eq!(view.awaiting_snapshot, vec![4]); + } + + /// A replica that only wants a snapshot keeps its data whole and can + /// lead, so it holds startup like any other group until it replays. + #[test] + fn a_group_wanting_a_snapshot_still_replays() { + let wanting = GroupProgress { + replica_state: ReplicaState::WantsSnapshot, + ..progress(1, 12, 12, 4) + }; + assert_eq!( + group_recovery(wanting), + GroupRecovery::Pending(GroupWait::ReplayLagging { + commit_index: 12, + host_applied: 4, + }) + ); } } diff --git a/nodedb/src/bridge/envelope/error_code_from.rs b/nodedb/src/bridge/envelope/error_code_from.rs index c94d78be3..c1862b0fa 100644 --- a/nodedb/src/bridge/envelope/error_code_from.rs +++ b/nodedb/src/bridge/envelope/error_code_from.rs @@ -298,6 +298,8 @@ impl From for ErrorCode { | crate::Error::CommittedResultUnavailable { .. } | crate::Error::ProposalOutcomeUnknown { .. } | crate::Error::MetadataLeaderUnavailable + | crate::Error::MetadataGroupGone { .. } + | crate::Error::MetadataDdlSuperseded { .. } | crate::Error::Wal(_) | crate::Error::Dispatch { .. } | crate::Error::Storage { .. } diff --git a/nodedb/src/config/server/config.rs b/nodedb/src/config/server/config.rs index b83609e69..4c09766e3 100644 --- a/nodedb/src/config/server/config.rs +++ b/nodedb/src/config/server/config.rs @@ -422,6 +422,24 @@ mod tests { assert!(msg.contains("at least 10 seconds"), "{msg}"); } + /// The RPC timeout is the QUIC idle timeout, so one under its floor is + /// refused at startup. + #[test] + fn from_file_rejects_an_rpc_timeout_below_the_floor() { + let path = write_temp_config( + "nodedb-domain-rpc-timeout.toml", + "[tuning.cluster_transport]\nrpc_timeout_secs = 0\n", + ); + let err = ServerConfig::from_file(&path).unwrap_err(); + std::fs::remove_file(&path).ok(); + let msg = err.to_string(); + assert!( + msg.contains("tuning.cluster_transport.rpc_timeout_secs"), + "{msg}" + ); + assert!(msg.contains("at least 500 ms"), "{msg}"); + } + #[test] fn from_file_rejects_a_wal_write_buffer_under_the_floor() { let path = write_temp_config( diff --git a/nodedb/src/config/server/domain.rs b/nodedb/src/config/server/domain.rs index b152a1300..375d79b4c 100644 --- a/nodedb/src/config/server/domain.rs +++ b/nodedb/src/config/server/domain.rs @@ -120,6 +120,7 @@ pub(super) fn validate_domain(config: &ServerConfig) -> crate::Result<()> { } validate_calvin(config)?; + validate_rpc_timeout(config)?; if let Some(cluster) = config.cluster.as_ref() { positive_u64( @@ -148,6 +149,21 @@ pub(super) fn validate_domain(config: &ServerConfig) -> crate::Result<()> { Ok(()) } +/// Checks the RPC timeout against its floor. It is also the QUIC idle +/// timeout, so a shorter one drops live peers. +fn validate_rpc_timeout(config: &ServerConfig) -> crate::Result<()> { + let floor = nodedb_cluster::transport::liveness::MIN_RPC_TIMEOUT; + let secs = config.tuning.cluster_transport.rpc_timeout_secs; + if std::time::Duration::from_secs(secs) < floor { + return Err(reject( + "tuning.cluster_transport.rpc_timeout_secs", + secs, + &format!("a timeout of at least {} ms", floor.as_millis()), + )); + } + Ok(()) +} + /// Checks `[tuning.calvin]`. A zero channel capacity panics the channel /// constructor, and a zero bound stalls every scheduler. fn validate_calvin(config: &ServerConfig) -> crate::Result<()> { diff --git a/nodedb/src/control/array_sync/inbound_propose.rs b/nodedb/src/control/array_sync/inbound_propose.rs index a11379cdc..c225da50c 100644 --- a/nodedb/src/control/array_sync/inbound_propose.rs +++ b/nodedb/src/control/array_sync/inbound_propose.rs @@ -96,8 +96,9 @@ impl OriginArrayInbound { )) })? .as_ref(); - let deadline = tokio::time::Instant::now() - + std::time::Duration::from_secs(self.shared().tuning.network.default_deadline_secs); + let deadline = + crate::control::server::shared::session::StatementDeadline::of_statement(self.shared()) + .tokio_instant(); match async_proposer(vshard_id, idempotency_key, data, deadline).await { Ok((_payload, _committed_version)) => Ok(()), Err(e) => { diff --git a/nodedb/src/control/backup/cut.rs b/nodedb/src/control/backup/cut.rs index 5ea8eed68..6d76047e3 100644 --- a/nodedb/src/control/backup/cut.rs +++ b/nodedb/src/control/backup/cut.rs @@ -48,12 +48,11 @@ //! //! [`ReplicatedWrite::CutBarrier`]: crate::control::wal_replication::ReplicatedWrite::CutBarrier -use std::time::Duration; - use nodedb_physical::physical_plan::CutCaptureRequest; use nodedb_types::Hlc; use crate::Error; +use crate::control::server::shared::session::StatementDeadline; use crate::control::state::SharedState; use super::cut_order::OrderedCut; @@ -72,7 +71,7 @@ pub(super) async fn consistent_cut(state: &SharedState) -> Result { /// Take the consistent cut at `watermark` on this node: wait until every /// write committed below it has a final outcome here. A remote source node /// runs it for the watermark the backup's coordinator picked. -pub(crate) async fn cut_at(state: &SharedState, watermark: u64) -> Result<(), Error> { +pub async fn cut_at(state: &SharedState, watermark: u64) -> Result<(), Error> { cut_at_point(state, watermark, 0).await } @@ -118,8 +117,9 @@ async fn cut_barriers(state: &SharedState, cut: OrderedCut) -> Result<(), Error> state .hlc_clock .update(Hlc::new(cut.hlc.saturating_add(1), 0)); - let timeout = Duration::from_secs(state.tuning.network.default_deadline_secs); - let deadline = tokio::time::Instant::now() + timeout; + let statement = StatementDeadline::of_statement(state); + let budget_ms = statement.remaining().as_millis(); + let deadline = statement.tokio_instant(); // Read after the watermark: a record minted after this read stamps its // write above the watermark. let target = state.outcome_floor.max_noted(); @@ -145,11 +145,11 @@ async fn cut_barriers(state: &SharedState, cut: OrderedCut) -> Result<(), Error> return Err(Error::Internal { detail: format!( "backup: writes minted at or below WAL LSN {} had no final outcome within \ - {}s, so the backup cannot take a consistent cut (outcome floor at {}). \ - Retry the backup. A write window held until restart keeps the floor \ - below it: restart the node if the floor does not move", + the statement deadline ({} ms), so the backup cannot take a consistent cut \ + (outcome floor at {}). Retry the backup. A write window held until restart \ + keeps the floor below it: restart the node if the floor does not move", target.as_u64(), - timeout.as_secs(), + budget_ms, state.outcome_floor.floor().as_u64(), ), }); diff --git a/nodedb/src/control/backup/cut_order/ordered_cut.rs b/nodedb/src/control/backup/cut_order/ordered_cut.rs index 29b6fa860..ec8406aeb 100644 --- a/nodedb/src/control/backup/cut_order/ordered_cut.rs +++ b/nodedb/src/control/backup/cut_order/ordered_cut.rs @@ -25,6 +25,7 @@ use nodedb_physical::physical_plan::CutCaptureRequest; use crate::control::state::SharedState; use crate::control::wal_replication::ReplicatedWrite; +use crate::types::CoveredCutBarrier; /// The clock skew between two nodes a cut's window tolerates. const CLOCK_SKEW_MARGIN: Duration = Duration::from_secs(30); @@ -93,6 +94,25 @@ impl OrderedCut { capture: self.capture.clone(), } } + + /// This cut's barrier at `log_index`, as a data-group snapshot carries it. + pub fn covered_at(&self, log_index: u64) -> CoveredCutBarrier { + CoveredCutBarrier { + hlc: self.hlc, + restore_point: self.restore_point, + capture: self.capture.clone(), + log_index, + } + } + + /// The cut a data-group snapshot's `barrier` names. + pub fn from_covered(barrier: &CoveredCutBarrier) -> Self { + Self { + hlc: barrier.hlc, + restore_point: barrier.restore_point, + capture: barrier.capture.clone(), + } + } } /// The ordering window of one cut on this node. @@ -165,6 +185,22 @@ mod tests { assert_ne!(plain.key(), cut.key(), "the capture keeps two cuts apart"); } + #[test] + fn a_cut_round_trips_through_a_snapshot() { + let cut = OrderedCut { + hlc: 7, + restore_point: 3, + capture: Some(CutCaptureRequest { + request_id: 11, + database_id: 2, + tenants: vec![1], + }), + }; + let covered = cut.covered_at(40); + assert_eq!(covered.log_index, 40); + assert_eq!(OrderedCut::from_covered(&covered), cut); + } + #[test] fn the_hold_outlasts_the_proposal_window_by_the_skew_margin() { let window = CutWindow::of(1_000, Duration::from_secs(30)); diff --git a/nodedb/src/control/backup/cut_order/registry.rs b/nodedb/src/control/backup/cut_order/registry.rs index dcdb15e71..d63a0a98e 100644 --- a/nodedb/src/control/backup/cut_order/registry.rs +++ b/nodedb/src/control/backup/cut_order/registry.rs @@ -117,6 +117,20 @@ impl CutBarriers { self.group(key, group).and_then(|barrier| barrier.applied) } + /// Every tracked cut this node applied a barrier of in `group`, with the + /// barrier's log index, in cut key order. + pub(crate) fn applied_in_group(&self, group: u64) -> Vec<(OrderedCut, u64)> { + self.cuts + .lock() + .unwrap_or_else(|p| p.into_inner()) + .values() + .filter_map(|tracked| { + let applied = tracked.groups.get(&group)?.applied?; + Some((tracked.cut.clone(), applied)) + }) + .collect() + } + /// Note that this node proposed `key`'s barrier into `group` while it /// led the group. pub(crate) fn note_proposed(&self, key: CutKey, group: u64, proposed: Proposed) { @@ -214,6 +228,21 @@ mod tests { assert_eq!(barriers.cut(key), Some(cut(5))); } + #[test] + fn a_group_lists_the_barriers_it_applied_in_key_order() { + let barriers = CutBarriers::new(); + barriers.note_applied(&cut(9), 1, 60); + barriers.note_applied(&cut(5), 1, 40); + barriers.note_applied(&cut(7), 2, 50); + barriers.claim_driver(&cut(8)); + assert_eq!( + barriers.applied_in_group(1), + [(cut(5), 40), (cut(9), 60)], + "only applied barriers of the group, by cut key" + ); + assert_eq!(barriers.applied_in_group(3), []); + } + /// A lost proposal is forgotten only when it is the one recorded. #[test] fn a_lost_proposal_is_forgotten_once() { diff --git a/nodedb/src/control/backup/restore/array_reissue.rs b/nodedb/src/control/backup/restore/array_reissue.rs index 87b176994..586722c50 100644 --- a/nodedb/src/control/backup/restore/array_reissue.rs +++ b/nodedb/src/control/backup/restore/array_reissue.rs @@ -326,7 +326,8 @@ impl CellWrite<'_> { state, proposer, entry, - crate::control::wal_replication::statement_propose_deadline(state), + crate::control::server::shared::session::StatementDeadline::of_statement(state) + .tokio_instant(), ) .await?; Ok(()) diff --git a/nodedb/src/control/backup/restore/crdt_reissue.rs b/nodedb/src/control/backup/restore/crdt_reissue.rs index 9016067ab..9caf721b9 100644 --- a/nodedb/src/control/backup/restore/crdt_reissue.rs +++ b/nodedb/src/control/backup/restore/crdt_reissue.rs @@ -51,7 +51,8 @@ async fn reissue_crdt_collection( })? .with_event_source(EventSource::Restore) .with_restore_id(target.restore_id); - let deadline = crate::control::wal_replication::statement_propose_deadline(state); + let deadline = crate::control::server::shared::session::StatementDeadline::of_statement(state) + .tokio_instant(); crate::control::wal_replication::propose_replicated_entry(state, proposer, entry, deadline) .await?; Ok(()) diff --git a/nodedb/src/control/backup/restore/durable.rs b/nodedb/src/control/backup/restore/durable.rs index 3fc14e7d1..2c7d19715 100644 --- a/nodedb/src/control/backup/restore/durable.rs +++ b/nodedb/src/control/backup/restore/durable.rs @@ -67,7 +67,8 @@ pub(crate) async fn reissue_plan_durably( // fire again for it. .with_event_source(crate::event::EventSource::Restore) .with_restore_id(target.restore_id); - let deadline = crate::control::wal_replication::statement_propose_deadline(state); + let deadline = crate::control::server::shared::session::StatementDeadline::of_statement(state) + .tokio_instant(); let (_, write_versions) = crate::control::wal_replication::propose_replicated_entry(state, proposer, entry, deadline) .await?; diff --git a/nodedb/src/control/backup/restore/guard.rs b/nodedb/src/control/backup/restore/guard.rs index 8e4c64467..2e8570039 100644 --- a/nodedb/src/control/backup/restore/guard.rs +++ b/nodedb/src/control/backup/restore/guard.rs @@ -37,6 +37,7 @@ use crate::Error; use crate::control::security::auth_fence::cluster::{ confirmed_read_index, hosts_group, routed_groups, wait_applied, }; +use crate::control::server::shared::session::StatementDeadline; use crate::control::state::SharedState; use crate::control::state::tenant_marks::GroupMark; use crate::types::{DatabaseId, TraceId}; @@ -92,8 +93,7 @@ pub(super) async fn newest_committed_write( } }; // The statement deadline, shared by every group and every attempt. - let deadline = tokio::time::Instant::now() - + Duration::from_secs(state.tuning.network.default_deadline_secs); + let deadline = StatementDeadline::of_statement(state).tokio_instant(); let groups: Vec = routed_groups(state) .into_iter() .filter(|group| *group != METADATA_GROUP_ID && *group != SEQUENCER_GROUP_ID) diff --git a/nodedb/src/control/backup/schedule/marks.rs b/nodedb/src/control/backup/schedule/marks.rs index bd98f7b9a..5217b88b3 100644 --- a/nodedb/src/control/backup/schedule/marks.rs +++ b/nodedb/src/control/backup/schedule/marks.rs @@ -12,12 +12,12 @@ //! lags therefore never mistakes a finished minute for a due one. use std::sync::Arc; -use std::time::{Duration, Instant}; +use std::time::Duration; use super::blocking::off_runtime; use crate::config::server::BackupScheduleSettings; use crate::control::catalog_entry::CatalogEntry; -use crate::control::cluster::linearizable_read::confirm_linearizable_read; +use crate::control::cluster::linearizable_read::{StatementDeadline, confirm_linearizable_read}; use crate::control::metadata_proposer::propose_catalog_entry_async; use crate::control::security::catalog::backup_schedule_marks::StoredBackupScheduleMark; use crate::control::state::SharedState; @@ -52,7 +52,7 @@ pub async fn settled_through_linearizable( confirm_linearizable_read( state, &[nodedb_cluster::METADATA_GROUP_ID], - Instant::now() + MARK_READ_TIMEOUT, + StatementDeadline::after(MARK_READ_TIMEOUT), ) .await?; let (state, schedule) = (Arc::clone(state), schedule.clone()); diff --git a/nodedb/src/control/catalog_entry/post_apply/async_dispatch/array.rs b/nodedb/src/control/catalog_entry/post_apply/async_dispatch/array.rs index d71b8f6bf..cc5c9bcd5 100644 --- a/nodedb/src/control/catalog_entry/post_apply/async_dispatch/array.rs +++ b/nodedb/src/control/catalog_entry/post_apply/async_dispatch/array.rs @@ -11,7 +11,7 @@ //! done: the tombstone fences a same-name open until the next `PutArray` //! purges it. //! - A `DeleteArray` with `moved_to` rekeys the store on every core instead. -//! Cells route by Hilbert prefix alone, so the vShard of every cell stays +//! Cells route by tile key alone, so the vShard of every cell stays //! the same, and only the store directory carries the database. //! - Every `DeleteArray` changes the mirror last, under the array //! incarnation's exclusive gate (see `control::write_gate`). diff --git a/nodedb/src/control/catalog_entry/post_apply/async_dispatch/collection.rs b/nodedb/src/control/catalog_entry/post_apply/async_dispatch/collection.rs index e80b09e54..7a8b0fdb1 100644 --- a/nodedb/src/control/catalog_entry/post_apply/async_dispatch/collection.rs +++ b/nodedb/src/control/catalog_entry/post_apply/async_dispatch/collection.rs @@ -186,7 +186,10 @@ pub(crate) async fn reclaim_collection_storage( // performance, requirement. let hold = (!drain_already_held).then(|| shared.quiesce.begin_drain(database_id, tenant_id, name)); - wait_drained_reporting_progress(shared, database_id, tenant_id, name).await; + shared + .quiesce + .wait_until_drained(database_id, tenant_id, name) + .await; // 4. Reclaim on local Data Plane. RESULT-CHECKED: the redb + // versioned engine purge is correctness-critical (the catalog @@ -282,41 +285,6 @@ pub(crate) async fn reclaim_collection_storage( } } -/// How often a quiesce drain checks for closed scans. -const DRAIN_PROGRESS_TICK: std::time::Duration = std::time::Duration::from_millis(100); - -/// Wait until every open scan of the collection closes. Each closed scan -/// counts as apply progress, so a proposer waiting on this purge keeps -/// waiting while scans close and times out only when none does. -/// -/// A draining collection admits no new scan, so the open count only falls. -async fn wait_drained_reporting_progress( - shared: &SharedState, - database_id: u64, - tenant_id: u64, - name: &str, -) { - let drained = shared - .quiesce - .wait_until_drained(database_id, tenant_id, name); - tokio::pin!(drained); - let mut open = shared.quiesce.open_scans(database_id, tenant_id, name); - loop { - tokio::select! { - () = &mut drained => return, - () = tokio::time::sleep(DRAIN_PROGRESS_TICK) => { - let now = shared.quiesce.open_scans(database_id, tenant_id, name); - if now < open { - shared - .metadata_apply_progress - .fetch_add(1, std::sync::atomic::Ordering::Release); - } - open = now; - } - } - } -} - /// Clear this node's storage under `name` before a new incarnation of the /// collection registers there. /// diff --git a/nodedb/src/control/cluster/array_cluster_exec/dispatch.rs b/nodedb/src/control/cluster/array_cluster_exec/dispatch.rs index b594003c9..4a14acece 100644 --- a/nodedb/src/control/cluster/array_cluster_exec/dispatch.rs +++ b/nodedb/src/control/cluster/array_cluster_exec/dispatch.rs @@ -15,9 +15,9 @@ use async_trait::async_trait; use nodedb_cluster::distributed_array::ArrayLocalExecutor; use nodedb_cluster::distributed_array::handler::handle_array_shard_rpc; use nodedb_cluster::distributed_array::rpc::ShardRpcDispatch; -use nodedb_cluster::error::Result as ClusterResult; -use nodedb_cluster::rpc_codec::RaftRpc; -use nodedb_cluster::wire::VShardEnvelope; +use nodedb_cluster::error::{ClusterError, Result as ClusterResult}; +use nodedb_cluster::rpc_codec::{RaftRpc, remaining_budget_ms}; +use nodedb_cluster::wire::{VShardEnvelope, VShardMessageType}; use nodedb_cluster::{NexarTransport, RoutingTable}; use crate::control::cluster::array_cluster_helpers::array_resp_msg_type; @@ -59,25 +59,59 @@ impl NexarArrayDispatch { } } +/// The sends one shard call makes at most: the first, and one resend. +const CALL_ATTEMPTS: usize = 2; + +/// The pause before a resend, so the metadata applier can heal a routing +/// entry that named a stale or fake leader. +const RESEND_PAUSE: std::time::Duration = std::time::Duration::from_millis(50); + #[async_trait] impl ShardRpcDispatch for NexarArrayDispatch { + /// Every send gets what is left of the `timeout_ms` budget. A live shard + /// can use all of it. A dead shard fails the send at the QUIC idle + /// timeout, and the resend reaches the shard's current leader. async fn call(&self, req: VShardEnvelope, timeout_ms: u64) -> ClusterResult { - match self.call_once(&req, timeout_ms).await { - Ok(resp) => Ok(resp), - Err(first_err) if is_transport_err(&first_err) => { - // Routing table may have pointed at a stale or fake leader. - // The metadata applier heals it asynchronously; wait briefly, - // re-read routing, and retry once. - tokio::time::sleep(std::time::Duration::from_millis(50)).await; - self.call_once(&req, timeout_ms).await + let deadline = tokio::time::Instant::now() + std::time::Duration::from_millis(timeout_ms); + let resend_safe = is_resend_safe(req.msg_type); + let mut attempts_left = CALL_ATTEMPTS; + loop { + match self.call_once(&req, remaining_budget_ms(deadline)).await { + Ok(resp) => return Ok(resp), + // A resend starts only while the budget has time left. + Err(e) + if attempts_left > 1 + && remaining_budget_ms(deadline) > 0 + && may_resend(&e, resend_safe) => + { + // `call_once` re-reads routing, so the resend reaches the + // shard's current leader. + attempts_left -= 1; + tokio::time::sleep(RESEND_PAUSE).await; + } + Err(e) => return Err(e), } - Err(e) => Err(e), } } } -fn is_transport_err(e: &nodedb_cluster::error::ClusterError) -> bool { - matches!(e, nodedb_cluster::error::ClusterError::Transport { .. }) +/// Whether a second run of the request leaves the same state and answer. A +/// slice, an aggregate and a surrogate bitmap only read the shard. A put or +/// a delete can have applied when its answer was lost. +fn is_resend_safe(msg_type: VShardMessageType) -> bool { + matches!( + msg_type, + VShardMessageType::ArrayShardSliceReq + | VShardMessageType::ArrayShardAggReq + | VShardMessageType::ArrayShardSurrogateBitmapReq + ) +} + +/// Whether a failed send can go again. A request that never reached the +/// shard always can. A lost answer can only for a resend-safe request: +/// any other one reports its unknown outcome. +fn may_resend(e: &ClusterError, resend_safe: bool) -> bool { + e.is_unsent() || (resend_safe && e.is_lost_answer()) } impl NexarArrayDispatch { @@ -130,8 +164,8 @@ impl NexarArrayDispatch { // Wrap in RaftRpc and send. The remote node's handler decodes the // envelope bytes and dispatches to the appropriate shard handler. let rpc = RaftRpc::VShardEnvelope(envelope_bytes); - // The shard's budget bounds the reply wait, never the transport's fixed - // RPC timeout. + // The call's remaining budget bounds the reply wait, never the + // transport's fixed RPC timeout. let reply_wait = std::time::Duration::from_millis(timeout_ms); let resp_rpc = tokio::time::timeout( reply_wait, @@ -139,16 +173,18 @@ impl NexarArrayDispatch { .send_rpc_with_read_timeout(node_id, rpc, reply_wait), ) .await - // The request can have reached the shard, so a timeout is never - // resent: `call` retries only a `Transport` error. + // The request can have reached the shard. The budget is spent, so + // `call` starts no resend. .map_err(|_| nodedb_cluster::error::ClusterError::ShardTimeout { vshard_id: req.vshard_id, elapsed_ms: timeout_ms, })??; + // The shard answered, so an answer this node cannot read is a codec + // error: a resend can run a put or a delete twice. match resp_rpc { RaftRpc::VShardEnvelope(bytes) => VShardEnvelope::from_bytes(&bytes).ok_or_else(|| { - nodedb_cluster::error::ClusterError::Transport { + nodedb_cluster::error::ClusterError::Codec { detail: "array shard response: failed to decode VShardEnvelope".into(), } }), @@ -158,7 +194,7 @@ impl NexarArrayDispatch { RaftRpc::VShardRefusal(refusal) => { Err(nodedb_cluster::error::ClusterError::from(refusal.error)) } - other => Err(nodedb_cluster::error::ClusterError::Transport { + other => Err(nodedb_cluster::error::ClusterError::Codec { detail: format!( "array shard RPC: unexpected response type {:?}", std::mem::discriminant(&other) @@ -167,3 +203,63 @@ impl NexarArrayDispatch { } } } + +#[cfg(test)] +mod tests { + use super::*; + + fn timed_out() -> ClusterError { + ClusterError::ShardTimeout { + vshard_id: 3, + elapsed_ms: 1_000, + } + } + + /// A slice, an aggregate and a surrogate bitmap only read the shard. + #[test] + fn only_array_reads_are_resend_safe() { + assert!(is_resend_safe(VShardMessageType::ArrayShardSliceReq)); + assert!(is_resend_safe(VShardMessageType::ArrayShardAggReq)); + assert!(is_resend_safe( + VShardMessageType::ArrayShardSurrogateBitmapReq + )); + assert!(!is_resend_safe(VShardMessageType::ArrayShardPutReq)); + assert!(!is_resend_safe(VShardMessageType::ArrayShardDeleteReq)); + } + + /// A read whose answer was lost goes again. + #[test] + fn a_timed_out_read_resends() { + assert!(may_resend(&timed_out(), true)); + let unanswered = ClusterError::Unanswered { + node_id: 2, + detail: "stream reset".into(), + }; + assert!(may_resend(&unanswered, true)); + } + + /// A put or a delete whose answer was lost reports its unknown outcome. + #[test] + fn a_timed_out_write_is_never_resent() { + assert!(!may_resend(&timed_out(), false)); + } + + /// A request that never reached the shard goes again, read or write. + #[test] + fn an_unsent_request_resends() { + let unsent = ClusterError::Transport { + detail: "connect refused".into(), + }; + assert!(may_resend(&unsent, false)); + assert!(may_resend(&unsent, true)); + } + + /// An answer the shard gave never goes again. + #[test] + fn an_answer_is_never_resent() { + let unreadable = ClusterError::Codec { + detail: "array shard response: failed to decode VShardEnvelope".into(), + }; + assert!(!may_resend(&unreadable, true)); + } +} diff --git a/nodedb/src/control/cluster/array_cluster_exec/executor.rs b/nodedb/src/control/cluster/array_cluster_exec/executor.rs index c3bc8ddbc..ea3680a59 100644 --- a/nodedb/src/control/cluster/array_cluster_exec/executor.rs +++ b/nodedb/src/control/cluster/array_cluster_exec/executor.rs @@ -17,7 +17,7 @@ use nodedb_cluster::routing::VSHARD_COUNT; use nodedb_cluster::{NexarTransport, RoutingTable}; use crate::control::cluster::array_cluster_helpers::{ - cluster_err, encode_err, finalize_agg_partials, + cluster_err, coord_err, encode_err, finalize_agg_partials, }; use crate::control::server::shared::session::pending_shard_reads::{ShardObservation, ShardReads}; use crate::control::server::shared::session::read_set::EngineTag; @@ -48,7 +48,7 @@ struct SliceArgs<'a> { slice_msgpack: &'a [u8], attr_projection: &'a [u32], limit: u32, - slice_hilbert_ranges: &'a [(u64, u64)], + slice_tile_keys: &'a [u64], prefix_bits: u8, system_time: nodedb_types::SystemTimeScope, valid_at_ms: Option, @@ -60,7 +60,7 @@ struct AggArgs<'a> { attr_idx: u32, reducer_msgpack: &'a [u8], group_by_dim: i32, - slice_hilbert_ranges: &'a [(u64, u64)], + slice_tile_keys: &'a [u64], prefix_bits: u8, system_as_of: Option, valid_at_ms: Option, @@ -114,7 +114,7 @@ impl ClusterArrayExecutor { slice_msgpack, attr_projection, limit, - slice_hilbert_ranges, + slice_tile_keys, prefix_bits, system_time, valid_at_ms, @@ -124,7 +124,7 @@ impl ClusterArrayExecutor { slice_msgpack, attr_projection, limit: *limit, - slice_hilbert_ranges, + slice_tile_keys, prefix_bits: *prefix_bits, system_time: *system_time, valid_at_ms: *valid_at_ms, @@ -138,7 +138,7 @@ impl ClusterArrayExecutor { attr_idx, reducer_msgpack, group_by_dim, - slice_hilbert_ranges, + slice_tile_keys, prefix_bits, system_as_of, valid_at_ms, @@ -148,7 +148,7 @@ impl ClusterArrayExecutor { attr_idx: *attr_idx, reducer_msgpack, group_by_dim: *group_by_dim, - slice_hilbert_ranges, + slice_tile_keys, prefix_bits: *prefix_bits, system_as_of: *system_as_of, valid_at_ms: *valid_at_ms, @@ -187,7 +187,7 @@ impl ClusterArrayExecutor { slice_msgpack, attr_projection, limit, - slice_hilbert_ranges, + slice_tile_keys, prefix_bits, system_time, valid_at_ms, @@ -196,7 +196,7 @@ impl ClusterArrayExecutor { let coordinator = ArrayCoordinator::for_slice( self.source_node, ARRAY_RPC_TIMEOUT_MS, - slice_hilbert_ranges, + slice_tile_keys, prefix_bits, self.total_shards, Arc::clone(&self.dispatch), @@ -211,13 +211,11 @@ impl ClusterArrayExecutor { attr_projection: attr_projection.to_vec(), limit, cell_filter_msgpack: vec![], - // prefix_bits, slice_hilbert_ranges, and shard_hilbert_range are - // stamped per-shard by coord_slice from ArrayCoordParams, which was - // populated by for_slice above. Initialise to defaults here; - // coord_slice overwrites them before serialising the request envelope. + // coord_slice stamps prefix_bits, slice_tile_keys and shard_bucket + // per shard from the ArrayCoordParams that for_slice built. prefix_bits: 0, - slice_hilbert_ranges: vec![], - shard_hilbert_range: None, + slice_tile_keys: vec![], + shard_bucket: None, system_time, valid_at_ms, txn_id: txn_id.map(TxnId::as_u64), @@ -225,7 +223,7 @@ impl ClusterArrayExecutor { let result = coordinator .coord_slice(req, limit, system_time) .await - .map_err(cluster_err)?; + .map_err(coord_err)?; note_array_reads(array_id, &result.read_versions); // Encode rows into the structured `ArraySliceResponse` — identical to @@ -265,7 +263,7 @@ impl ClusterArrayExecutor { attr_idx, reducer_msgpack, group_by_dim, - slice_hilbert_ranges, + slice_tile_keys, prefix_bits, system_as_of, valid_at_ms, @@ -274,7 +272,7 @@ impl ClusterArrayExecutor { let coordinator = ArrayCoordinator::for_slice( self.source_node, ARRAY_RPC_TIMEOUT_MS, - slice_hilbert_ranges, + slice_tile_keys, prefix_bits, self.total_shards, Arc::clone(&self.dispatch), @@ -289,12 +287,12 @@ impl ClusterArrayExecutor { reducer_msgpack: reducer_msgpack.to_vec(), group_by_dim, cell_filter_msgpack: vec![], - shard_hilbert_range: None, + shard_bucket: None, system_as_of, valid_at_ms, txn_id: txn_id.map(TxnId::as_u64), }; - let agg = coordinator.coord_agg(req).await.map_err(cluster_err)?; + let agg = coordinator.coord_agg(req).await.map_err(coord_err)?; note_array_reads(array_id, &agg.read_versions); // Decode the reducer so we know which field to finalize. @@ -342,7 +340,7 @@ impl ClusterArrayExecutor { &self.circuit_breaker, ) .await - .map_err(cluster_err)?; + .map_err(coord_err)?; // Sum each shard's real `affected` count — never `cells.len()`, which // counts cells named in the request, not cells the Data Plane @@ -373,7 +371,7 @@ impl ClusterArrayExecutor { &self.circuit_breaker, ) .await - .map_err(cluster_err)?; + .map_err(coord_err)?; // Sum each shard's real `affected` count — never `coords.len()`, // which counts coordinates named, not coordinates that existed and diff --git a/nodedb/src/control/cluster/array_cluster_helpers.rs b/nodedb/src/control/cluster/array_cluster_helpers.rs index 9300a5694..c8a4ddee8 100644 --- a/nodedb/src/control/cluster/array_cluster_helpers.rs +++ b/nodedb/src/control/cluster/array_cluster_helpers.rs @@ -5,6 +5,7 @@ //! fast path. use nodedb_cluster::distributed_array::merge::ArrayAggPartial; +use nodedb_cluster::distributed_array::scatter::{ArrayCoordError, ShardFailure}; use nodedb_cluster::error::ClusterError; use nodedb_cluster::wire::VShardMessageType; @@ -95,17 +96,33 @@ pub(super) fn finalize_agg_partials( rows } +/// Map an array coordinator error to an [`Error`]. +/// +/// A request that never reached its shard ran nothing. It answers the +/// definite, retryable `NoLeader` for that vShard. `NoLeader` never moves +/// the gateway's routing hint. Every other error maps as [`cluster_err`]. +pub(super) fn coord_err(e: ArrayCoordError) -> Error { + match e { + ArrayCoordError::Shard(ShardFailure { vshard_id, error }) if error.is_unsent() => { + tracing::debug!(vshard_id, %error, "array shard request never reached its shard"); + Error::NoLeader { + vshard_id: crate::types::VShardId::new(vshard_id), + } + } + ArrayCoordError::Shard(ShardFailure { error, .. }) + | ArrayCoordError::Coordinator(error) => cluster_err(error), + } +} + pub(super) fn cluster_err(e: ClusterError) -> Error { match e { // A shard did not answer within its timeout: surface as a deterministic // deadline rather than an opaque internal error, matching the // `TypedClusterError::DeadlineExceeded` mapping used elsewhere. A // sent request with no answer can have run, so it takes the same class. - ClusterError::ShardTimeout { .. } | ClusterError::Unanswered { .. } => { - Error::DeadlineExceeded { - request_id: crate::types::RequestId::new(0), - } - } + lost if lost.is_lost_answer() => Error::DeadlineExceeded { + request_id: crate::types::RequestId::new(0), + }, // A shard's Data-Plane verdict keeps its code, so the statement // renders the SQLSTATE a single-node execution renders. ClusterError::DataPlane { code } => Error::DataPlane(code.into()), @@ -168,6 +185,9 @@ pub(super) fn cluster_err(e: ClusterError) -> Error { | ClusterError::SpatialGather(_) | ClusterError::Bm25Gather(_) | ClusterError::TsGather(_) + // The guards above take these. They stay named for exhaustiveness. + | ClusterError::ShardTimeout { .. } + | ClusterError::Unanswered { .. } | ClusterError::ShufflePush(_) | ClusterError::RemoteUntyped { .. }) => Error::Internal { detail: format!("array cluster: {other}"), @@ -202,6 +222,45 @@ mod tests { use super::*; use crate::bridge::envelope::ErrorCode; + /// A shard request that never went out answers `NoLeader` for that + /// shard: nothing ran, so the refusal is definite. + #[test] + fn an_unsent_shard_request_is_no_leader_for_its_vshard() { + let unsent = ArrayCoordError::Shard(ShardFailure { + vshard_id: 41, + error: ClusterError::Transport { + detail: "connect refused".into(), + }, + }); + match coord_err(unsent) { + Error::NoLeader { vshard_id } => assert_eq!(vshard_id.as_u32(), 41), + other => panic!("expected NoLeader for vshard 41, got {other:?}"), + } + } + + /// Every other shard failure maps as `cluster_err` does. + #[test] + fn a_lost_shard_answer_stays_an_unknown_outcome() { + let lost = ArrayCoordError::Shard(ShardFailure { + vshard_id: 41, + error: ClusterError::Unanswered { + node_id: 2, + detail: "stream reset".into(), + }, + }); + assert!(matches!(coord_err(lost), Error::DeadlineExceeded { .. })); + } + + /// A coordinator-side transport error names no shard, so it keeps its + /// `cluster_err` mapping. + #[test] + fn a_coordinator_error_maps_as_cluster_err() { + let local = ArrayCoordError::Coordinator(ClusterError::Codec { + detail: "encode".into(), + }); + assert!(matches!(coord_err(local), Error::Internal { .. })); + } + /// A shard verdict that crossed the cluster keeps its code at the /// coordinator, never `Internal`. #[test] diff --git a/nodedb/src/control/cluster/array_executor/executor.rs b/nodedb/src/control/cluster/array_executor/executor.rs index 2383e4f39..20d4f8b48 100644 --- a/nodedb/src/control/cluster/array_executor/executor.rs +++ b/nodedb/src/control/cluster/array_executor/executor.rs @@ -12,7 +12,7 @@ use nodedb_cluster::rpc_codec::DataPlaneErrorCode; use super::refusal::execution_error; use crate::bridge::envelope::{Priority, Request, Response}; use crate::control::cluster::linearizable_read::{ - confirm_linearizable_read, groups_of_vshards, statement_read_deadline, + StatementDeadline, confirm_linearizable_read, groups_of_vshards, }; use crate::control::server::shared::write_admission::plan_is_write; use crate::control::state::SharedState; @@ -59,9 +59,13 @@ impl DataPlaneArrayExecutor { if !plan_is_write(&plan) { let groups = groups_of_vshards(&self.state, [local_vshard_id.as_u32()]) .map_err(|e| execution_error("array read confirmation", e))?; - confirm_linearizable_read(&self.state, &groups, statement_read_deadline(&self.state)) - .await - .map_err(|e| execution_error("array read confirmation", e))?; + confirm_linearizable_read( + &self.state, + &groups, + StatementDeadline::of_statement(&self.state), + ) + .await + .map_err(|e| execution_error("array read confirmation", e))?; } let request_id = self.state.next_request_id(); let request = local_request(request_id, array_id, local_vshard_id, plan, txn_id); diff --git a/nodedb/src/control/cluster/array_executor/read.rs b/nodedb/src/control/cluster/array_executor/read.rs index 6b6334177..596cd6224 100644 --- a/nodedb/src/control/cluster/array_executor/read.rs +++ b/nodedb/src/control/cluster/array_executor/read.rs @@ -48,7 +48,7 @@ impl DataPlaneArrayExecutor { attr_projection: req.attr_projection.clone(), limit: req.limit, cell_filter, - hilbert_range: req.shard_hilbert_range, + shard_bucket: req.shard_bucket, system_time: req.system_time, valid_at_ms: req.valid_at_ms, }); @@ -119,7 +119,7 @@ impl DataPlaneArrayExecutor { group_by_dim: req.group_by_dim, cell_filter, return_partial: true, - hilbert_range: req.shard_hilbert_range, + shard_bucket: req.shard_bucket, system_as_of: req.system_as_of, valid_at_ms: req.valid_at_ms, }); diff --git a/nodedb/src/control/cluster/array_executor/refusal.rs b/nodedb/src/control/cluster/array_executor/refusal.rs index 79e025fef..cd6805f66 100644 --- a/nodedb/src/control/cluster/array_executor/refusal.rs +++ b/nodedb/src/control/cluster/array_executor/refusal.rs @@ -123,6 +123,8 @@ pub(super) fn execution_error(context: &str, error: crate::Error) -> ClusterErro | crate::Error::GroupMarksUnavailable { .. } | crate::Error::BackupCaptureMoved { .. } | crate::Error::MetadataLeaderUnavailable + | crate::Error::MetadataGroupGone { .. } + | crate::Error::MetadataDdlSuperseded { .. } | crate::Error::AuthorizationStateBehind { .. } | crate::Error::LinearizableReadRefused { .. } | crate::Error::ExecutionLimitExceeded { .. } diff --git a/nodedb/src/control/cluster/array_executor/write.rs b/nodedb/src/control/cluster/array_executor/write.rs index 9be65ad3b..cc1812817 100644 --- a/nodedb/src/control/cluster/array_executor/write.rs +++ b/nodedb/src/control/cluster/array_executor/write.rs @@ -129,7 +129,10 @@ impl DataPlaneArrayExecutor { &self.state, proposer, entry, - crate::control::wal_replication::statement_propose_deadline(&self.state), + crate::control::server::shared::session::StatementDeadline::of_statement( + &self.state, + ) + .tokio_instant(), ) .await .map_err(|e| execution_error(&format!("{op_label} raft propose"), e))?; diff --git a/nodedb/src/control/cluster/bootstrap_listener.rs b/nodedb/src/control/cluster/bootstrap_listener.rs index 140b9f82d..0b526962d 100644 --- a/nodedb/src/control/cluster/bootstrap_listener.rs +++ b/nodedb/src/control/cluster/bootstrap_listener.rs @@ -61,14 +61,14 @@ impl nodedb_cluster::decommission::MetadataProposer for BootstrapMetadataPropose let index = raft_loop .propose_stamped_to_metadata_group_via_leader(bytes) .await?; - let watchers = Arc::clone(&self.watchers); - let outcome = tokio::task::spawn_blocking(move || { - watchers.wait_for(METADATA_GROUP_ID, index, Duration::from_secs(10)) - }) - .await - .map_err(|error| nodedb_cluster::ClusterError::Transport { - detail: format!("bootstrap metadata apply wait failed: {error}"), - })?; + let outcome = self + .watchers + .wait_for_async( + METADATA_GROUP_ID, + index, + std::time::Instant::now() + Duration::from_secs(10), + ) + .await; match outcome { WaitOutcome::Reached => Ok(index), WaitOutcome::TimedOut => Err(nodedb_cluster::ClusterError::Transport { diff --git a/nodedb/src/control/cluster/calvin/scheduler/barrier_store.rs b/nodedb/src/control/cluster/calvin/scheduler/barrier_store.rs index ff200246d..6cc18e7cb 100644 --- a/nodedb/src/control/cluster/calvin/scheduler/barrier_store.rs +++ b/nodedb/src/control/cluster/calvin/scheduler/barrier_store.rs @@ -7,7 +7,11 @@ //! durable applied floor of a group asserts that every entry at or below it //! is durable, and boot resumes Raft delivery above it. So an entry below //! the floor is never delivered again, and its row is what a barrier of the -//! txn reads after a restart: +//! txn reads after a restart. +//! +//! A saved event is visible at once and durable before the floor that +//! covers it: [`save_event`] commits without persisting, and the apply +//! loop's settle pass runs [`persist_pending`] before it saves any floor. //! //! - [`restore_at_boot`] marks the txn of every stored row stored in its //! vShard's buffer, and removes the rows of finished txns; @@ -30,6 +34,9 @@ use crate::types::CutBarrierLog; /// Fold `event` into the stored row of `txn` on `vshard_id`. An event the /// row's log does not take, a copy it holds or one after its timeout, /// writes nothing. +/// +/// The row is visible at once. It is durable after the next +/// [`persist_pending`]. pub fn save_event( catalog: &SystemCatalog, vshard_id: u32, @@ -49,6 +56,12 @@ pub fn save_event( }) } +/// Persist every row [`save_event`] wrote since the last persist, in one +/// durable commit. On an error the rows stay pending for the next call. +pub fn persist_pending(catalog: &SystemCatalog) -> crate::Result<()> { + catalog.persist_calvin_barrier_logs() +} + /// The stored log of `txn` on `vshard_id`. A txn with no row has an empty /// log. pub fn load_log(catalog: &SystemCatalog, vshard_id: u32, txn: TxnId) -> crate::Result { @@ -98,7 +111,10 @@ fn is_applied(shared: &SharedState, vshard_id: u32, epoch: u64, position: u32) - /// remove the rows of finished txns. Boot runs this once the applied /// ledgers are filled, before the data-group apply loop and the schedulers /// start. -pub fn restore_at_boot(shared: &SharedState) -> crate::Result<()> { +/// +/// Returns the vShards that still hold rows. The first scheduler reconcile +/// purges those this node no longer hosts. +pub fn restore_at_boot(shared: &SharedState) -> crate::Result> { let catalog = shared.credentials.catalog(); let mut finished = Vec::new(); let mut waiting: BTreeMap> = BTreeMap::new(); @@ -113,6 +129,7 @@ pub fn restore_at_boot(shared: &SharedState) -> crate::Result<()> { } } catalog.remove_calvin_barrier_logs(&finished)?; + let holding: BTreeSet = waiting.keys().copied().collect(); for (vshard_id, txns) in waiting { shared .calvin @@ -120,7 +137,7 @@ pub fn restore_at_boot(shared: &SharedState) -> crate::Result<()> { .vshard(vshard_id) .mark_stored(txns); } - Ok(()) + Ok(holding) } /// The stored rows of the unfinished txns of `vshards`, for a snapshot of @@ -280,7 +297,11 @@ mod tests { .applied .install(VSHARD, NOT_YET_APPLIED_EPOCH, BTreeSet::from([(6, 0)])); - restore_at_boot(&state).expect("restore"); + assert_eq!( + restore_at_boot(&state).expect("restore"), + BTreeSet::from([VSHARD]), + "boot names the vShards that still hold rows" + ); let buffer = state.calvin.read_results.vshard(VSHARD); assert_eq!(buffer.waiting_txns(), 1); diff --git a/nodedb/src/control/cluster/calvin/scheduler/driver/core/cut_marker.rs b/nodedb/src/control/cluster/calvin/scheduler/driver/core/cut_marker.rs index d4b954665..22187a099 100644 --- a/nodedb/src/control/cluster/calvin/scheduler/driver/core/cut_marker.rs +++ b/nodedb/src/control/cluster/calvin/scheduler/driver/core/cut_marker.rs @@ -233,6 +233,40 @@ mod tests { assert!(!scheduler.release_cut_holds(), "nothing left to end"); } + /// A scheduler holding for a cut whose barrier a snapshot install carried + /// is woken by the install and releases its hold at once. + #[tokio::test] + async fn an_installed_barrier_ends_the_hold() { + use futures::FutureExt; + + let (mut scheduler, _dir) = build_test_scheduler(test_coll_vshard()); + let cut = fresh_cut(&scheduler); + scheduler.process_scheduler_input(marker(&cut)); + assert!(scheduler.held_for_cut(0)); + let group_id = scheduler + .data_group() + .expect("the fixture routes every vShard"); + assert!(!scheduler.release_cut_holds()); + + let shared = Arc::clone(&scheduler.shared); + let woken = shared.calvin.cut_barriers.notified(); + tokio::pin!(woken); + woken.as_mut().enable(); + crate::control::cluster::calvin_snapshot::cut_barriers::install_cut_barriers( + &shared, + group_id, + &[cut.covered_at(12)], + &[], + ) + .expect("the install records the barrier"); + assert!( + woken.now_or_never().is_some(), + "the install wakes the run loop" + ); + assert!(scheduler.release_cut_holds()); + assert!(!scheduler.held_for_cut(0)); + } + /// A second copy of the marker keeps the first copy's place, and a copy /// of a cut whose barrier applied holds nothing for long. #[tokio::test] diff --git a/nodedb/src/control/cluster/calvin/scheduler/driver/core/install_gate.rs b/nodedb/src/control/cluster/calvin/scheduler/driver/core/install_gate.rs index f195d9abe..053614780 100644 --- a/nodedb/src/control/cluster/calvin/scheduler/driver/core/install_gate.rs +++ b/nodedb/src/control/cluster/calvin/scheduler/driver/core/install_gate.rs @@ -44,7 +44,8 @@ impl InstallGate { impl Scheduler { /// Record that this replica's Calvin state of the vShard has a hole, and /// make its data-group replica refuse log entries until a snapshot - /// brings the state back. + /// brings the state back. The group's data is whole here, so the + /// replica keeps campaigning and leading. pub(in crate::control::cluster::calvin::scheduler::driver::core) fn lose_calvin_base(&self) { if let Err(error) = self .shared @@ -60,10 +61,11 @@ impl Scheduler { } match group_of_vshard(&self.shared, self.vshard_id) { Ok(group_id) => { - self.multi_raft - .lock() - .unwrap_or_else(|p| p.into_inner()) - .set_snapshot_required(group_id, true); + let mut multi_raft = self.multi_raft.lock().unwrap_or_else(|p| p.into_inner()); + crate::control::cluster::calvin_snapshot::require_calvin_snapshot( + &mut multi_raft, + group_id, + ); } Err(error) => tracing::error!( vshard_id = self.vshard_id, diff --git a/nodedb/src/control/cluster/calvin/scheduler/driver/core/read_result.rs b/nodedb/src/control/cluster/calvin/scheduler/driver/core/read_result.rs index 53b0bc8bc..3a1917553 100644 --- a/nodedb/src/control/cluster/calvin/scheduler/driver/core/read_result.rs +++ b/nodedb/src/control/cluster/calvin/scheduler/driver/core/read_result.rs @@ -79,7 +79,7 @@ impl Scheduler { on the next pass" ); crate::diag::calvin_barrier_log_store_failed( - self.vshard_id, + Some(self.vshard_id), Some((txn_id.epoch, txn_id.position)), "load", &e, @@ -253,7 +253,7 @@ impl Scheduler { "calvin: the barrier rows of finished txns were not removed; retrying on \ the next stall tick" ); - crate::diag::calvin_barrier_log_store_failed(self.vshard_id, None, "remove", &e); + crate::diag::calvin_barrier_log_store_failed(Some(self.vshard_id), None, "remove", &e); } } @@ -281,7 +281,7 @@ impl Scheduler { "calvin: the barrier row of a finished txn was not removed" ); crate::diag::calvin_barrier_log_store_failed( - self.vshard_id, + Some(self.vshard_id), Some((txn_id.epoch, txn_id.position)), "remove", &e, diff --git a/nodedb/src/control/cluster/calvin/scheduler/driver/helpers.rs b/nodedb/src/control/cluster/calvin/scheduler/driver/helpers.rs index 7eaaf45e7..d7569e1dd 100644 --- a/nodedb/src/control/cluster/calvin/scheduler/driver/helpers.rs +++ b/nodedb/src/control/cluster/calvin/scheduler/driver/helpers.rs @@ -64,17 +64,18 @@ pub(super) fn decode_lock_key(wire: &LockKeyWire) -> LockKey { /// `Exclusive`, and its collection key is `Intent`. /// - A write [`EngineKeySet::Collection`] and a write array are `Exclusive` /// on the collection key. -/// - A read row key is `Shared`, with no collection key. -/// - A read [`EngineKeySet::Collection`] is `Shared` on the collection key. +/// - A read row key is `Shared`, and its collection key is `IntentShared`. +/// - A read [`EngineKeySet::Collection`] and a read array are `Shared` on the +/// collection key. /// /// A key named twice takes the merged mode (see [`LockMode::merge`]). pub(crate) fn expand_rw_set(txn: &SequencedTxn) -> BTreeMap { let mut keys = BTreeMap::new(); for ks in &txn.tx_class.read_set.0 { - add_key_set(&mut keys, ks, LockMode::Shared); + add_key_set(&mut keys, ks, Access::Read); } for ks in &txn.tx_class.write_set.0 { - add_key_set(&mut keys, ks, LockMode::Exclusive); + add_key_set(&mut keys, ks, Access::Write); } keys } @@ -86,14 +87,39 @@ pub(crate) fn expand_rw_set(txn: &SequencedTxn) -> BTreeMap { pub(crate) fn expand_write_key_sets(sets: &[EngineKeySet]) -> BTreeMap { let mut keys = BTreeMap::new(); for ks in sets { - add_key_set(&mut keys, ks, LockMode::Exclusive); + add_key_set(&mut keys, ks, Access::Write); } keys } -/// Add the keys of `ks` to `keys`. `row_mode` is `Shared` for a read set and -/// `Exclusive` for a write set. -fn add_key_set(keys: &mut BTreeMap, ks: &EngineKeySet, row_mode: LockMode) { +/// Whether a key set comes from a read set or a write set. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +enum Access { + Read, + Write, +} + +impl Access { + /// The mode of a row key, and of a whole-collection key set. + fn row_mode(self) -> LockMode { + match self { + Access::Read => LockMode::Shared, + Access::Write => LockMode::Exclusive, + } + } + + /// The mode a row key announces on its collection key. + fn announce_mode(self) -> LockMode { + match self { + Access::Read => LockMode::IntentShared, + Access::Write => LockMode::Intent, + } + } +} + +/// Add the keys of `ks` to `keys` for `access`. +fn add_key_set(keys: &mut BTreeMap, ks: &EngineKeySet, access: Access) { + let row_mode = access.row_mode(); let coll: Arc = Arc::from(ks.collection()); let whole = LockKey::Collection { collection: Arc::clone(&coll), @@ -145,11 +171,10 @@ fn add_key_set(keys: &mut BTreeMap, ks: &EngineKeySet, row_mo return; } } - // A row writer announces itself on its collection, so a collection-wide - // writer or a predicate reader of that collection orders against it. - if row_mode == LockMode::Exclusive { - request(keys, whole, LockMode::Intent); - } + // A row key announces itself on its collection, so a collection-wide + // writer orders against every row reader and writer of the collection, + // and a predicate reader orders against every row writer. + request(keys, whole, access.announce_mode()); } /// Request `key` in `mode`, merged with any mode already requested for it. diff --git a/nodedb/src/control/cluster/calvin/scheduler/lock/lock_entry.rs b/nodedb/src/control/cluster/calvin/scheduler/lock/lock_entry.rs index 314ae76aa..c25ee65d0 100644 --- a/nodedb/src/control/cluster/calvin/scheduler/lock/lock_entry.rs +++ b/nodedb/src/control/cluster/calvin/scheduler/lock/lock_entry.rs @@ -17,16 +17,22 @@ use super::lock_key::TxnId; /// /// Compatibility: /// -/// | held \ requested | Intent | Shared | Exclusive | -/// |------------------|--------|--------|-----------| -/// | Intent | yes | no | no | -/// | Shared | no | yes | no | -/// | Exclusive | no | no | no | +/// | held \ requested | IntentShared | Intent | Shared | Exclusive | +/// |------------------|--------------|--------|--------|-----------| +/// | IntentShared | yes | yes | yes | no | +/// | Intent | yes | yes | no | no | +/// | Shared | yes | no | yes | no | +/// | Exclusive | no | no | no | no | #[derive(Debug, Clone, Copy, PartialEq, Eq)] pub enum LockMode { - /// A row writer's hold on its collection key. Compatible only with other - /// `Intent` holders, so row writers run together and exclude a predicate - /// reader or a collection-wide writer. + /// A row reader's hold on its collection key. Excludes only a + /// collection-wide writer, so a TRUNCATE orders against the row reads of + /// its collection while row readers, row writers and predicate readers + /// share the key. + IntentShared, + /// A row writer's hold on its collection key. Compatible with `Intent` + /// and `IntentShared` holders, so row writers run together and exclude a + /// predicate reader or a collection-wide writer. Intent, /// Compatible with other shared holders on the same key. Shared, @@ -39,25 +45,39 @@ impl LockMode { /// together. pub fn compatible(self, other: LockMode) -> bool { match (self, other) { - (LockMode::Intent, LockMode::Intent) | (LockMode::Shared, LockMode::Shared) => true, - (LockMode::Intent, LockMode::Shared | LockMode::Exclusive) - | (LockMode::Shared, LockMode::Intent | LockMode::Exclusive) - | (LockMode::Exclusive, LockMode::Intent | LockMode::Shared | LockMode::Exclusive) => { - false - } + ( + LockMode::IntentShared, + LockMode::IntentShared | LockMode::Intent | LockMode::Shared, + ) + | (LockMode::Intent | LockMode::Shared, LockMode::IntentShared) + | (LockMode::Intent, LockMode::Intent) + | (LockMode::Shared, LockMode::Shared) => true, + (LockMode::IntentShared | LockMode::Intent | LockMode::Shared, LockMode::Exclusive) + | ( + LockMode::Exclusive, + LockMode::IntentShared | LockMode::Intent | LockMode::Shared | LockMode::Exclusive, + ) + | (LockMode::Intent, LockMode::Shared) + | (LockMode::Shared, LockMode::Intent) => false, } } /// The one mode that covers both `self` and `other` for one transaction. /// - /// Equal modes merge to themselves. Any other pair merges to `Exclusive`: - /// `Shared` plus `Intent` means the transaction both reads the whole - /// collection and writes rows of it. + /// Equal modes merge to themselves. `IntentShared` merges to the other + /// mode, which covers it. Any other pair merges to `Exclusive`: `Shared` + /// plus `Intent` means the transaction both reads the whole collection + /// and writes rows of it. pub fn merge(self, other: LockMode) -> LockMode { if self == other { - self - } else { - LockMode::Exclusive + return self; + } + match (self, other) { + (LockMode::IntentShared, stronger) | (stronger, LockMode::IntentShared) => stronger, + ( + LockMode::Intent | LockMode::Shared | LockMode::Exclusive, + LockMode::Intent | LockMode::Shared | LockMode::Exclusive, + ) => LockMode::Exclusive, } } } @@ -80,27 +100,87 @@ pub enum AcquireOutcome { /// Per-key lock state. /// /// Invariants maintained by [`LockManager`](super::manager::LockManager): -/// - Every holder holds the key in `mode`. Two modes never mix, because no -/// two distinct modes are compatible. +/// - Every pair of holder modes is compatible. `IntentShared` holders mix +/// with `Intent` or with `Shared` holders, never with both. +/// - `mode` is the merge of every holder's mode: the strongest one held. /// - `Exclusive` entries have exactly one holder. /// - `waiters` is FIFO; each waiter carries the mode it requested so that /// promotion on release is mode-aware. pub(super) struct LockEntry { - /// The mode the current holders hold this lock under. + /// The strongest mode a current holder holds this lock under. A request + /// is compatible with every holder iff it is compatible with this mode. pub(super) mode: LockMode, - /// The transactions currently holding this lock. Exclusive: exactly one; - /// Shared or Intent: one or more. - pub(super) holders: SmallVec<[TxnId; 2]>, + /// The transactions currently holding this lock, each with its own mode. + /// Exclusive: exactly one; otherwise one or more. + pub(super) holders: SmallVec<[(TxnId, LockMode); 2]>, /// Transactions waiting for this lock, in FIFO order, each tagged with the /// mode it requested. pub(super) waiters: VecDeque<(TxnId, LockMode)>, } +impl LockEntry { + /// Whether `txn` holds this lock. + pub(super) fn holds(&self, txn: TxnId) -> bool { + self.holders.iter().any(|(holder, _)| *holder == txn) + } + + /// The mode `txn` holds this lock in, `None` when it holds none. + pub(super) fn mode_of(&self, txn: TxnId) -> Option { + self.holders + .iter() + .find(|(holder, _)| *holder == txn) + .map(|(_, mode)| *mode) + } + + /// The holders' transactions, in grant order. + pub(super) fn holder_ids(&self) -> Vec { + self.holders.iter().map(|(holder, _)| *holder).collect() + } + + /// Whether every holder other than `txn` is compatible with `mode`. + pub(super) fn others_compatible(&self, txn: TxnId, mode: LockMode) -> bool { + self.holders + .iter() + .filter(|(holder, _)| *holder != txn) + .all(|(_, held)| held.compatible(mode)) + } + + /// Add `txn` in `mode`, or raise the mode it already holds to cover + /// `mode`, then recompute the group mode. + pub(super) fn hold(&mut self, txn: TxnId, mode: LockMode) { + match self.holders.iter_mut().find(|(holder, _)| *holder == txn) { + Some((_, held)) => *held = held.merge(mode), + None => self.holders.push((txn, mode)), + } + self.regroup(); + } + + /// Drop `txn` from the holders, then recompute the group mode. + pub(super) fn drop_holder(&mut self, txn: TxnId) { + self.holders.retain(|(holder, _)| *holder != txn); + self.regroup(); + } + + /// Set `mode` to the merge of every holder's mode. An entry with no + /// holder keeps its last mode until a waiter's grant replaces it. + fn regroup(&mut self) { + let mut held = self.holders.iter().map(|(_, mode)| *mode); + if let Some(first) = held.next() { + self.mode = held.fold(first, LockMode::merge); + } + } +} + #[cfg(test)] mod tests { use super::*; - const ALL: [LockMode; 3] = [LockMode::Intent, LockMode::Shared, LockMode::Exclusive]; + const ALL: [LockMode; 4] = [ + LockMode::IntentShared, + LockMode::Intent, + LockMode::Shared, + LockMode::Exclusive, + ]; #[test] fn compatibility_matrix() { @@ -111,9 +191,42 @@ mod tests { for mode in ALL { assert!(!mode.compatible(LockMode::Exclusive)); assert!(!LockMode::Exclusive.compatible(mode)); + assert_eq!( + mode.compatible(LockMode::IntentShared), + mode != LockMode::Exclusive + ); + assert_eq!( + LockMode::IntentShared.compatible(mode), + mode != LockMode::Exclusive + ); + } + } + + #[test] + fn intent_shared_merges_to_the_mode_that_covers_it() { + for mode in ALL { + assert_eq!(LockMode::IntentShared.merge(mode), mode); + assert_eq!(mode.merge(LockMode::IntentShared), mode); } } + #[test] + fn the_group_mode_drops_when_the_strongest_holder_leaves() { + let (reader, writer) = (TxnId::new(1, 0), TxnId::new(1, 1)); + let mut entry = LockEntry { + mode: LockMode::IntentShared, + holders: SmallVec::from_slice(&[(reader, LockMode::IntentShared)]), + waiters: VecDeque::new(), + }; + entry.hold(writer, LockMode::Intent); + assert_eq!(entry.mode, LockMode::Intent); + assert!(!entry.mode.compatible(LockMode::Shared)); + entry.drop_holder(writer); + assert_eq!(entry.mode, LockMode::IntentShared); + assert!(entry.mode.compatible(LockMode::Shared)); + assert_eq!(entry.holder_ids(), vec![reader]); + } + #[test] fn merge_keeps_the_strongest_mode() { for mode in ALL { diff --git a/nodedb/src/control/cluster/calvin/scheduler/lock/manager/acquire.rs b/nodedb/src/control/cluster/calvin/scheduler/lock/manager/acquire.rs index 0d13629bc..10c384087 100644 --- a/nodedb/src/control/cluster/calvin/scheduler/lock/manager/acquire.rs +++ b/nodedb/src/control/cluster/calvin/scheduler/lock/manager/acquire.rs @@ -163,7 +163,7 @@ mod tests { ); let entry = lm.table.get(&coll("c")).unwrap(); assert_eq!(entry.mode, LockMode::Intent); - assert_eq!(entry.holders.as_slice(), &[t1, t2]); + assert_eq!(entry.holder_ids(), vec![t1, t2]); } #[test] @@ -195,6 +195,85 @@ mod tests { } } + /// A row read: its row `Shared`, its collection `IntentShared`. + fn row_read(name: &str) -> BTreeMap { + BTreeMap::from([ + (key(name), LockMode::Shared), + (coll(name), LockMode::IntentShared), + ]) + } + + #[test] + fn a_row_reader_orders_against_a_collection_writer() { + // The reader holds first: the truncate waits for it. + let mut lm = LockManager::new(); + let (reader, truncate) = (txn(1, 0), txn(1, 1)); + assert_eq!(lm.acquire(reader, row_read("c")), AcquireOutcome::Ready); + assert_eq!( + lm.acquire(truncate, one(coll("c"), LockMode::Exclusive)), + AcquireOutcome::Blocked, + "a truncate waits for the row reads of its collection" + ); + assert_eq!(lm.release(reader), vec![truncate]); + + // The truncate holds first: the reader waits for it. + let mut lm = LockManager::new(); + let (truncate, reader) = (txn(1, 0), txn(1, 1)); + assert_eq!( + lm.acquire(truncate, one(coll("c"), LockMode::Exclusive)), + AcquireOutcome::Ready + ); + assert_eq!( + lm.acquire(reader, row_read("c")), + AcquireOutcome::Blocked, + "a row read waits for the truncate sequenced before it" + ); + assert_eq!(lm.release(truncate), vec![reader]); + } + + #[test] + fn a_row_reader_shares_its_collection_with_row_writers_and_predicate_readers() { + let mut lm = LockManager::new(); + let (reader, writer, scan) = (txn(1, 0), txn(1, 1), txn(1, 2)); + assert_eq!(lm.acquire(reader, row_read("c")), AcquireOutcome::Ready); + assert_eq!( + lm.acquire( + writer, + BTreeMap::from([ + (coll("c"), LockMode::Intent), + (key("d"), LockMode::Exclusive) + ]) + ), + AcquireOutcome::Ready, + "a row writer joins a row reader" + ); + assert_eq!( + lm.acquire(scan, one(coll("c"), LockMode::Shared)), + AcquireOutcome::Blocked, + "a predicate reader waits for the row writer" + ); + // The writer leaves: the group drops to the reader's mode, which the + // predicate reader shares. + assert_eq!(lm.release(writer), vec![scan]); + let entry = lm.table.get(&coll("c")).unwrap(); + assert_eq!(entry.mode, LockMode::Shared); + assert_eq!(entry.holder_ids(), vec![reader, scan]); + } + + #[test] + fn a_row_read_and_a_row_write_of_one_collection_merge_to_intent() { + let mut lm = LockManager::new(); + let (t, other) = (txn(1, 0), txn(1, 1)); + let mut keys = row_read("c"); + keys.insert(coll("c"), LockMode::IntentShared.merge(LockMode::Intent)); + assert_eq!(keys.get(&coll("c")), Some(&LockMode::Intent)); + assert_eq!(lm.acquire(t, keys), AcquireOutcome::Ready); + assert_eq!( + lm.acquire(other, one(coll("c"), LockMode::Intent)), + AcquireOutcome::Ready + ); + } + #[test] fn a_compatible_request_queues_behind_an_earlier_waiter() { let mut lm = LockManager::new(); @@ -325,7 +404,7 @@ mod tests { let entry = lm.table.get(&key("k")).unwrap(); assert_eq!(entry.mode, LockMode::Exclusive); - assert_eq!(entry.holders.as_slice(), &[t]); + assert_eq!(entry.holder_ids(), vec![t]); } #[test] @@ -352,7 +431,7 @@ mod tests { for k in ["k1", "k2"] { let entry = lm.table.get(&key(k)).unwrap(); assert_eq!(entry.mode, LockMode::Exclusive); - assert_eq!(entry.holders.as_slice(), &[t]); + assert_eq!(entry.holder_ids(), vec![t]); } } diff --git a/nodedb/src/control/cluster/calvin/scheduler/lock/manager/classify.rs b/nodedb/src/control/cluster/calvin/scheduler/lock/manager/classify.rs index 42492d644..77b406c5d 100644 --- a/nodedb/src/control/cluster/calvin/scheduler/lock/manager/classify.rs +++ b/nodedb/src/control/cluster/calvin/scheduler/lock/manager/classify.rs @@ -18,7 +18,8 @@ pub(super) enum KeyState { /// No entry: the requester can take the key. Free, /// The requester already holds the key in a mode that covers the request, - /// or holds it alone and can raise its mode in place. + /// or can raise its mode in place because every other holder is + /// compatible with the raised mode. Held, /// Other holders hold the key in a compatible mode and nobody waits: the /// requester can join them. @@ -34,8 +35,9 @@ impl LockManager { let Some(entry) = self.table.get(key) else { return KeyState::Free; }; - if entry.holders.contains(&txn) { - if entry.mode.merge(mode) == entry.mode || entry.held_solely_by(txn) { + if let Some(own) = entry.mode_of(txn) { + let raised = own.merge(mode); + if raised == own || entry.others_compatible(txn, raised) { return KeyState::Held; } return KeyState::Conflict; @@ -58,22 +60,14 @@ impl LockManager { key.clone(), LockEntry { mode, - holders: smallvec![txn], + holders: smallvec![(txn, mode)], waiters: VecDeque::new(), }, ); } - Some(entry) => { - if entry.holders.contains(&txn) { - // A sole holder raises its mode in place. A co-holder only - // reaches here when its mode already covers the request. - if entry.held_solely_by(txn) { - entry.mode = entry.mode.merge(mode); - } - } else { - entry.holders.push(txn); - } - } + // A holder raises its own mode in place. Classification admitted + // the raised mode against every other holder. + Some(entry) => entry.hold(txn, mode), } } @@ -89,11 +83,12 @@ impl LockManager { if entry.has_waiter(txn) { return; } - let wanted = if entry.holders.contains(&txn) { - entry.holders.retain(|h| *h != txn); - entry.mode.merge(mode) - } else { - mode + let wanted = match entry.mode_of(txn) { + Some(own) => { + entry.drop_holder(txn); + own.merge(mode) + } + None => mode, }; entry.waiters.push_back((txn, wanted)); } diff --git a/nodedb/src/control/cluster/calvin/scheduler/lock/manager/introspection.rs b/nodedb/src/control/cluster/calvin/scheduler/lock/manager/introspection.rs index d4b0018a5..480d35d24 100644 --- a/nodedb/src/control/cluster/calvin/scheduler/lock/manager/introspection.rs +++ b/nodedb/src/control/cluster/calvin/scheduler/lock/manager/introspection.rs @@ -35,7 +35,7 @@ impl LockManager { key: key.clone(), requested: *mode, held: entry.mode, - holders: entry.holders.to_vec(), + holders: entry.holder_ids(), waiters: entry.waiters.iter().copied().collect(), }) }) @@ -56,8 +56,8 @@ impl LockManager { pub fn is_ready(&self, txn: TxnId, keys: &BTreeMap) -> bool { keys.keys().all(|key| { match self.table.get(key) { - None => true, // key is free - Some(entry) => entry.holders.contains(&txn), // txn is a current holder + None => true, // key is free + Some(entry) => entry.holds(txn), // txn is a current holder } }) } @@ -69,7 +69,7 @@ impl LockManager { pub fn reservation_holder_count(&self, key: &LockKey) -> usize { self.table .get(key) - .map(|e| e.holders.iter().filter(|h| h.is_reservation()).count()) + .map(|e| e.holders.iter().filter(|(h, _)| h.is_reservation()).count()) .unwrap_or(0) } diff --git a/nodedb/src/control/cluster/calvin/scheduler/lock/manager/mod.rs b/nodedb/src/control/cluster/calvin/scheduler/lock/manager/mod.rs index 619b57359..4f850cfe0 100644 --- a/nodedb/src/control/cluster/calvin/scheduler/lock/manager/mod.rs +++ b/nodedb/src/control/cluster/calvin/scheduler/lock/manager/mod.rs @@ -7,10 +7,11 @@ //! The lock manager provides a deterministic, totally-ordered lock table over //! per-key entries keyed by //! [`LockKey`](crate::control::cluster::calvin::scheduler::lock::LockKey). -//! Locks come in three modes (see +//! Locks come in four modes (see //! [`LockMode`](crate::control::cluster::calvin::scheduler::lock::LockMode)): -//! `Exclusive` (one holder), `Shared` (many readers), and `Intent` (many row -//! writers of one collection). A transaction acquires one map of key to mode. +//! `Exclusive` (one holder), `Shared` (many readers), `Intent` (many row +//! writers of one collection), and `IntentShared` (many row readers of one +//! collection). A transaction acquires one map of key to mode. //! Read reservations take one `Shared` key via [`LockManager::acquire_shared`]. //! //! # Determinism diff --git a/nodedb/src/control/cluster/calvin/scheduler/lock/manager/release.rs b/nodedb/src/control/cluster/calvin/scheduler/lock/manager/release.rs index f7f1ca556..c47055bb2 100644 --- a/nodedb/src/control/cluster/calvin/scheduler/lock/manager/release.rs +++ b/nodedb/src/control/cluster/calvin/scheduler/lock/manager/release.rs @@ -29,7 +29,7 @@ impl LockManager { let Some(entry) = self.table.get_mut(key) else { continue; }; - entry.holders.retain(|h| *h != txn); + entry.drop_holder(txn); entry.waiters.retain(|(w, _)| *w != txn); self.promote_waiters(key, &mut newly_promoted); } @@ -56,10 +56,7 @@ impl LockManager { break; } entry.waiters.pop_front(); - if entry.holders.is_empty() { - entry.mode = mode; - } - entry.holders.push(next); + entry.hold(next, mode); granted.push(next); } if entry.holders.is_empty() && entry.waiters.is_empty() { @@ -70,11 +67,9 @@ impl LockManager { let Some(pending) = self.pending_keys.remove(&next) else { continue; }; - let all_held = pending.keys().all(|k| { - self.table - .get(k) - .is_some_and(|entry| entry.holders.contains(&next)) - }); + let all_held = pending + .keys() + .all(|k| self.table.get(k).is_some_and(|entry| entry.holds(next))); if all_held { self.finish_ready(next, pending); newly_promoted.insert(next); @@ -187,12 +182,12 @@ mod tests { // Release t1 — t2 should become holder (FIFO). lm.release(t1); - let holder = lm.table.get(&key("x")).unwrap().holders[0]; + let holder = lm.table.get(&key("x")).unwrap().holder_ids()[0]; assert_eq!(holder, t2); // Release t2 — t3 should become holder. lm.release(t2); - let holder = lm.table.get(&key("x")).unwrap().holders[0]; + let holder = lm.table.get(&key("x")).unwrap().holder_ids()[0]; assert_eq!(holder, t3); } @@ -228,8 +223,8 @@ mod tests { let entry = lm.table.get(&key("k")).unwrap(); assert_eq!(entry.mode, LockMode::Shared); - assert!(entry.holders.contains(&s1)); - assert!(entry.holders.contains(&s2)); + assert!(entry.holds(s1)); + assert!(entry.holds(s2)); } #[test] @@ -250,7 +245,7 @@ mod tests { let entry = lm.table.get(&key("k")).unwrap(); assert_eq!(entry.mode, LockMode::Exclusive); assert_eq!(entry.holders.len(), 1); - assert_eq!(entry.holders[0], x1); + assert_eq!(entry.holder_ids()[0], x1); // x2 is still waiting behind x1. assert!(entry.has_waiter(x2)); } @@ -268,8 +263,8 @@ mod tests { // Releasing one shared holder leaves the other holding the key. lm.release(t1); let entry = lm.table.get(&key("k")).unwrap(); - assert!(!entry.holders.contains(&t1)); - assert!(entry.holders.contains(&t2)); + assert!(!entry.holds(t1)); + assert!(entry.holds(t2)); assert_eq!(lm.lock_count(), 1); // Releasing the last shared holder frees the key. diff --git a/nodedb/src/control/cluster/calvin/scheduler/lock/manager/try_acquire.rs b/nodedb/src/control/cluster/calvin/scheduler/lock/manager/try_acquire.rs index 1455a67a9..838380a6c 100644 --- a/nodedb/src/control/cluster/calvin/scheduler/lock/manager/try_acquire.rs +++ b/nodedb/src/control/cluster/calvin/scheduler/lock/manager/try_acquire.rs @@ -60,7 +60,7 @@ mod tests { assert!(lm.try_acquire(b, BTreeMap::from([(coll(), LockMode::Intent)]))); assert!(!lm.try_acquire(c, BTreeMap::from([(coll(), LockMode::Exclusive)]))); let entry = lm.table.get(&coll()).unwrap(); - assert_eq!(entry.holders.as_slice(), &[a, b]); + assert_eq!(entry.holder_ids(), vec![a, b]); assert!(entry.waiters.is_empty(), "a refused probe leaves no waiter"); } } diff --git a/nodedb/src/control/cluster/calvin/scheduler/lock/manager/types.rs b/nodedb/src/control/cluster/calvin/scheduler/lock/manager/types.rs index 0ec4039d6..a21674224 100644 --- a/nodedb/src/control/cluster/calvin/scheduler/lock/manager/types.rs +++ b/nodedb/src/control/cluster/calvin/scheduler/lock/manager/types.rs @@ -81,11 +81,6 @@ impl Default for LockManager { } impl LockEntry { - /// Whether `txn` is the only holder of this entry. - pub(super) fn held_solely_by(&self, txn: TxnId) -> bool { - self.holders.len() == 1 && self.holders[0] == txn - } - /// Whether `txn` is already enqueued as a waiter on this entry. pub(super) fn has_waiter(&self, txn: TxnId) -> bool { self.waiters.iter().any(|(w, _)| *w == txn) diff --git a/nodedb/src/control/cluster/calvin/scheduler/lock/manager/wound_wait.rs b/nodedb/src/control/cluster/calvin/scheduler/lock/manager/wound_wait.rs index 00d2579c3..cb7da3fa8 100644 --- a/nodedb/src/control/cluster/calvin/scheduler/lock/manager/wound_wait.rs +++ b/nodedb/src/control/cluster/calvin/scheduler/lock/manager/wound_wait.rs @@ -36,14 +36,16 @@ impl LockManager { let Some(entry) = self.table.get(key) else { return ConflictResolution::Block; }; - if !entry.waiters.is_empty() || entry.mode != LockMode::Shared { + if !entry.waiters.is_empty() { return ConflictResolution::Block; } - let mut others = entry.holders.iter().filter(|h| **h != txn).peekable(); + let mut others = entry.holders.iter().filter(|(h, _)| *h != txn).peekable(); if others.peek().is_none() { return ConflictResolution::Block; } - if others.any(|holder| !holder.is_reservation() || txn >= *holder) { + if others.any(|(holder, held)| { + *held != LockMode::Shared || !holder.is_reservation() || txn >= *holder + }) { return ConflictResolution::Block; } } @@ -58,13 +60,9 @@ impl LockManager { let Some(entry) = self.table.get_mut(key) else { return; }; - let wanted = if entry.holders.contains(&txn) { - entry.mode.merge(mode) - } else { - mode - }; + let wanted = entry.mode_of(txn).map_or(mode, |own| own.merge(mode)); entry.holders.clear(); - entry.holders.push(txn); + entry.holders.push((txn, wanted)); entry.mode = wanted; } @@ -117,8 +115,8 @@ mod tests { let entry = lm.table.get(&key("s")).unwrap(); assert_eq!(entry.mode, LockMode::Shared); - assert!(entry.holders.contains(&r1)); - assert!(entry.holders.contains(&r2)); + assert!(entry.holds(r1)); + assert!(entry.holds(r2)); } #[test] @@ -162,7 +160,7 @@ mod tests { let entry = lm.table.get(&key("k")).unwrap(); assert_eq!(entry.mode, LockMode::Exclusive); - assert_eq!(entry.holders.as_slice(), &[writer]); + assert_eq!(entry.holder_ids(), vec![writer]); } #[test] @@ -181,7 +179,7 @@ mod tests { ); let entry = lm.table.get(&coll).unwrap(); assert_eq!(entry.mode, LockMode::Intent); - assert_eq!(entry.holders.as_slice(), &[writer]); + assert_eq!(entry.holder_ids(), vec![writer]); } #[test] @@ -197,8 +195,8 @@ mod tests { ); let entry = lm.table.get(&key("k")).unwrap(); - assert!(entry.holders.contains(&r), "the shared holder still holds"); - assert!(!entry.holders.contains(&writer)); + assert!(entry.holds(r), "the shared holder still holds"); + assert!(!entry.holds(writer)); assert!(entry.has_waiter(writer)); } @@ -212,7 +210,7 @@ mod tests { assert_eq!(lm.acquire(t1, exclusive(&["k"])), AcquireOutcome::Blocked); let entry = lm.table.get(&key("k")).unwrap(); - assert_eq!(entry.holders.as_slice(), &[t2]); + assert_eq!(entry.holder_ids(), vec![t2]); assert!(entry.has_waiter(t1)); } @@ -234,7 +232,7 @@ mod tests { for k in ["k1", "k2"] { let entry = lm.table.get(&key(k)).unwrap(); assert_eq!(entry.mode, LockMode::Exclusive); - assert_eq!(entry.holders.as_slice(), &[r]); + assert_eq!(entry.holder_ids(), vec![r]); } } @@ -256,7 +254,7 @@ mod tests { ); for (k, holder) in [("k1", s1), ("k2", s2)] { let entry = lm.table.get(&key(k)).unwrap(); - assert_eq!(entry.holders.as_slice(), &[holder]); + assert_eq!(entry.holder_ids(), vec![holder]); assert!(entry.has_waiter(r)); } } @@ -272,7 +270,7 @@ mod tests { assert_eq!(lm.acquire(r_old, exclusive(&["k"])), AcquireOutcome::Ready); let entry = lm.table.get(&key("k")).unwrap(); assert_eq!(entry.mode, LockMode::Exclusive); - assert_eq!(entry.holders.as_slice(), &[r_old]); + assert_eq!(entry.holder_ids(), vec![r_old]); // The younger one's self-upgrade drops its own hold and waits. let mut lm = LockManager::new(); @@ -284,13 +282,13 @@ mod tests { ); let entry = lm.table.get(&key("k")).unwrap(); assert_eq!(entry.mode, LockMode::Shared); - assert_eq!(entry.holders.as_slice(), &[r_old]); + assert_eq!(entry.holder_ids(), vec![r_old]); assert!(entry.has_waiter(r_young)); assert_eq!(lm.release(r_old), vec![r_young]); let entry = lm.table.get(&key("k")).unwrap(); assert_eq!(entry.mode, LockMode::Exclusive); - assert_eq!(entry.holders.as_slice(), &[r_young]); + assert_eq!(entry.holder_ids(), vec![r_young]); } #[test] @@ -311,6 +309,6 @@ mod tests { let k2 = lm.table.get(&key("k2")).unwrap(); assert_eq!(k2.mode, LockMode::Exclusive); - assert_eq!(k2.holders.as_slice(), &[writer]); + assert_eq!(k2.holder_ids(), vec![writer]); } } diff --git a/nodedb/src/control/cluster/calvin/scheduler/lock/reap.rs b/nodedb/src/control/cluster/calvin/scheduler/lock/reap.rs index a47ff6470..bc6ce9110 100644 --- a/nodedb/src/control/cluster/calvin/scheduler/lock/reap.rs +++ b/nodedb/src/control/cluster/calvin/scheduler/lock/reap.rs @@ -62,7 +62,8 @@ impl LockManager { keys.iter().all(|key| { self.table .get(key) - .is_none_or(|e| !e.holders.contains(&owner) || e.mode == LockMode::Shared) + .and_then(|e| e.mode_of(owner)) + .is_none_or(|mode| mode == LockMode::Shared) }) }); let waits_shared = self @@ -128,7 +129,7 @@ mod tests { let promoted = lm.reap_expired_shared(50); assert!(promoted.is_empty()); assert_eq!(lm.lock_count(), 1, "still within the lease, not reaped"); - assert!(lm.table.get(&key("k")).unwrap().holders.contains(&r)); + assert!(lm.table.get(&key("k")).unwrap().holds(r)); } #[test] @@ -144,7 +145,7 @@ mod tests { 1, "a real txn's exclusive lock is never reaped" ); - assert!(lm.table.get(&key("k")).unwrap().holders.contains(&t)); + assert!(lm.table.get(&key("k")).unwrap().holds(t)); } #[test] @@ -164,7 +165,7 @@ mod tests { "mid-commit reservation is left alone" ); assert_eq!(entry.holders.len(), 1); - assert_eq!(entry.holders[0], r); + assert_eq!(entry.holder_ids()[0], r); } #[test] @@ -196,7 +197,7 @@ mod tests { let entry = lm.table.get(&key("k")).unwrap(); assert_eq!(entry.mode, LockMode::Exclusive); assert_eq!(entry.holders.len(), 1); - assert_eq!(entry.holders[0], writer); + assert_eq!(entry.holder_ids()[0], writer); } #[test] diff --git a/nodedb/src/control/cluster/calvin_snapshot/capture.rs b/nodedb/src/control/cluster/calvin_snapshot/capture.rs index cc9fb8fea..064435443 100644 --- a/nodedb/src/control/cluster/calvin_snapshot/capture.rs +++ b/nodedb/src/control/cluster/calvin_snapshot/capture.rs @@ -18,6 +18,7 @@ use nodedb_cluster::calvin::SequencerEntry; use crate::Error; use crate::control::cluster::calvin::scheduler::barrier_store; +use crate::control::cluster::calvin_snapshot::cut_barriers::capture_cut_barriers; use crate::control::security::catalog::calvin_base::CalvinBase; use crate::control::state::SharedState; use crate::types::{GroupCalvinCut, VShardCalvinState}; @@ -103,18 +104,22 @@ pub async fn await_calvin_cut( } } -/// The group's Calvin cut through `through`: the applied positions of every -/// group vShard here, and the stored barrier logs of their unfinished txns. -/// The caller holds the group's apply gate exclusive, so no Calvin install -/// of these vShards runs and no barrier entry of the group applies. +/// The group's Calvin cut through `through`, for a snapshot through the +/// group's log index `cut_index`: the applied positions of every group +/// vShard here, the stored barrier logs of their unfinished txns, and the +/// group's backup cut barriers and floors. The caller holds the group's +/// apply gate exclusive, so no Calvin install of these vShards runs and no +/// entry of the group applies. /// -/// Fails when the barrier logs cannot be read whole (see +/// Fails when the barrier logs or cut floors cannot be read whole (see /// [`barrier_store::capture_group`]). The build retries on the next /// heartbeat. pub fn capture_calvin_cut( shared: &SharedState, + group_id: u64, group_vshards: &HashSet, through: u64, + cut_index: u64, ) -> Result { let ledgers = &shared.calvin.applied; let ids: BTreeSet = group_vshards.iter().copied().collect(); @@ -130,9 +135,12 @@ pub fn capture_calvin_cut( }) .collect(); let barrier_logs = barrier_store::capture_group(shared, &ids)?; + let (cut_barriers, cut_floors) = capture_cut_barriers(shared, group_id, cut_index)?; Ok(GroupCalvinCut { through, vshards, barrier_logs, + cut_barriers, + cut_floors, }) } diff --git a/nodedb/src/control/cluster/calvin_snapshot/cut_barriers.rs b/nodedb/src/control/cluster/calvin_snapshot/cut_barriers.rs new file mode 100644 index 000000000..b8aaf0d25 --- /dev/null +++ b/nodedb/src/control/cluster/calvin_snapshot/cut_barriers.rs @@ -0,0 +1,195 @@ +// SPDX-License-Identifier: BUSL-1.1 + +//! The backup cut barriers a data-group snapshot carries. +//! +//! A cut barrier applied in a group is replicated state, yet a replica learns +//! of it only from the entry. A receiver never applies the entries a snapshot +//! covers, so the snapshot carries two things for them: +//! +//! - every barrier of a cut whose window is open, with its log index. A cut +//! waiting on the receiver and a scheduler holding redo for the cut read it +//! as applied. +//! - the group's durable cut floors. Every later entry records a commit HLC +//! above them, on the receiver as on the builder. + +use crate::Error; +use crate::control::backup::cut_order::{CutWindow, OrderedCut}; +use crate::control::pitr::restore_point::{group_cut_floors, install_cut_floors}; +use crate::control::state::SharedState; +use crate::types::{CoveredCutBarrier, CutFloor}; + +/// The cut barriers of `group_id` a snapshot through `cut_index` carries. +/// The caller holds the group's apply gate exclusive, so no barrier of the +/// group applies meanwhile, and every applied one sits at or below the cut. +pub fn capture_cut_barriers( + shared: &SharedState, + group_id: u64, + cut_index: u64, +) -> Result<(Vec, Vec), Error> { + // no-determinism: the window bounds only which cuts still wait anywhere. + let now = shared.hlc_clock.now().wall_ns; + let barriers = shared + .calvin + .cut_barriers + .applied_in_group(group_id) + .into_iter() + .filter(|(cut, log_index)| { + *log_index <= cut_index && now <= CutWindow::on(shared, cut.hlc).hold_until + }) + .map(|(cut, log_index)| cut.covered_at(log_index)) + .collect(); + let floors = group_cut_floors(shared, group_id, cut_index)?; + Ok((barriers, floors)) +} + +/// Install the cut barriers a snapshot of `group_id` carried: the floors +/// first, durably, then each barrier as applied here, which wakes every +/// cut wait and every scheduler holding redo for it. +pub fn install_cut_barriers( + shared: &SharedState, + group_id: u64, + barriers: &[CoveredCutBarrier], + floors: &[CutFloor], +) -> crate::Result<()> { + install_cut_floors(shared, group_id, floors)?; + for barrier in barriers { + shared.calvin.cut_barriers.note_applied( + &OrderedCut::from_covered(barrier), + group_id, + barrier.log_index, + ); + } + Ok(()) +} + +#[cfg(test)] +mod tests { + use std::collections::HashSet; + use std::sync::{Arc, RwLock}; + use std::time::Duration; + + use nodedb_cluster::RoutingTable; + + use super::*; + use crate::bridge::dispatch::Dispatcher; + use crate::control::backup::cut_order::wait::await_group_barriers; + use crate::control::cluster::calvin_snapshot::{capture_calvin_cut, install_calvin_cut}; + use crate::wal::WalManager; + + const NODE: u64 = 1; + + /// A fresh node that hosts every group of its routing table, and one + /// of those groups. + fn node() -> (Arc, u64, tempfile::TempDir) { + let dir = tempfile::tempdir().unwrap(); + let wal = Arc::new(WalManager::open_for_testing(&dir.path().join("test.wal")).unwrap()); + let (dispatcher, _sides) = Dispatcher::new(1, 8); + let mut state = SharedState::new(dispatcher, wal).unwrap(); + let routing = RoutingTable::uniform(1, &[NODE], 1); + let group_id = routing.group_ids()[0]; + let shared = Arc::get_mut(&mut state).unwrap(); + shared.node_id = NODE; + shared.cluster_routing = Some(Arc::new(RwLock::new(routing))); + (state, group_id, dir) + } + + fn open_cut(state: &SharedState) -> OrderedCut { + OrderedCut { + hlc: state.hlc_clock.now().wall_ns, + restore_point: 0, + capture: None, + } + } + + #[test] + fn a_snapshot_carries_the_barriers_at_or_below_its_cut() { + let (leader, group_id, _dir) = node(); + let cut = open_cut(&leader); + let closed = OrderedCut { + hlc: 1, + ..cut.clone() + }; + let later = OrderedCut { + hlc: cut.hlc + 1, + ..cut.clone() + }; + leader.calvin.cut_barriers.note_applied(&cut, group_id, 40); + leader + .calvin + .cut_barriers + .note_applied(&closed, group_id, 30); + leader + .calvin + .cut_barriers + .note_applied(&later, group_id, 60); + crate::control::pitr::restore_point::persist_cut_floor(&leader, group_id, 40, cut.hlc) + .unwrap(); + crate::control::pitr::restore_point::persist_cut_floor(&leader, group_id, 60, later.hlc) + .unwrap(); + + let captured = capture_calvin_cut(&leader, group_id, &HashSet::new(), 7, 50).unwrap(); + assert_eq!( + captured.cut_barriers, + [cut.covered_at(40)], + "a closed window and a barrier above the cut stay behind" + ); + assert_eq!( + captured.cut_floors, + [CutFloor { + index: 40, + watermark: cut.hlc + }] + ); + } + + #[tokio::test] + async fn an_installed_barrier_ends_a_pending_cut_wait() { + let (leader, group_id, _leader_dir) = node(); + let cut = open_cut(&leader); + leader.calvin.cut_barriers.note_applied(&cut, group_id, 40); + crate::control::pitr::restore_point::persist_cut_floor(&leader, group_id, 40, cut.hlc) + .unwrap(); + let captured = capture_calvin_cut(&leader, group_id, &HashSet::new(), 7, 50).unwrap(); + + let (follower, follower_group, _follower_dir) = node(); + assert_eq!(follower_group, group_id); + let waiting = { + let follower = Arc::clone(&follower); + let cut = cut.clone(); + tokio::spawn(async move { + let deadline = tokio::time::Instant::now() + Duration::from_secs(10); + await_group_barriers(&follower, &cut, &[group_id], deadline).await + }) + }; + tokio::task::yield_now().await; + assert!(!waiting.is_finished(), "no barrier applied on the follower"); + + // The install raises the follower's applied index past the snapshot. + follower.group_watchers.bump(group_id, 50); + install_calvin_cut(&follower, group_id, &HashSet::new(), captured).unwrap(); + assert_eq!( + follower.calvin.cut_barriers.applied(cut.key(), group_id), + Some(40) + ); + assert_eq!( + follower.pitr.recorded_cuts().watermark_below(group_id, 51), + Some(cut.hlc), + "the apply loop raises later entries above the cut" + ); + assert_eq!( + follower + .credentials + .catalog() + .group_cut_floors(group_id) + .unwrap() + .len(), + 1, + "the floor outlives a restart" + ); + tokio::time::timeout(Duration::from_secs(5), waiting) + .await + .expect("the install wakes the wait") + .expect("the wait runs") + .expect("the cut wait succeeds"); + } +} diff --git a/nodedb/src/control/cluster/calvin_snapshot/install.rs b/nodedb/src/control/cluster/calvin_snapshot/install.rs index 582ec801d..a959aef7f 100644 --- a/nodedb/src/control/cluster/calvin_snapshot/install.rs +++ b/nodedb/src/control/cluster/calvin_snapshot/install.rs @@ -13,6 +13,8 @@ //! //! The stored dependent-read barrier logs of the group's vShards become the //! cut's too: the snapshot covers barrier entries this node never applies. +//! So do the group's backup cut barriers and cut floors (see +//! [`super::cut_barriers`]). //! //! A WAL `SnapshotInstalled` record of the group, written before this runs, //! makes boot recovery drop the vShards' applied markers from before it. @@ -20,6 +22,7 @@ use std::collections::{BTreeSet, HashSet}; use crate::control::cluster::calvin::scheduler::barrier_store; +use crate::control::cluster::calvin_snapshot::cut_barriers::install_cut_barriers; use crate::control::cluster::snapshot_install::SnapshotInstallError; use crate::control::security::catalog::calvin_applied::StoredCalvinApplied; use crate::control::state::SharedState; @@ -80,5 +83,12 @@ pub fn install_calvin_cut( .calvin .bases .record_snapshot(catalog, &ids, cut.through) - .map_err(settle_error) + .map_err(settle_error)?; + install_cut_barriers(shared, group_id, &cut.cut_barriers, &cut.cut_floors).map_err(|source| { + SnapshotInstallError::Settle { + group_id, + step: crate::control::cluster::snapshot_install::SettleStep::CutBarriers, + source, + } + }) } diff --git a/nodedb/src/control/cluster/calvin_snapshot/mod.rs b/nodedb/src/control/cluster/calvin_snapshot/mod.rs index 1f7b75a92..4473a2ede 100644 --- a/nodedb/src/control/cluster/calvin_snapshot/mod.rs +++ b/nodedb/src/control/cluster/calvin_snapshot/mod.rs @@ -1,11 +1,13 @@ // SPDX-License-Identifier: BUSL-1.1 pub mod capture; +pub mod cut_barriers; pub mod install; pub mod requirement; pub use capture::{await_calvin_cut, capture_calvin_cut}; pub use install::install_calvin_cut; pub use requirement::{ - forget_left, install_snapshot_requirement, may_start, note_kept, rebased, retain_mounted, + forget_left, install_snapshot_requirement, may_start, note_kept, rebased, + require_calvin_snapshot, retain_mounted, }; diff --git a/nodedb/src/control/cluster/calvin_snapshot/requirement.rs b/nodedb/src/control/cluster/calvin_snapshot/requirement.rs index a337335f8..144bfcb13 100644 --- a/nodedb/src/control/cluster/calvin_snapshot/requirement.rs +++ b/nodedb/src/control/cluster/calvin_snapshot/requirement.rs @@ -22,11 +22,20 @@ //! A data group whose chunked redo streams this node dropped when it left //! the group requires a snapshot too, until one installs (see //! `wal_replication::transaction_redo::chunks::owed`). +//! +//! The two needs differ in kind: +//! - A missing Calvin base leaves the group's data whole. The replica wants +//! a snapshot ([`ReplicaState::WantsSnapshot`]) and may still lead, so a +//! group whose every replica lost the base keeps a leader, and Calvin +//! stays stopped there. +//! - A snapshot debt means this replica's data diverged from the log. It +//! awaits a snapshot ([`ReplicaState::AwaitingSnapshot`]) and never leads. use std::collections::BTreeSet; use std::sync::{Arc, Mutex, RwLock}; use nodedb_cluster::multi_raft::MultiRaft; +use nodedb_raft::ReplicaState; use crate::control::security::catalog::calvin_base::CalvinBase; use crate::control::state::SharedState; @@ -50,7 +59,7 @@ pub fn install_snapshot_requirement( let weak = Arc::downgrade(shared); multi_raft.set_snapshot_requirement(Arc::new(move |group_id, sequencer_first| { let Some(shared) = weak.upgrade() else { - return false; + return ReplicaState::Whole; }; let vshards = routing .read() @@ -58,7 +67,13 @@ pub fn install_snapshot_requirement( .vshards_for_group(group_id); // Evaluated first and always: it registers the vShards as waiting. let reaches = shared.calvin.bases.group_reaches(&vshards, sequencer_first); - !reaches || shared.redo_chunks.owes_snapshot(group_id) + if shared.redo_chunks.owes_snapshot(group_id) { + ReplicaState::AwaitingSnapshot + } else if reaches { + ReplicaState::Whole + } else { + ReplicaState::WantsSnapshot + } })); Ok(()) } @@ -95,11 +110,11 @@ pub fn may_start( } if let Some(group_id) = group_id { let mut multi_raft = multi_raft.lock().unwrap_or_else(|p| p.into_inner()); - multi_raft.set_snapshot_required(group_id, true); + require_calvin_snapshot(&mut multi_raft, group_id); // Without a scheduler here the vShard casts no vote, so a leader here - // blocks every Calvin transaction on it. A replica whose state is - // whole takes the leadership. When none qualifies, the transactions - // end at their deadline with an error. + // blocks every Calvin transaction on it. A replica whose Calvin state + // is whole takes the leadership. When none qualifies, the + // transactions end at their deadline with an error. let sole_replica = multi_raft .group_membership(group_id) .is_some_and(|m| m.voters.len() + m.learners.len() <= 1); @@ -134,6 +149,13 @@ pub fn may_start( false } +/// Make this node's replica of `group_id` refuse log entries until a +/// snapshot brings its Calvin state back. Its data is whole, so it keeps +/// campaigning and leading. +pub fn require_calvin_snapshot(multi_raft: &mut MultiRaft, group_id: u64) { + multi_raft.require_snapshot(group_id, ReplicaState::WantsSnapshot); +} + /// Record that the schedulers of `vshards`, each `(vshard_id, from)`, /// started from bases that reach the sequencer log and caught up from /// index `from`. They keep them whole from here. @@ -166,9 +188,11 @@ pub fn rebased(shared: &SharedState, served: &[u32]) -> Vec { /// Forget the Calvin state of `vshards`: this node left their groups. A /// later return catches up from nothing, so neither their bases nor their /// applied positions can survive. -pub fn forget_left(shared: &SharedState, vshards: &[u32]) { +/// +/// On an error the stored state stays, and the caller retries the forget. +pub fn forget_left(shared: &SharedState, vshards: &[u32]) -> crate::Result<()> { if vshards.is_empty() { - return; + return Ok(()); } let catalog = shared.credentials.catalog(); let resets: Vec = @@ -189,13 +213,15 @@ pub fn forget_left(shared: &SharedState, vshards: &[u32]) { let forgotten = catalog .replace_calvin_applied(&resets) .and_then(|()| shared.calvin.bases.forget(catalog, vshards)); - if let Err(error) = forgotten { + if let Err(error) = &forgotten { tracing::error!( ?vshards, %error, "calvin: the left vShards' Calvin state was not forgotten" ); + crate::diag::calvin_left_state_not_forgotten(vshards, error); } + forgotten } /// Keep the sequencer log range only for vShards of the data groups this diff --git a/nodedb/src/control/cluster/execution_error_wire.rs b/nodedb/src/control/cluster/execution_error_wire.rs index 89323fa95..a37053b3a 100644 --- a/nodedb/src/control/cluster/execution_error_wire.rs +++ b/nodedb/src/control/cluster/execution_error_wire.rs @@ -133,6 +133,8 @@ pub(crate) fn execution_error_to_typed(err: crate::Error) -> TypedClusterError { | crate::Error::GroupMarksUnavailable { .. } | crate::Error::BackupCaptureMoved { .. } | crate::Error::MetadataLeaderUnavailable + | crate::Error::MetadataGroupGone { .. } + | crate::Error::MetadataDdlSuperseded { .. } | crate::Error::AuthorizationStateBehind { .. } | crate::Error::LinearizableReadRefused { .. } | crate::Error::ExecutionLimitExceeded { .. } diff --git a/nodedb/src/control/cluster/forwarded_budget.rs b/nodedb/src/control/cluster/forwarded_budget.rs new file mode 100644 index 000000000..82b9911a7 --- /dev/null +++ b/nodedb/src/control/cluster/forwarded_budget.rs @@ -0,0 +1,53 @@ +// SPDX-License-Identifier: BUSL-1.1 + +//! The deadline a leader-side hook works to for a forwarded request. + +use nodedb_cluster::TypedClusterError; +use nodedb_cluster::rpc_codec::forwarded_deadline; + +use crate::bridge::envelope::ErrorCode; +use crate::control::cluster::execution_error_wire::execution_error_to_typed; + +/// The deadline for a forwarded request that carries `deadline_remaining_ms`. +/// +/// A spent budget refuses the request before it runs. The refusal is +/// `ExpiredBeforeExecution`, so the coordinator reports a definite refusal, +/// never an unknown outcome. +pub(crate) fn hook_deadline( + deadline_remaining_ms: u64, +) -> Result { + match forwarded_deadline(deadline_remaining_ms) { + Some(deadline) => Ok(deadline), + None => Err(execution_error_to_typed(crate::Error::DataPlane( + ErrorCode::ExpiredBeforeExecution, + ))), + } +} + +#[cfg(test)] +mod tests { + use std::time::Duration; + + use super::*; + use crate::control::server::dispatch_utils::write_definitely_not_applied; + + /// A spent budget comes back to the coordinator as a definite refusal. + #[test] + fn a_spent_budget_is_a_definite_refusal() { + let refusal = hook_deadline(0).expect_err("a spent budget is refused"); + let crate::Error::DataPlane(code) = crate::Error::from(refusal) else { + panic!("expected a Data-Plane verdict"); + }; + assert_eq!(code, ErrorCode::ExpiredBeforeExecution); + assert!(write_definitely_not_applied(&code)); + } + + /// A live budget is used as it arrived, never stretched or floored. + #[tokio::test] + async fn a_live_budget_ends_within_what_arrived() { + let before = tokio::time::Instant::now(); + let deadline = hook_deadline(50).expect("a live budget"); + assert!(deadline >= before + Duration::from_millis(50)); + assert!(deadline <= tokio::time::Instant::now() + Duration::from_millis(50)); + } +} diff --git a/nodedb/src/control/cluster/leased_read.rs b/nodedb/src/control/cluster/leased_read.rs index 0d7150fd3..9e0db6e6c 100644 --- a/nodedb/src/control/cluster/leased_read.rs +++ b/nodedb/src/control/cluster/leased_read.rs @@ -10,11 +10,9 @@ //! misses. A group whose lease this node does not hold is refused, with the //! leader and term its routing table names. -use std::time::Instant; - use crate::control::state::SharedState; -use super::linearizable_read::wait_applied_through; +use super::linearizable_read::{StatementDeadline, wait_applied_through}; /// A group this node does not serve a leased read of. #[derive(Debug, Clone, Copy, PartialEq, Eq)] @@ -31,13 +29,13 @@ pub struct LeaseRefusal { /// leases. /// /// Returns the groups whose lease this node does not hold. Every other group -/// is applied here through its lease read index before this returns. A node -/// with no routing table serves every group: without a cluster there is one -/// copy and nothing to prove. +/// is applied here through its lease read index, within `deadline`, before +/// this returns. A node with no routing table serves every group: without a +/// cluster there is one copy and nothing to prove. pub async fn confirm_leased_read( state: &SharedState, groups: &[u64], - deadline: Instant, + deadline: StatementDeadline, ) -> crate::Result> { let Some(routing) = state.cluster_routing.as_ref() else { return Ok(Vec::new()); @@ -91,7 +89,7 @@ pub async fn confirm_leased_read( mod tests { use std::sync::atomic::{AtomicBool, Ordering}; use std::sync::{Arc, RwLock}; - use std::time::Duration; + use std::time::{Duration, Instant}; use async_trait::async_trait; use nodedb_cluster::RoutingTable; @@ -130,7 +128,12 @@ mod tests { Err(ReadIndexRefusal::NotLeader) } - fn within_staleness_bound(&self, _group_id: u64, _max_staleness: Duration) -> bool { + fn within_staleness_bound( + &self, + _group_id: u64, + _max_staleness: Duration, + _applied: u64, + ) -> bool { self.probe(); false } @@ -178,7 +181,7 @@ mod tests { ); let groups = routing.read().expect("routing lock").group_ids(); - let refusals = confirm_leased_read(&state, &groups, Instant::now()) + let refusals = confirm_leased_read(&state, &groups, StatementDeadline::at(Instant::now())) .await .expect("a refused lease is not an error"); assert!( diff --git a/nodedb/src/control/cluster/left_vshard_forget.rs b/nodedb/src/control/cluster/left_vshard_forget.rs new file mode 100644 index 000000000..265ce2f53 --- /dev/null +++ b/nodedb/src/control/cluster/left_vshard_forget.rs @@ -0,0 +1,413 @@ +// SPDX-License-Identifier: BUSL-1.1 + +//! Forget the stored Calvin state of the vShards this node left. +//! +//! A vShard that left keeps no applied positions, base or dependent-read +//! barrier rows here. A forget that fails leaves the vShard in +//! `CalvinLocalState::forget_pending`, and every scheduler reconcile pass +//! retries it. Boot adds every vShard that holds any stored Calvin state. +//! +//! A pending vShard whose group lists this node as a voter or learner +//! leaves the set with no forget: its state belongs to that membership. +//! +//! No pass forgets before boot recovery sets +//! `SharedState::data_groups_replayed`. Until every mounted data group +//! applied its committed log, a group can still re-apply a conf change, so +//! the routing table can omit a group this node replicates. + +use std::collections::BTreeSet; +use std::sync::RwLock; + +use nodedb_cluster::RoutingTable; + +use crate::control::cluster::calvin::scheduler::{NOT_YET_APPLIED_EPOCH, barrier_store}; +use crate::control::cluster::calvin_snapshot; +use crate::control::state::SharedState; + +/// Add to the pending set every vShard with stored Calvin state at boot: +/// `holding_rows`, the vShards with barrier rows, and every vShard with an +/// applied ledger or a base. Boot runs this once both are loaded. +pub fn seed_at_boot(shared: &SharedState, holding_rows: BTreeSet) { + let mut seeded = holding_rows; + seeded.extend( + shared + .calvin + .applied + .snapshot_all() + .into_iter() + // A reset ledger holds no applied position. + .filter(|state| { + state.fully_applied_epoch != NOT_YET_APPLIED_EPOCH || !state.tail.is_empty() + }) + .map(|state| state.vshard_id), + ); + seeded.extend(shared.calvin.bases.vshards()); + shared + .calvin + .forget_pending + .lock() + .unwrap_or_else(|p| p.into_inner()) + .extend(seeded); +} + +/// The vShards of every Raft group that lists `node_id` as a voter or a +/// learner, sorted. A learner applies the group's entries, so it holds the +/// group's Calvin state too. +pub fn replicated_vshards(routing: &RwLock, node_id: u64) -> Vec { + let routing = routing.read().unwrap_or_else(|p| p.into_inner()); + let mut vshards = Vec::new(); + for (group_id, info) in routing.group_members() { + if info.members.contains(&node_id) || info.learners.contains(&node_id) { + vshards.extend(routing.vshards_for_group(*group_id)); + } + } + vshards.sort_unstable(); + vshards.dedup(); + vshards +} + +/// Whether the routing table names every group this node replicates: boot +/// recovery saw every mounted data group apply its committed log. +fn membership_settled(shared: &SharedState) -> bool { + shared + .data_groups_replayed + .load(std::sync::atomic::Ordering::Acquire) +} + +/// Forget the stored Calvin state of `left`, the vShards this node no +/// longer hosts. Each vShard whose forget fails waits in the pending set. +pub fn forget_left_vshards(shared: &SharedState, left: &[u32]) { + if left.is_empty() { + return; + } + let state_forgotten = calvin_snapshot::forget_left(shared, left).is_ok(); + let mut failed = Vec::new(); + for &vshard_id in left { + let rows_forgotten = forget_rows(shared, vshard_id); + if !state_forgotten || !rows_forgotten { + failed.push(vshard_id); + } + } + if failed.is_empty() { + return; + } + shared + .calvin + .forget_pending + .lock() + .unwrap_or_else(|p| p.into_inner()) + .extend(failed); +} + +/// Retry the forget of every pending vShard. `replicated` is sorted: the +/// vShards of every group that lists this node as a voter or learner. A +/// replicated vShard leaves the set with no forget, and a forgotten one +/// leaves it. Before membership settles, the pass changes nothing. +pub fn retry_pending_forgets(shared: &SharedState, replicated: &[u32]) { + if !membership_settled(shared) { + return; + } + let pending: Vec = shared + .calvin + .forget_pending + .lock() + .unwrap_or_else(|p| p.into_inner()) + .iter() + .copied() + .collect(); + if pending.is_empty() { + return; + } + let mut settled = Vec::new(); + for vshard_id in pending { + if replicated.binary_search(&vshard_id).is_ok() { + settled.push(vshard_id); + continue; + } + // Both forgets are idempotent, so a retry runs both. + let state_forgotten = calvin_snapshot::forget_left(shared, &[vshard_id]).is_ok(); + let rows_forgotten = forget_rows(shared, vshard_id); + if state_forgotten && rows_forgotten { + tracing::info!( + vshard_id, + "calvin: the stored state of a vShard that left is forgotten" + ); + settled.push(vshard_id); + } + } + let mut set = shared + .calvin + .forget_pending + .lock() + .unwrap_or_else(|p| p.into_inner()); + for vshard_id in settled { + set.remove(&vshard_id); + } +} + +/// Remove every barrier row and the buffer of `vshard_id`. Returns whether +/// the rows went. +fn forget_rows(shared: &SharedState, vshard_id: u32) -> bool { + match barrier_store::forget_vshard(shared, vshard_id) { + Ok(()) => true, + Err(e) => { + tracing::warn!( + vshard_id, + error = %e, + "calvin: the barrier rows of a vShard that left were not removed; the next \ + reconcile pass retries" + ); + crate::diag::calvin_barrier_log_store_failed(Some(vshard_id), None, "forget", &e); + false + } + } +} + +#[cfg(test)] +mod tests { + use std::collections::BTreeSet; + use std::sync::Arc; + + use super::*; + use crate::bridge::dispatch::Dispatcher; + use crate::control::cluster::calvin::BarrierEvent; + use crate::control::cluster::calvin::scheduler::lock_manager::TxnId; + use crate::wal::WalManager; + + const VSHARD: u32 = 5; + const NODE: u64 = 1; + const OTHER: u64 = 2; + + /// Routing whose one data group holds every vShard, with `voters`. + fn routing(voters: &[u64]) -> RwLock { + RwLock::new(RoutingTable::uniform(1, voters, 1)) + } + + /// A state whose boot recovery has not run. + fn booting_state() -> (Arc, tempfile::TempDir) { + let dir = tempfile::tempdir().expect("tempdir"); + let wal = Arc::new(WalManager::open_for_testing(&dir.path().join("wal")).expect("wal")); + let (dispatcher, _sides) = Dispatcher::new(1, 64); + ( + SharedState::new(dispatcher, wal).expect("shared state"), + dir, + ) + } + + /// A state whose boot recovery finished. + fn state() -> (Arc, tempfile::TempDir) { + let (state, dir) = booting_state(); + state + .data_groups_replayed + .store(true, std::sync::atomic::Ordering::Release); + (state, dir) + } + + fn save_row(state: &SharedState, vshard_id: u32) { + barrier_store::save_event( + state.credentials.catalog(), + vshard_id, + TxnId::new(3, 0), + &BarrierEvent::Timeout, + ) + .expect("save the row"); + } + + fn row_vshards(state: &SharedState) -> BTreeSet { + state + .credentials + .catalog() + .load_calvin_barrier_logs(None) + .expect("load rows") + .into_iter() + .map(|row| row.vshard_id) + .collect() + } + + fn pending(state: &SharedState) -> BTreeSet { + state + .calvin + .forget_pending + .lock() + .expect("pending set") + .clone() + } + + /// A failed row removal keeps the vShard pending. The next pass that + /// finds it unhosted removes its rows and drops it. + #[test] + fn a_failed_forget_retries_on_the_next_pass() { + let (state, _dir) = state(); + save_row(&state, VSHARD); + save_row(&state, VSHARD + 1); + + state + .credentials + .catalog() + .fail_next_calvin_barrier_replace_for_test(); + forget_left_vshards(&state, &[VSHARD]); + assert_eq!(pending(&state), BTreeSet::from([VSHARD])); + assert_eq!(row_vshards(&state), BTreeSet::from([VSHARD, VSHARD + 1])); + + retry_pending_forgets(&state, &[VSHARD + 1]); + assert!(pending(&state).is_empty()); + assert_eq!( + row_vshards(&state), + BTreeSet::from([VSHARD + 1]), + "the hosted vShard keeps its rows" + ); + } + + /// A pending vShard hosted again leaves the set, and its rows stay. + #[test] + fn a_rehosted_vshard_leaves_the_set_without_a_purge() { + let (state, _dir) = state(); + save_row(&state, VSHARD); + state + .credentials + .catalog() + .fail_next_calvin_barrier_replace_for_test(); + forget_left_vshards(&state, &[VSHARD]); + assert_eq!(pending(&state), BTreeSet::from([VSHARD])); + + retry_pending_forgets(&state, &[VSHARD]); + assert!(pending(&state).is_empty()); + assert_eq!(row_vshards(&state), BTreeSet::from([VSHARD])); + } + + /// Boot names every vShard that holds rows. The first pass removes the + /// rows of the vShards this node does not host and keeps the rest. + #[test] + fn boot_rows_of_an_unhosted_vshard_go_on_the_first_pass() { + let (state, _dir) = state(); + save_row(&state, VSHARD); + save_row(&state, VSHARD + 1); + + let holding = barrier_store::restore_at_boot(&state).expect("restore"); + assert_eq!(holding, BTreeSet::from([VSHARD, VSHARD + 1])); + state + .calvin + .forget_pending + .lock() + .expect("pending set") + .extend(holding); + + retry_pending_forgets(&state, &[VSHARD + 1]); + assert!(pending(&state).is_empty()); + assert_eq!(row_vshards(&state), BTreeSet::from([VSHARD + 1])); + assert_eq!( + state.calvin.read_results.waiting_txns(VSHARD), + 0, + "the unhosted vShard's buffer goes with its rows" + ); + assert_eq!(state.calvin.read_results.waiting_txns(VSHARD + 1), 1); + } + + /// A pass that runs before boot recovery forgets nothing, even when its + /// routing omits a group this node replicates. Recovery waits for the + /// group's leader. The first pass after it keeps the state of every + /// replicated vShard. + #[tokio::test] + async fn an_early_pass_never_purges_a_replicated_vshard() { + let (state, _dir) = booting_state(); + let leader = Arc::new(std::sync::atomic::AtomicU64::new(0)); + let status_leader = Arc::clone(&leader); + let status_fn: Arc Vec + Send + Sync> = + Arc::new(move || { + vec![nodedb_cluster::GroupStatus { + group_id: 1, + role: "Leader".into(), + leader_id: status_leader.load(std::sync::atomic::Ordering::Acquire), + term: 1, + commit_index: 0, + last_applied: 0, + last_log_index: 0, + snapshot_index: 0, + replica_state: nodedb_raft::ReplicaState::Whole, + member_count: 1, + learner_count: 0, + vshard_count: 1, + }] + }); + assert!(state.raft_status_fn.set(status_fn).is_ok()); + + save_row(&state, VSHARD); + state.calvin.applied.install(VSHARD, 2, BTreeSet::new()); + let holding = barrier_store::restore_at_boot(&state).expect("restore"); + seed_at_boot(&state, holding); + assert_eq!(pending(&state), BTreeSet::from([VSHARD])); + + // The routing omits this node: membership has not settled. + retry_pending_forgets(&state, &replicated_vshards(&routing(&[OTHER]), NODE)); + assert_eq!(pending(&state), BTreeSet::from([VSHARD])); + assert_eq!(row_vshards(&state), BTreeSet::from([VSHARD])); + assert!(state.calvin.applied.get(VSHARD).is_some()); + + // The group elects, and boot recovery finishes. + leader.store(NODE, std::sync::atomic::Ordering::Release); + crate::bootstrap::data_group_recovery::await_data_group_recovery(&state) + .await + .expect("recovery"); + assert!( + state + .data_groups_replayed + .load(std::sync::atomic::Ordering::Acquire) + ); + retry_pending_forgets(&state, &replicated_vshards(&routing(&[NODE]), NODE)); + assert!(pending(&state).is_empty()); + assert_eq!(row_vshards(&state), BTreeSet::from([VSHARD])); + assert!(state.calvin.applied.get(VSHARD).is_some()); + } + + /// A learner replicates its group, so its vShards count as replicated. + #[test] + fn a_learner_replicates_its_groups_vshards() { + let mut table = RoutingTable::uniform(1, &[OTHER], 1); + table.set_group_learners(1, vec![NODE]); + assert!( + replicated_vshards(&RwLock::new(table), NODE) + .binary_search(&VSHARD) + .is_ok() + ); + assert!(replicated_vshards(&routing(&[OTHER]), NODE).is_empty()); + } + + /// An unreplicated vShard with only an applied ledger and a base is + /// forgotten on the first pass after the data groups replay. + #[test] + fn an_unreplicated_ledger_and_base_are_forgotten() { + let (state, _dir) = state(); + let catalog = state.credentials.catalog(); + state.calvin.applied.install(VSHARD, 4, BTreeSet::new()); + state + .calvin + .bases + .record_kept(catalog, &[(VSHARD, 1)]) + .expect("record the base"); + seed_at_boot(&state, BTreeSet::new()); + assert_eq!(pending(&state), BTreeSet::from([VSHARD])); + + retry_pending_forgets(&state, &replicated_vshards(&routing(&[OTHER]), NODE)); + assert!(pending(&state).is_empty()); + assert!(state.calvin.applied.get(VSHARD).is_none()); + assert!(state.calvin.bases.base(VSHARD).is_none()); + assert!( + catalog + .load_calvin_bases() + .expect("load bases") + .iter() + .all(|(vshard_id, _)| *vshard_id != VSHARD) + ); + assert!( + catalog + .load_all_calvin_applied() + .expect("load ledgers") + .iter() + .filter(|stored| stored.vshard_id == VSHARD) + .all(|stored| { + stored.fully_applied_epoch == NOT_YET_APPLIED_EPOCH && stored.tail.is_empty() + }), + "a stored ledger of the vShard is a reset one" + ); + } +} diff --git a/nodedb/src/control/cluster/linearizable_read.rs b/nodedb/src/control/cluster/linearizable_read.rs index 369026938..d63f7feb1 100644 --- a/nodedb/src/control/cluster/linearizable_read.rs +++ b/nodedb/src/control/cluster/linearizable_read.rs @@ -8,18 +8,19 @@ //! leads (under its lease or a quorum round), the leader over the transport //! otherwise. Every path that serves a strong read calls //! [`confirm_linearizable_read`] on the serving node before it reads. - -use std::time::{Duration, Instant}; +//! +//! The two steps run under different bounds (see [`super::read_deadline`]): +//! - The quorum round for the read index ends at the transport RPC timeout, +//! so a partition refuses the read quickly. +//! - This node's wait to apply through the index follows the statement +//! deadline, so a lagging apply delays the read instead of refusing it. use nodedb_cluster::WaitOutcome; use crate::control::cluster::read_index::ReadIndexRefusal; use crate::control::state::SharedState; -/// Budget for one linearizable read to get each group's read index and apply -/// through it. Several production election timeouts (150-300ms), so an -/// ordinary round trip fits and a partition is refused rather than hung on. -pub const LINEARIZABLE_READ_TIMEOUT: Duration = Duration::from_millis(750); +pub use super::read_deadline::{ReadIndexDeadline, StatementDeadline}; /// The Raft groups that hold `vshard_ids`, deduplicated. Empty on a node with /// no routing table: without a cluster there is one copy and nothing to prove. @@ -87,29 +88,16 @@ fn hosts_group(state: &SharedState, routing: &nodedb_cluster::RoutingTable, grou }) } -/// The deadline a linearizable read in the running statement gets: never past -/// what is left of the statement's budget. -pub fn statement_read_deadline(state: &SharedState) -> Instant { - let remaining_ms = crate::control::server::shared::session::statement_deadline_ms( - state.tuning.network.default_deadline_secs, - ); - linearizable_read_deadline(Duration::from_millis(remaining_ms)) -} - -/// The deadline a linearizable read gets, never past `statement_budget`. -pub fn linearizable_read_deadline(statement_budget: Duration) -> Instant { - Instant::now() + LINEARIZABLE_READ_TIMEOUT.min(statement_budget) -} - /// Make a linearizable read of `groups` safe to serve on this node. /// /// Each group gets a read index taken after this call, then this node waits /// until it has applied the group through that index. The groups run -/// concurrently under one `deadline`. +/// concurrently. Each quorum round ends at the transport RPC timeout, and the +/// apply waits end at `deadline`. pub async fn confirm_linearizable_read( state: &SharedState, groups: &[u64], - deadline: Instant, + deadline: StatementDeadline, ) -> crate::Result<()> { if groups.is_empty() { return Ok(()); @@ -118,44 +106,55 @@ pub async fn confirm_linearizable_read( // A routing table without a gate: `start_raft` has not published it. return Err(refused(groups[0], "raft is not serving on this node yet")); }; + let round = ReadIndexDeadline::of_round(state, deadline); let reads = groups.iter().map(|&group_id| async move { - let remaining = deadline.saturating_duration_since(Instant::now()); - let read_index = - gate.read_index(group_id, remaining) - .await - .map_err(|refusal| match refusal { - ReadIndexRefusal::NotLeader => { - refused(group_id, "no leader confirmed a read index") - } - ReadIndexRefusal::Timeout { waited_ms } => refused( - group_id, - &format!("no quorum confirmed a read index within {waited_ms}ms"), - ), - })?; + let read_index = read_index_within(&**gate, group_id, round).await?; wait_applied_through(state, group_id, read_index, deadline).await }); futures::future::try_join_all(reads).await.map(|_| ()) } +/// A read index of `group_id` from its leader, refused once `round` passes. +async fn read_index_within( + gate: &dyn crate::control::cluster::read_index::RaftReadGate, + group_id: u64, + round: ReadIndexDeadline, +) -> crate::Result { + let asked = gate.read_index(group_id, round.remaining()); + let answer = tokio::time::timeout_at(tokio::time::Instant::from_std(round.instant()), asked) + .await + .map_err(|_| refused(group_id, "no quorum confirmed a read index in time"))?; + answer.map_err(|refusal| match refusal { + ReadIndexRefusal::NotLeader => refused(group_id, "no leader confirmed a read index"), + ReadIndexRefusal::Timeout { waited_ms } => refused( + group_id, + &format!("no quorum confirmed a read index within {waited_ms}ms"), + ), + }) +} + /// Wait until this node has applied `group_id` through `read_index`, or /// refuse at `deadline`. +/// +/// A replica here that awaits a snapshot install applies nothing until one +/// installs, so its read is refused at once. pub(crate) async fn wait_applied_through( state: &SharedState, group_id: u64, read_index: u64, - deadline: Instant, + deadline: StatementDeadline, ) -> crate::Result<()> { let watcher = state.applied_index_watcher(group_id); if watcher.current() >= read_index { return Ok(()); } - let remaining = deadline.saturating_duration_since(Instant::now()); - // The watcher parks its caller on a condition variable, so the wait runs - // on the blocking pool. - let outcome = tokio::task::spawn_blocking(move || watcher.wait_for(read_index, remaining)) - .await - .map_err(|e| refused(group_id, &format!("the apply wait did not finish: {e}")))?; - match outcome { + if awaits_snapshot(state, group_id) { + return Err(refused( + group_id, + "this replica awaits a snapshot install and applies nothing until one installs", + )); + } + match watcher.wait_for_async(read_index, deadline.instant()).await { WaitOutcome::Reached => Ok(()), WaitOutcome::TimedOut => Err(refused( group_id, @@ -165,6 +164,18 @@ pub(crate) async fn wait_applied_through( } } +/// Whether this node's replica of `group_id` awaits a snapshot install: its +/// data diverged from the log. +fn awaits_snapshot(state: &SharedState, group_id: u64) -> bool { + state.multi_raft.get().is_some_and(|multi_raft| { + multi_raft + .lock() + .unwrap_or_else(|p| p.into_inner()) + .replica_state(group_id) + == Some(nodedb_raft::ReplicaState::AwaitingSnapshot) + }) +} + fn refused(group_id: u64, detail: &str) -> crate::Error { crate::Error::LinearizableReadRefused { group_id, @@ -174,20 +185,122 @@ fn refused(group_id: u64, detail: &str) -> crate::Error { #[cfg(test)] mod tests { - use std::time::Duration; + use std::sync::Arc; + use std::time::{Duration, Instant}; - use super::{LINEARIZABLE_READ_TIMEOUT, linearizable_read_deadline}; + use async_trait::async_trait; - #[test] - fn the_deadline_never_exceeds_the_statement_budget() { - let short = Duration::from_millis(10); - let before = std::time::Instant::now(); - let deadline = linearizable_read_deadline(short); - assert!(deadline <= std::time::Instant::now() + short); - assert!(deadline >= before + short); + use super::*; + use crate::bridge::dispatch::Dispatcher; + use crate::control::cluster::read_index::RaftReadGate; + use crate::wal::WalManager; + + const GROUP: u64 = 3; + + fn test_state(dir: &tempfile::TempDir) -> Arc { + let wal = Arc::new( + WalManager::open_for_testing(&dir.path().join("linearizable.wal")).expect("wal"), + ); + let (dispatcher, _sides) = Dispatcher::new(1, 64); + SharedState::new(dispatcher, wal).expect("shared state") + } + + /// A gate whose quorum rounds never answer. + struct SilentGate; + + #[async_trait] + impl RaftReadGate for SilentGate { + async fn confirm_leader( + &self, + _group_id: u64, + _timeout: Duration, + ) -> Result { + std::future::pending().await + } + + fn within_staleness_bound( + &self, + _group_id: u64, + _max_staleness: Duration, + _applied: u64, + ) -> bool { + false + } + + fn holds_leader_lease(&self, _group_id: u64) -> bool { + false + } + + fn leader_lease_term(&self, _group_id: u64) -> Option { + None + } + + fn lease_read_index(&self, _group_id: u64) -> Option { + None + } + } + + /// A lagging apply delays the read: the wait follows the statement + /// deadline, not a short fixed bound. + #[tokio::test] + async fn the_apply_wait_follows_the_statement_deadline() { + let dir = tempfile::tempdir().expect("tempdir"); + let state = test_state(&dir); + let applier = { + let state = Arc::clone(&state); + tokio::spawn(async move { + tokio::time::sleep(Duration::from_millis(1_500)).await; + state.applied_index_watcher(GROUP).bump(9); + }) + }; + let deadline = StatementDeadline::after(Duration::from_secs(5)); + let waited = wait_applied_through(&state, GROUP, 9, deadline).await; + applier.await.expect("join applier"); + waited.expect("the read waits out the lag inside the statement deadline"); + } + + /// An apply that never reaches the read index is refused at the + /// statement deadline. + #[tokio::test] + async fn an_apply_that_never_arrives_is_refused_at_the_deadline() { + let dir = tempfile::tempdir().expect("tempdir"); + let state = test_state(&dir); + let started = Instant::now(); + let budget = Duration::from_millis(300); + let waited = wait_applied_through(&state, GROUP, 9, StatementDeadline::after(budget)).await; + assert!( + matches!(waited, Err(crate::Error::LinearizableReadRefused { .. })), + "{waited:?}" + ); + assert!(started.elapsed() >= budget); + } - let long = LINEARIZABLE_READ_TIMEOUT * 10; - let deadline = linearizable_read_deadline(long); - assert!(deadline <= std::time::Instant::now() + LINEARIZABLE_READ_TIMEOUT); + /// A read index no quorum confirms is refused at the transport RPC + /// timeout, long before a 30 s statement deadline. + #[tokio::test] + async fn a_read_index_with_no_quorum_is_refused_at_the_rpc_bound() { + let dir = tempfile::tempdir().expect("tempdir"); + let mut state = test_state(&dir); + Arc::get_mut(&mut state) + .expect("sole owner of fresh state") + .tuning + .cluster_transport + .rpc_timeout_secs = 1; + assert!( + state + .raft_read_gate + .set(Arc::new(SilentGate) as Arc) + .is_ok() + ); + let started = Instant::now(); + let deadline = StatementDeadline::after(Duration::from_secs(30)); + let read = confirm_linearizable_read(&state, &[GROUP], deadline).await; + let waited = started.elapsed(); + assert!( + matches!(read, Err(crate::Error::LinearizableReadRefused { .. })), + "{read:?}" + ); + assert!(waited >= Duration::from_secs(1), "{waited:?}"); + assert!(waited < Duration::from_secs(10), "{waited:?}"); } } diff --git a/nodedb/src/control/cluster/metadata_applier/audit.rs b/nodedb/src/control/cluster/metadata_applier/audit.rs index bffc2e766..2b98bc492 100644 --- a/nodedb/src/control/cluster/metadata_applier/audit.rs +++ b/nodedb/src/control/cluster/metadata_applier/audit.rs @@ -121,9 +121,10 @@ pub(super) fn emit_ddl_audit( // mutex lets audit-log contention stall EVERY metadata apply, and with it // collection materialization, DDL, and any proposer waiting on the applied // index. That is a liveness bug rather than a slow path: one contended - // acquisition here was measured parking the loop for a full 5s propose - // timeout, so peer `CollectionSchema` announces never materialized and the - // engine writes that followed them were rejected as unknown collections. + // acquisition here was measured parking the loop past a proposer's whole + // apply wait, so peer `CollectionSchema` announces never materialized and + // the engine writes that followed them were rejected as unknown + // collections. // // The record's payload is fully built above and needs nothing further from // the apply loop, so deferring the emit keeps the compliance row (no silent diff --git a/nodedb/src/control/cluster/metadata_applier/catalog_ddl.rs b/nodedb/src/control/cluster/metadata_applier/catalog_ddl.rs index c71d79712..536642f21 100644 --- a/nodedb/src/control/cluster/metadata_applier/catalog_ddl.rs +++ b/nodedb/src/control/cluster/metadata_applier/catalog_ddl.rs @@ -81,6 +81,14 @@ impl MetadataCommitApplier { }; let shared = self.shared_state()?; + // Slows one collection's create on one node, as a long apply does. + #[cfg(feature = "failpoints")] + if let catalog_entry::CatalogEntry::PutCollection(stored) + | catalog_entry::CatalogEntry::PutCollectionIfAbsent(stored) = &stamped + { + crate::control::fail_gate::before_catalog_ddl_apply(shared.node_id, &stored.name).await; + } + // Holds back one node's apply of backup schedule marks with a // transient error, so a test can make that node's catalog lag the // metadata group. Raft re-delivers the entry once it is cleared. diff --git a/nodedb/src/control/cluster/metadata_applier/dispatch.rs b/nodedb/src/control/cluster/metadata_applier/dispatch.rs index fdf8343f5..d9b92ff80 100644 --- a/nodedb/src/control/cluster/metadata_applier/dispatch.rs +++ b/nodedb/src/control/cluster/metadata_applier/dispatch.rs @@ -275,14 +275,8 @@ impl MetadataCommitApplier { entries: &[MetadataEntry], raft_index: u64, ) -> Result<(), crate::Error> { - let shared = self.shared.get().and_then(std::sync::Weak::upgrade); for sub in entries { self.apply_host_side_effects(sub, raft_index).await?; - if let Some(shared) = &shared { - shared - .metadata_apply_progress - .fetch_add(1, std::sync::atomic::Ordering::Release); - } } Ok(()) } diff --git a/nodedb/src/control/cluster/metadata_applier/wedge.rs b/nodedb/src/control/cluster/metadata_applier/wedge.rs index 95d7ff076..021f502d7 100644 --- a/nodedb/src/control/cluster/metadata_applier/wedge.rs +++ b/nodedb/src/control/cluster/metadata_applier/wedge.rs @@ -132,6 +132,8 @@ pub fn classify(error: &crate::Error) -> ApplyFailureClass { | crate::Error::GroupMarksUnavailable { .. } | crate::Error::BackupCaptureMoved { .. } | crate::Error::MetadataLeaderUnavailable + | crate::Error::MetadataGroupGone { .. } + | crate::Error::MetadataDdlSuperseded { .. } | crate::Error::AuthorizationStateBehind { .. } | crate::Error::LinearizableReadRefused { .. } | crate::Error::ExecutionLimitExceeded { .. } diff --git a/nodedb/src/control/cluster/mod.rs b/nodedb/src/control/cluster/mod.rs index de6a5717e..5dcf47912 100644 --- a/nodedb/src/control/cluster/mod.rs +++ b/nodedb/src/control/cluster/mod.rs @@ -17,14 +17,17 @@ pub mod data_plane_error_wire; pub mod data_plane_fault_wire; pub mod decommission_bridge; pub mod execution_error_wire; +pub mod forwarded_budget; pub mod handle; pub mod init; pub mod leased_read; +pub mod left_vshard_forget; pub mod linearizable_read; pub mod metadata_applier; pub mod metadata_image; pub mod metadata_stamp; pub mod pem_io; +pub mod read_deadline; pub mod read_index; pub mod recovery_check; pub mod redo_stream_membership; diff --git a/nodedb/src/control/cluster/read_deadline.rs b/nodedb/src/control/cluster/read_deadline.rs new file mode 100644 index 000000000..d244dfb17 --- /dev/null +++ b/nodedb/src/control/cluster/read_deadline.rs @@ -0,0 +1,67 @@ +// SPDX-License-Identifier: BUSL-1.1 + +//! The two bounds a linearizable read runs under. +//! +//! - [`StatementDeadline`] bounds the whole read. This node's wait to apply +//! through the read index follows it. +//! - [`ReadIndexDeadline`] bounds one quorum round for a read index: the +//! transport RPC timeout, never past the statement. A partitioned leader +//! is refused at this bound, not at the statement deadline. +//! +//! Each is its own type, so a caller cannot pass one where the other belongs. + +use std::time::{Duration, Instant}; + +pub use crate::control::server::shared::session::StatementDeadline; +use crate::control::state::SharedState; + +/// When one quorum round for a read index gives up. +#[derive(Debug, Clone, Copy, PartialEq, Eq, PartialOrd, Ord)] +pub struct ReadIndexDeadline(Instant); + +impl ReadIndexDeadline { + /// `rpc_timeout` from now, never past `statement`. + pub fn within(statement: StatementDeadline, rpc_timeout: Duration) -> Self { + Self(statement.instant().min(Instant::now() + rpc_timeout)) + } + + /// The bound `state`'s transport tuning gives a round started now, never + /// past `statement`. + pub fn of_round(state: &SharedState, statement: StatementDeadline) -> Self { + Self::within( + statement, + Duration::from_secs(state.tuning.cluster_transport.rpc_timeout_secs), + ) + } + + /// The deadline as an instant. + pub fn instant(self) -> Instant { + self.0 + } + + /// Time left until the deadline, zero once it passed. + pub fn remaining(self) -> Duration { + self.0.saturating_duration_since(Instant::now()) + } +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn the_round_bound_is_the_rpc_timeout_inside_a_long_statement() { + let statement = StatementDeadline::after(Duration::from_secs(30)); + let before = Instant::now(); + let round = ReadIndexDeadline::within(statement, Duration::from_secs(1)); + assert!(round.instant() >= before + Duration::from_secs(1)); + assert!(round.instant() < statement.instant()); + } + + #[test] + fn the_round_bound_never_passes_the_statement() { + let statement = StatementDeadline::after(Duration::from_millis(10)); + let round = ReadIndexDeadline::within(statement, Duration::from_secs(5)); + assert_eq!(round.instant(), statement.instant()); + } +} diff --git a/nodedb/src/control/cluster/read_index.rs b/nodedb/src/control/cluster/read_index.rs index 86493a307..711378ecb 100644 --- a/nodedb/src/control/cluster/read_index.rs +++ b/nodedb/src/control/cluster/read_index.rs @@ -40,8 +40,10 @@ pub trait RaftReadGate: Send + Sync { ) -> Result; /// Whether this node's replica of `group_id` is within `max_staleness` of - /// the leader. Local state only — no quorum round, so this does not block. - fn within_staleness_bound(&self, group_id: u64, max_staleness: Duration) -> bool; + /// the leader, with reads here seeing state through `applied`: the + /// group's host applied watermark. Local state only — no quorum round, so + /// this does not block. + fn within_staleness_bound(&self, group_id: u64, max_staleness: Duration, applied: u64) -> bool; /// Whether this node leads `group_id` under a leader lease that is valid /// now. Local state only, so this does not block. @@ -173,11 +175,11 @@ impl RaftReadGate for MultiRaftReadGate { .map_err(refusal_of) } - fn within_staleness_bound(&self, group_id: u64, max_staleness: Duration) -> bool { + fn within_staleness_bound(&self, group_id: u64, max_staleness: Duration, applied: u64) -> bool { self.multi_raft .lock() .unwrap_or_else(|p| p.into_inner()) - .within_staleness_bound(group_id, max_staleness) + .within_staleness_bound(group_id, max_staleness, applied) } fn holds_leader_lease(&self, group_id: u64) -> bool { diff --git a/nodedb/src/control/cluster/redo_stream_membership.rs b/nodedb/src/control/cluster/redo_stream_membership.rs index fdcc58ee0..c3d4dea3e 100644 --- a/nodedb/src/control/cluster/redo_stream_membership.rs +++ b/nodedb/src/control/cluster/redo_stream_membership.rs @@ -8,13 +8,14 @@ //! install before it applies here again. A remount resumes its old log, //! which can still name a dropped stream. //! - A mounted group that owes a snapshot refuses log entries until one -//! installs, and its leadership here moves to a replica that can lead. -//! This covers a group that mounted again, and a group whose apply here -//! held an entry of a dropped stream. +//! installs. Its replica here steps down from leadership and never +//! campaigns. This covers a group that mounted again, and a group whose +//! apply here holds an entry. use std::sync::Mutex; use nodedb_cluster::multi_raft::MultiRaft; +use nodedb_raft::ReplicaState; use crate::control::state::SharedState; @@ -56,8 +57,9 @@ fn drop_unmounted(shared: &SharedState, multi_raft: &Mutex) { } } -/// Make every mounted group that owes a snapshot refuse log entries, and -/// move a leadership of it this node holds: a leader takes no snapshot. +/// Make every mounted group that owes a snapshot refuse log entries. The +/// requirement steps down a leadership of it this node holds: a leader +/// takes no snapshot. /// /// The debt is read under the `MultiRaft` lock. An install settles the debt /// before it refreshes the group's requirement under the same lock, so this @@ -72,20 +74,12 @@ fn require_owed_snapshots(shared: &SharedState, multi_raft: &Mutex) { if !multi_raft.contains_group(group_id) || !shared.redo_chunks.owes_snapshot(group_id) { continue; } - if !multi_raft.snapshot_required(group_id) { - multi_raft.set_snapshot_required(group_id, true); + if multi_raft.replica_state(group_id) != Some(ReplicaState::AwaitingSnapshot) { + multi_raft.require_snapshot(group_id, ReplicaState::AwaitingSnapshot); tracing::warn!( group_id, "redo chunks: the group's replica here owes a snapshot install; it refuses \ - log entries until one installs" - ); - } - if let Some(target) = multi_raft.hand_off_leadership(group_id) { - tracing::info!( - group_id, - target, - "redo chunks: the group's replica here owes a snapshot install; its \ - leadership moves to a replica that can lead" + log entries and leads nothing until one installs" ); } } diff --git a/nodedb/src/control/cluster/snapshot_applier.rs b/nodedb/src/control/cluster/snapshot_applier.rs index 76d1def3e..399c4b147 100644 --- a/nodedb/src/control/cluster/snapshot_applier.rs +++ b/nodedb/src/control/cluster/snapshot_applier.rs @@ -283,16 +283,18 @@ impl DataPlaneSnapshotApplier { if floor == 0 || watcher.current() >= floor { return Ok(()); } - let waiting = Arc::clone(&watcher); - let outcome = tokio::task::spawn_blocking(move || waiting.wait_for(floor, limit)).await; + let outcome = watcher + .wait_for_async(floor, std::time::Instant::now() + limit) + .await; match outcome { - Ok(nodedb_cluster::WaitOutcome::Reached) => Ok(()), - Ok(nodedb_cluster::WaitOutcome::TimedOut | nodedb_cluster::WaitOutcome::GroupGone) - | Err(_) => Err(SnapshotInstallError::MetadataBehind { - group_id, - floor, - applied: watcher.current(), - }), + nodedb_cluster::WaitOutcome::Reached => Ok(()), + nodedb_cluster::WaitOutcome::TimedOut | nodedb_cluster::WaitOutcome::GroupGone => { + Err(SnapshotInstallError::MetadataBehind { + group_id, + floor, + applied: watcher.current(), + }) + } } } diff --git a/nodedb/src/control/cluster/snapshot_builder.rs b/nodedb/src/control/cluster/snapshot_builder.rs index c67b429ca..1f4771afd 100644 --- a/nodedb/src/control/cluster/snapshot_builder.rs +++ b/nodedb/src/control/cluster/snapshot_builder.rs @@ -304,6 +304,8 @@ impl nodedb_cluster::SnapshotBuilder for DataPlaneSnapshotBuilder { last_included_index: u64, _last_included_term: u64, ) -> Result> { + #[cfg(feature = "failpoints")] + crate::control::fail_gate::before_snapshot_build(self.shared.node_id, group_id).await; // No routing table or an empty group ships nothing: the sender falls // back to the stub chunk, at the index asked for. let nothing = nodedb_cluster::BuiltGroupSnapshot { @@ -358,8 +360,14 @@ impl nodedb_cluster::SnapshotBuilder for DataPlaneSnapshotBuilder { .map_err(|e| Box::new(e) as Box)?; // Read under the fence: the storage captured below holds exactly the // Calvin positions this names. - let calvin_cut = capture_calvin_cut(&self.shared, &group_vshards, calvin_through) - .map_err(|e| Box::new(e) as Box)?; + let calvin_cut = capture_calvin_cut( + &self.shared, + group_id, + &group_vshards, + calvin_through, + cut_index, + ) + .map_err(|e| Box::new(e) as Box)?; // Enumerate tenants and their databases from the system catalog — the // same source the backup orchestrator's catalog sections use. Every diff --git a/nodedb/src/control/cluster/snapshot_cut.rs b/nodedb/src/control/cluster/snapshot_cut.rs index c28fce096..946d02374 100644 --- a/nodedb/src/control/cluster/snapshot_cut.rs +++ b/nodedb/src/control/cluster/snapshot_cut.rs @@ -52,16 +52,9 @@ pub(crate) async fn settled_cut( let watcher = shared.applied_index_watcher(group_id); let cut = cut_of(tracker.started_through(group_id), watcher.current()); if watcher.current() < cut { - let waiting = std::sync::Arc::clone(&watcher); - let outcome = - tokio::task::spawn_blocking(move || waiting.wait_for(cut, CUT_SETTLE_TIMEOUT)) - .await - .map_err(|e| Error::Internal { - detail: format!( - "snapshot build: group {group_id}: the settle wait for cut {cut} \ - did not finish: {e}" - ), - })?; + let outcome = watcher + .wait_for_async(cut, std::time::Instant::now() + CUT_SETTLE_TIMEOUT) + .await; match outcome { WaitOutcome::Reached => {} WaitOutcome::TimedOut | WaitOutcome::GroupGone => { diff --git a/nodedb/src/control/cluster/snapshot_install/error.rs b/nodedb/src/control/cluster/snapshot_install/error.rs index a433c4b04..f24ffe12d 100644 --- a/nodedb/src/control/cluster/snapshot_install/error.rs +++ b/nodedb/src/control/cluster/snapshot_install/error.rs @@ -18,6 +18,8 @@ pub enum SettleStep { CalvinState, /// Replacing the group's open chunked redo streams. RedoStreams, + /// Recording the group's backup cut floors and barriers. + CutBarriers, } impl std::fmt::Display for SettleStep { @@ -29,6 +31,7 @@ impl std::fmt::Display for SettleStep { Self::EventLane => "event lane install", Self::CalvinState => "Calvin state install", Self::RedoStreams => "redo stream install", + Self::CutBarriers => "cut barrier install", }) } } diff --git a/nodedb/src/control/cluster/start_raft/hooks.rs b/nodedb/src/control/cluster/start_raft/hooks.rs index a68f13243..2709c23f2 100644 --- a/nodedb/src/control/cluster/start_raft/hooks.rs +++ b/nodedb/src/control/cluster/start_raft/hooks.rs @@ -119,7 +119,11 @@ pub(super) async fn build_hooks( // at or below a group's durable applied floor is never delivered again, // so its row is what a barrier reads from now on. A boot completion of a // snapshot install below replaces its group's rows. - crate::control::cluster::calvin::scheduler::barrier_store::restore_at_boot(shared)?; + let holding = + crate::control::cluster::calvin::scheduler::barrier_store::restore_at_boot(shared)?; + // The first reconcile pass after the data groups replay forgets the + // stored Calvin state of each vShard this node no longer replicates. + crate::control::cluster::left_vshard_forget::seed_at_boot(shared, holding); // Per-group snapshot builder for the SEND path: on the leader, build the // real serialized engine state for a lagging follower's group vshards diff --git a/nodedb/src/control/cluster/start_raft/propose_error.rs b/nodedb/src/control/cluster/start_raft/propose_error.rs index 54001390e..ea053b6d6 100644 --- a/nodedb/src/control/cluster/start_raft/propose_error.rs +++ b/nodedb/src/control/cluster/start_raft/propose_error.rs @@ -34,11 +34,13 @@ pub(super) fn async_propose_error(vshard_id: u32, error: ClusterError) -> crate: ) | ClusterError::ReadIndexNotLeader { .. } | ClusterError::MigrationInProgress { .. } - | ClusterError::WrongOwner { .. } + | ClusterError::WrongOwner { .. } => crate::Error::NoLeader { + vshard_id: VShardId::new(vshard_id), + }, // The forward never reached the leader whole: the leader is not in // this node's topology, or the connect, stream open or write failed. // Nothing was proposed. A written forward ends as `Unanswered`. - | ClusterError::Transport { .. } => crate::Error::NoLeader { + unsent if unsent.is_unsent() => crate::Error::NoLeader { vshard_id: VShardId::new(vshard_id), }, // The forward did not answer before its timeout. The leader can have @@ -46,7 +48,7 @@ pub(super) fn async_propose_error(vshard_id: u32, error: ClusterError) -> crate: // deadline class, the same class the array fan-out gives it. A // forward whose stream failed after the request was written can // also have been proposed, so it takes the same class. - error @ (ClusterError::ShardTimeout { .. } | ClusterError::Unanswered { .. }) => { + error if error.is_lost_answer() => { tracing::warn!( vshard_id, %error, @@ -110,7 +112,11 @@ pub(super) fn async_propose_error(vshard_id: u32, error: ClusterError) -> crate: | ClusterError::Bm25Gather(_) | ClusterError::TsGather(_) | ClusterError::ShufflePush(_) - | ClusterError::RemoteUntyped { .. }) => crate::Error::Internal { + | ClusterError::RemoteUntyped { .. } + // The guards above take these. They stay named for exhaustiveness. + | ClusterError::Transport { .. } + | ClusterError::ShardTimeout { .. } + | ClusterError::Unanswered { .. }) => crate::Error::Internal { detail: format!("raft propose (async): {other}"), }, } diff --git a/nodedb/src/control/cluster/start_raft/proposer_wiring.rs b/nodedb/src/control/cluster/start_raft/proposer_wiring.rs index 264ae0d08..2b79defe7 100644 --- a/nodedb/src/control/cluster/start_raft/proposer_wiring.rs +++ b/nodedb/src/control/cluster/start_raft/proposer_wiring.rs @@ -225,8 +225,8 @@ fn install_async_proposer( // Held weakly for the same cycle-breaking reason as `raft_proposer` above: // the proposer lives on `SharedState`. let state_for_proposer = Arc::downgrade(shared); - let async_submit: Arc = - Arc::new(move |vshard_id, idempotency_key, data, deadline| { + let async_submit: Arc = Arc::new( + move |vshard_id, idempotency_key, data, deadline| { let rl_weak = raft_loop_async.clone(); let tk = tracker_for_proposer.clone(); let state_weak = state_for_proposer.clone(); @@ -307,6 +307,7 @@ fn install_async_proposer( group_id, through: log_index, }], + crate::control::server::shared::session::StatementDeadline::of_statement(&state).instant(), ) .await?; } @@ -320,7 +321,8 @@ fn install_async_proposer( applied, }) }) - }); + }, + ); crate::control::vshard_admission::install_async_raft_proposer(shared, async_submit) } diff --git a/nodedb/src/control/cluster/start_raft_helpers.rs b/nodedb/src/control/cluster/start_raft_helpers.rs index 04be91339..a4888f1e8 100644 --- a/nodedb/src/control/cluster/start_raft_helpers.rs +++ b/nodedb/src/control/cluster/start_raft_helpers.rs @@ -11,6 +11,7 @@ use crate::control::cluster::calvin::{ }; use crate::control::cluster::calvin_snapshot; use crate::control::cluster::handle::ClusterHandle; +use crate::control::cluster::left_vshard_forget; use crate::control::state::SharedState; /// The vShards whose scheduler this node started and has not stopped. @@ -41,8 +42,11 @@ fn hosted_vshards(routing: &RwLock, node_id: u64) /// from. Dropping the vShard's sequencer sender closes the scheduler's /// intake, and the scheduler exits. Its other channels and its lock table go /// too. It hands the barrier events of the txns it did not finish back to -/// the vShard's read-result buffer. The next reconcile that finds the vShard hosted spawns a new -/// scheduler from the vShard's base. +/// the vShard's read-result buffer. The next reconcile that finds the vShard +/// hosted spawns a new scheduler from the vShard's base. +/// +/// A vShard that left has its stored Calvin state forgotten (see +/// [`left_vshard_forget`]). fn retire_vshard_schedulers( hosted: &[u32], shared: &Arc, @@ -67,24 +71,10 @@ fn retire_vshard_schedulers( } (left, rebased) }; - calvin_snapshot::forget_left(shared, &left); - // A vShard that left this node keeps no barrier events here, in memory - // or stored. A rebased vShard keeps them for the scheduler that starts - // again. - for &vshard in &left { - if let Err(e) = - crate::control::cluster::calvin::scheduler::barrier_store::forget_vshard(shared, vshard) - { - // The rows stay. Boot marks them stored for a vShard with no - // scheduler, and a snapshot install of the vShard replaces them. - tracing::warn!( - vshard_id = vshard, - error = %e, - "calvin: the barrier rows of a vShard that left were not removed" - ); - crate::diag::calvin_barrier_log_store_failed(vshard, None, "remove", &e); - } - } + // A vShard that left this node keeps no Calvin state here, in memory or + // stored. A rebased vShard keeps its barrier events for the scheduler + // that starts again. + left_vshard_forget::forget_left_vshards(shared, &left); for &vshard in &rebased { tracing::info!( vshard_id = vshard, @@ -164,6 +154,11 @@ fn reconcile_vshard_schedulers(params: ReconcileSchedulersParams<'_>) -> crate:: } = params; let hosted = hosted_vshards(routing, node_id); + // Forgets that failed on an earlier pass, or wait since boot, go first. + left_vshard_forget::retry_pending_forgets( + shared, + &left_vshard_forget::replicated_vshards(routing, node_id), + ); retire_vshard_schedulers( &hosted, shared, diff --git a/nodedb/src/control/crdt_admission.rs b/nodedb/src/control/crdt_admission.rs deleted file mode 100644 index 560ff47ed..000000000 --- a/nodedb/src/control/crdt_admission.rs +++ /dev/null @@ -1,1333 +0,0 @@ -// SPDX-License-Identifier: BUSL-1.1 - -//! Serialized preview, policy, and fenced CRDT apply admission. - -use std::time::Duration; - -use nodedb_physical::physical_plan::CrdtOp; -use nodedb_types::CrdtPreviewResult; - -use crate::bridge::envelope::{ErrorCode, PhysicalPlan, Status}; -use crate::control::server::shared::authorization::AuthorizedTask; -use crate::control::state::SharedState; -use crate::control::wal_replication::{ReplicableWrite, to_replicated_entry}; -use crate::event::EventSource; -use crate::types::{DatabaseId, TenantId, VShardId}; - -pub trait CrdtPostImagePolicy: Send + Sync { - fn evaluate(&self, preview: &CrdtPreviewResult) -> crate::Result<()>; -} - -/// Explicit policy for trusted internal callers. -pub struct TrustedInternalCrdtPolicy; - -impl CrdtPostImagePolicy for TrustedInternalCrdtPolicy { - fn evaluate(&self, _preview: &CrdtPreviewResult) -> crate::Result<()> { - Ok(()) - } -} - -const FRONTIER_RETRY_LIMIT: usize = 8; - -pub struct AuthorizedCrdtApplyAdmissionRequest<'a> { - pub authorized: AuthorizedTask, - /// The db-qualified collection (`QualifiedCollection::as_str`), the same - /// string the plan's `CrdtOp::Apply` carries. The preview and the apply - /// address the Data Plane by it, and the catalog lookup de-qualifies it. - pub collection: &'a str, - pub timeout: Duration, - pub event_source: EventSource, - pub policy: &'a dyn CrdtPostImagePolicy, -} - -pub struct CrdtApplyAdmissionRequest<'a> { - pub tenant_id: TenantId, - pub database_id: DatabaseId, - /// The db-qualified collection, equal to the plan's `CrdtOp::Apply` - /// collection. - pub collection: &'a str, - pub plan: PhysicalPlan, - pub timeout: Duration, - pub event_source: EventSource, - pub policy: &'a dyn CrdtPostImagePolicy, -} - -pub struct CrdtAdmissionOutcome { - pub payload: Vec, - /// The versions the apply stamped: the entry's log position on the - /// written vShard. - pub write_versions: crate::types::ReadVersions, - /// Operations the admitted delta encoded that the target document already - /// knew, measured by the preview that fenced this apply. Distinguishes a - /// client whose writes land from one whose writes are being absorbed — - /// both otherwise produce the same successful payload. - pub trimmed_ops: u64, -} - -impl CrdtAdmissionOutcome { - /// Attach the trim count measured by the preview that fenced this apply. - fn with_trimmed_ops(self, trimmed_ops: u64) -> Self { - Self { - trimmed_ops, - ..self - } - } -} - -pub struct CrdtRestoreAdmissionRequest<'a> { - pub tenant_id: TenantId, - pub database_id: DatabaseId, - /// The db-qualified collection the generated restore ops address. - pub collection: &'a str, - pub document_id: &'a str, - pub target_version_json: &'a str, - pub surrogate: nodedb_types::Surrogate, - pub peer_id: u64, - pub timeout: Duration, - pub event_source: EventSource, - pub policy: &'a dyn CrdtPostImagePolicy, -} - -struct CrdtAdmissionWorkflow<'a> { - state: &'a SharedState, - tenant_id: TenantId, - database_id: DatabaseId, - vshard_id: VShardId, - collection: &'a str, - timeout: Duration, - event_source: EventSource, - policy: &'a dyn CrdtPostImagePolicy, -} - -/// Whether an operation changes the Loro frontier and must serialize with an -/// admission preview when it is dispatched to this node's cores without a -/// proposal. -pub fn changes_crdt_frontier(op: &CrdtOp) -> bool { - match op { - CrdtOp::Apply { .. } - | CrdtOp::ApplyAuthenticated { .. } - | CrdtOp::ImportSnapshot { .. } - | CrdtOp::ListInsert { .. } - | CrdtOp::ListDelete { .. } - | CrdtOp::ListMove { .. } - | CrdtOp::DocUpsert { .. } - | CrdtOp::DocDelete { .. } - | CrdtOp::RestoreToVersion { .. } => true, - CrdtOp::Read { .. } - | CrdtOp::PreviewApply { .. } - | CrdtOp::GetVersionVector { .. } - | CrdtOp::ExportDelta { .. } - | CrdtOp::CompactAtVersion { .. } - | CrdtOp::SetConstraints { .. } - | CrdtOp::DropConstraints { .. } - | CrdtOp::ReadConstraints { .. } - | CrdtOp::SetPolicy { .. } - | CrdtOp::GetPolicy { .. } - | CrdtOp::ReadAtVersion { .. } => false, - } -} - -/// Preview, authorize, fence, and durably apply one externally authorized delta. -pub async fn dispatch_authorized_crdt_apply_admitted_outcome( - state: &SharedState, - request: AuthorizedCrdtApplyAdmissionRequest<'_>, -) -> crate::Result { - let AuthorizedCrdtApplyAdmissionRequest { - authorized, - collection, - timeout, - event_source, - policy, - } = request; - enforce_external_signing_policy(state, &authorized, collection)?; - let task = authorized.into_physical_task(); - dispatch_crdt_apply_admitted_outcome( - state, - CrdtApplyAdmissionRequest { - tenant_id: task.tenant_id, - database_id: task.database_id, - collection, - plan: task.plan, - timeout, - event_source, - policy, - }, - ) - .await -} - -/// `collection` is db-qualified. The catalog keys collections by the bare -/// name, so the lookup de-qualifies it first. -fn enforce_external_signing_policy( - state: &SharedState, - authorized: &AuthorizedTask, - collection: &str, -) -> crate::Result<()> { - let bare = crate::control::target_identity::naming::bare_collection_name( - authorized.database_id(), - collection, - ); - let stored = state - .credentials - .catalog() - .get_collection( - authorized.database_id(), - authorized.tenant_id().as_u64(), - &bare, - )? - .ok_or_else(|| crate::Error::CollectionNotFound { - tenant_id: authorized.tenant_id(), - collection: bare.clone(), - })?; - if stored.crdt_signing_required - && matches!(authorized.plan(), PhysicalPlan::Crdt(CrdtOp::Apply { .. })) - { - return Err(crate::Error::RejectedAuthz { - tenant_id: authorized.tenant_id(), - resource: format!("collection:{bare}:unsigned_crdt_delta_requires_authenticated_sync"), - }); - } - Ok(()) -} - -/// Preview, authorize, fence, and durably apply one trusted-internal CRDT delta. -pub(crate) async fn dispatch_crdt_apply_admitted_outcome( - state: &SharedState, - request: CrdtApplyAdmissionRequest<'_>, -) -> crate::Result { - let CrdtApplyAdmissionRequest { - tenant_id, - database_id, - collection, - plan, - timeout, - event_source, - policy, - } = request; - // Every CRDT apply converges here; the lease gates it on a drained - // collection and lives until the apply's outcome. - let _lease = crate::control::server::shared::clone_write::write_lease( - state, - tenant_id, - database_id, - &plan, - ) - .await?; - let (document_id, delta) = match &plan { - PhysicalPlan::Crdt( - CrdtOp::Apply { - collection: plan_collection, - document_id, - delta, - expected_frontier_digest: None, - .. - } - | CrdtOp::ApplyAuthenticated { - collection: plan_collection, - document_id, - delta, - expected_frontier_digest: None, - .. - }, - ) if plan_collection.as_str() == collection => (document_id.clone(), delta.clone()), - PhysicalPlan::Crdt( - CrdtOp::Apply { - expected_frontier_digest: Some(_), - .. - } - | CrdtOp::ApplyAuthenticated { - expected_frontier_digest: Some(_), - .. - }, - ) => return Err(crate::Error::CrdtAdmissionCallerFence), - _ => { - return Err(crate::Error::CrdtAdmissionInvalidPlan { - reason: "expected an unfenced CRDT Apply for the supplied collection", - }); - } - }; - // `collection` is the plan's database-qualified name. - let vshard_id = - nodedb_types::CollectionKey::from_qualified_str(database_id, collection)?.vshard(); - let workflow = CrdtAdmissionWorkflow { - state, - tenant_id, - database_id, - vshard_id, - collection, - timeout, - event_source, - policy, - }; - tokio::time::timeout( - timeout, - state.vshard_admission_sequencer.run_after_proposed( - vshard_id, - |last_proposed| async move { - caught_up(&workflow, last_proposed).await?; - admit_apply_locked(&workflow, plan, &document_id, &delta).await - }, - ), - ) - .await - .map_err(|_| crate::Error::CrdtAdmissionTimeout { - vshard_id, - timeout_ms: timeout_ms(timeout), - })? -} - -/// Wait until this node applied every write admitted to the workflow's -/// vShard before this admission took the slot, so its preview reads them. -async fn caught_up( - workflow: &CrdtAdmissionWorkflow<'_>, - last_proposed: Option, -) -> crate::Result<()> { - crate::control::vshard_admission::await_admitted_applies( - workflow.state, - workflow.vshard_id, - last_proposed, - std::time::Instant::now() + workflow.timeout, - ) - .await -} - -async fn admit_apply_locked( - workflow: &CrdtAdmissionWorkflow<'_>, - plan: PhysicalPlan, - document_id: &str, - delta: &[u8], -) -> crate::Result { - for _attempt in 0..FRONTIER_RETRY_LIMIT { - let preview = preview(workflow, document_id, delta).await?; - workflow.policy.evaluate(&preview)?; - let fenced = stamp_fence(plan.clone(), preview.frontier_digest)?; - match apply_fenced(workflow, fenced).await { - Err(crate::Error::DataPlane(ErrorCode::CrdtFrontierMismatch { .. })) => {} - // The trim count belongs to the preview that fenced *this* attempt. - // A retry re-previews against the advanced frontier and produces its - // own count, so the two are never mixed. - result => { - return result.map(|outcome| outcome.with_trimmed_ops(preview.trimmed_ops)); - } - } - } - Err(crate::Error::CrdtAdmissionRetriesExhausted { - vshard_id: workflow.vshard_id, - attempts: FRONTIER_RETRY_LIMIT, - }) -} - -async fn preview( - workflow: &CrdtAdmissionWorkflow<'_>, - document_id: &str, - delta: &[u8], -) -> crate::Result { - let response = tokio::time::timeout( - workflow.timeout, - crate::control::server::shared::ddl::sync_dispatch::dispatch_system_response_with_source( - workflow.state, - crate::control::server::shared::ddl::sync_dispatch::SystemTask::new( - crate::control::server::shared::ddl::sync_dispatch::SystemReason::AdmittedContinuation, - workflow.tenant_id, - nodedb_types::CollectionKey::from_qualified_str(workflow.database_id, workflow.collection)?, - PhysicalPlan::Crdt(CrdtOp::PreviewApply { - collection: nodedb_types::QualifiedCollection::from_stored( - workflow.collection.to_owned(), - ), - document_id: document_id.to_owned(), - delta: delta.to_vec(), - }), - ), - workflow.timeout, - workflow.event_source, - ), - ) - .await - .map_err(|_| crate::Error::CrdtAdmissionTimeout { - vshard_id: workflow.vshard_id, - timeout_ms: timeout_ms(workflow.timeout), - })??; - if response.status != Status::Ok { - return Err(response_error(&response)); - } - zerompk::from_msgpack(response.payload.as_bytes()).map_err(|error| crate::Error::Internal { - detail: format!("CRDT preview payload decode: {error}"), - }) -} - -fn stamp_fence(plan: PhysicalPlan, digest: [u8; 32]) -> crate::Result { - match plan { - PhysicalPlan::Crdt(CrdtOp::Apply { - collection, - document_id, - delta, - peer_id, - mutation_id, - surrogate, - provenance, - constraint_version_required, - expected_frontier_digest: None, - }) => Ok(PhysicalPlan::Crdt(CrdtOp::Apply { - collection, - document_id, - delta, - peer_id, - mutation_id, - surrogate, - provenance, - constraint_version_required, - expected_frontier_digest: Some(digest), - })), - PhysicalPlan::Crdt(CrdtOp::ApplyAuthenticated { - collection, - document_id, - delta, - peer_id, - mutation_id, - surrogate, - provenance, - constraint_version_required, - expected_frontier_digest: None, - auth_user_id, - auth_device_id, - auth_seq_no, - delta_signature, - signing_required, - }) => Ok(PhysicalPlan::Crdt(CrdtOp::ApplyAuthenticated { - collection, - document_id, - delta, - peer_id, - mutation_id, - surrogate, - provenance, - constraint_version_required, - expected_frontier_digest: Some(digest), - auth_user_id, - auth_device_id, - auth_seq_no, - delta_signature, - signing_required, - })), - _ => Err(crate::Error::CrdtAdmissionInvalidPlan { - reason: "validated CRDT Apply plan was changed before fence stamping", - }), - } -} - -/// Payload-only wrapper for an externally authorized caller. -pub async fn dispatch_authorized_crdt_apply_admitted( - state: &SharedState, - request: AuthorizedCrdtApplyAdmissionRequest<'_>, -) -> crate::Result> { - Ok( - dispatch_authorized_crdt_apply_admitted_outcome(state, request) - .await? - .payload, - ) -} - -/// Payload-only compatibility wrapper for trusted internal test callers. -#[cfg(test)] -pub(crate) async fn dispatch_crdt_apply_admitted( - state: &SharedState, - request: CrdtApplyAdmissionRequest<'_>, -) -> crate::Result> { - Ok(dispatch_crdt_apply_admitted_outcome(state, request) - .await? - .payload) -} - -/// Generate a restore delta and admit it without releasing the vShard slot. -/// A stale apply fence regenerates the delta from authoritative state before -/// retrying, so neither historical projection nor policy output is reused. -pub(crate) async fn dispatch_crdt_restore_admitted( - state: &SharedState, - request: CrdtRestoreAdmissionRequest<'_>, -) -> crate::Result> { - let CrdtRestoreAdmissionRequest { - tenant_id, - database_id, - collection, - document_id, - target_version_json, - surrogate, - peer_id, - timeout, - event_source, - policy, - } = request; - // The restore writes a delta like any apply; the lease gates it on a - // drained collection and lives until its outcome. - let _lease = crate::control::server::shared::clone_write::collections_write_lease( - state, - tenant_id, - database_id, - [collection.to_owned()], - ) - .await?; - // `collection` is the plan's database-qualified name. - let vshard_id = - nodedb_types::CollectionKey::from_qualified_str(database_id, collection)?.vshard(); - let workflow = CrdtAdmissionWorkflow { - state, - tenant_id, - database_id, - vshard_id, - collection, - timeout, - event_source, - policy, - }; - tokio::time::timeout( - timeout, - state.vshard_admission_sequencer.run_after_proposed( - vshard_id, - |last_proposed| async move { - caught_up(&workflow, last_proposed).await?; - for _attempt in 0..FRONTIER_RETRY_LIMIT { - let delta = generate_restore_delta( - &workflow, - document_id, - target_version_json, - surrogate, - ) - .await?; - if delta.is_empty() { - return Ok(None); - } - let plan = PhysicalPlan::Crdt(CrdtOp::Apply { - collection: nodedb_types::QualifiedCollection::from_stored( - collection.to_owned(), - ), - document_id: document_id.to_owned(), - delta: delta.clone(), - peer_id, - mutation_id: 0, - surrogate, - provenance: None, - constraint_version_required: 0, - expected_frontier_digest: None, - }); - let preview = preview(&workflow, document_id, &delta).await?; - workflow.policy.evaluate(&preview)?; - match apply_fenced(&workflow, stamp_fence(plan, preview.frontier_digest)?).await - { - Err(crate::Error::DataPlane(ErrorCode::CrdtFrontierMismatch { - .. - })) => {} - result => return result.map(Some), - } - } - Err(crate::Error::CrdtAdmissionRetriesExhausted { - vshard_id, - attempts: FRONTIER_RETRY_LIMIT, - }) - }, - ), - ) - .await - .map_err(|_| crate::Error::CrdtAdmissionTimeout { - vshard_id, - timeout_ms: timeout_ms(timeout), - })? -} - -async fn generate_restore_delta( - workflow: &CrdtAdmissionWorkflow<'_>, - document_id: &str, - target_version_json: &str, - surrogate: nodedb_types::Surrogate, -) -> crate::Result> { - let response = - crate::control::server::shared::ddl::sync_dispatch::dispatch_system_response_with_source( - workflow.state, - crate::control::server::shared::ddl::sync_dispatch::SystemTask::new( - crate::control::server::shared::ddl::sync_dispatch::SystemReason::AdmittedContinuation, - workflow.tenant_id, - nodedb_types::CollectionKey::from_qualified_str(workflow.database_id, workflow.collection)?, - PhysicalPlan::Crdt(CrdtOp::RestoreToVersion { - collection: nodedb_types::QualifiedCollection::from_stored( - workflow.collection.to_owned(), - ), - document_id: document_id.to_owned(), - target_version_json: target_version_json.to_owned(), - surrogate, - }), - ), - workflow.timeout, - workflow.event_source, - ) - .await?; - if response.status != Status::Ok { - return Err(response_error(&response)); - } - Ok(response.payload.to_vec()) -} - -async fn apply_fenced( - workflow: &CrdtAdmissionWorkflow<'_>, - plan: PhysicalPlan, -) -> crate::Result { - let raw = workflow.state.raw_async_raft_proposer()?; - let entry = to_replicated_entry( - workflow.tenant_id, - workflow.database_id, - workflow.vshard_id, - &ReplicableWrite::decide_for_replication(&plan)?, - )? - .ok_or(crate::Error::CrdtAdmissionInvalidPlan { - reason: "admitted CRDT Apply has no replicated form", - })? - // Every replica gives the write the source this node dispatches it - // with. - .with_event_source(workflow.event_source); - let outcome = tokio::time::timeout( - workflow.timeout, - crate::control::wal_replication::propose_replicated_entry( - workflow.state, - raw, - entry, - tokio::time::Instant::now() + workflow.timeout, - ), - ) - .await - .map_err(|_| crate::Error::CrdtAdmissionTimeout { - vshard_id: workflow.vshard_id, - timeout_ms: timeout_ms(workflow.timeout), - })??; - // This node's apply of the entry recorded its commit HLC. - Ok(CrdtAdmissionOutcome { - payload: outcome.0, - write_versions: outcome.1, - trimmed_ops: 0, - }) -} - -fn timeout_ms(timeout: Duration) -> u64 { - u64::try_from(timeout.as_millis()).unwrap_or(u64::MAX) -} - -fn response_error(response: &crate::bridge::envelope::Response) -> crate::Error { - match response.error_code.as_deref() { - Some(code) => crate::Error::DataPlane(code.clone()), - None => crate::Error::Internal { - detail: String::from_utf8_lossy(&response.payload).into_owned(), - }, - } -} - -#[cfg(test)] -mod tests { - use std::sync::atomic::{AtomicUsize, Ordering}; - use std::sync::{Arc, Mutex}; - use std::time::Instant; - - use nodedb_types::{CrdtPreviewResult, Surrogate}; - use tokio::time::Duration; - - use super::*; - use crate::bridge::dispatch::{BridgeResponse, CoreChannelDataSide, Dispatcher}; - use crate::bridge::envelope::{Response, Status}; - use crate::control::state::SharedState; - use crate::types::{Lsn, RequestId}; - use crate::wal::WalManager; - use nodedb_types::QualifiedCollection; - - fn collection() -> String { - "docs".to_owned() - } - - fn admission_request<'a>(policy: &'a dyn CrdtPostImagePolicy) -> CrdtApplyAdmissionRequest<'a> { - CrdtApplyAdmissionRequest { - tenant_id: TenantId::new(1), - database_id: DatabaseId::DEFAULT, - collection: "docs", - plan: apply_plan(), - timeout: Duration::from_secs(1), - event_source: EventSource::User, - policy, - } - } - - fn apply_plan() -> PhysicalPlan { - PhysicalPlan::Crdt(CrdtOp::Apply { - collection: QualifiedCollection::new(DatabaseId::DEFAULT, &collection()), - document_id: "doc-1".into(), - delta: vec![0x91, 0x01], - peer_id: 7, - mutation_id: 9, - surrogate: Surrogate::new(1), - provenance: None, - constraint_version_required: 0, - expected_frontier_digest: None, - }) - } - - fn fixture() -> (Arc, CoreChannelDataSide, tempfile::TempDir) { - let directory = tempfile::tempdir().expect("temporary WAL directory"); - let wal = Arc::new( - WalManager::open_for_testing(&directory.path().join("admission.wal")) - .expect("test WAL"), - ); - let (dispatcher, mut sides) = Dispatcher::new(1, 64); - let side = sides.pop().expect("one data side"); - let state = SharedState::new(dispatcher, wal).expect("shared state"); - (state, side, directory) - } - - fn response(request_id: RequestId, payload: Vec) -> Response { - Response { - request_id, - status: Status::Ok, - attempt: 1, - partial: false, - payload: payload.into(), - watermark_lsn: Lsn::ZERO, - error_code: None, - stage_vote: None, - read_versions: crate::types::ReadVersions::new(), - write_set: Vec::new(), - } - } - - async fn respond_n( - state: Arc, - mut side: CoreChannelDataSide, - n: usize, - mut f: F, - ) where - F: FnMut(crate::bridge::envelope::Request) -> Response + Send + 'static, - { - let deadline = Instant::now() + Duration::from_secs(2); - let mut handled = 0; - while handled < n && Instant::now() < deadline { - if let Ok(request) = side.request_rx.try_pop() { - let response = f(request.inner); - side.response_tx - .try_push(BridgeResponse { inner: response }) - .expect("fake data-plane response queue has capacity"); - handled += 1; - } - state.poll_and_route_responses(); - tokio::task::yield_now().await; - } - assert_eq!(handled, n, "fake data-plane received expected requests"); - state.poll_and_route_responses(); - } - - struct RecordingPolicy { - seen: Arc>>>, - reject: bool, - } - - impl CrdtPostImagePolicy for RecordingPolicy { - fn evaluate(&self, preview: &CrdtPreviewResult) -> crate::Result<()> { - self.seen - .lock() - .expect("test policy lock") - .push(preview.post_image_msgpack.clone()); - if self.reject { - Err(crate::Error::CrdtAdmissionCallerFence) - } else { - Ok(()) - } - } - } - - #[tokio::test] - async fn admitted_apply_previews_exact_post_image_before_fenced_apply() { - let (state, side, _directory) = fixture(); - let digest = [0x4d; 32]; - let post_image = vec![0x81, 0xa2, b'o', b'k', 0xc3]; - let preview_payload = zerompk::to_msgpack_vec(&CrdtPreviewResult { - post_image_msgpack: post_image.clone(), - imported_ops: 1, - trimmed_ops: 0, - frontier_digest: digest, - }) - .expect("preview payload"); - let preview_fields = Arc::new(Mutex::new(Vec::new())); - let fields = Arc::clone(&preview_fields); - let responder = - tokio::spawn(respond_n( - Arc::clone(&state), - side, - 1, - move |request| match request.plan { - PhysicalPlan::Crdt(CrdtOp::PreviewApply { - collection, - document_id, - delta, - }) => { - fields - .lock() - .expect("fields lock") - .push((collection, document_id, delta)); - response(request.request_id, preview_payload.clone()) - } - other => panic!("unexpected request: {other:?}"), - }, - )); - let apply_fences = Arc::new(Mutex::new(Vec::new())); - let fences = Arc::clone(&apply_fences); - let raw: Arc = - Arc::new(move |_shard, _key, bytes, _deadline| { - let fences = Arc::clone(&fences); - Box::pin(async move { - let entry = - crate::control::wal_replication::ReplicatedEntry::from_bytes(&bytes) - .expect("replicated entry"); - if let crate::control::wal_replication::ReplicatedWrite::CrdtApplyFenced { - expected_frontier_digest, - .. - } = entry.write - { - fences - .lock() - .expect("fences lock") - .push(expected_frontier_digest); - } - Ok((Vec::new(), crate::types::ReadVersions::new())) - }) - }); - crate::control::vshard_admission::install_async_raft_proposer( - &state, - crate::control::vshard_admission::applying_submit(raw), - ) - .expect("install raw/sequenced proposer pair"); - let seen = Arc::new(Mutex::new(Vec::new())); - let policy = RecordingPolicy { - seen: Arc::clone(&seen), - reject: false, - }; - dispatch_crdt_apply_admitted(&state, admission_request(&policy)) - .await - .expect("admitted apply"); - responder.await.expect("responder completes"); - assert_eq!(*seen.lock().expect("seen lock"), vec![post_image]); - assert_eq!(preview_fields.lock().expect("fields lock").len(), 1); - assert_eq!(*apply_fences.lock().expect("fences lock"), vec![digest]); - } - - #[tokio::test] - async fn denied_policy_dispatches_only_preview_and_does_not_advance_hlc() { - let (state, side, _directory) = fixture(); - let preview_payload = zerompk::to_msgpack_vec(&CrdtPreviewResult { - post_image_msgpack: vec![0xc0], - imported_ops: 0, - trimmed_ops: 0, - frontier_digest: [7; 32], - }) - .expect("preview payload"); - let responder = tokio::spawn(respond_n(Arc::clone(&state), side, 1, move |request| { - assert!(matches!( - request.plan, - PhysicalPlan::Crdt(CrdtOp::PreviewApply { .. }) - )); - response(request.request_id, preview_payload.clone()) - })); - let seen = Arc::new(Mutex::new(Vec::new())); - let policy = RecordingPolicy { seen, reject: true }; - let before_hlc = state.tenant_write_mark(1); - let result = dispatch_crdt_apply_admitted(&state, admission_request(&policy)).await; - responder.await.expect("responder completes"); - assert!(matches!( - result, - Err(crate::Error::CrdtAdmissionCallerFence) - )); - assert_eq!( - state.tenant_write_mark(1), - before_hlc, - "policy rejection must not advance the tenant write HLC" - ); - state.wal.sync().expect("sync wal"); - assert!( - state.wal.replay().expect("replay wal").is_empty(), - "policy rejection must not append a durable CRDT delta" - ); - } - - #[tokio::test] - async fn admitted_cluster_apply_uses_raw_proposer_with_fenced_entry() { - let (state, side, _directory) = fixture(); - let digest = [0xa5; 32]; - let preview_payload = zerompk::to_msgpack_vec(&CrdtPreviewResult { - post_image_msgpack: vec![0xc0], - imported_ops: 1, - trimmed_ops: 0, - frontier_digest: digest, - }) - .expect("preview payload"); - let responder = tokio::spawn(respond_n(Arc::clone(&state), side, 1, move |request| { - assert!(matches!( - request.plan, - PhysicalPlan::Crdt(CrdtOp::PreviewApply { .. }) - )); - response(request.request_id, preview_payload.clone()) - })); - let fenced = Arc::new(Mutex::new(Vec::new())); - let observed = Arc::clone(&fenced); - let raw: Arc = - Arc::new(move |_shard, _key, bytes, _deadline| { - let observed = Arc::clone(&observed); - Box::pin(async move { - let entry = - crate::control::wal_replication::ReplicatedEntry::from_bytes(&bytes) - .expect("replicated entry"); - match entry.write { - crate::control::wal_replication::ReplicatedWrite::CrdtApplyFenced { - expected_frontier_digest, - .. - } => { - observed - .lock() - .expect("fence lock") - .push(expected_frontier_digest); - } - other => panic!("unexpected replicated write: {other:?}"), - } - Ok((Vec::new(), crate::types::ReadVersions::new())) - }) - }); - crate::control::vshard_admission::install_async_raft_proposer( - &state, - crate::control::vshard_admission::applying_submit(raw), - ) - .expect("install raw/sequenced proposer pair"); - let seen = Arc::new(Mutex::new(Vec::new())); - let policy = RecordingPolicy { - seen, - reject: false, - }; - dispatch_crdt_apply_admitted(&state, admission_request(&policy)) - .await - .expect("cluster admitted apply"); - responder.await.expect("responder completes"); - assert_eq!(*fenced.lock().expect("fence lock"), vec![digest]); - } - - #[tokio::test] - async fn stale_frontier_retries_preview_and_policy_before_success() { - let (state, side, _directory) = fixture(); - let digest = [0x82; 32]; - let preview_payload = zerompk::to_msgpack_vec(&CrdtPreviewResult { - post_image_msgpack: vec![0xc0], - imported_ops: 0, - trimmed_ops: 0, - frontier_digest: digest, - }) - .expect("preview payload"); - let responder = tokio::spawn(respond_n(Arc::clone(&state), side, 2, move |request| { - assert!(matches!( - request.plan, - PhysicalPlan::Crdt(CrdtOp::PreviewApply { .. }) - )); - response(request.request_id, preview_payload.clone()) - })); - let fences = Arc::new(AtomicUsize::new(0)); - let count = Arc::clone(&fences); - let raw: Arc = - Arc::new(move |_shard, _key, _bytes, _deadline| { - let count = Arc::clone(&count); - Box::pin(async move { - if count.fetch_add(1, Ordering::SeqCst) == 0 { - Err(crate::Error::DataPlane(ErrorCode::CrdtFrontierMismatch { - expected: digest, - actual: [0; 32], - })) - } else { - Ok((Vec::new(), crate::types::ReadVersions::new())) - } - }) - }); - crate::control::vshard_admission::install_async_raft_proposer( - &state, - crate::control::vshard_admission::applying_submit(raw), - ) - .expect("install raw/sequenced proposer pair"); - let seen = Arc::new(Mutex::new(Vec::new())); - let policy = RecordingPolicy { - seen: Arc::clone(&seen), - reject: false, - }; - dispatch_crdt_apply_admitted(&state, admission_request(&policy)) - .await - .expect("second fence succeeds"); - responder.await.expect("responder completes"); - assert_eq!(fences.load(Ordering::SeqCst), 2); - assert_eq!(seen.lock().expect("policy lock").len(), 2); - } - - #[tokio::test] - async fn stale_restore_regenerates_before_retrying_admission() { - let (state, side, _directory) = fixture(); - let delta = vec![0x91, 0x02]; - let preview_payload = zerompk::to_msgpack_vec(&CrdtPreviewResult { - post_image_msgpack: vec![0xc0], - imported_ops: 1, - trimmed_ops: 0, - frontier_digest: [0x84; 32], - }) - .expect("preview payload"); - let request_index = Arc::new(AtomicUsize::new(0)); - let index = Arc::clone(&request_index); - let responder = tokio::spawn(respond_n(Arc::clone(&state), side, 4, move |request| { - let is_restore = matches!( - &request.plan, - PhysicalPlan::Crdt(CrdtOp::RestoreToVersion { .. }) - ); - match index.fetch_add(1, Ordering::SeqCst) { - 0 | 2 => assert!(is_restore), - 1 | 3 => assert!(matches!( - &request.plan, - PhysicalPlan::Crdt(CrdtOp::PreviewApply { .. }) - )), - _ => unreachable!("responder only handles four requests"), - } - let payload = if is_restore { - delta.clone() - } else { - preview_payload.clone() - }; - response(request.request_id, payload) - })); - let attempts = Arc::new(AtomicUsize::new(0)); - let raw: Arc = { - let attempts = Arc::clone(&attempts); - Arc::new(move |_shard, _key, _bytes, _deadline| { - let attempts = Arc::clone(&attempts); - Box::pin(async move { - if attempts.fetch_add(1, Ordering::SeqCst) == 0 { - Err(crate::Error::DataPlane(ErrorCode::CrdtFrontierMismatch { - expected: [0x84; 32], - actual: [0; 32], - })) - } else { - Ok((Vec::new(), crate::types::ReadVersions::new())) - } - }) - }) - }; - crate::control::vshard_admission::install_async_raft_proposer( - &state, - crate::control::vshard_admission::applying_submit(raw), - ) - .expect("install proposer"); - let policy = RecordingPolicy { - seen: Arc::new(Mutex::new(Vec::new())), - reject: false, - }; - dispatch_crdt_restore_admitted( - &state, - CrdtRestoreAdmissionRequest { - tenant_id: TenantId::new(1), - database_id: DatabaseId::DEFAULT, - collection: "docs", - document_id: "doc-1", - target_version_json: "{}", - surrogate: Surrogate::new(1), - peer_id: 1, - timeout: Duration::from_secs(1), - event_source: EventSource::User, - policy: &policy, - }, - ) - .await - .expect("second restore admission succeeds"); - responder.await.expect("responder completes"); - assert_eq!(request_index.load(Ordering::SeqCst), 4); - assert_eq!(attempts.load(Ordering::SeqCst), 2); - } - - #[tokio::test] - async fn stale_frontier_retries_preview_and_policy_then_exhausts_typed_error() { - let (state, side, _directory) = fixture(); - let digest = [0x81; 32]; - let preview_payload = zerompk::to_msgpack_vec(&CrdtPreviewResult { - post_image_msgpack: vec![0xc0], - imported_ops: 0, - trimmed_ops: 0, - frontier_digest: digest, - }) - .expect("preview payload"); - let responder = tokio::spawn(respond_n( - Arc::clone(&state), - side, - FRONTIER_RETRY_LIMIT, - move |request| { - assert!(matches!( - request.plan, - PhysicalPlan::Crdt(CrdtOp::PreviewApply { .. }) - )); - response(request.request_id, preview_payload.clone()) - }, - )); - let fenced_count = Arc::new(AtomicUsize::new(0)); - let count = Arc::clone(&fenced_count); - let raw: Arc = - Arc::new(move |_shard, _key, _bytes, _deadline| { - let count = Arc::clone(&count); - Box::pin(async move { - count.fetch_add(1, Ordering::SeqCst); - Err(crate::Error::DataPlane(ErrorCode::CrdtFrontierMismatch { - expected: digest, - actual: [0; 32], - })) - }) - }); - crate::control::vshard_admission::install_async_raft_proposer( - &state, - crate::control::vshard_admission::applying_submit(raw), - ) - .expect("install raw/sequenced proposer pair"); - let seen = Arc::new(Mutex::new(Vec::new())); - let policy = RecordingPolicy { - seen: Arc::clone(&seen), - reject: false, - }; - let result = dispatch_crdt_apply_admitted(&state, admission_request(&policy)).await; - responder.await.expect("responder completes"); - assert!(matches!( - result, - Err(crate::Error::CrdtAdmissionRetriesExhausted { - attempts: FRONTIER_RETRY_LIMIT, - .. - }) - )); - assert_eq!(fenced_count.load(Ordering::SeqCst), FRONTIER_RETRY_LIMIT); - assert_eq!( - seen.lock().expect("policy lock").len(), - FRONTIER_RETRY_LIMIT - ); - } - - #[tokio::test] - async fn signed_delta_collection_rejects_plain_external_apply_before_preview() { - let (state, _side, _directory) = fixture(); - let tenant_id = TenantId::new(1); - let mut collection = crate::control::security::catalog::StoredCollection::stamped_for_test( - 1, "docs", "owner", - ); - collection.crdt = true; - collection.crdt_signing_required = true; - state - .credentials - .catalog() - .put_collection(DatabaseId::DEFAULT, &collection) - .expect("store signed-delta collection"); - let task = nodedb_physical::physical_task::PhysicalTask { - tenant_id, - database_id: DatabaseId::DEFAULT, - vshard_id: nodedb_types::CollectionKey::from_bare(DatabaseId::DEFAULT, "docs").vshard(), - plan: apply_plan(), - post_set_op: nodedb_physical::physical_task::PostSetOp::None, - txn_id: None, - }; - let identity = - crate::control::security::identity::AuthenticatedIdentity::new_internal_service( - 1, - "crdt-test", - tenant_id, - Vec::new(), - true, - None, - crate::control::security::identity::AuthenticatedIdentity::default_database_set( - true, - ), - ); - let authorized = crate::control::server::shared::authorization::authorize_task_set( - &identity, - std::slice::from_ref(&task), - &state.permissions, - &state.roles, - &crate::control::security::audit::NoopAuditEmitter, - ) - .expect("authorize test task") - .into_tasks() - .into_iter() - .next() - .expect("one authorized task"); - let result = dispatch_authorized_crdt_apply_admitted( - &state, - AuthorizedCrdtApplyAdmissionRequest { - authorized, - collection: "docs", - timeout: Duration::from_millis(10), - event_source: EventSource::User, - policy: &TrustedInternalCrdtPolicy, - }, - ) - .await; - assert!(matches!(result, Err(crate::Error::RejectedAuthz { .. }))); - } - - #[tokio::test] - async fn generic_replicated_dispatch_rejects_unadmitted_apply() { - let (state, _side, _directory) = fixture(); - let tenant_id = TenantId::new(1); - let task = nodedb_physical::physical_task::PhysicalTask { - tenant_id, - database_id: DatabaseId::DEFAULT, - vshard_id: nodedb_types::CollectionKey::from_bare(DatabaseId::DEFAULT, "docs").vshard(), - plan: apply_plan(), - post_set_op: nodedb_physical::physical_task::PostSetOp::None, - txn_id: None, - }; - let identity = - crate::control::security::identity::AuthenticatedIdentity::new_internal_service( - 1, - "crdt-test", - tenant_id, - Vec::new(), - true, - None, - crate::control::security::identity::AuthenticatedIdentity::default_database_set( - true, - ), - ); - let authorized = crate::control::server::shared::authorization::authorize_task_set( - &identity, - std::slice::from_ref(&task), - &state.permissions, - &state.roles, - &crate::control::security::audit::NoopAuditEmitter, - ) - .expect("authorize test task") - .into_tasks() - .into_iter() - .next() - .expect("one authorized task"); - let result = crate::control::server::sync::raft_dispatch::dispatch_write_replicated( - &state, - "docs", - authorized, - EventSource::User, - ) - .await; - assert!(matches!( - result, - Err(crate::Error::CrdtApplyRequiresAdmission) - )); - } - - fn assert_frontier_mutation(op: CrdtOp) { - assert!(changes_crdt_frontier(&op), "{op:?} must serialize"); - } - - fn assert_frontier_read(op: CrdtOp) { - assert!(!changes_crdt_frontier(&op), "{op:?} must not serialize"); - } - - #[test] - fn frontier_classifier_covers_every_crdt_operation_category() { - let surrogate = Surrogate::new(1); - assert_frontier_mutation(CrdtOp::Apply { - collection: QualifiedCollection::new(DatabaseId::DEFAULT, &collection()), - document_id: "id".into(), - delta: Vec::new(), - peer_id: 1, - mutation_id: 1, - surrogate, - provenance: None, - constraint_version_required: 0, - expected_frontier_digest: None, - }); - assert_frontier_mutation(CrdtOp::ImportSnapshot { - tenant_id: 1, - collection: QualifiedCollection::new(DatabaseId::DEFAULT, &collection()), - bytes: Vec::new(), - }); - assert_frontier_mutation(CrdtOp::RestoreToVersion { - collection: QualifiedCollection::new(DatabaseId::DEFAULT, &collection()), - document_id: "id".into(), - target_version_json: "{}".into(), - surrogate, - }); - assert_frontier_mutation(CrdtOp::ListInsert { - collection: QualifiedCollection::new(DatabaseId::DEFAULT, &collection()), - document_id: "id".into(), - list_path: "blocks".into(), - index: 0, - fields_json: "{}".into(), - surrogate, - }); - assert_frontier_mutation(CrdtOp::ListDelete { - collection: QualifiedCollection::new(DatabaseId::DEFAULT, &collection()), - document_id: "id".into(), - list_path: "blocks".into(), - index: 0, - surrogate, - }); - assert_frontier_mutation(CrdtOp::ListMove { - collection: QualifiedCollection::new(DatabaseId::DEFAULT, &collection()), - document_id: "id".into(), - list_path: "blocks".into(), - from_index: 0, - to_index: 1, - surrogate, - }); - assert_frontier_mutation(CrdtOp::DocUpsert { - collection: QualifiedCollection::new(DatabaseId::DEFAULT, &collection()), - document_id: "id".into(), - fields_json: "{}".into(), - surrogate, - partial: false, - verb: nodedb_physical::physical_plan::CrdtWriteVerb::Insert, - returning: None, - rls_filters: Vec::new(), - }); - assert_frontier_mutation(CrdtOp::DocDelete { - collection: QualifiedCollection::new(DatabaseId::DEFAULT, &collection()), - document_id: "id".into(), - surrogate: Some(surrogate), - returning: None, - rls_filters: Vec::new(), - }); - - assert_frontier_read(CrdtOp::Read { - collection: QualifiedCollection::new(DatabaseId::DEFAULT, &collection()), - document_id: "id".into(), - }); - assert_frontier_read(CrdtOp::PreviewApply { - collection: QualifiedCollection::new(DatabaseId::DEFAULT, &collection()), - document_id: "id".into(), - delta: Vec::new(), - }); - assert_frontier_read(CrdtOp::GetVersionVector { - collection: QualifiedCollection::new(DatabaseId::DEFAULT, &collection()), - }); - assert_frontier_read(CrdtOp::ExportDelta { - collection: QualifiedCollection::new(DatabaseId::DEFAULT, &collection()), - from_version_json: "{}".into(), - }); - assert_frontier_read(CrdtOp::CompactAtVersion { - collection: QualifiedCollection::new(DatabaseId::DEFAULT, &collection()), - target_version_json: "{}".into(), - }); - assert_frontier_read(CrdtOp::SetConstraints { - collection: QualifiedCollection::new(DatabaseId::DEFAULT, &collection()), - constraint_version: 1, - constraints: Vec::new(), - }); - assert_frontier_read(CrdtOp::DropConstraints { - collection: QualifiedCollection::new(DatabaseId::DEFAULT, &collection()), - constraint_version: 1, - }); - assert_frontier_read(CrdtOp::ReadConstraints { - collection: QualifiedCollection::new(DatabaseId::DEFAULT, &collection()), - }); - assert_frontier_read(CrdtOp::SetPolicy { - collection: QualifiedCollection::new(DatabaseId::DEFAULT, &collection()), - policy_json: "{}".into(), - }); - assert_frontier_read(CrdtOp::GetPolicy { - collection: QualifiedCollection::new(DatabaseId::DEFAULT, &collection()), - }); - assert_frontier_read(CrdtOp::ReadAtVersion { - collection: QualifiedCollection::new(DatabaseId::DEFAULT, &collection()), - document_id: "id".into(), - version_vector_json: "{}".into(), - }); - } -} diff --git a/nodedb/src/control/crdt_admission/apply.rs b/nodedb/src/control/crdt_admission/apply.rs new file mode 100644 index 000000000..dc4921d74 --- /dev/null +++ b/nodedb/src/control/crdt_admission/apply.rs @@ -0,0 +1,680 @@ +// SPDX-License-Identifier: BUSL-1.1 + +//! Preview, authorize, fence, and durably apply one CRDT delta. + +use nodedb_physical::physical_plan::CrdtOp; + +use crate::bridge::envelope::{ErrorCode, PhysicalPlan}; +use crate::control::server::shared::authorization::AuthorizedTask; +use crate::control::state::SharedState; + +use super::fence::{apply_fenced, stamp_fence}; +use super::legs::preview; +use super::request::{ + AuthorizedCrdtApplyAdmissionRequest, CrdtAdmissionOutcome, CrdtApplyAdmissionRequest, +}; +use super::workflow::{CrdtAdmissionWorkflow, FRONTIER_RETRY_LIMIT}; + +/// Preview, authorize, fence, and durably apply one externally authorized delta. +pub async fn dispatch_authorized_crdt_apply_admitted_outcome( + state: &SharedState, + request: AuthorizedCrdtApplyAdmissionRequest<'_>, +) -> crate::Result { + let AuthorizedCrdtApplyAdmissionRequest { + authorized, + collection, + deadline, + event_source, + policy, + } = request; + enforce_external_signing_policy(state, &authorized, collection)?; + let task = authorized.into_physical_task(); + dispatch_crdt_apply_admitted_outcome( + state, + CrdtApplyAdmissionRequest { + tenant_id: task.tenant_id, + database_id: task.database_id, + collection, + plan: task.plan, + deadline, + event_source, + policy, + }, + ) + .await +} + +/// `collection` is db-qualified. The catalog keys collections by the bare +/// name, so the lookup de-qualifies it first. +fn enforce_external_signing_policy( + state: &SharedState, + authorized: &AuthorizedTask, + collection: &str, +) -> crate::Result<()> { + let bare = crate::control::target_identity::naming::bare_collection_name( + authorized.database_id(), + collection, + ); + let stored = state + .credentials + .catalog() + .get_collection( + authorized.database_id(), + authorized.tenant_id().as_u64(), + &bare, + )? + .ok_or_else(|| crate::Error::CollectionNotFound { + tenant_id: authorized.tenant_id(), + collection: bare.clone(), + })?; + if stored.crdt_signing_required + && matches!(authorized.plan(), PhysicalPlan::Crdt(CrdtOp::Apply { .. })) + { + return Err(crate::Error::RejectedAuthz { + tenant_id: authorized.tenant_id(), + resource: format!("collection:{bare}:unsigned_crdt_delta_requires_authenticated_sync"), + }); + } + Ok(()) +} + +/// Preview, authorize, fence, and durably apply one trusted-internal CRDT delta. +pub(crate) async fn dispatch_crdt_apply_admitted_outcome( + state: &SharedState, + request: CrdtApplyAdmissionRequest<'_>, +) -> crate::Result { + let CrdtApplyAdmissionRequest { + tenant_id, + database_id, + collection, + plan, + deadline, + event_source, + policy, + } = request; + // Every CRDT apply converges here; the lease gates it on a drained + // collection and lives until the apply's outcome. + let _lease = crate::control::server::shared::clone_write::write_lease( + state, + tenant_id, + database_id, + &plan, + ) + .await?; + let (document_id, delta) = match &plan { + PhysicalPlan::Crdt( + CrdtOp::Apply { + collection: plan_collection, + document_id, + delta, + expected_frontier_digest: None, + .. + } + | CrdtOp::ApplyAuthenticated { + collection: plan_collection, + document_id, + delta, + expected_frontier_digest: None, + .. + }, + ) if plan_collection.as_str() == collection => (document_id.clone(), delta.clone()), + PhysicalPlan::Crdt( + CrdtOp::Apply { + expected_frontier_digest: Some(_), + .. + } + | CrdtOp::ApplyAuthenticated { + expected_frontier_digest: Some(_), + .. + }, + ) => return Err(crate::Error::CrdtAdmissionCallerFence), + _ => { + return Err(crate::Error::CrdtAdmissionInvalidPlan { + reason: "expected an unfenced CRDT Apply for the supplied collection", + }); + } + }; + let workflow = CrdtAdmissionWorkflow::new( + state, + tenant_id, + database_id, + collection, + deadline, + event_source, + policy, + )?; + workflow + .run(|| admit_apply_locked(&workflow, plan, &document_id, &delta)) + .await +} + +async fn admit_apply_locked( + workflow: &CrdtAdmissionWorkflow<'_>, + plan: PhysicalPlan, + document_id: &str, + delta: &[u8], +) -> crate::Result { + for _attempt in 0..FRONTIER_RETRY_LIMIT { + let preview = preview(workflow, document_id, delta).await?; + workflow.policy.evaluate(&preview)?; + let fenced = stamp_fence(plan.clone(), preview.frontier_digest)?; + match apply_fenced(workflow, fenced).await { + Err(crate::Error::DataPlane(ErrorCode::CrdtFrontierMismatch { .. })) => {} + // The trim count belongs to the preview that fenced *this* attempt. + // A retry re-previews against the advanced frontier and produces its + // own count, so the two are never mixed. + result => { + return result.map(|outcome| outcome.with_trimmed_ops(preview.trimmed_ops)); + } + } + } + Err(crate::Error::CrdtAdmissionRetriesExhausted { + vshard_id: workflow.vshard_id, + attempts: FRONTIER_RETRY_LIMIT, + }) +} + +/// Payload-only wrapper for an externally authorized caller. +pub async fn dispatch_authorized_crdt_apply_admitted( + state: &SharedState, + request: AuthorizedCrdtApplyAdmissionRequest<'_>, +) -> crate::Result> { + Ok( + dispatch_authorized_crdt_apply_admitted_outcome(state, request) + .await? + .payload, + ) +} + +#[cfg(test)] +mod tests { + use std::sync::atomic::{AtomicUsize, Ordering}; + use std::sync::{Arc, Mutex}; + use std::time::Duration; + + use nodedb_types::CrdtPreviewResult; + + use super::*; + use crate::control::crdt_admission::policy::TrustedInternalCrdtPolicy; + use crate::control::crdt_admission::test_support::{ + RecordingPolicy, admission_request, admission_request_until, apply_plan, + dispatch_crdt_apply_admitted, fixture, preview_payload, respond_n, response, + }; + use crate::control::server::shared::session::StatementDeadline; + use crate::event::EventSource; + use crate::types::{DatabaseId, TenantId}; + + #[tokio::test] + async fn admitted_apply_previews_exact_post_image_before_fenced_apply() { + let (state, side, _directory) = fixture(); + let digest = [0x4d; 32]; + let post_image = vec![0x81, 0xa2, b'o', b'k', 0xc3]; + let preview_payload = zerompk::to_msgpack_vec(&CrdtPreviewResult { + post_image_msgpack: post_image.clone(), + imported_ops: 1, + trimmed_ops: 0, + frontier_digest: digest, + }) + .expect("preview payload"); + let preview_fields = Arc::new(Mutex::new(Vec::new())); + let fields = Arc::clone(&preview_fields); + let responder = + tokio::spawn(respond_n( + Arc::clone(&state), + side, + 1, + move |request| match request.plan { + PhysicalPlan::Crdt(CrdtOp::PreviewApply { + collection, + document_id, + delta, + }) => { + fields + .lock() + .expect("fields lock") + .push((collection, document_id, delta)); + response(request.request_id, preview_payload.clone()) + } + other => panic!("unexpected request: {other:?}"), + }, + )); + let apply_fences = Arc::new(Mutex::new(Vec::new())); + let fences = Arc::clone(&apply_fences); + let raw: Arc = + Arc::new(move |_shard, _key, bytes, _deadline| { + let fences = Arc::clone(&fences); + Box::pin(async move { + let entry = + crate::control::wal_replication::ReplicatedEntry::from_bytes(&bytes) + .expect("replicated entry"); + if let crate::control::wal_replication::ReplicatedWrite::CrdtApplyFenced { + expected_frontier_digest, + .. + } = entry.write + { + fences + .lock() + .expect("fences lock") + .push(expected_frontier_digest); + } + Ok((Vec::new(), crate::types::ReadVersions::new())) + }) + }); + crate::control::vshard_admission::install_async_raft_proposer( + &state, + crate::control::vshard_admission::applying_submit(raw), + ) + .expect("install raw/sequenced proposer pair"); + let seen = Arc::new(Mutex::new(Vec::new())); + let policy = RecordingPolicy { + seen: Arc::clone(&seen), + reject: false, + }; + dispatch_crdt_apply_admitted(&state, admission_request(&policy)) + .await + .expect("admitted apply"); + responder.await.expect("responder completes"); + assert_eq!(*seen.lock().expect("seen lock"), vec![post_image]); + assert_eq!(preview_fields.lock().expect("fields lock").len(), 1); + assert_eq!(*apply_fences.lock().expect("fences lock"), vec![digest]); + } + + #[tokio::test] + async fn denied_policy_dispatches_only_preview_and_does_not_advance_hlc() { + let (state, side, _directory) = fixture(); + let preview_payload = zerompk::to_msgpack_vec(&CrdtPreviewResult { + post_image_msgpack: vec![0xc0], + imported_ops: 0, + trimmed_ops: 0, + frontier_digest: [7; 32], + }) + .expect("preview payload"); + let responder = tokio::spawn(respond_n(Arc::clone(&state), side, 1, move |request| { + assert!(matches!( + request.plan, + PhysicalPlan::Crdt(CrdtOp::PreviewApply { .. }) + )); + response(request.request_id, preview_payload.clone()) + })); + let seen = Arc::new(Mutex::new(Vec::new())); + let policy = RecordingPolicy { seen, reject: true }; + let before_hlc = state.tenant_write_mark(1); + let result = dispatch_crdt_apply_admitted(&state, admission_request(&policy)).await; + responder.await.expect("responder completes"); + assert!(matches!( + result, + Err(crate::Error::CrdtAdmissionCallerFence) + )); + assert_eq!( + state.tenant_write_mark(1), + before_hlc, + "policy rejection must not advance the tenant write HLC" + ); + state.wal.sync().expect("sync wal"); + assert!( + state.wal.replay().expect("replay wal").is_empty(), + "policy rejection must not append a durable CRDT delta" + ); + } + + #[tokio::test] + async fn admitted_cluster_apply_uses_raw_proposer_with_fenced_entry() { + let (state, side, _directory) = fixture(); + let digest = [0xa5; 32]; + let preview_payload = zerompk::to_msgpack_vec(&CrdtPreviewResult { + post_image_msgpack: vec![0xc0], + imported_ops: 1, + trimmed_ops: 0, + frontier_digest: digest, + }) + .expect("preview payload"); + let responder = tokio::spawn(respond_n(Arc::clone(&state), side, 1, move |request| { + assert!(matches!( + request.plan, + PhysicalPlan::Crdt(CrdtOp::PreviewApply { .. }) + )); + response(request.request_id, preview_payload.clone()) + })); + let fenced = Arc::new(Mutex::new(Vec::new())); + let observed = Arc::clone(&fenced); + let raw: Arc = + Arc::new(move |_shard, _key, bytes, _deadline| { + let observed = Arc::clone(&observed); + Box::pin(async move { + let entry = + crate::control::wal_replication::ReplicatedEntry::from_bytes(&bytes) + .expect("replicated entry"); + match entry.write { + crate::control::wal_replication::ReplicatedWrite::CrdtApplyFenced { + expected_frontier_digest, + .. + } => { + observed + .lock() + .expect("fence lock") + .push(expected_frontier_digest); + } + other => panic!("unexpected replicated write: {other:?}"), + } + Ok((Vec::new(), crate::types::ReadVersions::new())) + }) + }); + crate::control::vshard_admission::install_async_raft_proposer( + &state, + crate::control::vshard_admission::applying_submit(raw), + ) + .expect("install raw/sequenced proposer pair"); + let seen = Arc::new(Mutex::new(Vec::new())); + let policy = RecordingPolicy { + seen, + reject: false, + }; + dispatch_crdt_apply_admitted(&state, admission_request(&policy)) + .await + .expect("cluster admitted apply"); + responder.await.expect("responder completes"); + assert_eq!(*fenced.lock().expect("fence lock"), vec![digest]); + } + + #[tokio::test] + async fn stale_frontier_retries_preview_and_policy_before_success() { + let (state, side, _directory) = fixture(); + let digest = [0x82; 32]; + let preview_payload = zerompk::to_msgpack_vec(&CrdtPreviewResult { + post_image_msgpack: vec![0xc0], + imported_ops: 0, + trimmed_ops: 0, + frontier_digest: digest, + }) + .expect("preview payload"); + let responder = tokio::spawn(respond_n(Arc::clone(&state), side, 2, move |request| { + assert!(matches!( + request.plan, + PhysicalPlan::Crdt(CrdtOp::PreviewApply { .. }) + )); + response(request.request_id, preview_payload.clone()) + })); + let fences = Arc::new(AtomicUsize::new(0)); + let count = Arc::clone(&fences); + let raw: Arc = + Arc::new(move |_shard, _key, _bytes, _deadline| { + let count = Arc::clone(&count); + Box::pin(async move { + if count.fetch_add(1, Ordering::SeqCst) == 0 { + Err(crate::Error::DataPlane(ErrorCode::CrdtFrontierMismatch { + expected: digest, + actual: [0; 32], + })) + } else { + Ok((Vec::new(), crate::types::ReadVersions::new())) + } + }) + }); + crate::control::vshard_admission::install_async_raft_proposer( + &state, + crate::control::vshard_admission::applying_submit(raw), + ) + .expect("install raw/sequenced proposer pair"); + let seen = Arc::new(Mutex::new(Vec::new())); + let policy = RecordingPolicy { + seen: Arc::clone(&seen), + reject: false, + }; + dispatch_crdt_apply_admitted(&state, admission_request(&policy)) + .await + .expect("second fence succeeds"); + responder.await.expect("responder completes"); + assert_eq!(fences.load(Ordering::SeqCst), 2); + assert_eq!(seen.lock().expect("policy lock").len(), 2); + } + + #[tokio::test] + async fn stale_frontier_retries_preview_and_policy_then_exhausts_typed_error() { + let (state, side, _directory) = fixture(); + let digest = [0x81; 32]; + let preview_payload = zerompk::to_msgpack_vec(&CrdtPreviewResult { + post_image_msgpack: vec![0xc0], + imported_ops: 0, + trimmed_ops: 0, + frontier_digest: digest, + }) + .expect("preview payload"); + let responder = tokio::spawn(respond_n( + Arc::clone(&state), + side, + FRONTIER_RETRY_LIMIT, + move |request| { + assert!(matches!( + request.plan, + PhysicalPlan::Crdt(CrdtOp::PreviewApply { .. }) + )); + response(request.request_id, preview_payload.clone()) + }, + )); + let fenced_count = Arc::new(AtomicUsize::new(0)); + let count = Arc::clone(&fenced_count); + let raw: Arc = + Arc::new(move |_shard, _key, _bytes, _deadline| { + let count = Arc::clone(&count); + Box::pin(async move { + count.fetch_add(1, Ordering::SeqCst); + Err(crate::Error::DataPlane(ErrorCode::CrdtFrontierMismatch { + expected: digest, + actual: [0; 32], + })) + }) + }); + crate::control::vshard_admission::install_async_raft_proposer( + &state, + crate::control::vshard_admission::applying_submit(raw), + ) + .expect("install raw/sequenced proposer pair"); + let seen = Arc::new(Mutex::new(Vec::new())); + let policy = RecordingPolicy { + seen: Arc::clone(&seen), + reject: false, + }; + let result = dispatch_crdt_apply_admitted(&state, admission_request(&policy)).await; + responder.await.expect("responder completes"); + assert!(matches!( + result, + Err(crate::Error::CrdtAdmissionRetriesExhausted { + attempts: FRONTIER_RETRY_LIMIT, + .. + }) + )); + assert_eq!(fenced_count.load(Ordering::SeqCst), FRONTIER_RETRY_LIMIT); + assert_eq!( + seen.lock().expect("policy lock").len(), + FRONTIER_RETRY_LIMIT + ); + } + + #[tokio::test] + async fn signed_delta_collection_rejects_plain_external_apply_before_preview() { + let (state, _side, _directory) = fixture(); + let tenant_id = TenantId::new(1); + let mut collection = crate::control::security::catalog::StoredCollection::stamped_for_test( + 1, "docs", "owner", + ); + collection.crdt = true; + collection.crdt_signing_required = true; + state + .credentials + .catalog() + .put_collection(DatabaseId::DEFAULT, &collection) + .expect("store signed-delta collection"); + let task = nodedb_physical::physical_task::PhysicalTask { + tenant_id, + database_id: DatabaseId::DEFAULT, + vshard_id: nodedb_types::CollectionKey::from_bare(DatabaseId::DEFAULT, "docs").vshard(), + plan: apply_plan(), + post_set_op: nodedb_physical::physical_task::PostSetOp::None, + txn_id: None, + }; + let identity = + crate::control::security::identity::AuthenticatedIdentity::new_internal_service( + 1, + "crdt-test", + tenant_id, + Vec::new(), + true, + None, + crate::control::security::identity::AuthenticatedIdentity::default_database_set( + true, + ), + ); + let authorized = crate::control::server::shared::authorization::authorize_task_set( + &identity, + std::slice::from_ref(&task), + &state.permissions, + &state.roles, + &crate::control::security::audit::NoopAuditEmitter, + ) + .expect("authorize test task") + .into_tasks() + .into_iter() + .next() + .expect("one authorized task"); + let result = dispatch_authorized_crdt_apply_admitted( + &state, + AuthorizedCrdtApplyAdmissionRequest { + authorized, + collection: "docs", + deadline: StatementDeadline::after(Duration::from_millis(10)), + event_source: EventSource::User, + policy: &TrustedInternalCrdtPolicy, + }, + ) + .await; + assert!(matches!(result, Err(crate::Error::RejectedAuthz { .. }))); + } + + #[tokio::test] + async fn generic_replicated_dispatch_rejects_unadmitted_apply() { + let (state, _side, _directory) = fixture(); + let tenant_id = TenantId::new(1); + let task = nodedb_physical::physical_task::PhysicalTask { + tenant_id, + database_id: DatabaseId::DEFAULT, + vshard_id: nodedb_types::CollectionKey::from_bare(DatabaseId::DEFAULT, "docs").vshard(), + plan: apply_plan(), + post_set_op: nodedb_physical::physical_task::PostSetOp::None, + txn_id: None, + }; + let identity = + crate::control::security::identity::AuthenticatedIdentity::new_internal_service( + 1, + "crdt-test", + tenant_id, + Vec::new(), + true, + None, + crate::control::security::identity::AuthenticatedIdentity::default_database_set( + true, + ), + ); + let authorized = crate::control::server::shared::authorization::authorize_task_set( + &identity, + std::slice::from_ref(&task), + &state.permissions, + &state.roles, + &crate::control::security::audit::NoopAuditEmitter, + ) + .expect("authorize test task") + .into_tasks() + .into_iter() + .next() + .expect("one authorized task"); + let result = crate::control::server::sync::raft_dispatch::dispatch_write_replicated( + &state, + "docs", + authorized, + StatementDeadline::of_statement(&state), + EventSource::User, + ) + .await; + assert!(matches!( + result, + Err(crate::Error::CrdtApplyRequiresAdmission) + )); + } + + /// A deadline that passes while the preview is unanswered refuses the + /// delta, and nothing is proposed. + #[tokio::test] + async fn a_deadline_before_the_proposal_is_a_definite_refusal() { + let (state, _side, _directory) = fixture(); + let proposals = Arc::new(AtomicUsize::new(0)); + let count = Arc::clone(&proposals); + let raw: Arc = + Arc::new(move |_shard, _key, _bytes, _deadline| { + count.fetch_add(1, Ordering::SeqCst); + Box::pin(async { Ok((Vec::new(), crate::types::ReadVersions::new())) }) + }); + crate::control::vshard_admission::install_async_raft_proposer( + &state, + crate::control::vshard_admission::applying_submit(raw), + ) + .expect("install proposer"); + let policy = TrustedInternalCrdtPolicy; + + let result = dispatch_crdt_apply_admitted( + &state, + admission_request_until(&policy, StatementDeadline::after(Duration::from_millis(50))), + ) + .await; + + assert!( + matches!(result, Err(crate::Error::CrdtAdmissionTimeout { .. })), + "got {result:?}" + ); + assert_eq!(proposals.load(Ordering::SeqCst), 0, "nothing was proposed"); + } + + /// A deadline that passes while the proposal is in flight leaves the + /// outcome unknown. + #[tokio::test] + async fn a_deadline_during_the_proposal_is_an_unknown_outcome() { + let (state, side, _directory) = fixture(); + let payload = preview_payload([0x5a; 32]); + let responder = tokio::spawn(respond_n(Arc::clone(&state), side, 1, move |request| { + assert!(matches!( + request.plan, + PhysicalPlan::Crdt(CrdtOp::PreviewApply { .. }) + )); + response(request.request_id, payload.clone()) + })); + let proposals = Arc::new(AtomicUsize::new(0)); + let count = Arc::clone(&proposals); + let raw: Arc = + Arc::new(move |_shard, _key, _bytes, _deadline| { + count.fetch_add(1, Ordering::SeqCst); + Box::pin(std::future::pending::< + crate::Result<(Vec, crate::types::ReadVersions)>, + >()) + }); + crate::control::vshard_admission::install_async_raft_proposer( + &state, + crate::control::vshard_admission::applying_submit(raw), + ) + .expect("install proposer"); + let policy = TrustedInternalCrdtPolicy; + + let result = dispatch_crdt_apply_admitted( + &state, + admission_request_until( + &policy, + StatementDeadline::after(Duration::from_millis(300)), + ), + ) + .await; + responder.await.expect("responder completes"); + + assert!( + matches!(result, Err(crate::Error::DeadlineExceeded { .. })), + "got {result:?}" + ); + assert_eq!(proposals.load(Ordering::SeqCst), 1, "the proposal was sent"); + } +} diff --git a/nodedb/src/control/crdt_admission/classify.rs b/nodedb/src/control/crdt_admission/classify.rs new file mode 100644 index 000000000..000905515 --- /dev/null +++ b/nodedb/src/control/crdt_admission/classify.rs @@ -0,0 +1,165 @@ +// SPDX-License-Identifier: BUSL-1.1 + +//! Which CRDT operations move a document's Loro frontier. + +use nodedb_physical::physical_plan::CrdtOp; + +/// Whether an operation changes the Loro frontier and must serialize with an +/// admission preview when it is dispatched to this node's cores without a +/// proposal. +pub fn changes_crdt_frontier(op: &CrdtOp) -> bool { + match op { + CrdtOp::Apply { .. } + | CrdtOp::ApplyAuthenticated { .. } + | CrdtOp::ImportSnapshot { .. } + | CrdtOp::ListInsert { .. } + | CrdtOp::ListDelete { .. } + | CrdtOp::ListMove { .. } + | CrdtOp::DocUpsert { .. } + | CrdtOp::DocDelete { .. } + | CrdtOp::RestoreToVersion { .. } => true, + CrdtOp::Read { .. } + | CrdtOp::PreviewApply { .. } + | CrdtOp::GetVersionVector { .. } + | CrdtOp::ExportDelta { .. } + | CrdtOp::CompactAtVersion { .. } + | CrdtOp::SetConstraints { .. } + | CrdtOp::DropConstraints { .. } + | CrdtOp::ReadConstraints { .. } + | CrdtOp::SetPolicy { .. } + | CrdtOp::GetPolicy { .. } + | CrdtOp::ReadAtVersion { .. } => false, + } +} + +#[cfg(test)] +mod tests { + use nodedb_types::{QualifiedCollection, Surrogate}; + + use super::*; + use crate::types::DatabaseId; + + fn collection() -> String { + "docs".to_owned() + } + + fn assert_frontier_mutation(op: CrdtOp) { + assert!(changes_crdt_frontier(&op), "{op:?} must serialize"); + } + + fn assert_frontier_read(op: CrdtOp) { + assert!(!changes_crdt_frontier(&op), "{op:?} must not serialize"); + } + + #[test] + fn frontier_classifier_covers_every_crdt_operation_category() { + let surrogate = Surrogate::new(1); + assert_frontier_mutation(CrdtOp::Apply { + collection: QualifiedCollection::new(DatabaseId::DEFAULT, &collection()), + document_id: "id".into(), + delta: Vec::new(), + peer_id: 1, + mutation_id: 1, + surrogate, + provenance: None, + constraint_version_required: 0, + expected_frontier_digest: None, + }); + assert_frontier_mutation(CrdtOp::ImportSnapshot { + tenant_id: 1, + collection: QualifiedCollection::new(DatabaseId::DEFAULT, &collection()), + bytes: Vec::new(), + }); + assert_frontier_mutation(CrdtOp::RestoreToVersion { + collection: QualifiedCollection::new(DatabaseId::DEFAULT, &collection()), + document_id: "id".into(), + target_version_json: "{}".into(), + surrogate, + }); + assert_frontier_mutation(CrdtOp::ListInsert { + collection: QualifiedCollection::new(DatabaseId::DEFAULT, &collection()), + document_id: "id".into(), + list_path: "blocks".into(), + index: 0, + fields_json: "{}".into(), + surrogate, + }); + assert_frontier_mutation(CrdtOp::ListDelete { + collection: QualifiedCollection::new(DatabaseId::DEFAULT, &collection()), + document_id: "id".into(), + list_path: "blocks".into(), + index: 0, + surrogate, + }); + assert_frontier_mutation(CrdtOp::ListMove { + collection: QualifiedCollection::new(DatabaseId::DEFAULT, &collection()), + document_id: "id".into(), + list_path: "blocks".into(), + from_index: 0, + to_index: 1, + surrogate, + }); + assert_frontier_mutation(CrdtOp::DocUpsert { + collection: QualifiedCollection::new(DatabaseId::DEFAULT, &collection()), + document_id: "id".into(), + fields_json: "{}".into(), + surrogate, + partial: false, + verb: nodedb_physical::physical_plan::CrdtWriteVerb::Insert, + returning: None, + rls_filters: Vec::new(), + }); + assert_frontier_mutation(CrdtOp::DocDelete { + collection: QualifiedCollection::new(DatabaseId::DEFAULT, &collection()), + document_id: "id".into(), + surrogate: Some(surrogate), + returning: None, + rls_filters: Vec::new(), + }); + + assert_frontier_read(CrdtOp::Read { + collection: QualifiedCollection::new(DatabaseId::DEFAULT, &collection()), + document_id: "id".into(), + }); + assert_frontier_read(CrdtOp::PreviewApply { + collection: QualifiedCollection::new(DatabaseId::DEFAULT, &collection()), + document_id: "id".into(), + delta: Vec::new(), + }); + assert_frontier_read(CrdtOp::GetVersionVector { + collection: QualifiedCollection::new(DatabaseId::DEFAULT, &collection()), + }); + assert_frontier_read(CrdtOp::ExportDelta { + collection: QualifiedCollection::new(DatabaseId::DEFAULT, &collection()), + from_version_json: "{}".into(), + }); + assert_frontier_read(CrdtOp::CompactAtVersion { + collection: QualifiedCollection::new(DatabaseId::DEFAULT, &collection()), + target_version_json: "{}".into(), + }); + assert_frontier_read(CrdtOp::SetConstraints { + collection: QualifiedCollection::new(DatabaseId::DEFAULT, &collection()), + constraint_version: 1, + constraints: Vec::new(), + }); + assert_frontier_read(CrdtOp::DropConstraints { + collection: QualifiedCollection::new(DatabaseId::DEFAULT, &collection()), + constraint_version: 1, + }); + assert_frontier_read(CrdtOp::ReadConstraints { + collection: QualifiedCollection::new(DatabaseId::DEFAULT, &collection()), + }); + assert_frontier_read(CrdtOp::SetPolicy { + collection: QualifiedCollection::new(DatabaseId::DEFAULT, &collection()), + policy_json: "{}".into(), + }); + assert_frontier_read(CrdtOp::GetPolicy { + collection: QualifiedCollection::new(DatabaseId::DEFAULT, &collection()), + }); + assert_frontier_read(CrdtOp::ReadAtVersion { + collection: QualifiedCollection::new(DatabaseId::DEFAULT, &collection()), + document_id: "id".into(), + version_vector_json: "{}".into(), + }); + } +} diff --git a/nodedb/src/control/crdt_admission/fence.rs b/nodedb/src/control/crdt_admission/fence.rs new file mode 100644 index 000000000..6c70aaa90 --- /dev/null +++ b/nodedb/src/control/crdt_admission/fence.rs @@ -0,0 +1,106 @@ +// SPDX-License-Identifier: BUSL-1.1 + +//! Stamp the previewed frontier on a delta and propose it fenced. + +use nodedb_physical::physical_plan::CrdtOp; + +use crate::bridge::envelope::PhysicalPlan; +use crate::control::wal_replication::{ReplicableWrite, to_replicated_entry}; + +use super::request::CrdtAdmissionOutcome; +use super::workflow::CrdtAdmissionWorkflow; + +/// `plan` with `digest` as the frontier its apply expects. +pub(super) fn stamp_fence(plan: PhysicalPlan, digest: [u8; 32]) -> crate::Result { + match plan { + PhysicalPlan::Crdt(CrdtOp::Apply { + collection, + document_id, + delta, + peer_id, + mutation_id, + surrogate, + provenance, + constraint_version_required, + expected_frontier_digest: None, + }) => Ok(PhysicalPlan::Crdt(CrdtOp::Apply { + collection, + document_id, + delta, + peer_id, + mutation_id, + surrogate, + provenance, + constraint_version_required, + expected_frontier_digest: Some(digest), + })), + PhysicalPlan::Crdt(CrdtOp::ApplyAuthenticated { + collection, + document_id, + delta, + peer_id, + mutation_id, + surrogate, + provenance, + constraint_version_required, + expected_frontier_digest: None, + auth_user_id, + auth_device_id, + auth_seq_no, + delta_signature, + signing_required, + }) => Ok(PhysicalPlan::Crdt(CrdtOp::ApplyAuthenticated { + collection, + document_id, + delta, + peer_id, + mutation_id, + surrogate, + provenance, + constraint_version_required, + expected_frontier_digest: Some(digest), + auth_user_id, + auth_device_id, + auth_seq_no, + delta_signature, + signing_required, + })), + _ => Err(crate::Error::CrdtAdmissionInvalidPlan { + reason: "validated CRDT Apply plan was changed before fence stamping", + }), + } +} + +/// Propose the fenced apply and wait for this node's apply of it. +pub(super) async fn apply_fenced( + workflow: &CrdtAdmissionWorkflow<'_>, + plan: PhysicalPlan, +) -> crate::Result { + let raw = workflow.state.raw_async_raft_proposer()?; + let entry = to_replicated_entry( + workflow.tenant_id, + workflow.database_id, + workflow.vshard_id, + &ReplicableWrite::decide_for_replication(&plan)?, + )? + .ok_or(crate::Error::CrdtAdmissionInvalidPlan { + reason: "admitted CRDT Apply has no replicated form", + })? + // Every replica gives the write the source this node dispatches it + // with. + .with_event_source(workflow.event_source); + let (payload, write_versions) = workflow + .propose(crate::control::wal_replication::propose_replicated_entry( + workflow.state, + raw, + entry, + workflow.deadline.tokio_instant(), + )) + .await?; + // This node's apply of the entry recorded its commit HLC. + Ok(CrdtAdmissionOutcome { + payload, + write_versions, + trimmed_ops: 0, + }) +} diff --git a/nodedb/src/control/crdt_admission/legs.rs b/nodedb/src/control/crdt_admission/legs.rs new file mode 100644 index 000000000..5a4614fb0 --- /dev/null +++ b/nodedb/src/control/crdt_admission/legs.rs @@ -0,0 +1,98 @@ +// SPDX-License-Identifier: BUSL-1.1 + +//! The Data-Plane reads an admission runs before it proposes: the preview of +//! a delta, and the generation of a restore delta. +//! +//! Both are legs of a request whose capability was consumed at its entry +//! point, so they go through the system door. + +use nodedb_physical::physical_plan::CrdtOp; +use nodedb_types::CrdtPreviewResult; + +use crate::bridge::envelope::{PhysicalPlan, Status}; +use crate::control::server::shared::ddl::sync_dispatch::{ + SystemReason, SystemTask, dispatch_system_response_with_source, +}; + +use super::workflow::CrdtAdmissionWorkflow; + +/// The post-image and frontier digest `delta` will produce on the document. +pub(super) async fn preview( + workflow: &CrdtAdmissionWorkflow<'_>, + document_id: &str, + delta: &[u8], +) -> crate::Result { + let response = dispatch_system_response_with_source( + workflow.state, + SystemTask::new( + SystemReason::AdmittedContinuation, + workflow.tenant_id, + nodedb_types::CollectionKey::from_qualified_str( + workflow.database_id, + workflow.collection, + )?, + PhysicalPlan::Crdt(CrdtOp::PreviewApply { + collection: nodedb_types::QualifiedCollection::from_stored( + workflow.collection.to_owned(), + ), + document_id: document_id.to_owned(), + delta: delta.to_vec(), + }), + ), + workflow.deadline, + workflow.event_source, + ) + .await + .map_err(|error| workflow.before_propose(error))?; + if response.status != Status::Ok { + return Err(response_error(&response)); + } + zerompk::from_msgpack(response.payload.as_bytes()).map_err(|error| crate::Error::Internal { + detail: format!("CRDT preview payload decode: {error}"), + }) +} + +/// The delta that takes the document back to `target_version_json`. +pub(super) async fn generate_restore_delta( + workflow: &CrdtAdmissionWorkflow<'_>, + document_id: &str, + target_version_json: &str, + surrogate: nodedb_types::Surrogate, +) -> crate::Result> { + let response = dispatch_system_response_with_source( + workflow.state, + SystemTask::new( + SystemReason::AdmittedContinuation, + workflow.tenant_id, + nodedb_types::CollectionKey::from_qualified_str( + workflow.database_id, + workflow.collection, + )?, + PhysicalPlan::Crdt(CrdtOp::RestoreToVersion { + collection: nodedb_types::QualifiedCollection::from_stored( + workflow.collection.to_owned(), + ), + document_id: document_id.to_owned(), + target_version_json: target_version_json.to_owned(), + surrogate, + }), + ), + workflow.deadline, + workflow.event_source, + ) + .await + .map_err(|error| workflow.before_propose(error))?; + if response.status != Status::Ok { + return Err(response_error(&response)); + } + Ok(response.payload.to_vec()) +} + +fn response_error(response: &crate::bridge::envelope::Response) -> crate::Error { + match response.error_code.as_deref() { + Some(code) => crate::Error::DataPlane(code.clone()), + None => crate::Error::Internal { + detail: String::from_utf8_lossy(&response.payload).into_owned(), + }, + } +} diff --git a/nodedb/src/control/crdt_admission/mod.rs b/nodedb/src/control/crdt_admission/mod.rs new file mode 100644 index 000000000..d88ec1843 --- /dev/null +++ b/nodedb/src/control/crdt_admission/mod.rs @@ -0,0 +1,25 @@ +// SPDX-License-Identifier: BUSL-1.1 + +//! Serialized preview, policy, and fenced CRDT apply admission. + +pub(crate) mod apply; +mod classify; +mod fence; +mod legs; +mod policy; +mod request; +mod restore; +#[cfg(test)] +mod test_support; +mod workflow; + +pub use apply::{ + dispatch_authorized_crdt_apply_admitted, dispatch_authorized_crdt_apply_admitted_outcome, +}; +pub use classify::changes_crdt_frontier; +pub use policy::{CrdtPostImagePolicy, TrustedInternalCrdtPolicy}; +pub use request::{ + AuthorizedCrdtApplyAdmissionRequest, CrdtAdmissionOutcome, CrdtApplyAdmissionRequest, + CrdtRestoreAdmissionRequest, +}; +pub(crate) use restore::dispatch_crdt_restore_admitted; diff --git a/nodedb/src/control/crdt_admission/policy.rs b/nodedb/src/control/crdt_admission/policy.rs new file mode 100644 index 000000000..c4ddea0ff --- /dev/null +++ b/nodedb/src/control/crdt_admission/policy.rs @@ -0,0 +1,18 @@ +// SPDX-License-Identifier: BUSL-1.1 + +//! The post-image policy an admission evaluates before it proposes. + +use nodedb_types::CrdtPreviewResult; + +pub trait CrdtPostImagePolicy: Send + Sync { + fn evaluate(&self, preview: &CrdtPreviewResult) -> crate::Result<()>; +} + +/// Explicit policy for trusted internal callers. +pub struct TrustedInternalCrdtPolicy; + +impl CrdtPostImagePolicy for TrustedInternalCrdtPolicy { + fn evaluate(&self, _preview: &CrdtPreviewResult) -> crate::Result<()> { + Ok(()) + } +} diff --git a/nodedb/src/control/crdt_admission/request.rs b/nodedb/src/control/crdt_admission/request.rs new file mode 100644 index 000000000..4b80d2549 --- /dev/null +++ b/nodedb/src/control/crdt_admission/request.rs @@ -0,0 +1,73 @@ +// SPDX-License-Identifier: BUSL-1.1 + +//! What a caller hands a CRDT admission, and what it gets back. + +use crate::bridge::envelope::PhysicalPlan; +use crate::control::server::shared::authorization::AuthorizedTask; +use crate::control::server::shared::session::StatementDeadline; +use crate::event::EventSource; +use crate::types::{DatabaseId, TenantId}; + +use super::policy::CrdtPostImagePolicy; + +pub struct AuthorizedCrdtApplyAdmissionRequest<'a> { + pub authorized: AuthorizedTask, + /// The db-qualified collection (`QualifiedCollection::as_str`), the same + /// string the plan's `CrdtOp::Apply` carries. The preview and the apply + /// address the Data Plane by it, and the catalog lookup de-qualifies it. + pub collection: &'a str, + /// The deadline every phase of the admission ends at. + pub deadline: StatementDeadline, + pub event_source: EventSource, + pub policy: &'a dyn CrdtPostImagePolicy, +} + +pub struct CrdtApplyAdmissionRequest<'a> { + pub tenant_id: TenantId, + pub database_id: DatabaseId, + /// The db-qualified collection, equal to the plan's `CrdtOp::Apply` + /// collection. + pub collection: &'a str, + pub plan: PhysicalPlan, + /// The deadline every phase of the admission ends at. + pub deadline: StatementDeadline, + pub event_source: EventSource, + pub policy: &'a dyn CrdtPostImagePolicy, +} + +pub struct CrdtAdmissionOutcome { + pub payload: Vec, + /// The versions the apply stamped: the entry's log position on the + /// written vShard. + pub write_versions: crate::types::ReadVersions, + /// Operations the admitted delta encoded that the target document already + /// knew, measured by the preview that fenced this apply. Distinguishes a + /// client whose writes land from one whose writes are being absorbed — + /// both otherwise produce the same successful payload. + pub trimmed_ops: u64, +} + +impl CrdtAdmissionOutcome { + /// Attach the trim count measured by the preview that fenced this apply. + pub(super) fn with_trimmed_ops(self, trimmed_ops: u64) -> Self { + Self { + trimmed_ops, + ..self + } + } +} + +pub struct CrdtRestoreAdmissionRequest<'a> { + pub tenant_id: TenantId, + pub database_id: DatabaseId, + /// The db-qualified collection the generated restore ops address. + pub collection: &'a str, + pub document_id: &'a str, + pub target_version_json: &'a str, + pub surrogate: nodedb_types::Surrogate, + pub peer_id: u64, + /// The deadline every phase of the admission ends at. + pub deadline: StatementDeadline, + pub event_source: EventSource, + pub policy: &'a dyn CrdtPostImagePolicy, +} diff --git a/nodedb/src/control/crdt_admission/restore.rs b/nodedb/src/control/crdt_admission/restore.rs new file mode 100644 index 000000000..a0cd29533 --- /dev/null +++ b/nodedb/src/control/crdt_admission/restore.rs @@ -0,0 +1,201 @@ +// SPDX-License-Identifier: BUSL-1.1 + +//! Admit a restore delta generated inside the vShard's admission slot. + +use nodedb_physical::physical_plan::CrdtOp; + +use crate::bridge::envelope::{ErrorCode, PhysicalPlan}; +use crate::control::state::SharedState; + +use super::fence::{apply_fenced, stamp_fence}; +use super::legs::{generate_restore_delta, preview}; +use super::request::{CrdtAdmissionOutcome, CrdtRestoreAdmissionRequest}; +use super::workflow::{CrdtAdmissionWorkflow, FRONTIER_RETRY_LIMIT}; + +/// Generate a restore delta and admit it without releasing the vShard slot. +/// A stale apply fence regenerates the delta from authoritative state before +/// retrying, so neither historical projection nor policy output is reused. +pub(crate) async fn dispatch_crdt_restore_admitted( + state: &SharedState, + request: CrdtRestoreAdmissionRequest<'_>, +) -> crate::Result> { + let CrdtRestoreAdmissionRequest { + tenant_id, + database_id, + collection, + document_id, + target_version_json, + surrogate, + peer_id, + deadline, + event_source, + policy, + } = request; + // The restore writes a delta like any apply; the lease gates it on a + // drained collection and lives until its outcome. + let _lease = crate::control::server::shared::clone_write::collections_write_lease( + state, + tenant_id, + database_id, + [collection.to_owned()], + ) + .await?; + let workflow = CrdtAdmissionWorkflow::new( + state, + tenant_id, + database_id, + collection, + deadline, + event_source, + policy, + )?; + workflow + .run(|| { + restore_locked( + &workflow, + document_id, + target_version_json, + surrogate, + peer_id, + ) + }) + .await +} + +/// Generate, preview and propose the restore delta, inside the slot. +async fn restore_locked( + workflow: &CrdtAdmissionWorkflow<'_>, + document_id: &str, + target_version_json: &str, + surrogate: nodedb_types::Surrogate, + peer_id: u64, +) -> crate::Result> { + for _attempt in 0..FRONTIER_RETRY_LIMIT { + let delta = + generate_restore_delta(workflow, document_id, target_version_json, surrogate).await?; + if delta.is_empty() { + return Ok(None); + } + let plan = PhysicalPlan::Crdt(CrdtOp::Apply { + collection: nodedb_types::QualifiedCollection::from_stored( + workflow.collection.to_owned(), + ), + document_id: document_id.to_owned(), + delta: delta.clone(), + peer_id, + mutation_id: 0, + surrogate, + provenance: None, + constraint_version_required: 0, + expected_frontier_digest: None, + }); + let preview = preview(workflow, document_id, &delta).await?; + workflow.policy.evaluate(&preview)?; + match apply_fenced(workflow, stamp_fence(plan, preview.frontier_digest)?).await { + Err(crate::Error::DataPlane(ErrorCode::CrdtFrontierMismatch { .. })) => {} + result => return result.map(Some), + } + } + Err(crate::Error::CrdtAdmissionRetriesExhausted { + vshard_id: workflow.vshard_id, + attempts: FRONTIER_RETRY_LIMIT, + }) +} + +#[cfg(test)] +mod tests { + use std::sync::atomic::{AtomicUsize, Ordering}; + use std::sync::{Arc, Mutex}; + use std::time::Duration; + + use nodedb_types::{CrdtPreviewResult, Surrogate}; + + use super::*; + use crate::control::crdt_admission::test_support::{ + RecordingPolicy, fixture, respond_n, response, + }; + use crate::control::server::shared::session::StatementDeadline; + use crate::event::EventSource; + use crate::types::{DatabaseId, TenantId}; + + #[tokio::test] + async fn stale_restore_regenerates_before_retrying_admission() { + let (state, side, _directory) = fixture(); + let delta = vec![0x91, 0x02]; + let preview_payload = zerompk::to_msgpack_vec(&CrdtPreviewResult { + post_image_msgpack: vec![0xc0], + imported_ops: 1, + trimmed_ops: 0, + frontier_digest: [0x84; 32], + }) + .expect("preview payload"); + let request_index = Arc::new(AtomicUsize::new(0)); + let index = Arc::clone(&request_index); + let responder = tokio::spawn(respond_n(Arc::clone(&state), side, 4, move |request| { + let is_restore = matches!( + &request.plan, + PhysicalPlan::Crdt(CrdtOp::RestoreToVersion { .. }) + ); + match index.fetch_add(1, Ordering::SeqCst) { + 0 | 2 => assert!(is_restore), + 1 | 3 => assert!(matches!( + &request.plan, + PhysicalPlan::Crdt(CrdtOp::PreviewApply { .. }) + )), + _ => unreachable!("responder only handles four requests"), + } + let payload = if is_restore { + delta.clone() + } else { + preview_payload.clone() + }; + response(request.request_id, payload) + })); + let attempts = Arc::new(AtomicUsize::new(0)); + let raw: Arc = { + let attempts = Arc::clone(&attempts); + Arc::new(move |_shard, _key, _bytes, _deadline| { + let attempts = Arc::clone(&attempts); + Box::pin(async move { + if attempts.fetch_add(1, Ordering::SeqCst) == 0 { + Err(crate::Error::DataPlane(ErrorCode::CrdtFrontierMismatch { + expected: [0x84; 32], + actual: [0; 32], + })) + } else { + Ok((Vec::new(), crate::types::ReadVersions::new())) + } + }) + }) + }; + crate::control::vshard_admission::install_async_raft_proposer( + &state, + crate::control::vshard_admission::applying_submit(raw), + ) + .expect("install proposer"); + let policy = RecordingPolicy { + seen: Arc::new(Mutex::new(Vec::new())), + reject: false, + }; + dispatch_crdt_restore_admitted( + &state, + CrdtRestoreAdmissionRequest { + tenant_id: TenantId::new(1), + database_id: DatabaseId::DEFAULT, + collection: "docs", + document_id: "doc-1", + target_version_json: "{}", + surrogate: Surrogate::new(1), + peer_id: 1, + deadline: StatementDeadline::after(Duration::from_secs(1)), + event_source: EventSource::User, + policy: &policy, + }, + ) + .await + .expect("second restore admission succeeds"); + responder.await.expect("responder completes"); + assert_eq!(request_index.load(Ordering::SeqCst), 4); + assert_eq!(attempts.load(Ordering::SeqCst), 2); + } +} diff --git a/nodedb/src/control/crdt_admission/test_support.rs b/nodedb/src/control/crdt_admission/test_support.rs new file mode 100644 index 000000000..7a4d6c94f --- /dev/null +++ b/nodedb/src/control/crdt_admission/test_support.rs @@ -0,0 +1,149 @@ +// SPDX-License-Identifier: BUSL-1.1 + +//! Fixtures shared by the admission tests: a one-core state, a fake Data +//! Plane responder, and a recording policy. + +use std::sync::{Arc, Mutex}; +use std::time::{Duration, Instant}; + +use nodedb_physical::physical_plan::CrdtOp; +use nodedb_types::{CrdtPreviewResult, QualifiedCollection, Surrogate}; + +use crate::bridge::dispatch::{BridgeResponse, CoreChannelDataSide, Dispatcher}; +use crate::bridge::envelope::{PhysicalPlan, Response, Status}; +use crate::control::server::shared::session::StatementDeadline; +use crate::control::state::SharedState; +use crate::event::EventSource; +use crate::types::{DatabaseId, Lsn, RequestId, TenantId}; +use crate::wal::WalManager; + +use super::apply::dispatch_crdt_apply_admitted_outcome; +use super::policy::CrdtPostImagePolicy; +use super::request::CrdtApplyAdmissionRequest; + +/// Payload-only wrapper over the trusted-internal admission. +pub(super) async fn dispatch_crdt_apply_admitted( + state: &SharedState, + request: CrdtApplyAdmissionRequest<'_>, +) -> crate::Result> { + Ok(dispatch_crdt_apply_admitted_outcome(state, request) + .await? + .payload) +} + +pub(super) fn admission_request<'a>( + policy: &'a dyn CrdtPostImagePolicy, +) -> CrdtApplyAdmissionRequest<'a> { + admission_request_until(policy, StatementDeadline::after(Duration::from_secs(1))) +} + +pub(super) fn admission_request_until<'a>( + policy: &'a dyn CrdtPostImagePolicy, + deadline: StatementDeadline, +) -> CrdtApplyAdmissionRequest<'a> { + CrdtApplyAdmissionRequest { + tenant_id: TenantId::new(1), + database_id: DatabaseId::DEFAULT, + collection: "docs", + plan: apply_plan(), + deadline, + event_source: EventSource::User, + policy, + } +} + +pub(super) fn apply_plan() -> PhysicalPlan { + PhysicalPlan::Crdt(CrdtOp::Apply { + collection: QualifiedCollection::new(DatabaseId::DEFAULT, "docs"), + document_id: "doc-1".into(), + delta: vec![0x91, 0x01], + peer_id: 7, + mutation_id: 9, + surrogate: Surrogate::new(1), + provenance: None, + constraint_version_required: 0, + expected_frontier_digest: None, + }) +} + +pub(super) fn fixture() -> (Arc, CoreChannelDataSide, tempfile::TempDir) { + let directory = tempfile::tempdir().expect("temporary WAL directory"); + let wal = Arc::new( + WalManager::open_for_testing(&directory.path().join("admission.wal")).expect("test WAL"), + ); + let (dispatcher, mut sides) = Dispatcher::new(1, 64); + let side = sides.pop().expect("one data side"); + let state = SharedState::new(dispatcher, wal).expect("shared state"); + (state, side, directory) +} + +pub(super) fn response(request_id: RequestId, payload: Vec) -> Response { + Response { + request_id, + status: Status::Ok, + attempt: 1, + partial: false, + payload: payload.into(), + watermark_lsn: Lsn::ZERO, + error_code: None, + stage_vote: None, + read_versions: crate::types::ReadVersions::new(), + write_set: Vec::new(), + } +} + +/// An encoded preview answer with frontier `digest`. +pub(super) fn preview_payload(digest: [u8; 32]) -> Vec { + zerompk::to_msgpack_vec(&CrdtPreviewResult { + post_image_msgpack: vec![0xc0], + imported_ops: 1, + trimmed_ops: 0, + frontier_digest: digest, + }) + .expect("preview payload") +} + +/// Answer `n` Data-Plane requests with `f`, within two seconds. +pub(super) async fn respond_n( + state: Arc, + mut side: CoreChannelDataSide, + n: usize, + mut f: F, +) where + F: FnMut(crate::bridge::envelope::Request) -> Response + Send + 'static, +{ + let deadline = Instant::now() + Duration::from_secs(2); + let mut handled = 0; + while handled < n && Instant::now() < deadline { + if let Ok(request) = side.request_rx.try_pop() { + let response = f(request.inner); + side.response_tx + .try_push(BridgeResponse { inner: response }) + .expect("fake data-plane response queue has capacity"); + handled += 1; + } + state.poll_and_route_responses(); + tokio::task::yield_now().await; + } + assert_eq!(handled, n, "fake data-plane received expected requests"); + state.poll_and_route_responses(); +} + +pub(super) struct RecordingPolicy { + pub(super) seen: Arc>>>, + pub(super) reject: bool, +} + +impl CrdtPostImagePolicy for RecordingPolicy { + fn evaluate(&self, preview: &CrdtPreviewResult) -> crate::Result<()> { + self.seen + .lock() + .expect("test policy lock") + .push(preview.post_image_msgpack.clone()); + if self.reject { + Err(crate::Error::CrdtAdmissionCallerFence) + } else { + Ok(()) + } + } +} diff --git a/nodedb/src/control/crdt_admission/workflow.rs b/nodedb/src/control/crdt_admission/workflow.rs new file mode 100644 index 000000000..4861de843 --- /dev/null +++ b/nodedb/src/control/crdt_admission/workflow.rs @@ -0,0 +1,129 @@ +// SPDX-License-Identifier: BUSL-1.1 + +//! One CRDT admission and the deadline it runs under. +//! +//! Every phase ends at one deadline: the slot wait, the catch-up, the +//! preview and the proposal. A deadline that passes with no proposal in +//! flight refuses the delta, and nothing applied. A deadline that passes +//! while a proposal is in flight leaves the outcome unknown. + +use std::future::Future; +use std::sync::atomic::{AtomicBool, Ordering}; + +use crate::control::server::shared::session::StatementDeadline; +use crate::control::state::SharedState; +use crate::event::EventSource; +use crate::types::{DatabaseId, RequestId, TenantId, VShardId}; + +use super::policy::CrdtPostImagePolicy; + +/// Frontier mismatches an admission retries before it gives up. +pub(super) const FRONTIER_RETRY_LIMIT: usize = 8; + +pub(super) struct CrdtAdmissionWorkflow<'a> { + pub(super) state: &'a SharedState, + pub(super) tenant_id: TenantId, + pub(super) database_id: DatabaseId, + pub(super) vshard_id: VShardId, + /// The plan's database-qualified collection name. + pub(super) collection: &'a str, + pub(super) deadline: StatementDeadline, + pub(super) event_source: EventSource, + pub(super) policy: &'a dyn CrdtPostImagePolicy, + /// The budget left when the admission started, for the refusal message. + budget_ms: u64, + /// Set while a proposal is in flight. + proposing: AtomicBool, +} + +impl<'a> CrdtAdmissionWorkflow<'a> { + pub(super) fn new( + state: &'a SharedState, + tenant_id: TenantId, + database_id: DatabaseId, + collection: &'a str, + deadline: StatementDeadline, + event_source: EventSource, + policy: &'a dyn CrdtPostImagePolicy, + ) -> crate::Result { + let vshard_id = + nodedb_types::CollectionKey::from_qualified_str(database_id, collection)?.vshard(); + Ok(Self { + state, + tenant_id, + database_id, + vshard_id, + collection, + deadline, + event_source, + policy, + budget_ms: u64::try_from(deadline.remaining().as_millis()).unwrap_or(u64::MAX), + proposing: AtomicBool::new(false), + }) + } + + /// Run `operation` in the vShard's admission slot, once this node applied + /// every write admitted before it. The whole run ends at the deadline. + pub(super) async fn run(&self, operation: F) -> crate::Result + where + F: FnOnce() -> Fut, + Fut: Future>, + { + let admission = self.state.vshard_admission_sequencer.run_after_proposed( + self.vshard_id, + |last_proposed| async move { + crate::control::vshard_admission::await_admitted_applies( + self.state, + self.vshard_id, + last_proposed, + self.deadline.instant(), + ) + .await?; + operation().await + }, + ); + match tokio::time::timeout_at(self.deadline.tokio_instant(), admission).await { + Ok(result) => result, + Err(_) => Err(self.expired()), + } + } + + /// Await `proposal` with the in-flight mark set. A deadline that already + /// passed refuses before anything is proposed. + pub(super) async fn propose( + &self, + proposal: impl Future>, + ) -> crate::Result { + if self.deadline.remaining().is_zero() { + return Err(self.expired()); + } + self.proposing.store(true, Ordering::Release); + let outcome = proposal.await; + self.proposing.store(false, Ordering::Release); + outcome + } + + /// The error of a passed deadline. With no proposal in flight it is a + /// definite refusal. With one in flight the outcome is unknown. + pub(super) fn expired(&self) -> crate::Error { + if self.proposing.load(Ordering::Acquire) { + crate::Error::DeadlineExceeded { + request_id: RequestId::new(0), + } + } else { + crate::Error::CrdtAdmissionTimeout { + vshard_id: self.vshard_id, + timeout_ms: self.budget_ms, + } + } + } + + /// `error` of a dispatch that runs before any proposal. A passed deadline + /// there refuses the delta. + pub(super) fn before_propose(&self, error: crate::Error) -> crate::Error { + match error { + crate::Error::DeadlineExceeded { .. } => self.expired(), + other => other, + } + } +} diff --git a/nodedb/src/control/distributed_applier/apply_loop/calvin_read_result.rs b/nodedb/src/control/distributed_applier/apply_loop/calvin_read_result.rs index 2ad1f2fca..c541d6a37 100644 --- a/nodedb/src/control/distributed_applier/apply_loop/calvin_read_result.rs +++ b/nodedb/src/control/distributed_applier/apply_loop/calvin_read_result.rs @@ -13,9 +13,11 @@ //! //! The entry counts as durably applied once its row holds the event. Boot //! resumes Raft delivery above the group's durable applied floor, so the -//! row is what a barrier reads after a restart. A row write that fails -//! leaves the event in memory alone, and the entry breaks the applied -//! prefix: a restart delivers it again. +//! row is what a barrier reads after a restart. The row is visible at once +//! and durable before the floor that covers it: the settle pass persists +//! every saved row before it saves a floor. A row write that fails leaves +//! the event in memory alone, and the entry breaks the applied prefix: a +//! restart delivers it again. use crate::control::array_sync::raft_apply::AppliedPosition; use crate::control::cluster::calvin::BarrierEvent; @@ -253,7 +255,7 @@ fn drop_finished(ctx: ApplyContext<'_>, vshard_id: u32, txn: TxnId) { "calvin: the barrier row of a finished txn was not removed" ); crate::diag::calvin_barrier_log_store_failed( - vshard_id, + Some(vshard_id), Some((txn.epoch, txn.position)), "remove", &e, @@ -262,7 +264,8 @@ fn drop_finished(ctx: ApplyContext<'_>, vshard_id: u32, txn: TxnId) { } /// Save `event` in the row of `txn` on `vshard_id`, then hand it to the -/// vShard's buffer. Returns whether the row holds it. +/// vShard's buffer. Returns whether the row holds it. The settle pass +/// persists the row before a floor covers its entry. /// /// A txn with an event its row lacks skips the row: the row stays a prefix /// of the txn's events, so a restart that delivers the missing entry again @@ -282,7 +285,7 @@ fn save_and_buffer(ctx: ApplyContext<'_>, vshard_id: u32, txn: TxnId, event: Bar "calvin: the barrier event's row write failed; the event stays in memory" ); crate::diag::calvin_barrier_log_store_failed( - vshard_id, + Some(vshard_id), Some((txn.epoch, txn.position)), "save", &e, diff --git a/nodedb/src/control/distributed_applier/apply_loop/context.rs b/nodedb/src/control/distributed_applier/apply_loop/context.rs index 3c7996901..6e2d65016 100644 --- a/nodedb/src/control/distributed_applier/apply_loop/context.rs +++ b/nodedb/src/control/distributed_applier/apply_loop/context.rs @@ -69,6 +69,12 @@ pub(super) enum LoopEvent<'a> { entry: StartedEntry<'a>, }, Finished(FinishedApply), + /// The entry's enqueue or apply panicked. `detail` is the panic payload. + Panicked { + group_id: u64, + log_index: u64, + detail: String, + }, } /// A future the loop collects. diff --git a/nodedb/src/control/distributed_applier/apply_loop/driver.rs b/nodedb/src/control/distributed_applier/apply_loop/driver.rs index abdba3b87..d74984f8b 100644 --- a/nodedb/src/control/distributed_applier/apply_loop/driver.rs +++ b/nodedb/src/control/distributed_applier/apply_loop/driver.rs @@ -5,8 +5,10 @@ //! and every started entry concluded. use std::sync::Arc; +use std::time::Duration; use tokio::sync::{mpsc, watch}; +use tokio::time::MissedTickBehavior; use crate::control::distributed_applier::applier::ApplyBatch; use crate::control::distributed_applier::proposal_ledger::{ @@ -55,7 +57,23 @@ pub async fn run_apply_loop( tracker: &tracker, }; let mut pipeline = Pipeline::new(ctx, ledger); + drive(&mut pipeline, &mut apply_rx, &state).await; +} + +/// Feed `pipeline` from `apply_rx` and its wakes until the channel closes and +/// every started entry concluded. +pub(super) async fn drive( + pipeline: &mut Pipeline<'_>, + apply_rx: &mut mpsc::Receiver, + state: &SharedState, +) { let mut install_released = state.raft_apply_gates.get().map(|g| g.subscribe_released()); + // A held group whose snapshot debt did not persist gets no batches. The + // loop wakes on each Raft tick to write the debt again. + let mut debt_retry = tokio::time::interval(Duration::from_millis( + state.tuning.cluster_transport.raft_tick_interval_ms, + )); + debt_retry.set_missed_tick_behavior(MissedTickBehavior::Delay); let mut accepting = true; loop { if !accepting && !pipeline.has_running() { @@ -64,13 +82,15 @@ pub async fn run_apply_loop( return; } let running = pipeline.has_running(); - let blocked = pipeline.install_blocked(); + let blocked = pipeline.awaits_install(); + let retry_debt = pipeline.retries_debt(); tokio::select! { biased; Some(event) = pipeline.next_event(), if running => { pipeline.handle(event); } () = wait_install_released(&mut install_released), if blocked => {} + _ = debt_retry.tick(), if retry_debt => {} batch = apply_rx.recv(), if accepting => match batch { Some(batch) => { pipeline.accept(batch); diff --git a/nodedb/src/control/distributed_applier/apply_loop/group_watch.rs b/nodedb/src/control/distributed_applier/apply_loop/group_watch.rs index 9d85000cc..1b661d4b3 100644 --- a/nodedb/src/control/distributed_applier/apply_loop/group_watch.rs +++ b/nodedb/src/control/distributed_applier/apply_loop/group_watch.rs @@ -5,8 +5,9 @@ //! floors, which raise the commit HLC of every entry after a cut barrier. use std::collections::{BTreeMap, HashMap}; +use std::sync::Arc; -use crate::control::pitr::restore_point::RecordedCut; +use crate::control::pitr::restore_point::RecordedCuts; /// Per-group apply state. #[derive(Debug, Default)] @@ -15,23 +16,29 @@ pub(super) struct GroupWatch { /// Per group, barrier log index to the lowest commit HLC an entry after /// that barrier records: one above the barrier's watermark. cut_floors: HashMap>, + /// The cut barriers this loop never applied: those of an earlier node + /// life, and those a snapshot install carried. + recorded: Arc, + /// The generation of `recorded` merged into `cut_floors`. + merged: u64, } impl GroupWatch { - /// A watch that knows the restore point cuts this node applied before a - /// restart: the entries it applies again after one of them record above - /// it, as they did the first time. - pub(super) fn with_cuts(cuts: &[RecordedCut]) -> Self { - let mut watch = Self::default(); - for cut in cuts { - watch.raise_cut(cut.group_id, cut.barrier_index, cut.hlc); - } + /// A watch that knows the cuts of `recorded`: the entries it applies + /// after one of them record above it, as on every other replica. + pub(super) fn with_cuts(recorded: Arc) -> Self { + let mut watch = Self { + recorded, + ..Self::default() + }; + watch.merge_recorded(); watch } /// Note that the apply loop applies `(group_id, log_index)`. Reports an /// index at or below one it already applied as a second apply. pub(super) fn note_apply(&mut self, group_id: u64, log_index: u64) { + self.merge_recorded(); let highest = self.highest_applied.entry(group_id).or_insert(0); if log_index <= *highest { crate::diag::raft_entry_reapplied(group_id, log_index, *highest); @@ -69,7 +76,28 @@ impl GroupWatch { .cut_floors .get(&group_id) .and_then(|floors| floors.range(..log_index).map(|(_, floor)| *floor).max()); - floor.map_or(write_hlc, |floor| write_hlc.max(floor)) + // A cut recorded since the last merge binds this entry too. + let unmerged = (self.recorded.generation() != self.merged) + .then(|| self.recorded.watermark_below(group_id, log_index)) + .flatten() + .map(|watermark| watermark.saturating_add(1)); + floor + .into_iter() + .chain(unmerged) + .max() + .map_or(write_hlc, |floor| write_hlc.max(floor)) + } + + /// Merge the cuts `recorded` holds once its generation moved. + fn merge_recorded(&mut self) { + let generation = self.recorded.generation(); + if generation == self.merged { + return; + } + for cut in self.recorded.all() { + self.raise_cut(cut.group_id, cut.barrier_index, cut.hlc); + } + self.merged = generation; } /// Fold every barrier below `log_index` into the highest of them: entries @@ -96,6 +124,7 @@ impl GroupWatch { #[cfg(test)] mod tests { use super::*; + use crate::control::pitr::restore_point::RecordedCut; #[test] fn an_entry_after_a_cut_records_above_the_cut() { @@ -117,6 +146,42 @@ mod tests { ); } + fn watch_of(cuts: &[RecordedCut]) -> GroupWatch { + let recorded = RecordedCuts::new(); + recorded.record(cuts); + GroupWatch::with_cuts(Arc::new(recorded)) + } + + /// A snapshot install carries a barrier the follower never applied. The + /// follower's next entries record above it, as the leader's do. + #[test] + fn an_installed_floor_binds_like_an_applied_one() { + let mut leader = GroupWatch::default(); + leader.note_apply(1, 10); + leader.raise_cut(1, 10, 100); + + let recorded = Arc::new(RecordedCuts::new()); + let mut follower = GroupWatch::with_cuts(Arc::clone(&recorded)); + assert_eq!(follower.commit_hlc(1, 11, 50), 50, "nothing installed yet"); + recorded.replace_group( + 1, + &[RecordedCut { + group_id: 1, + barrier_index: 10, + hlc: 100, + }], + ); + // Read before the entry's apply notes it, as the pipeline stamps it. + assert_eq!(follower.commit_hlc(1, 11, 50), leader.commit_hlc(1, 11, 50)); + follower.note_apply(1, 11); + leader.note_apply(1, 11); + assert_eq!(follower.commit_hlc(1, 11, 50), 101); + assert_eq!( + follower.commit_hlc(1, 12, 300), + leader.commit_hlc(1, 12, 300) + ); + } + #[test] fn an_entry_applied_again_after_a_restart_records_as_before() { let cuts = [ @@ -131,7 +196,7 @@ mod tests { hlc: 200, }, ]; - let mut watch = GroupWatch::with_cuts(&cuts); + let mut watch = watch_of(&cuts); // Delivery resumes between the two barriers. watch.note_apply(1, 15); assert_eq!(watch.commit_hlc(1, 15, 50), 101); @@ -205,7 +270,7 @@ mod tests { // The restart reads the floors back from the catalog. let catalog = SystemCatalog::open(&catalog_path).unwrap(); - let mut watch = GroupWatch::with_cuts(&load_recorded_cuts(&catalog).unwrap()); + let mut watch = watch_of(&load_recorded_cuts(&catalog).unwrap()); watch.note_apply(1, 11); assert_eq!(watch.commit_hlc(1, 11, 50), 101); } diff --git a/nodedb/src/control/distributed_applier/apply_loop/held_entry.rs b/nodedb/src/control/distributed_applier/apply_loop/held_entry.rs new file mode 100644 index 000000000..7cc642bbc --- /dev/null +++ b/nodedb/src/control/distributed_applier/apply_loop/held_entry.rs @@ -0,0 +1,292 @@ +// SPDX-License-Identifier: BUSL-1.1 + +//! A committed entry this replica cannot apply as its peers do holds its +//! group's lane (see [`super::lane`]). Later entries applied on state +//! without it would diverge from the other replicas, and reads would see +//! that state. +//! +//! While the lane holds, the group's replica here owes a snapshot install: +//! - The debt is durable, so a restart keeps it. +//! - The replica refuses log entries, steps down from a leadership it +//! holds, and never campaigns. Its leader sends a snapshot. +//! - The install that covers the held entry releases it and settles the +//! debt. Backlog entries at or below the install conclude as covered. +//! +//! The debt is kept once per install of the group. A debt write that +//! failed is written again on the apply loop's timed wake. + +use std::collections::{BTreeSet, HashMap}; + +use crate::control::state::SharedState; + +use super::context::ApplyContext; +use super::lane::Lane; +use super::proposal_gate::{HoldReason, ProposalGate}; +use super::snapshot_gate::{EntryAdmission, admit_entry}; + +/// Where a group's lane stands against a hold. +pub(super) enum HoldCheck { + /// No entry holds the lane: it can start entries. + Free, + /// An entry holds the lane and no install covers it yet. + Held, + /// An install holds or awaits the group's apply gate. + Installing, +} + +/// Note that the entry at `log_index` of `group_id` entered a hold. The +/// group's debt is recorded by [`keep_snapshot_owed`], under the group's +/// apply gate. +pub(super) fn note_hold(group_id: u64, log_index: u64, reason: HoldReason) { + tracing::error!( + group_id, + log_index, + reason = reason.as_str(), + "apply loop: this replica cannot apply the committed entry as its peers do; the \ + group's apply holds at it until a snapshot of the group installs" + ); +} + +/// The snapshot debt this loop kept for each held group. +#[derive(Default)] +pub(super) struct HeldDebts { + groups: HashMap, +} + +#[derive(Clone, Copy)] +struct KeptDebt { + /// The group's installed index when the debt was kept. + installed_through: u64, + /// Whether the debt write persisted. + recorded: bool, +} + +/// What a pass does for the debt of a held group. +#[derive(Debug, PartialEq, Eq)] +enum DebtStep { + /// The debt persisted after the group's last install. + Kept, + /// Write the debt. + Write { + /// Raise the replica's snapshot requirement too: no install lifted it + /// since this loop last raised it. + raise: bool, + /// The previous write failed: this write continues that failure run. + failing: bool, + }, +} + +impl HeldDebts { + /// An install settles the debt and can lift the requirement, so a hold + /// that outlives one keeps both again. + fn step(&self, group_id: u64, installed_through: u64) -> DebtStep { + match self.groups.get(&group_id) { + Some(kept) if kept.installed_through == installed_through && kept.recorded => { + DebtStep::Kept + } + Some(kept) => DebtStep::Write { + raise: kept.installed_through != installed_through, + failing: !kept.recorded, + }, + None => DebtStep::Write { + raise: true, + failing: false, + }, + } + } + + fn note(&mut self, group_id: u64, installed_through: u64, recorded: bool) { + self.groups.insert( + group_id, + KeptDebt { + installed_through, + recorded, + }, + ); + } + + /// Drop the record of `group_id`: no entry holds its lane. + fn forget(&mut self, group_id: u64) { + self.groups.remove(&group_id); + } + + /// Whether a held group's debt write failed. The apply loop wakes on a + /// timer to write it again. + pub fn retry_due(&self) -> bool { + self.groups.values().any(|kept| !kept.recorded) + } +} + +/// Record the group's snapshot debt and raise its replica's snapshot +/// requirement, once per install of the group. `installed_through` is the +/// group's installed index as the caller read it. +/// +/// The caller holds the group's apply gate or read it open below the held +/// entry, so no install that covers the entry settled the debt first. +pub(super) fn keep_snapshot_owed( + state: &SharedState, + debts: &mut HeldDebts, + group_id: u64, + log_index: u64, + installed_through: u64, +) { + let DebtStep::Write { raise, failing } = debts.step(group_id, installed_through) else { + return; + }; + let written = if state.redo_chunks.owes_snapshot(group_id) { + Ok(()) + } else { + state + .redo_chunks + .record_owed(state.credentials.catalog(), &BTreeSet::from([group_id])) + }; + let recorded = match written { + Ok(()) => { + if failing { + tracing::info!( + group_id, + log_index, + "apply loop: the snapshot debt of a held group persisted" + ); + } + true + } + // One report per failure run: the timed retries collapse into it. + Err(error) if failing => { + tracing::debug!( + group_id, + log_index, + %error, + "apply loop: the snapshot debt of a held group still did not persist" + ); + false + } + Err(error) => { + tracing::error!( + group_id, + log_index, + %error, + "apply loop: the snapshot debt of a held group did not persist; the loop \ + writes it again on each Raft tick" + ); + crate::diag::held_entry_debt_not_recorded(group_id, log_index, &error); + false + } + }; + debts.note(group_id, installed_through, recorded); + // Once the debt persists, the redo-stream membership pass holds the + // requirement up. Until then only this loop does. + if !raise && recorded { + return; + } + let Some(multi_raft) = state.multi_raft.get() else { + return; + }; + // The data here diverged from the log: the replica must not lead. + multi_raft + .lock() + .unwrap_or_else(|p| p.into_inner()) + .require_snapshot(group_id, nodedb_raft::ReplicaState::AwaitingSnapshot); +} + +/// Release the entry that holds `lane` once an installed snapshot covers +/// it. The install's index is read under the group's apply gate, as an +/// entry's admission reads it. An entry still held keeps its group owing a +/// snapshot. +pub(super) fn release_covered_hold( + ctx: ApplyContext<'_>, + lane: &mut Lane, + gate: &mut ProposalGate, + debts: &mut HeldDebts, + group_id: u64, +) -> HoldCheck { + let Some(held) = lane.held_index() else { + debts.forget(group_id); + return HoldCheck::Free; + }; + let gates = ctx.state.raft_apply_gates.get().map(|gates| &**gates); + let covered_through = ctx.tracker.covered_through(group_id); + let admission = admit_entry(gates, group_id, held, covered_through); + let installed_through = admission.installed_through; + // The permit lives to the end: no install starts before the debt is + // kept. + let (through, _permit) = match admission.decision { + EntryAdmission::Installing => return HoldCheck::Installing, + EntryAdmission::Start(permit) => (admission.installed_through, permit), + EntryAdmission::Covered => (admission.installed_through, None), + // Above the install's Raft index and at or below its cut: the + // installed state holds the entry. + EntryAdmission::CoveredByCut => (covered_through, None), + }; + match lane.cover_through(through) { + Some(proposal_key) => { + gate.note_covered(proposal_key); + debts.forget(group_id); + tracing::info!( + group_id, + log_index = held, + installed_through = through, + "apply loop: an installed snapshot covers the held entry; the group's apply \ + resumes" + ); + HoldCheck::Free + } + None => { + keep_snapshot_owed(ctx.state, debts, group_id, held, installed_through); + HoldCheck::Held + } + } +} + +#[cfg(test)] +mod tests { + use super::*; + + const GROUP: u64 = 4; + + /// The debt is written once per install of the group: an install + /// settles it and can lift the requirement. + #[test] + fn a_held_debt_is_kept_once_per_install() { + let mut debts = HeldDebts::default(); + let first = DebtStep::Write { + raise: true, + failing: false, + }; + assert_eq!(debts.step(GROUP, 10), first); + debts.note(GROUP, 10, true); + assert_eq!(debts.step(GROUP, 10), DebtStep::Kept); + assert!(!debts.retry_due()); + assert_eq!(debts.step(GROUP, 12), first, "an install settled the debt"); + } + + /// A failed write is retried as one failure run, across installs, until + /// a write persists or the hold ends. + #[test] + fn a_failed_debt_write_retries_as_one_failure_run() { + let mut debts = HeldDebts::default(); + debts.note(GROUP, 10, false); + assert!(debts.retry_due()); + assert_eq!( + debts.step(GROUP, 10), + DebtStep::Write { + raise: false, + failing: true, + } + ); + assert_eq!( + debts.step(GROUP, 12), + DebtStep::Write { + raise: true, + failing: true, + } + ); + + debts.note(GROUP, 10, true); + assert!(!debts.retry_due()); + + debts.note(GROUP, 10, false); + debts.forget(GROUP); + assert!(!debts.retry_due(), "a released hold retries nothing"); + } +} diff --git a/nodedb/src/control/distributed_applier/apply_loop/lane.rs b/nodedb/src/control/distributed_applier/apply_loop/lane.rs index 9cf667612..04b3a3e6f 100644 --- a/nodedb/src/control/distributed_applier/apply_loop/lane.rs +++ b/nodedb/src/control/distributed_applier/apply_loop/lane.rs @@ -9,6 +9,10 @@ //! order: the group's applied index is the highest index with every earlier //! entry finished, and its durable floor is the highest index with every //! earlier entry durable. +//! +//! An entry this replica cannot apply as its peers do holds the lane. It +//! never settles, the applied index stays below it, and no later entry +//! starts. Only a snapshot install that covers it releases it. use std::collections::VecDeque; @@ -21,7 +25,7 @@ use crate::control::state::tenant_marks::{MarkSite, TenantMarks}; use crate::control::wal_replication::types::CalvinRedoMeta; use crate::control::wal_replication::{ReplicatedEntry, ReplicatedWrite, decode_replicated_entry}; -use super::proposal_gate::PrefixStep; +use super::proposal_gate::{PrefixStep, Settlement}; /// A committed entry handed to the loop and not yet started. pub(super) struct QueuedEntry { @@ -146,6 +150,18 @@ pub(super) enum SlotState { Barrier, /// The entry concluded. Concluded(PrefixStep), + /// The entry holds the lane until a snapshot install covers it. Its + /// reason is reported when the hold starts. + Held, +} + +impl From for SlotState { + fn from(settlement: Settlement) -> Self { + match settlement { + Settlement::Settles(step) => Self::Concluded(step), + Settlement::Holds(_) => Self::Held, + } + } } /// A started entry not yet settled. @@ -199,6 +215,11 @@ impl Lane { } } + /// The group this lane applies. + pub fn group_id(&self) -> u64 { + self.group_id + } + /// The first started entry not yet settled. pub fn front_index(&self) -> Option { self.slots.front().map(|slot| slot.log_index) @@ -209,6 +230,31 @@ impl Lane { self.last_settled } + /// The entry that holds the lane, if any. No entry after it starts, so + /// it is the last started entry. + pub fn held_index(&self) -> Option { + self.slots.iter().find_map(|slot| match slot.state { + SlotState::Held => Some(slot.log_index), + SlotState::Starting + | SlotState::Running + | SlotState::Barrier + | SlotState::Concluded(_) => None, + }) + } + + /// Whether an entry holds the lane. + pub fn is_held(&self) -> bool { + self.held_index().is_some() + } + + /// The proposal key of the entry at `log_index` while it starts or runs. + pub fn running_key(&self, log_index: u64) -> Option { + self.slots.iter().find_map(|slot| { + let running = matches!(slot.state, SlotState::Starting | SlotState::Running); + (slot.log_index == log_index && running).then_some(slot.proposal_key) + }) + } + /// Whether any started entry of the group has not concluded. pub fn has_running(&self) -> bool { self.slots @@ -257,7 +303,7 @@ impl Lane { } /// Conclude the running entry at `log_index`: `conclude` receives its - /// proposal key and returns how it moves the prefix. `wrote_rows` says + /// proposal key and returns how it stands in the lane. `wrote_rows` says /// whether the apply wrote the entry's rows; an entry that wrote none /// raises no tenant write mark. Returns `false` when no running entry /// has that index. @@ -265,7 +311,7 @@ impl Lane { &mut self, log_index: u64, wrote_rows: bool, - conclude: impl FnOnce(u64) -> PrefixStep, + conclude: impl FnOnce(u64) -> Settlement, ) -> bool { let Some(slot) = self.slot_mut(log_index) else { return false; @@ -274,7 +320,7 @@ impl Lane { return false; } slot.user_write &= wrote_rows; - slot.state = SlotState::Concluded(conclude(slot.proposal_key)); + slot.state = SlotState::from(conclude(slot.proposal_key)); if self.blocking == Some(log_index) { self.blocking = None; } @@ -285,12 +331,13 @@ impl Lane { /// barrier's waiter, raise the tenant's write mark in this group, extend /// or break the durable prefix, and advance the applied watermark. The /// mark rises first, so a reader that waited for the applied index sees - /// it. Returns how many entries settled. + /// it. A held entry stops the pass: the applied index stays below it. + /// Returns how many entries settled. pub fn settle(&mut self, tracker: &ProposeTracker, marks: &TenantMarks) -> usize { let mut settled = 0; while let Some(front) = self.slots.front() { let step = match front.state { - SlotState::Starting | SlotState::Running => break, + SlotState::Starting | SlotState::Running | SlotState::Held => break, SlotState::Barrier => { // Every entry before the barrier finished; a waiting // backup can snapshot this group now. @@ -352,13 +399,19 @@ impl Lane { /// Count every entry of the group through `installed_through` durable. /// An installed snapshot holds their state, so a break at or below it - /// heals. + /// heals, and a held entry at or below it concludes. Returns the held + /// entry's proposal key when one concludes. /// /// The caller passes the index it read under the group's apply gate, /// before it admits an entry. Every entry above the index reaches the /// lane after the install, so none settles before this call. - pub fn cover_through(&mut self, installed_through: u64) { + pub fn cover_through(&mut self, installed_through: u64) -> Option { self.prefix.cover_through(installed_through); + let slot = self.slots.iter_mut().find(|slot| { + slot.log_index <= installed_through && matches!(slot.state, SlotState::Held) + })?; + slot.state = SlotState::Concluded(PrefixStep::Record(true)); + Some(slot.proposal_key) } /// Whether the durable floor moved past the floor last saved. @@ -381,8 +434,17 @@ impl Lane { #[cfg(test)] mod tests { + use super::super::proposal_gate::HoldReason; use super::*; + fn settles(durable: bool) -> Settlement { + Settlement::Settles(PrefixStep::Record(durable)) + } + + fn holds() -> Settlement { + Settlement::Holds(HoldReason::BodyMissing) + } + fn slot(log_index: u64, state: SlotState) -> Slot { Slot { log_index, @@ -402,10 +464,10 @@ mod tests { lane.push(slot(6, SlotState::Running)); lane.push(slot(7, SlotState::Running)); - assert!(lane.conclude(7, true, |_| PrefixStep::Record(true))); - assert!(lane.conclude(6, true, |_| PrefixStep::Record(true))); + assert!(lane.conclude(7, true, |_| settles(true))); + assert!(lane.conclude(6, true, |_| settles(true))); assert!( - !lane.conclude(6, true, |_| PrefixStep::Record(true)), + !lane.conclude(6, true, |_| settles(true)), "6 concluded already" ); assert_eq!( @@ -420,24 +482,86 @@ mod tests { "the timeout diagnostic names the entry the group waits behind" ); - assert!(lane.conclude(5, true, |_| PrefixStep::Record(true))); + assert!(lane.conclude(5, true, |_| settles(true))); assert_eq!(lane.settle(&tracker, &TenantMarks::default()), 3); assert_eq!(lane.take_floor_to_save(), Some(7)); assert_eq!(lane.take_floor_to_save(), None, "a saved floor saves once"); assert!(tracker.applying(1).is_none()); } + /// A held entry neither settles nor moves the applied index. Entries + /// before it settle. #[test] - fn a_non_durable_entry_holds_the_floor_across_later_settles() { + fn a_held_entry_neither_settles_nor_advances_the_applied_index() { + let watchers = std::sync::Arc::new(nodedb_cluster::GroupAppliedWatchers::new()); + let tracker = ProposeTracker::new().with_group_watchers(std::sync::Arc::clone(&watchers)); + let marks = TenantMarks::default(); + let mut lane = Lane::new(1); + lane.push(slot(1, SlotState::Running)); + lane.push(slot(2, SlotState::Running)); + assert!(lane.conclude(2, false, |_| holds())); + assert_eq!(lane.held_index(), Some(2)); + assert!(lane.has_running(), "entry 1 still runs"); + + assert!(lane.conclude(1, true, |_| settles(true))); + assert_eq!(lane.settle(&tracker, &marks), 1); + assert_eq!(watchers.get_or_create(1).current(), 1); + assert_eq!(lane.take_floor_to_save(), Some(1)); + + assert_eq!(lane.settle(&tracker, &marks), 0, "entry 2 holds the lane"); + assert_eq!( + watchers.get_or_create(1).current(), + 1, + "the applied index stays below the held entry" + ); + assert_eq!( + tracker.applying(1).map(|entry| entry.log_index), + Some(2), + "the group waits behind the held entry" + ); + assert_eq!(lane.take_floor_to_save(), None); + } + + /// A snapshot that installs below the held entry leaves it held. One + /// that covers it concludes it, and the floor passes it. + #[test] + fn only_a_covering_snapshot_releases_a_held_entry() { + let tracker = ProposeTracker::new(); + let marks = TenantMarks::default(); + let mut lane = Lane::new(1); + lane.push(slot(1, SlotState::Concluded(PrefixStep::Record(true)))); + let mut held = slot(2, SlotState::Running); + held.proposal_key = 77; + lane.push(held); + assert!(lane.conclude(2, false, |_| holds())); + lane.settle(&tracker, &marks); + assert_eq!(lane.take_floor_to_save(), Some(1)); + + assert_eq!(lane.cover_through(1), None, "the install is below entry 2"); + assert!(lane.is_held()); + assert_eq!(lane.settle(&tracker, &marks), 0); + + assert_eq!(lane.cover_through(4), Some(77)); + assert!(!lane.is_held()); + assert_eq!(lane.settle(&tracker, &marks), 1); + assert_eq!(lane.last_settled(), Some(2)); + assert_eq!(lane.take_floor_to_save(), Some(4)); + } + + /// A transient failure breaks the floor and the lane moves on. A later + /// entry still settles, and the floor holds below the break. + #[test] + fn a_non_durable_entry_holds_the_floor_but_not_the_lane() { let tracker = ProposeTracker::new(); let mut lane = Lane::new(1); lane.push(slot(1, SlotState::Concluded(PrefixStep::Record(true)))); lane.push(slot(2, SlotState::Concluded(PrefixStep::Record(false)))); - lane.settle(&tracker, &TenantMarks::default()); + assert_eq!(lane.settle(&tracker, &TenantMarks::default()), 2); assert_eq!(lane.take_floor_to_save(), Some(1)); lane.push(slot(3, SlotState::Concluded(PrefixStep::Record(true)))); - lane.settle(&tracker, &TenantMarks::default()); + assert_eq!(lane.settle(&tracker, &TenantMarks::default()), 1); + assert!(!lane.is_held()); assert_eq!( lane.take_floor_to_save(), None, @@ -446,7 +570,7 @@ mod tests { } #[test] - fn a_covering_snapshot_lets_the_floor_pass_a_held_entry() { + fn a_covering_snapshot_lets_the_floor_pass_a_broken_entry() { let tracker = ProposeTracker::new(); let marks = TenantMarks::default(); let mut lane = Lane::new(1); @@ -457,10 +581,10 @@ mod tests { assert_eq!(lane.take_floor_to_save(), Some(1)); // A snapshot at index 4 installs while entry 3 still runs. - lane.cover_through(4); + assert_eq!(lane.cover_through(4), None); assert_eq!(lane.take_floor_to_save(), Some(4)); - assert!(lane.conclude(3, false, |_| PrefixStep::Record(false))); + assert!(lane.conclude(3, false, |_| settles(false))); lane.push(slot(5, SlotState::Concluded(PrefixStep::Record(true)))); lane.settle(&tracker, &marks); assert_eq!( @@ -471,11 +595,11 @@ mod tests { } #[test] - fn a_held_entry_above_the_snapshot_still_holds_the_floor() { + fn a_broken_entry_above_the_snapshot_still_holds_the_floor() { let tracker = ProposeTracker::new(); let marks = TenantMarks::default(); let mut lane = Lane::new(1); - lane.cover_through(4); + assert_eq!(lane.cover_through(4), None); lane.push(slot(5, SlotState::Concluded(PrefixStep::Record(false)))); lane.push(slot(6, SlotState::Concluded(PrefixStep::Record(true)))); lane.settle(&tracker, &marks); @@ -559,7 +683,7 @@ mod tests { refused.collection = Some("docs".to_owned()); lane.push(refused); - assert!(lane.conclude(1, false, |_| PrefixStep::Record(true))); + assert!(lane.conclude(1, false, |_| settles(true))); lane.settle(&tracker, &marks); assert_eq!( marks.get(4, 7), @@ -579,7 +703,7 @@ mod tests { lane.settle(&tracker, &TenantMarks::default()); assert!(barrier.try_recv().is_err(), "entry 1 still runs"); - assert!(lane.conclude(1, true, |_| PrefixStep::Record(true))); + assert!(lane.conclude(1, true, |_| settles(true))); lane.settle(&tracker, &TenantMarks::default()); assert!(matches!(barrier.try_recv(), Ok(Ok(_)))); } diff --git a/nodedb/src/control/distributed_applier/apply_loop/metadata_floor.rs b/nodedb/src/control/distributed_applier/apply_loop/metadata_floor.rs index 9194ee802..180afd7c2 100644 --- a/nodedb/src/control/distributed_applier/apply_loop/metadata_floor.rs +++ b/nodedb/src/control/distributed_applier/apply_loop/metadata_floor.rs @@ -85,17 +85,11 @@ async fn await_metadata_floor(state: &SharedState, held: HeldEntry) -> crate::Re metadata_applied = watcher.current(), "a replicated write holds its group's lane for this node's metadata apply" ); + let floor = held.metadata_floor; loop { - let waiting = Arc::clone(&watcher); - let floor = held.metadata_floor; - let outcome = tokio::task::spawn_blocking(move || waiting.wait_for(floor, WAIT_SLICE)) - .await - .map_err(|e| crate::Error::Internal { - detail: format!( - "raft group {} entry {}: the metadata catch-up wait did not finish: {e}", - held.group_id, held.log_index - ), - })?; + let outcome = watcher + .wait_for_async(floor, std::time::Instant::now() + WAIT_SLICE) + .await; match outcome { WaitOutcome::Reached => return Ok(()), WaitOutcome::TimedOut => tracing::warn!( diff --git a/nodedb/src/control/distributed_applier/apply_loop/mod.rs b/nodedb/src/control/distributed_applier/apply_loop/mod.rs index 031510b6d..2f1c023f5 100644 --- a/nodedb/src/control/distributed_applier/apply_loop/mod.rs +++ b/nodedb/src/control/distributed_applier/apply_loop/mod.rs @@ -18,6 +18,9 @@ //! applies. //! - [`pipeline`]: every group's lane and the applies that run. //! - [`lane`]: one group's queued and started entries, settled in log order. +//! - [`settle`]: settles every group's concluded entries and saves the +//! floors that moved. +//! - [`panicked`]: an entry whose enqueue or apply panicked holds its group. //! - [`start`]: prepares one entry and routes it to its apply path. //! - [`cut_barrier`]: a backup cut's barrier: the group's cut floor, the //! restore point's place, and a database backup's capture. @@ -41,6 +44,8 @@ //! and records each applied proposal in the ledger. //! - [`snapshot_gate`]: skips an entry an installed snapshot covers, and //! orders each write before a later snapshot restore. +//! - [`held_entry`]: an entry this replica cannot apply as its peers do +//! holds its group's lane until a snapshot install covers it. //! - [`group_watch`]: per-group second-apply detection and backup cut floors. //! - [`bookkeeping`]: applied-floor persistence + Raft log compaction trigger. //! - [`helpers`]: shared response/result classification helpers. @@ -61,12 +66,15 @@ mod context; mod cut_barrier; mod driver; mod group_watch; +mod held_entry; mod helpers; mod lane; mod metadata_floor; +mod panicked; mod pipeline; mod proposal_gate; mod redo_chunk; +mod settle; mod snapshot_gate; mod start; mod surrogate_bind; diff --git a/nodedb/src/control/distributed_applier/apply_loop/panicked.rs b/nodedb/src/control/distributed_applier/apply_loop/panicked.rs new file mode 100644 index 000000000..4bedfec57 --- /dev/null +++ b/nodedb/src/control/distributed_applier/apply_loop/panicked.rs @@ -0,0 +1,96 @@ +// SPDX-License-Identifier: BUSL-1.1 + +//! An entry whose enqueue or apply panicked. +//! +//! The loop polls every running entry in one task, so an unguarded panic +//! ends the loop for every group and leaves each waiter to its deadline. +//! A guarded panic concludes its own entry instead: +//! - Its effect on this replica is unknown, so the entry holds its group's +//! lane like an entry this replica cannot apply as its peers do. No later +//! entry of the group starts until a snapshot install covers it. +//! - Its waiter receives an error at once. +//! - The panic files a report. +//! +//! Other groups keep applying. + +use std::panic::AssertUnwindSafe; + +use futures::FutureExt; + +use crate::data::panic_payload::panic_payload_to_string; + +use super::context::{ApplyContext, ApplyFuture, LoopEvent, LoopFuture}; +use super::lane::{Lane, SlotState}; +use super::proposal_gate::{EntryOutcome, HoldReason, ProposalGate}; + +/// Run `future`, the enqueue or apply of the entry at `log_index` of +/// `group_id`. A panic inside it becomes [`LoopEvent::Panicked`]. +pub(super) fn guarded(group_id: u64, log_index: u64, future: LoopFuture<'_>) -> LoopFuture<'_> { + Box::pin( + AssertUnwindSafe(future) + .catch_unwind() + .map(move |polled| match polled { + Ok(event) => event, + Err(payload) => LoopEvent::Panicked { + group_id, + log_index, + detail: panic_payload_to_string(payload.as_ref()), + }, + }), + ) +} + +/// The guarded event of an entry's running apply. +pub(super) fn finished_event( + group_id: u64, + log_index: u64, + apply: ApplyFuture<'_>, +) -> LoopFuture<'_> { + guarded( + group_id, + log_index, + Box::pin(apply.map(LoopEvent::Finished)), + ) +} + +/// Conclude the entry at `log_index` whose enqueue or apply panicked: report +/// it, answer its waiter with an error, and hold `lane`. Returns `false` when +/// no entry at that index starts or runs. +pub(super) fn conclude_panicked( + ctx: ApplyContext<'_>, + lane: &mut Lane, + gate: &mut ProposalGate, + log_index: u64, + detail: &str, +) -> bool { + let group_id = lane.group_id(); + let Some(proposal_key) = lane.running_key(log_index) else { + return false; + }; + tracing::error!( + group_id, + log_index, + "apply loop: the committed entry's apply panicked; the group's apply holds at it \ + until a snapshot of the group installs" + ); + crate::diag::apply_panicked(group_id, log_index, detail); + ctx.tracker.complete( + group_id, + log_index, + proposal_key, + Err(crate::Error::Internal { + detail: format!( + "the apply of group {group_id} entry {log_index} panicked on this replica; \ + the group's apply holds until a snapshot installs" + ), + }), + ); + ctx.tracker.note_concluded(group_id, log_index); + // A panicked enqueue leaves its entry starting. A panicked apply leaves + // it running, and this is a no-op. + lane.enqueued(log_index, SlotState::Running, None, false); + let outcome = EntryOutcome::Held { + reason: HoldReason::ApplyPanicked, + }; + lane.conclude(log_index, false, |key| gate.conclude(key, true, outcome)) +} diff --git a/nodedb/src/control/distributed_applier/apply_loop/pipeline.rs b/nodedb/src/control/distributed_applier/apply_loop/pipeline.rs index f99e02c5e..ad5dd1fc7 100644 --- a/nodedb/src/control/distributed_applier/apply_loop/pipeline.rs +++ b/nodedb/src/control/distributed_applier/apply_loop/pipeline.rs @@ -9,6 +9,10 @@ //! and no other: later entries of its group start and finish, and other //! groups never wait on it. Only the group's applied index and durable floor //! wait for it to finish. +//! +//! An entry this replica cannot apply as its peers do holds its group: no +//! later entry of the group starts until a snapshot install covers it (see +//! [`super::held_entry`]). use std::collections::HashMap; use std::sync::Arc; @@ -18,15 +22,17 @@ use futures::stream::{FuturesUnordered, StreamExt}; use crate::control::distributed_applier::applier::ApplyBatch; use crate::control::distributed_applier::proposal_ledger::ProposalLedger; -use crate::control::server::dispatch_utils::publish_settled_changes; use crate::control::state::SharedState; -use super::bookkeeping::record_durable_apply; -use super::context::{ApplyContext, ApplyFuture, LoopEvent, LoopFuture, Started, StartedEntry}; +use super::context::{ApplyContext, LoopEvent, LoopFuture, Started, StartedEntry}; use super::group_watch::GroupWatch; +use super::held_entry::{ + HeldDebts, HoldCheck, keep_snapshot_owed, note_hold, release_covered_hold, +}; use super::lane::{Lane, QueuedEntry, Slot, SlotState}; use super::metadata_floor::{HeldEntry, hold_for_metadata}; -use super::proposal_gate::{EntryOutcome, ProposalGate}; +use super::panicked::{conclude_panicked, finished_event, guarded}; +use super::proposal_gate::{EntryOutcome, HoldReason, ProposalGate}; use super::snapshot_gate::{EntryAdmission, admit_entry, hold_permit}; use super::start::{Prepared, prepare_entry}; @@ -37,6 +43,7 @@ pub(super) struct Pipeline<'a> { gate: ProposalGate, watch: GroupWatch, running: FuturesUnordered>, + debts: HeldDebts, /// Set by a pump that left an entry queued because a snapshot install /// held its group's apply gate. install_blocked: bool, @@ -50,14 +57,22 @@ impl<'a> Pipeline<'a> { gate: ProposalGate::new(ledger), watch: GroupWatch::with_cuts(ctx.state.pitr.recorded_cuts()), running: FuturesUnordered::new(), + debts: HeldDebts::default(), install_blocked: false, } } - /// Whether the last pump left an entry queued behind a snapshot install. - /// The loop pumps again once an install releases its gate. - pub fn install_blocked(&self) -> bool { - self.install_blocked + /// Whether a group waits for a snapshot install: the last pump left an + /// entry queued behind one, or an entry holds its group's lane. The loop + /// pumps again once an install releases its gate. + pub fn awaits_install(&self) -> bool { + self.install_blocked || self.lanes.values().any(Lane::is_held) + } + + /// Whether a held group's snapshot debt did not persist. The loop wakes + /// on a timer to write it again. + pub fn retries_debt(&self) -> bool { + self.debts.retry_due() } /// Queue a batch the applier handed off, behind its group's earlier @@ -116,13 +131,34 @@ impl<'a> Pipeline<'a> { .note_concluded(finished.group_id, finished.log_index); let gate = &mut self.gate; let wrote_rows = finished.outcome.wrote_rows(); + let hold = finished.outcome.hold_reason(); let handled = self.lanes.get_mut(&finished.group_id).is_some_and(|lane| { lane.conclude(finished.log_index, wrote_rows, |proposal_key| { gate.conclude(proposal_key, true, finished.outcome) }) }); + // The pump that follows records the group's debt under its + // apply gate. + if handled && let Some(reason) = hold { + note_hold(finished.group_id, finished.log_index, reason); + } (finished.group_id, finished.log_index, handled) } + LoopEvent::Panicked { + group_id, + log_index, + detail, + } => { + let gate = &mut self.gate; + let handled = self.lanes.get_mut(&group_id).is_some_and(|lane| { + conclude_panicked(self.ctx, lane, gate, log_index, &detail) + }); + // The pump that follows records the group's debt. + if handled { + note_hold(group_id, log_index, HoldReason::ApplyPanicked); + } + (group_id, log_index, handled) + } }; // The entry's enqueue returned, or its exclusive apply finished: // every entry of the group through it reached its core. @@ -150,7 +186,8 @@ impl<'a> Pipeline<'a> { }; match started { Started::Running(apply) => { - self.running.push(finished_event(apply)); + self.running + .push(finished_event(group_id, log_index, apply)); lane.enqueued(log_index, SlotState::Running, collection, user_write) } Started::Concluded(outcome) => { @@ -159,10 +196,15 @@ impl<'a> Pipeline<'a> { self.ctx.tracker.note_concluded(group_id, log_index); let gate = &mut self.gate; let wrote_rows = outcome.wrote_rows(); - lane.enqueued(log_index, SlotState::Running, collection, user_write) + let hold = outcome.hold_reason(); + let handled = lane.enqueued(log_index, SlotState::Running, collection, user_write) && lane.conclude(log_index, wrote_rows, |proposal_key| { gate.conclude(proposal_key, true, outcome) - }) + }); + if handled && let Some(reason) = hold { + note_hold(group_id, log_index, reason); + } + handled } } } @@ -178,7 +220,7 @@ impl<'a> Pipeline<'a> { let groups: Vec = self .lanes .iter() - .filter(|(_, lane)| !lane.backlog.is_empty()) + .filter(|(_, lane)| !lane.backlog.is_empty() || lane.is_held()) .map(|(group_id, _)| *group_id) .collect(); for group_id in groups { @@ -196,6 +238,17 @@ impl<'a> Pipeline<'a> { fn pump_group(&mut self, group_id: u64) { let shared: &'a Arc = self.ctx.state; let gates = shared.raft_apply_gates.get().map(|gates| &**gates); + let Some(lane) = self.lanes.get_mut(&group_id) else { + return; + }; + match release_covered_hold(self.ctx, lane, &mut self.gate, &mut self.debts, group_id) { + HoldCheck::Free => {} + HoldCheck::Held => return, + HoldCheck::Installing => { + self.install_blocked = true; + return; + } + } while let Some(queued) = self.next_startable(group_id) { let log_index = queued.entry.index; let proposal_key = queued.proposal_key(); @@ -203,8 +256,12 @@ impl<'a> Pipeline<'a> { let admission = admit_entry(gates, group_id, log_index, covered_through); // Before the entry joins the lane: no entry above the installed // index settles before the lane covers it. - if let Some(lane) = self.lanes.get_mut(&group_id) { - lane.cover_through(admission.installed_through); + if let Some(lane) = self.lanes.get_mut(&group_id) + && let Some(proposal_key) = lane.cover_through(admission.installed_through) + { + // The install holds every entry through its index: a held + // one concludes. + self.gate.note_covered(proposal_key); } let permit = match admission.decision { EntryAdmission::Installing => { @@ -282,12 +339,14 @@ impl<'a> Pipeline<'a> { held, prepare_entry(self.ctx, &mut self.watch, &self.gate, group_id, queued), ); + let mut hold = None; let (state, blocks, user_write) = match prepared { Prepared::Concluded(outcome) => { release_admission_holds(shared, group_id, log_index); let user_write = matches!(outcome, EntryOutcome::Repeat) && repeat_writes; + hold = outcome.hold_reason(); ( - SlotState::Concluded(self.gate.conclude(proposal_key, false, outcome)), + SlotState::from(self.gate.conclude(proposal_key, false, outcome)), false, user_write, ) @@ -300,12 +359,12 @@ impl<'a> Pipeline<'a> { self.gate.open(proposal_key); self.ctx.tracker.note_dispatched(group_id, log_index); let enqueue = hold_permit(permit, enqueue); - self.running - .push(Box::pin(enqueue.map(move |entry| LoopEvent::Enqueued { - group_id, - log_index, - entry, - }))); + let enqueued = Box::pin(enqueue.map(move |entry| LoopEvent::Enqueued { + group_id, + log_index, + entry, + })); + self.running.push(guarded(group_id, log_index, enqueued)); // The enqueue reports whether the plan writes user data. (SlotState::Starting, true, false) } @@ -313,8 +372,11 @@ impl<'a> Pipeline<'a> { // An array op or cell write: user data. self.gate.open(proposal_key); self.ctx.tracker.note_dispatched(group_id, log_index); - self.running - .push(finished_event(hold_permit(permit, apply))); + self.running.push(finished_event( + group_id, + log_index, + hold_permit(permit, apply), + )); (SlotState::Running, true, true) } }; @@ -332,6 +394,19 @@ impl<'a> Pipeline<'a> { user_write, state, }); + // The entry's admission permit is still held: no install that + // covers the entry settles the debt first. + if let Some(reason) = hold { + note_hold(group_id, log_index, reason); + let installed_through = admission.installed_through; + keep_snapshot_owed( + shared, + &mut self.debts, + group_id, + log_index, + installed_through, + ); + } } } @@ -350,7 +425,7 @@ impl<'a> Pipeline<'a> { self.ctx .tracker .complete_covered(group_id, log_index, proposal_key); - let concluded = SlotState::Concluded(self.gate.conclude(proposal_key, false, outcome)); + let concluded = SlotState::from(self.gate.conclude(proposal_key, false, outcome)); let Some(lane) = self.lanes.get_mut(&group_id) else { return false; }; @@ -367,13 +442,14 @@ impl<'a> Pipeline<'a> { /// Take the next entry of `group_id` when it can start now. /// - /// It waits while the group's previous write is in its enqueue or an - /// exclusive entry of the group runs, while it is exclusive and an - /// earlier entry of the group has not concluded, and while a copy of its - /// proposal runs: the ledger decides it once that copy concludes. + /// It waits while an entry holds the group's lane, while the group's + /// previous write is in its enqueue or an exclusive entry of the group + /// runs, while it is exclusive and an earlier entry of the group has not + /// concluded, and while a copy of its proposal runs: the ledger decides + /// it once that copy concludes. fn next_startable(&mut self, group_id: u64) -> Option { let lane = self.lanes.get_mut(&group_id)?; - if lane.blocking.is_some() { + if lane.blocking.is_some() || lane.is_held() { return None; } let front = lane.backlog.front()?; @@ -386,49 +462,10 @@ impl<'a> Pipeline<'a> { lane.backlog.pop_front() } - /// Settle every group's concluded entries in log order, release their - /// window, and save each durable floor that moved. - /// - /// The tenant write marks of the settled entries are persisted before any - /// floor that covers them. An entry above the saved floor is delivered - /// again after a restart and records its mark again, so every committed - /// write keeps a durable mark. + /// Settle every group's concluded entries in log order. See + /// [`super::settle::settle_lanes`]. pub fn settle(&mut self) { - let state = self.ctx.state; - let tracker = self.ctx.tracker; - for (group_id, lane) in &mut self.lanes { - let first = lane.front_index(); - let settled = lane.settle(tracker, &state.tenant_marks); - if settled > 0 { - tracker.window().release(*group_id, settled); - // The entries' change events publish in log order, at their - // log positions, on every replica. - if let (Some(first), Some(last)) = (first, lane.last_settled()) { - publish_settled_changes(state, *group_id, first, last); - } - } - if !lane.floor_pending() { - continue; - } - // One save per pass that moved the floor, never one per entry: - // each save is an fsync, and the pass coalesces every apply that - // finished before it. - if let Err(error) = state.tenant_marks.persist(state.credentials.catalog()) { - // The floor stays where it is, so every entry above it keeps - // its place in the log. The next pass persists and saves again. - tracing::error!( - group_id = *group_id, - %error, - "apply loop: tenant write marks did not persist; the applied floor waits" - ); - continue; - } - if let Some(floor) = lane.take_floor_to_save() { - // Every entry the floor covers keeps its changes on disk. - state.change_stream.journal_group(*group_id, floor); - record_durable_apply(state, *group_id, floor); - } - } + super::settle::settle_lanes(self.ctx, &mut self.lanes); } } @@ -441,6 +478,233 @@ fn release_admission_holds(state: &SharedState, group_id: u64, log_index: u64) { .release_through(group_id, log_index); } -fn finished_event(apply: ApplyFuture<'_>) -> LoopFuture<'_> { - Box::pin(apply.map(LoopEvent::Finished)) +#[cfg(test)] +mod tests { + use nodedb_cluster::{GroupAppliedWatchers, GroupApplyGates}; + use nodedb_raft::message::LogEntry; + + use super::*; + use crate::bridge::dispatch::Dispatcher; + use crate::control::distributed_applier::proposal_ledger::PROPOSAL_LEDGER_CAPACITY; + use crate::control::distributed_applier::propose_tracker::ProposeTracker; + use crate::wal::WalManager; + + use super::super::context::FinishedApply; + use super::super::proposal_gate::PrefixStep; + + const GROUP: u64 = 4; + + fn slot(log_index: u64, state: SlotState) -> Slot { + Slot { + log_index, + proposal_key: 0, + collection: None, + write_mark: None, + user_write: false, + state, + } + } + + /// Entry 5 holds its group with entry 6 queued behind it. Entry 6 never + /// starts and the applied index stays at 4 until an install covers 6. + /// Then 5 and 6 conclude covered and the applied index moves to 6. + #[tokio::test] + async fn a_held_entry_stops_its_group_until_an_install_covers_it() { + let dir = tempfile::tempdir().expect("tempdir"); + let wal = Arc::new(WalManager::open_for_testing(&dir.path().join("wal")).expect("wal")); + let (dispatcher, _sides) = Dispatcher::new(1, 64); + let state = SharedState::new(dispatcher, wal).expect("shared state"); + let gates = Arc::new(GroupApplyGates::new()); + gates.mount(GROUP); + assert!(state.raft_apply_gates.set(Arc::clone(&gates)).is_ok()); + let watchers = Arc::new(GroupAppliedWatchers::new()); + let tracker = Arc::new(ProposeTracker::new().with_group_watchers(Arc::clone(&watchers))); + let ctx = ApplyContext { + state: &state, + tracker: &tracker, + }; + let mut pipeline = Pipeline::new(ctx, ProposalLedger::new(PROPOSAL_LEDGER_CAPACITY)); + + let mut lane = Lane::new(GROUP); + lane.push(slot(4, SlotState::Concluded(PrefixStep::Record(true)))); + lane.push(slot(5, SlotState::Held)); + lane.backlog.push_back(QueuedEntry::new(LogEntry { + term: 1, + index: 6, + data: Vec::new(), + })); + pipeline.lanes.insert(GROUP, lane); + + pipeline.pump(); + pipeline.settle(); + assert_eq!( + pipeline.lanes[&GROUP].backlog.len(), + 1, + "entry 6 waits behind the held entry" + ); + assert!( + tracker.started_through(GROUP) < 6, + "entry 6 never starts while entry 5 holds" + ); + assert_eq!(watchers.get_or_create(GROUP).current(), 4); + assert!(pipeline.awaits_install()); + assert!( + state.redo_chunks.owes_snapshot(GROUP), + "a held group owes a snapshot install" + ); + assert!(!pipeline.retries_debt(), "the debt persisted"); + + // A snapshot through entry 4 installs: entry 5 still holds. + gates.install(GROUP).await.adopted(4); + pipeline.pump(); + pipeline.settle(); + assert!(pipeline.lanes[&GROUP].is_held()); + assert_eq!(watchers.get_or_create(GROUP).current(), 4); + + // A snapshot through entry 6 installs. + gates.install(GROUP).await.adopted(6); + pipeline.pump(); + pipeline.settle(); + let lane = &pipeline.lanes[&GROUP]; + assert!(!lane.is_held()); + assert!(lane.backlog.is_empty()); + assert_eq!(tracker.started_through(GROUP), 6); + assert_eq!( + watchers.get_or_create(GROUP).current(), + 6, + "entries 5 and 6 settle once the install covers them" + ); + assert!(!pipeline.awaits_install()); + } + + /// The apply of a running entry that never returns a result. + fn exploding_apply() -> FinishedApply { + panic!("the apply of entry 5 exploded") + } + + /// Entry 5's apply panics with entry 6 queued behind it. Its waiter + /// receives an error at once, entry 5 holds the group, entry 6 never + /// starts, and the applied index stays at 4. + #[tokio::test] + async fn a_panicking_apply_answers_its_waiter_and_holds_its_group() { + let dir = tempfile::tempdir().expect("tempdir"); + let wal = Arc::new(WalManager::open_for_testing(&dir.path().join("wal")).expect("wal")); + let (dispatcher, _sides) = Dispatcher::new(1, 64); + let state = SharedState::new(dispatcher, wal).expect("shared state"); + let gates = Arc::new(GroupApplyGates::new()); + gates.mount(GROUP); + assert!(state.raft_apply_gates.set(Arc::clone(&gates)).is_ok()); + let watchers = Arc::new(GroupAppliedWatchers::new()); + let tracker = Arc::new(ProposeTracker::new().with_group_watchers(Arc::clone(&watchers))); + let ctx = ApplyContext { + state: &state, + tracker: &tracker, + }; + let mut pipeline = Pipeline::new(ctx, ProposalLedger::new(PROPOSAL_LEDGER_CAPACITY)); + + const KEY: u64 = 77; + let waiter = tracker.register(GROUP, 5, KEY); + let mut lane = Lane::new(GROUP); + lane.push(slot(4, SlotState::Concluded(PrefixStep::Record(true)))); + lane.push(Slot { + proposal_key: KEY, + ..slot(5, SlotState::Running) + }); + lane.backlog.push_back(QueuedEntry::new(LogEntry { + term: 1, + index: 6, + data: Vec::new(), + })); + pipeline.lanes.insert(GROUP, lane); + pipeline.running.push(finished_event( + GROUP, + 5, + Box::pin(async { exploding_apply() }), + )); + + let event = pipeline.next_event().await.expect("the apply concludes"); + assert!(matches!(event, LoopEvent::Panicked { log_index: 5, .. })); + pipeline.handle(event); + pipeline.pump(); + pipeline.settle(); + + let answered = waiter.await.expect("the waiter is answered"); + assert!(answered.is_err(), "a panicked apply answers with an error"); + let lane = &pipeline.lanes[&GROUP]; + assert_eq!(lane.held_index(), Some(5), "entry 5 holds its group"); + assert_eq!(lane.backlog.len(), 1, "entry 6 waits behind entry 5"); + assert_eq!(watchers.get_or_create(GROUP).current(), 4); + assert!(pipeline.awaits_install()); + assert!( + state.redo_chunks.owes_snapshot(GROUP), + "a group held by a panic owes a snapshot install" + ); + } + + /// A held group's debt write fails twice and files one report. The + /// loop's Raft-tick wake writes it again, with no batch or install, and + /// the debt persists. + /// + /// The recorder is process-wide: nextest runs this test in its own + /// process. + #[tokio::test(start_paused = true)] + async fn a_failed_debt_write_persists_on_a_later_tick() { + let reports = tempfile::tempdir().expect("reports dir"); + assert!( + faultbox::init( + faultbox::Config::new("nodedb-test", "0.0.0", reports.path()) + .install_panic_hook(false), + ), + "this test must be the first initializer of the recorder" + ); + let dir = tempfile::tempdir().expect("tempdir"); + let wal = Arc::new(WalManager::open_for_testing(&dir.path().join("wal")).expect("wal")); + let (dispatcher, _sides) = Dispatcher::new(1, 64); + let state = SharedState::new(dispatcher, wal).expect("shared state"); + let gates = Arc::new(GroupApplyGates::new()); + gates.mount(GROUP); + assert!(state.raft_apply_gates.set(Arc::clone(&gates)).is_ok()); + let watchers = Arc::new(GroupAppliedWatchers::new()); + let tracker = Arc::new(ProposeTracker::new().with_group_watchers(watchers)); + let ctx = ApplyContext { + state: &state, + tracker: &tracker, + }; + let mut pipeline = Pipeline::new(ctx, ProposalLedger::new(PROPOSAL_LEDGER_CAPACITY)); + let mut lane = Lane::new(GROUP); + lane.push(slot(4, SlotState::Concluded(PrefixStep::Record(true)))); + lane.push(slot(5, SlotState::Held)); + pipeline.lanes.insert(GROUP, lane); + let catalog = state.credentials.catalog(); + + catalog.fail_next_redo_snapshot_owed_write_for_test(); + pipeline.pump(); + assert!(pipeline.retries_debt()); + catalog.fail_next_redo_snapshot_owed_write_for_test(); + pipeline.pump(); + assert!(pipeline.retries_debt(), "the second write failed too"); + assert!(!state.redo_chunks.owes_snapshot(GROUP)); + + // The sender stays open: only the timed wake can pump. + let (_tx, mut rx) = tokio::sync::mpsc::channel::(1); + let ran = tokio::time::timeout( + std::time::Duration::from_secs(1), + super::super::driver::drive(&mut pipeline, &mut rx, &state), + ) + .await; + assert!(ran.is_err(), "the loop waits for the held entry's install"); + assert!(!pipeline.retries_debt(), "a tick wrote the debt"); + assert!(state.redo_chunks.owes_snapshot(GROUP)); + assert!(pipeline.lanes[&GROUP].is_held()); + + let debt_reports: u64 = faultbox::reader::list(reports.path()) + .expect("list reports") + .iter() + .filter(|group| { + group.first.domain_kind.as_deref() == Some("nodedb.held_entry_debt_not_recorded") + }) + .map(faultbox::reader::Group::occurrences) + .sum(); + assert_eq!(debt_reports, 1, "one report for the failure run"); + } } diff --git a/nodedb/src/control/distributed_applier/apply_loop/proposal_gate.rs b/nodedb/src/control/distributed_applier/apply_loop/proposal_gate.rs index 7eda3df9d..f881d6d4e 100644 --- a/nodedb/src/control/distributed_applier/apply_loop/proposal_gate.rs +++ b/nodedb/src/control/distributed_applier/apply_loop/proposal_gate.rs @@ -34,16 +34,50 @@ pub(super) enum EntryOutcome { durable: bool, result: Option, }, + /// This replica cannot apply the entry as its peers do. The entry holds + /// its group's lane until a snapshot install covers it. + Held { reason: HoldReason }, +} + +/// Why this replica holds a committed entry it cannot apply as its peers do. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub(super) enum HoldReason { + /// The redo body is missing here: this replica dropped the chunk stream + /// the entry names. + BodyMissing, + /// The entry's bytes do not decode as the redo they name. + Malformed, + /// The entry's enqueue or apply panicked here, so its effect on this + /// replica is unknown. + ApplyPanicked, +} + +impl HoldReason { + pub fn as_str(self) -> &'static str { + match self { + Self::BodyMissing => "body_missing", + Self::Malformed => "malformed", + Self::ApplyPanicked => "apply_panicked", + } + } } impl EntryOutcome { + /// Why the entry holds its group's lane, when it does. + pub fn hold_reason(&self) -> Option { + match self { + Self::Held { reason } => Some(*reason), + Self::Skipped | Self::Repeat | Self::Covered | Self::Applied { .. } => None, + } + } + /// Whether the apply wrote the entry's rows. A refused or failed apply /// wrote none, so it raises no tenant write mark. A second copy writes /// nothing itself: its mark follows its first copy (see /// [`ProposalGate::prior_wrote_rows`]). pub fn wrote_rows(&self) -> bool { match self { - Self::Skipped | Self::Repeat | Self::Covered => false, + Self::Skipped | Self::Repeat | Self::Covered | Self::Held { .. } => false, Self::Applied { result: Some(outcome), .. @@ -66,6 +100,16 @@ pub(super) enum PrefixStep { Record(bool), } +/// How a concluded entry stands in its group's lane. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub(super) enum Settlement { + /// The entry settles in log order and moves the durable prefix by its + /// step. + Settles(PrefixStep), + /// The entry holds its group's lane until a snapshot install covers it. + Holds(HoldReason), +} + /// The outcome a waiter received, in the form the ledger keeps. A refusal is /// kept as its typed code; an error with no code keeps its message. pub(super) fn ledger_outcome(result: &crate::Result) -> AppliedOutcome { @@ -147,6 +191,14 @@ impl ProposalGate { self.ledger.note(proposal_key, None); } + /// Note a proposal an installed snapshot holds: a later copy of it is a + /// duplicate here too. + pub fn note_covered(&mut self, proposal_key: u64) { + if proposal_key != 0 { + self.ledger.note(proposal_key, None); + } + } + /// Note that a copy of `proposal_key` started and has not concluded. pub fn open(&mut self, proposal_key: u64) { if proposal_key != 0 { @@ -156,7 +208,7 @@ impl ProposalGate { /// Conclude an entry of `proposal_key` with `outcome`: note a durable /// apply in the ledger, close the copy `open` noted when `opened`, and - /// return how the entry moves its group's prefix. + /// return how the entry stands in its group's lane. /// /// No separate marker is written: the entry's own records carry its key, /// durable in the same write as its effect. @@ -165,7 +217,7 @@ impl ProposalGate { proposal_key: u64, opened: bool, outcome: EntryOutcome, - ) -> PrefixStep { + ) -> Settlement { if opened && proposal_key != 0 && let Some(copies) = self.in_flight.get_mut(&proposal_key) @@ -176,21 +228,21 @@ impl ProposalGate { } } match outcome { - EntryOutcome::Skipped => PrefixStep::Neutral, - EntryOutcome::Repeat => PrefixStep::Record(true), + EntryOutcome::Skipped => Settlement::Settles(PrefixStep::Neutral), + EntryOutcome::Repeat => Settlement::Settles(PrefixStep::Record(true)), EntryOutcome::Covered => { - // A later copy of the proposal is a duplicate here too. - if proposal_key != 0 { - self.ledger.note(proposal_key, None); - } - PrefixStep::Record(true) + self.note_covered(proposal_key); + Settlement::Settles(PrefixStep::Record(true)) } EntryOutcome::Applied { durable, result } => { if durable { self.ledger.note(proposal_key, result); } - PrefixStep::Record(durable) + Settlement::Settles(PrefixStep::Record(durable)) } + // No outcome is recorded: the install that releases the entry + // notes its proposal. + EntryOutcome::Held { reason } => Settlement::Holds(reason), } } } @@ -216,7 +268,7 @@ mod tests { result: Some(Ok(AppliedWrite::unversioned(Vec::new()))), }, ); - assert_eq!(step, PrefixStep::Record(true)); + assert_eq!(step, Settlement::Settles(PrefixStep::Record(true))); assert!(!gate.in_flight(42)); assert!(gate.skip_duplicate(&tracker, 1, 9, 42)); } @@ -234,7 +286,7 @@ mod tests { result: None, }, ); - assert_eq!(step, PrefixStep::Record(false)); + assert_eq!(step, Settlement::Settles(PrefixStep::Record(false))); assert!(!gate.in_flight(7)); assert!(!gate.skip_duplicate(&tracker, 1, 9, 7)); } @@ -246,8 +298,28 @@ mod tests { let tracker = ProposeTracker::new(); let mut gate = ProposalGate::new(ProposalLedger::new(PROPOSAL_LEDGER_CAPACITY)); let step = gate.conclude(11, false, EntryOutcome::Covered); - assert_eq!(step, PrefixStep::Record(true)); + assert_eq!(step, Settlement::Settles(PrefixStep::Record(true))); assert!(!EntryOutcome::Covered.wrote_rows()); assert!(gate.skip_duplicate(&tracker, 1, 14, 11)); } + + /// A held entry closes its in-flight copy and records no outcome, so a + /// later copy of its proposal is not skipped as applied. + #[test] + fn a_held_entry_holds_its_lane_and_records_no_outcome() { + let tracker = ProposeTracker::new(); + let mut gate = ProposalGate::new(ProposalLedger::new(PROPOSAL_LEDGER_CAPACITY)); + gate.open(9); + let held = EntryOutcome::Held { + reason: HoldReason::BodyMissing, + }; + assert!(!held.wrote_rows()); + let step = gate.conclude(9, true, held); + assert_eq!(step, Settlement::Holds(HoldReason::BodyMissing)); + assert!(!gate.in_flight(9)); + assert!(!gate.skip_duplicate(&tracker, 1, 12, 9)); + + gate.note_covered(9); + assert!(gate.skip_duplicate(&tracker, 1, 12, 9)); + } } diff --git a/nodedb/src/control/distributed_applier/apply_loop/redo_chunk.rs b/nodedb/src/control/distributed_applier/apply_loop/redo_chunk.rs index 335ed0c61..7e976e9c1 100644 --- a/nodedb/src/control/distributed_applier/apply_loop/redo_chunk.rs +++ b/nodedb/src/control/distributed_applier/apply_loop/redo_chunk.rs @@ -12,8 +12,8 @@ //! //! A group whose replica here owes a snapshot install is the exception (see //! `chunks::owed`). This node dropped streams of it that the other replicas -//! still hold. A refusal there can be this replica's alone, so it holds the -//! group's durable floor until the snapshot installs. +//! still hold. A refusal there can be this replica's alone: the entry's body +//! is missing here. It holds the group's lane until the snapshot installs. use crate::control::array_sync::raft_apply::AppliedPosition; use crate::control::distributed_applier::propose_tracker::{AppliedWrite, ProposeTracker}; @@ -28,7 +28,7 @@ use crate::wal::manager::RedoChunkPlacement; use super::calvin_redo::{ClaimOutcome, claim_chunked}; use super::context::{ApplyContext, FinishedApply}; -use super::proposal_gate::{EntryOutcome, ledger_outcome}; +use super::proposal_gate::{EntryOutcome, HoldReason, ledger_outcome}; use super::start::Prepared; use super::transaction_redo::{ RedoApply, conclude_held, conclude_held_claimed, conclude_installed_copy, enqueue_payload, @@ -65,6 +65,8 @@ pub(super) fn prepare_stream_entry<'a>( bytes, }; Prepared::Exclusive(Box::pin(async move { + #[cfg(feature = "failpoints")] + crate::control::fail_gate::before_chunk_apply(ctx.state.node_id, index).await; let applied = ctx.state.redo_chunks.apply_chunk(chunk).await; // The chunk's records are durable here, its entry not yet // concluded. @@ -86,6 +88,7 @@ pub(super) fn prepare_stream_entry<'a>( other => conclude_held( ctx, pos, + HoldReason::Malformed, crate::Error::Internal { detail: format!("redo stream apply called with another write: {other:?}"), }, @@ -139,13 +142,13 @@ fn concluded( outcome } -/// Hold a chunk entry this replica cannot decide as the other replicas do. -/// No ledger outcome records it, so a re-delivery applies it again. +/// Hold a chunk entry whose stream this replica dropped: the other +/// replicas decide it from a stream it lacks. No ledger outcome records it, +/// so a re-delivery applies it again. fn held(tracker: &ProposeTracker, pos: AppliedPosition, error: crate::Error) -> EntryOutcome { tracker.complete(pos.group_id, pos.log_index, pos.applied_key, Err(error)); - EntryOutcome::Applied { - durable: false, - result: None, + EntryOutcome::Held { + reason: HoldReason::BodyMissing, } } @@ -217,7 +220,9 @@ pub(super) fn prepare_chunked_final<'a>( // Another copy of the position installed, or installs now: this // copy's stream drops. ClaimOutcome::Refused => return conclude_installed_copy(ctx, pos, taken), - ClaimOutcome::Malformed(error) => return conclude_held(ctx, pos, error, taken), + ClaimOutcome::Malformed(error) => { + return conclude_held(ctx, pos, HoldReason::Malformed, error, taken); + } }; // Every chunk is durable and the final entry committed; nothing of the // final entry applied yet. @@ -235,9 +240,10 @@ pub(super) fn prepare_chunked_final<'a>( Ok(bytes) => bytes, Err(refusal) if ctx.state.redo_chunks.owes_snapshot(pos.group_id) => { let error = stream_lost_here(pos, chunked.stream, LostEntry::Final, &refusal); + let reason = HoldReason::BodyMissing; return match calvin { - Some(claim) => conclude_held_claimed(ctx, pos, error, taken, claim), - None => conclude_held(ctx, pos, error, taken), + Some(claim) => conclude_held_claimed(ctx, pos, reason, error, taken, claim), + None => conclude_held(ctx, pos, reason, error, taken), }; } Err(refusal) => { @@ -276,10 +282,13 @@ pub(super) fn prepare_chunked_final<'a>( calvin, }, )), - Err(error) => match calvin { - Some(claim) => conclude_held_claimed(ctx, pos, error, taken, claim), - None => conclude_held(ctx, pos, error, taken), - }, + Err(error) => { + let reason = HoldReason::Malformed; + match calvin { + Some(claim) => conclude_held_claimed(ctx, pos, reason, error, taken, claim), + None => conclude_held(ctx, pos, reason, error, taken), + } + } } } @@ -368,9 +377,8 @@ mod tests { fn holds(outcome: &EntryOutcome) -> bool { matches!( outcome, - EntryOutcome::Applied { - durable: false, - result: None + EntryOutcome::Held { + reason: HoldReason::BodyMissing } ) } @@ -378,7 +386,7 @@ mod tests { /// This node applied a stream's first chunk, left the group, and /// mounted it again on the group's old log. The next chunk and the /// final entry refuse here while every other replica installs the - /// redo. Each holds the group's durable floor instead of passing. + /// redo. Each holds the group's lane instead of passing. #[tokio::test] async fn a_final_entry_of_a_stream_this_replica_dropped_holds_the_apply() { let dir = tempfile::tempdir().expect("tempdir"); diff --git a/nodedb/src/control/distributed_applier/apply_loop/settle.rs b/nodedb/src/control/distributed_applier/apply_loop/settle.rs new file mode 100644 index 000000000..d7b0d6aac --- /dev/null +++ b/nodedb/src/control/distributed_applier/apply_loop/settle.rs @@ -0,0 +1,162 @@ +// SPDX-License-Identifier: BUSL-1.1 + +//! Settle every group's concluded entries and save the floors that moved. + +use std::collections::HashMap; + +use crate::control::cluster::calvin::scheduler::barrier_store; +use crate::control::server::dispatch_utils::publish_settled_changes; +use crate::control::state::SharedState; + +use super::bookkeeping::record_durable_apply; +use super::context::ApplyContext; +use super::lane::Lane; + +/// Settle every group's concluded entries in log order, release their +/// window, and save each durable floor that moved. +/// +/// The tenant write marks of the settled entries are persisted before any +/// floor that covers them. An entry above the saved floor is delivered +/// again after a restart and records its mark again, so every committed +/// write keeps a durable mark. +/// +/// The dependent-read barrier rows saved before the pass persist before any +/// floor too: a floor covers a barrier entry only once its row is durable. +pub(super) fn settle_lanes(ctx: ApplyContext<'_>, lanes: &mut HashMap) { + let state = ctx.state; + let tracker = ctx.tracker; + // Persisted once per pass, at the first floor that moved. + let mut barrier_rows_durable: Option = None; + for (group_id, lane) in lanes { + let first = lane.front_index(); + let settled = lane.settle(tracker, &state.tenant_marks); + if settled > 0 { + tracker.window().release(*group_id, settled); + // The entries' change events publish in log order, at their + // log positions, on every replica. + if let (Some(first), Some(last)) = (first, lane.last_settled()) { + publish_settled_changes(state, *group_id, first, last); + } + } + if !lane.floor_pending() { + continue; + } + if !*barrier_rows_durable.get_or_insert_with(|| persist_barrier_rows(state)) { + // Every floor waits: one may cover a barrier entry whose row a + // crash loses. The next pass persists again. + continue; + } + // One save per pass that moved the floor, never one per entry: + // each save is an fsync, and the pass coalesces every apply that + // finished before it. + if let Err(error) = state.tenant_marks.persist(state.credentials.catalog()) { + // The floor stays where it is, so every entry above it keeps + // its place in the log. The next pass persists and saves again. + tracing::error!( + group_id = *group_id, + %error, + "apply loop: tenant write marks did not persist; the applied floor waits" + ); + continue; + } + if let Some(floor) = lane.take_floor_to_save() { + // Every entry the floor covers keeps its changes on disk. + state.change_stream.journal_group(*group_id, floor); + record_durable_apply(state, *group_id, floor); + } + } +} + +/// Persist the barrier rows saved since the last persist. Returns whether +/// they are durable. +fn persist_barrier_rows(state: &SharedState) -> bool { + match barrier_store::persist_pending(state.credentials.catalog()) { + Ok(()) => true, + Err(error) => { + tracing::error!( + %error, + "apply loop: the dependent-read barrier rows did not persist; every applied \ + floor waits" + ); + crate::diag::calvin_barrier_log_store_failed(None, None, "persist", &error); + false + } + } +} + +#[cfg(test)] +mod tests { + use std::sync::{Arc, Mutex}; + + use nodedb_cluster::GroupAppliedWatchers; + + use super::*; + use crate::bridge::dispatch::Dispatcher; + use crate::control::cluster::calvin::BarrierEvent; + use crate::control::cluster::calvin::scheduler::lock_manager::TxnId; + use crate::control::distributed_applier::propose_tracker::ProposeTracker; + use crate::wal::WalManager; + + use super::super::lane::{Slot, SlotState}; + use super::super::proposal_gate::PrefixStep; + + const GROUP: u64 = 4; + + /// A concluded barrier entry at `log_index` whose row holds its event. + fn concluded(log_index: u64) -> Slot { + Slot { + log_index, + proposal_key: 0, + collection: None, + write_mark: None, + user_write: false, + state: SlotState::Concluded(PrefixStep::Record(true)), + } + } + + /// A barrier row persist fails: the pass saves no floor, so the floor + /// never passes the barrier entry. The next pass persists and saves it. + #[test] + fn the_floor_waits_for_the_barrier_rows_to_persist() { + let dir = tempfile::tempdir().expect("tempdir"); + let wal = Arc::new(WalManager::open_for_testing(&dir.path().join("wal")).expect("wal")); + let (dispatcher, _sides) = Dispatcher::new(1, 64); + let state = SharedState::new(dispatcher, wal).expect("shared state"); + let saved: Arc>> = Arc::new(Mutex::new(Vec::new())); + let sink_saved = Arc::clone(&saved); + let sink: Arc = + Arc::new(move |group_id, applied_index| { + sink_saved + .lock() + .expect("saved floors") + .push((group_id, applied_index)); + Ok::<(), crate::Error>(()) + }); + assert!(state.raft_applied_index_sink.set(sink).is_ok()); + let watchers = Arc::new(GroupAppliedWatchers::new()); + let tracker = Arc::new(ProposeTracker::new().with_group_watchers(watchers)); + let ctx = ApplyContext { + state: &state, + tracker: &tracker, + }; + + let catalog = state.credentials.catalog(); + barrier_store::save_event(catalog, 7, TxnId::new(3, 0), &BarrierEvent::Timeout) + .expect("save the row"); + let mut lane = Lane::new(GROUP); + lane.push(concluded(5)); + let mut lanes = HashMap::from([(GROUP, lane)]); + + catalog.fail_next_calvin_barrier_persist_for_test(); + settle_lanes(ctx, &mut lanes); + assert!( + saved.lock().expect("saved floors").is_empty(), + "no floor passes a row that is not durable" + ); + assert!(lanes[&GROUP].floor_pending()); + + settle_lanes(ctx, &mut lanes); + assert_eq!(*saved.lock().expect("saved floors"), vec![(GROUP, 5)]); + assert!(!lanes[&GROUP].floor_pending()); + } +} diff --git a/nodedb/src/control/distributed_applier/apply_loop/start.rs b/nodedb/src/control/distributed_applier/apply_loop/start.rs index f7563d87c..6356e560f 100644 --- a/nodedb/src/control/distributed_applier/apply_loop/start.rs +++ b/nodedb/src/control/distributed_applier/apply_loop/start.rs @@ -15,6 +15,7 @@ use crate::control::array_sync::raft_apply::{ use nodedb_raft::message::LogEntry; use crate::control::backup::cut_order::OrderedCut; +use crate::control::distributed_applier::propose_tracker::ProposeTracker; use crate::control::wal_replication::{ReplicatedEntry, ReplicatedWrite}; use crate::types::{DatabaseId, TenantId}; @@ -58,7 +59,8 @@ pub(super) fn prepare_entry<'a>( watch.note_apply(group_id, log_index); if entry.data.is_empty() { - return conclude_leader_change_noop(ctx, group_id, log_index); + conclude_leader_change_noop(ctx.tracker, group_id, log_index); + return Prepared::Concluded(EntryOutcome::Skipped); } // `0` for unparseable / pre-key entries; the tracker treats 0 as "no key" @@ -94,21 +96,15 @@ pub(super) fn prepare_entry<'a>( prepare_replicated(ctx, watch, pos, log_term, entry, scope, replicated) } -/// A leader-change no-op committed where a proposer can wait. The -/// proposer's data is gone; firing an empty success tells it the -/// write applied. `RetryableLeaderChange` makes the gateway re-propose. -fn conclude_leader_change_noop<'a>( - ctx: ApplyContext<'a>, - group_id: u64, - log_index: u64, -) -> Prepared<'a> { - tracing::error!( - group_id, - log_index, - "leader-change no-op committed at index where a proposer was waiting; \ - surfacing RetryableLeaderChange so the gateway re-proposes" - ); - ctx.tracker.complete( +/// Conclude the leader-change no-op at `log_index`. Every new leader commits +/// one. Returns whether a proposer waited at the index. +/// +/// A proposer whose entry held the index lost it to the leader change. It +/// gets `RetryableLeaderChange`, so the gateway re-proposes; an empty success +/// would tell it the write applied. A proposer that registers later finds +/// the same answer stored. Only an overwritten waiter is logged. +fn conclude_leader_change_noop(tracker: &ProposeTracker, group_id: u64, log_index: u64) -> bool { + let overwritten = tracker.complete( group_id, log_index, 0, @@ -117,7 +113,15 @@ fn conclude_leader_change_noop<'a>( log_index, }), ); - Prepared::Concluded(EntryOutcome::Skipped) + if overwritten { + tracing::warn!( + group_id, + log_index, + "a leader-change no-op committed where a proposer was waiting; surfacing \ + RetryableLeaderChange so the gateway re-proposes" + ); + } + overwritten } /// The scope a decoded entry applies under. The entry's incarnations move @@ -368,3 +372,36 @@ fn prepare_array_op<'a>( } })) } + +#[cfg(test)] +mod tests { + use super::*; + + /// A no-op at an index a proposer waits on answers it with + /// `RetryableLeaderChange`. + #[tokio::test] + async fn a_noop_over_a_waiter_surfaces_a_leader_change() { + let tracker = ProposeTracker::new(); + let waiter = tracker.register(4, 9, 77); + assert!(conclude_leader_change_noop(&tracker, 4, 9)); + let answer = waiter.await.expect("the waiter is answered"); + assert!( + matches!( + answer, + Err(crate::Error::RetryableLeaderChange { + group_id: 4, + log_index: 9 + }) + ), + "{answer:?}" + ); + } + + /// A no-op at an index no proposer waits on, as after every election, + /// finds no waiter and logs nothing. + #[test] + fn a_noop_with_no_waiter_finds_none() { + let tracker = ProposeTracker::new(); + assert!(!conclude_leader_change_noop(&tracker, 4, 1)); + } +} diff --git a/nodedb/src/control/distributed_applier/apply_loop/transaction_redo.rs b/nodedb/src/control/distributed_applier/apply_loop/transaction_redo.rs index 34e59d9de..0eec50834 100644 --- a/nodedb/src/control/distributed_applier/apply_loop/transaction_redo.rs +++ b/nodedb/src/control/distributed_applier/apply_loop/transaction_redo.rs @@ -38,7 +38,7 @@ use crate::types::{DatabaseId, TenantId, VShardId}; use super::calvin_redo::{CalvinClaim, ClaimOutcome, claim_inline}; use super::context::{ApplyContext, EnqueueFuture, FinishedApply, Started, StartedEntry}; use super::helpers::committed_response_result; -use super::proposal_gate::{EntryOutcome, ledger_outcome}; +use super::proposal_gate::{EntryOutcome, HoldReason, ledger_outcome}; use super::start::Prepared; /// Prepare one committed `TransactionRedo` entry. Its enqueue appends the @@ -57,7 +57,7 @@ pub(super) fn prepare_transaction_redo_entry<'a>( ) -> Prepared<'a> { let decoded = match decode_transaction_redo(&entry.write) { Ok(decoded) => decoded, - Err(error) => return conclude_held(ctx, pos, error, None), + Err(error) => return conclude_held(ctx, pos, HoldReason::Malformed, error, None), }; let target = RedoTarget { tenant_id: TenantId::new(entry.tenant_id), @@ -75,7 +75,9 @@ pub(super) fn prepare_transaction_redo_entry<'a>( ClaimOutcome::NotCalvin => None, ClaimOutcome::Claimed(claim) => Some(claim), ClaimOutcome::Refused => return conclude_installed_copy(ctx, pos, None), - ClaimOutcome::Malformed(error) => return conclude_held(ctx, pos, error, None), + ClaimOutcome::Malformed(error) => { + return conclude_held(ctx, pos, HoldReason::Malformed, error, None); + } }; Prepared::Enqueue(enqueue_payload( ctx, @@ -129,15 +131,16 @@ pub(super) struct RedoApply { /// Conclude an entry this replica cannot apply as the other replicas do: /// its own bytes are malformed, or its stream here is not the one the log -/// built. It holds the floor rather than skipping a committed transaction. -/// A taken stream stays on disk for it. +/// built. It holds its group's lane rather than skipping a committed +/// transaction. A taken stream stays on disk for it. pub(super) fn conclude_held<'a>( ctx: ApplyContext<'a>, pos: AppliedPosition, + reason: HoldReason, error: crate::Error, stream: Option, ) -> Prepared<'a> { - Prepared::Concluded(held_outcome(ctx, pos, error, stream)) + Prepared::Concluded(held_outcome(ctx, pos, reason, error, stream)) } /// [`conclude_held`] for an entry that claimed a Calvin position: the claim @@ -145,12 +148,13 @@ pub(super) fn conclude_held<'a>( pub(super) fn conclude_held_claimed<'a>( ctx: ApplyContext<'a>, pos: AppliedPosition, + reason: HoldReason, error: crate::Error, stream: Option, claim: CalvinClaim, ) -> Prepared<'a> { let event = claim.not_applied(error.to_string()); - let outcome = held_outcome(ctx, pos, error, stream); + let outcome = held_outcome(ctx, pos, reason, error, stream); report_then_conclude(ctx, pos, claim, event, outcome) } @@ -175,19 +179,32 @@ pub(super) fn report_then_conclude<'a>( } /// Resolve the waiter of an entry this replica holds, and settle its -/// stream. +/// stream. A malformed entry is reported here, where it is detected. A +/// missing body is reported where its stream refused. fn held_outcome( ctx: ApplyContext<'_>, pos: AppliedPosition, + reason: HoldReason, error: crate::Error, stream: Option, ) -> EntryOutcome { + match reason { + HoldReason::Malformed => { + tracing::error!( + group_id = pos.group_id, + log_index = pos.log_index, + %error, + "committed redo entry does not decode on this replica; the group's apply holds \ + at it until a snapshot of the group installs" + ); + crate::diag::redo_entry_malformed(pos.group_id, pos.log_index, &error); + } + // Each is reported at the site that detects it. + HoldReason::BodyMissing | HoldReason::ApplyPanicked => {} + } ctx.tracker .complete(pos.group_id, pos.log_index, pos.applied_key, Err(error)); - let outcome = EntryOutcome::Applied { - durable: false, - result: None, - }; + let outcome = EntryOutcome::Held { reason }; settle_stream(ctx, stream, &outcome); outcome } @@ -199,7 +216,10 @@ fn settle_stream(ctx: ApplyContext<'_>, stream: Option, outcome: &En let Some(stream) = stream else { return; }; - if matches!(outcome, EntryOutcome::Applied { durable: false, .. }) { + if matches!( + outcome, + EntryOutcome::Held { .. } | EntryOutcome::Applied { durable: false, .. } + ) { ctx.state.redo_chunks.park(stream); } } diff --git a/nodedb/src/control/exec_receiver/executor.rs b/nodedb/src/control/exec_receiver/executor.rs index 05ef7fed3..25501aabc 100644 --- a/nodedb/src/control/exec_receiver/executor.rs +++ b/nodedb/src/control/exec_receiver/executor.rs @@ -122,6 +122,8 @@ impl LocalPlanExecutor { ), TypedClusterError, > { + #[cfg(feature = "failpoints")] + crate::control::fail_gate::before_execute(self.state.node_id).await; let (deadline, database_id) = validate_request(&self.state, req)?; let plan = decode_plan(&self.state, database_id, req.tenant_id, &req.plan_bytes)?; // A backup's snapshot plan takes the backup's cut on this node first. diff --git a/nodedb/src/control/exec_receiver/metadata_applied.rs b/nodedb/src/control/exec_receiver/metadata_applied.rs index 56e2d99ea..19f88734e 100644 --- a/nodedb/src/control/exec_receiver/metadata_applied.rs +++ b/nodedb/src/control/exec_receiver/metadata_applied.rs @@ -25,17 +25,10 @@ pub(super) async fn answer_applied_plan( return None; }; let watcher = state.applied_index_watcher(METADATA_GROUP_ID); - let reached = match crate::control::metadata_proposer::wait::wait_applied( - watcher, *index, budget, - ) - .await - { - Ok(outcome) => outcome.is_reached(), - Err(error) => { - tracing::warn!(index = *index, %error, "metadata-applied wait did not finish"); - false - } - }; + let reached = watcher + .wait_for_async(*index, std::time::Instant::now() + budget) + .await + .is_reached(); Some(if reached { ExecuteResponse::ok(vec![Vec::new()], 0, Vec::new()) } else { diff --git a/nodedb/src/control/fail_gate.rs b/nodedb/src/control/fail_gate.rs index 264a84e1a..62d87f273 100644 --- a/nodedb/src/control/fail_gate.rs +++ b/nodedb/src/control/fail_gate.rs @@ -61,6 +61,20 @@ pub(crate) async fn before_dispatch(node_id: u64, plan: &PhysicalPlan, wal_lsn: } } +/// The execute gate of node `node_id`, `exec::before_execute`: a received +/// `ExecuteRequest` or `ExecuteStreamRequest` parks here before it is checked +/// or run. Armed with `wait_file()`, one node takes a request and +/// never answers. Armed with `sleep()`, it runs the request late. +pub(crate) async fn before_execute(node_id: u64) { + let scope = FailScope::Node(node_id); + let name = "exec::before_execute"; + if let Some(FailAction::Sleep(pause)) = lookup(scope, name) { + tokio::time::sleep(pause).await; + return; + } + wait(scope, name).await; +} + /// The Calvin submit gate of node `node_id`: after a transaction's collection /// incarnations are stamped and before the sequencer admits it. Named per /// collection, `calvin::after_stamp::`. A test parks a @@ -206,3 +220,39 @@ pub(crate) fn holds_metadata_apply(node_id: u64) -> bool { crate::control::cluster::metadata_applier::METADATA_APPLY_HOLD_POINT, ) } + +/// The redo chunk gate of node `node_id`: a committed `RedoChunk` entry +/// parks here before its chunk applies. Named per chunk index, +/// `redo_chunk::before_apply::`. The chunk's entry applies with +/// nothing else of its group in flight, so every later entry of the group +/// waits behind it. +pub(crate) async fn before_chunk_apply(node_id: u64, index: u32) { + wait( + FailScope::Node(node_id), + &format!("redo_chunk::before_apply::{index}"), + ) + .await; +} + +/// The data-group snapshot build gate of node `node_id`: a leader's build +/// of a snapshot of `group_id` parks here before it reads any state. Named +/// `snapshot::before_build::`. No snapshot of the group leaves +/// the node while it is parked. +pub(crate) async fn before_snapshot_build(node_id: u64, group_id: u64) { + wait( + FailScope::Node(node_id), + &format!("snapshot::before_build::{group_id}"), + ) + .await; +} + +/// The catalog DDL apply gate of node `node_id`: a `CatalogDdl` entry that +/// puts collection `collection` pauses here before it applies. Named +/// `catalog_ddl::before_apply::`, armed with `sleep()`. +/// The pause awaits a Tokio timer, as a slow apply awaits its effects. +pub(crate) async fn before_catalog_ddl_apply(node_id: u64, collection: &str) { + let name = format!("catalog_ddl::before_apply::{collection}"); + if let Some(FailAction::Sleep(pause)) = lookup(FailScope::Node(node_id), &name) { + tokio::time::sleep(pause).await; + } +} diff --git a/nodedb/src/control/gateway/cluster_error.rs b/nodedb/src/control/gateway/cluster_error.rs index 32c462e10..1be61531b 100644 --- a/nodedb/src/control/gateway/cluster_error.rs +++ b/nodedb/src/control/gateway/cluster_error.rs @@ -1,7 +1,9 @@ // SPDX-License-Identifier: BUSL-1.1 -//! Map a remote node's [`TypedClusterError`] to the gateway's [`Error`]. +//! Map a remote node's [`TypedClusterError`], and a failed send to it, to +//! the gateway's [`Error`]. +use nodedb_cluster::ClusterError; use nodedb_cluster::rpc_codec::TypedClusterError; use crate::Error; @@ -74,10 +76,191 @@ pub(super) fn map_typed_cluster_error(err: TypedClusterError, vshard_id: u64) -> } } +/// One `ExecuteRequest` or `ExecuteStreamRequest` send, as its failure +/// mapping needs it. +#[derive(Debug, Clone, Copy)] +pub(super) struct RemoteSend { + pub node_id: u64, + pub vshard_id: u64, + /// A second run of the plan is harmless: the plan only reads. + pub resend_safe: bool, + /// The statement's deadline for this send. + pub deadline: tokio::time::Instant, +} + +/// The error of a failed `ExecuteRequest` or `ExecuteStreamRequest` send. +/// +/// A request the target never received, and a lost answer to a read, retry as +/// `NotLeader` with `leader_node = 0`. The unreachable node is then not kept as +/// leader, and the next try routes locally to the real leader. +/// +/// These end the statement as `DeadlineExceeded` instead: +/// - a lost answer to any other plan: it can have run, and a second send +/// would run it again; +/// - a lost answer once the deadline passed: the wait used the whole budget, +/// and no retry has time left; +/// - a `ShardTimeout`: the wait used the whole budget. +pub(super) fn send_error(e: ClusterError, send: &RemoteSend) -> Error { + if matches!(e, ClusterError::ShardTimeout { .. }) { + return deadline_exceeded(send, &e); + } + if e.is_lost_answer() && (!send.resend_safe || tokio::time::Instant::now() >= send.deadline) { + return deadline_exceeded(send, &e); + } + Error::NotLeader { + vshard_id: VShardId::new((send.vshard_id % VShardId::COUNT as u64) as u32), + leader_node: 0, + leader_addr: format!("node-{} (transport error: {e})", send.node_id), + leader_term: 0, + } +} + +/// The error of a remote execute whose reply wait expired. +/// +/// The wait covers the statement's whole remaining budget plus the reply +/// margin, so no retry has time left: the statement ends as `DeadlineExceeded`. +pub(super) fn reply_wait_expired(send: &RemoteSend, reply_wait: std::time::Duration) -> Error { + let timed_out = ClusterError::ShardTimeout { + vshard_id: (send.vshard_id % VShardId::COUNT as u64) as u32, + elapsed_ms: u64::try_from(reply_wait.as_millis()).unwrap_or(u64::MAX), + }; + deadline_exceeded(send, &timed_out) +} + +fn deadline_exceeded(send: &RemoteSend, cause: &ClusterError) -> Error { + tracing::warn!( + node_id = send.node_id, + vshard_id = send.vshard_id, + error = %cause, + "gateway: a remote execute ended with no answer within the statement budget" + ); + Error::DeadlineExceeded { + request_id: crate::types::RequestId::new(0), + } +} + #[cfg(test)] mod tests { use super::*; + /// A send to node 2 for vShard 7 whose statement has time left. + fn send(resend_safe: bool) -> RemoteSend { + RemoteSend { + node_id: 2, + vshard_id: 7, + resend_safe, + deadline: tokio::time::Instant::now() + std::time::Duration::from_secs(30), + } + } + + /// The same send once its statement deadline passed. + fn spent(resend_safe: bool) -> RemoteSend { + RemoteSend { + deadline: tokio::time::Instant::now(), + ..send(resend_safe) + } + } + + fn unanswered() -> ClusterError { + ClusterError::Unanswered { + node_id: 2, + detail: "stream reset".into(), + } + } + + /// A write whose answer was lost can have applied: it is never sent again. + #[test] + fn a_lost_answer_to_a_write_is_an_unknown_outcome() { + assert!(matches!( + send_error(unanswered(), &send(false)), + Error::DeadlineExceeded { .. } + )); + } + + /// A read whose answer was lost retries through the leader lookup. + #[test] + fn a_lost_answer_to_a_read_retries() { + assert!(matches!( + send_error(unanswered(), &send(true)), + Error::NotLeader { leader_node: 0, .. } + )); + } + + /// A send that timed out used the statement's whole budget: a read and a + /// write both end as `DeadlineExceeded`, with no retry. + #[test] + fn a_timed_out_read_ends_the_statement() { + let timed_out = || ClusterError::ShardTimeout { + vshard_id: 7, + elapsed_ms: 1_000, + }; + assert!(matches!( + send_error(timed_out(), &send(true)), + Error::DeadlineExceeded { .. } + )); + assert!(matches!( + send_error(timed_out(), &send(false)), + Error::DeadlineExceeded { .. } + )); + } + + /// A live node holds a read past its budget: the transport's read timeout + /// ends the wait as a lost answer once the deadline passed. No retry has + /// time left, so the read ends as `DeadlineExceeded`, never `NotLeader`. + #[test] + fn a_lost_answer_to_a_read_past_its_deadline_ends_the_statement() { + assert!(matches!( + send_error(unanswered(), &spent(true)), + Error::DeadlineExceeded { .. } + )); + } + + /// A request that never went out leaves nothing to wait for, so it + /// retries even at the deadline. The retry loop decides whether time is left. + #[test] + fn an_unsent_read_past_its_deadline_stays_retryable() { + let unsent = ClusterError::Transport { + detail: "connect refused".into(), + }; + assert!(matches!( + send_error(unsent, &spent(true)), + Error::NotLeader { leader_node: 0, .. } + )); + } + + /// An expired reply wait ends the statement, never a retry. + #[test] + fn an_expired_reply_wait_is_deadline_exceeded() { + assert!(matches!( + reply_wait_expired(&send(true), std::time::Duration::from_secs(14)), + Error::DeadlineExceeded { .. } + )); + } + + /// A transport error is unsent: a read and a write both retry. + #[test] + fn an_unsent_read_retries() { + let unsent = ClusterError::Transport { + detail: "handshake timeout".into(), + }; + assert!(matches!( + send_error(unsent, &send(true)), + Error::NotLeader { leader_node: 0, .. } + )); + } + + /// A request that never reached the target retries, read or write. + #[test] + fn an_unsent_request_retries() { + let unsent = ClusterError::Transport { + detail: "connect refused".into(), + }; + assert!(matches!( + send_error(unsent, &send(false)), + Error::NotLeader { leader_node: 0, .. } + )); + } + #[test] fn map_not_leader() { let err = TypedClusterError::NotLeader { diff --git a/nodedb/src/control/gateway/core.rs b/nodedb/src/control/gateway/core.rs index 64aaca526..a47d6b5ca 100644 --- a/nodedb/src/control/gateway/core.rs +++ b/nodedb/src/control/gateway/core.rs @@ -8,7 +8,7 @@ //! version mapping). //! 2. Routes the plan via [`route_plan`] to `Local` or `Remote` task routes. //! 3. Dispatches each route (local SPSC or `ExecuteRequest` RPC) with typed -//! `NotLeader` retry (up to 3 attempts). +//! `NotLeader` retry until the statement deadline. //! 4. Handles `RetryableSchemaChanged` (descriptor cache miss) by fetching a //! fresh lease and re-planning once. //! 5. Fuses multiple vShard payloads for broadcast scans. @@ -31,16 +31,17 @@ use crate::control::trace_export::EmitSpanParams; use crate::types::{DatabaseId, Lsn, TenantId, TraceId, TxnId, VShardId}; use nodedb_physical::physical_plan::PhysicalPlan; -use super::dispatcher::{DispatchRouteParams, dispatch_route, statement_deadline_ms}; +use super::dispatcher::{DispatchRouteParams, dispatch_route}; use super::fuser::fuse_payloads; use super::key_extractor::UnwiredKeyExtractor; use super::live_leaders::resolve_live_decision; use super::outcome::GatewayOutcome; use super::plan_cache::PlanCache; -use super::retry::retry_not_leader; +use super::retry::{RetryPacing, retry_not_leader}; use super::route::TaskRoute; use super::router::route_plan; use super::version_set::GatewayVersionSet; +use crate::control::server::shared::session::StatementDeadline; /// Context passed to [`Gateway::execute`]. pub struct QueryContext { @@ -248,7 +249,9 @@ impl Gateway { let single_route = routes.len() == 1; let mut not_found = false; - let deadline_ms = statement_deadline_ms(&shared); + // One deadline for every route and attempt of the statement. + let deadline = StatementDeadline::of_statement(&shared).tokio_instant(); + let pacing = RetryPacing::from_tuning(&shared.tuning.cluster_transport); // Gateway-level byte ceiling: per-route `dispatch_to_data_plane` // already caps each shard's payload; this additionally caps the // scatter-gather *sum* so an N-shard fan-out can't accumulate @@ -271,8 +274,8 @@ impl Gateway { let retry_counter = Arc::clone(&self.not_leader_retry_count); let version_set_for_route = version_set.clone(); let shared_for_route = Arc::clone(&shared); - let outcome = retry_not_leader(routing_ref, move |attempt| { - if attempt > 0 { + let outcome = retry_not_leader(routing_ref, deadline, pacing, move |attempt| { + if attempt.index > 0 { retry_counter.fetch_add(1, Ordering::Relaxed); } let plan = plan_for_retry.clone(); @@ -296,7 +299,7 @@ impl Gateway { tenant_id, database_id, trace_id, - deadline_ms, + deadline_ms: attempt.remaining_ms(), version_set: &version_set, txn_id, linearizable, diff --git a/nodedb/src/control/gateway/dispatch_local.rs b/nodedb/src/control/gateway/dispatch_local.rs index fc26c5325..a8e530dbe 100644 --- a/nodedb/src/control/gateway/dispatch_local.rs +++ b/nodedb/src/control/gateway/dispatch_local.rs @@ -98,7 +98,8 @@ pub(super) async fn dispatch_local( shared, proposer, entry, - crate::control::wal_replication::statement_propose_deadline(shared), + crate::control::server::shared::session::StatementDeadline::of_statement(shared) + .tokio_instant(), ) .await?; return Ok(DispatchOutcome { diff --git a/nodedb/src/control/gateway/dispatch_remote.rs b/nodedb/src/control/gateway/dispatch_remote.rs index 94b6de798..2053e95a4 100644 --- a/nodedb/src/control/gateway/dispatch_remote.rs +++ b/nodedb/src/control/gateway/dispatch_remote.rs @@ -8,7 +8,6 @@ use std::sync::Arc; -use nodedb_cluster::ClusterError; use nodedb_cluster::rpc_codec::{ExecuteRequest, RaftRpc}; use tracing::debug; @@ -18,7 +17,7 @@ use crate::control::state::SharedState; use crate::types::{DatabaseId, Lsn, TenantId, TraceId, TxnId, VShardId}; use nodedb_physical::physical_plan::wire as plan_wire; -use super::cluster_error::map_typed_cluster_error; +use super::cluster_error::{RemoteSend, map_typed_cluster_error, reply_wait_expired, send_error}; use super::dispatcher::DispatchOutcome; use super::version_set::GatewayVersionSet; @@ -32,6 +31,7 @@ pub(super) struct RemoteDispatchArgs<'a> { pub tenant_id: TenantId, pub database_id: DatabaseId, pub trace_id: TraceId, + /// Milliseconds left on the statement when this attempt starts. pub deadline_ms: u64, pub version_set: &'a GatewayVersionSet, /// Session-transaction context forwarded to the remote executor, or `None` @@ -63,6 +63,8 @@ pub(super) async fn dispatch_remote( linearizable, read_groups, } = args; + // The statement's deadline, fixed before any wait of this attempt. + let deadline = tokio::time::Instant::now() + std::time::Duration::from_millis(deadline_ms); let transport = shared.cluster_transport.as_ref().ok_or(Error::Internal { detail: "gateway: cluster transport not available for remote dispatch".into(), })?; @@ -132,7 +134,12 @@ pub(super) async fn dispatch_remote( // A read can be sent again after a lost answer. Any other plan can have // run on the target, so a lost answer is an unknown outcome. - let resend_safe = required_permission(&plan) == Permission::Read; + let send = RemoteSend { + node_id, + vshard_id, + resend_safe: required_permission(&plan) == Permission::Read, + deadline, + }; // Encode the plan. let plan_bytes = plan_wire::encode(&plan).map_err(|e| Error::Internal { @@ -170,9 +177,9 @@ pub(super) async fn dispatch_remote( "gateway: dispatching ExecuteRequest to remote node" ); - // The remote handler works within `deadline_ms`: a write can wait at the - // leader's gate or in a Calvin queue for that long. The reply wait follows - // that budget, never the transport's fixed RPC timeout. + // The remote handler works within the statement's whole remaining budget. + // A live node gets all of it. A dead node fails the send at the QUIC idle + // timeout, and a read then retries elsewhere. let reply_wait = nodedb_cluster::rpc_codec::reply_wait(deadline_ms); let resp_rpc = match tokio::time::timeout( reply_wait, @@ -180,18 +187,9 @@ pub(super) async fn dispatch_remote( ) .await { - Ok(answer) => answer.map_err(|e| send_error(e, resend_safe, node_id, vshard_id))?, - Err(_) => { - return Err(send_error( - ClusterError::ShardTimeout { - vshard_id: (vshard_id % VShardId::COUNT as u64) as u32, - elapsed_ms: u64::try_from(reply_wait.as_millis()).unwrap_or(u64::MAX), - }, - resend_safe, - node_id, - vshard_id, - )); - } + Ok(answer) => answer.map_err(|e| send_error(e, &send))?, + // The wait was the statement's whole budget: no retry has time left. + Err(_) => return Err(reply_wait_expired(&send, reply_wait)), }; match resp_rpc { @@ -249,79 +247,11 @@ pub(super) fn scoped_vshard( Ok(Some(VShardId::new(raw))) } -/// The error of a failed `ExecuteRequest` send. -/// -/// A request the target never received, and a lost answer to a read, retry as -/// `NotLeader` with `leader_node = 0`. The unreachable node is then not kept as -/// leader, and the next try routes locally to the real leader. A lost answer -/// to any other plan is an unknown outcome: the plan can have run, and a -/// second send would run it again. -fn send_error(e: ClusterError, resend_safe: bool, node_id: u64, vshard_id: u64) -> Error { - let outcome_unknown = matches!( - e, - ClusterError::Unanswered { .. } | ClusterError::ShardTimeout { .. } - ); - if outcome_unknown && !resend_safe { - tracing::warn!( - node_id, - vshard_id, - error = %e, - "gateway: a remote execute reached its deadline with no answer" - ); - return Error::DeadlineExceeded { - request_id: crate::types::RequestId::new(0), - }; - } - Error::NotLeader { - vshard_id: VShardId::new((vshard_id % VShardId::COUNT as u64) as u32), - leader_node: 0, - leader_addr: format!("node-{node_id} (transport error: {e})"), - leader_term: 0, - } -} - #[cfg(test)] mod tests { use super::*; use nodedb_physical::physical_plan::{MetaOp, PhysicalPlan}; - fn unanswered() -> ClusterError { - ClusterError::Unanswered { - node_id: 2, - detail: "stream reset".into(), - } - } - - /// A write whose answer was lost can have applied: it is never sent again. - #[test] - fn a_lost_answer_to_a_write_is_an_unknown_outcome() { - assert!(matches!( - send_error(unanswered(), false, 2, 7), - Error::DeadlineExceeded { .. } - )); - } - - /// A read whose answer was lost retries through the leader lookup. - #[test] - fn a_lost_answer_to_a_read_retries() { - assert!(matches!( - send_error(unanswered(), true, 2, 7), - Error::NotLeader { leader_node: 0, .. } - )); - } - - /// A request that never reached the target retries, read or write. - #[test] - fn an_unsent_request_retries() { - let unsent = ClusterError::Transport { - detail: "connect refused".into(), - }; - assert!(matches!( - send_error(unsent, false, 2, 7), - Error::NotLeader { leader_node: 0, .. } - )); - } - #[test] fn a_transaction_meta_op_carries_its_route_vshard() { let plan = PhysicalPlan::Meta(MetaOp::DropTxnOverlay { diff --git a/nodedb/src/control/gateway/dispatch_remote_stream.rs b/nodedb/src/control/gateway/dispatch_remote_stream.rs index 3dd04f63a..03c88a10a 100644 --- a/nodedb/src/control/gateway/dispatch_remote_stream.rs +++ b/nodedb/src/control/gateway/dispatch_remote_stream.rs @@ -11,10 +11,11 @@ use nodedb_physical::physical_plan::wire as plan_wire; use tracing::debug; use crate::Error; +use crate::control::security::identity::{Permission, required_permission}; use crate::control::server::result_stream::{ResultStream, RowBatch}; use crate::types::{Lsn, VShardId}; -use super::cluster_error::map_typed_cluster_error; +use super::cluster_error::{RemoteSend, map_typed_cluster_error, reply_wait_expired, send_error}; use super::dispatch_remote::{RemoteDispatchArgs, scoped_vshard}; /// Remote streaming dispatch via the multi-frame `ExecuteStreamRequest` RPC. @@ -35,6 +36,10 @@ use super::dispatch_remote::{RemoteDispatchArgs, scoped_vshard}; /// `Err` and never retried (re-running the plan duplicates the rows /// already streamed to the client). /// +/// The remote runs the stream within the statement's whole remaining budget. +/// The open and the first frame wait at most `reply_wait` of that budget. A +/// dead node fails the open at the QUIC idle timeout, and a read retries. +/// /// The returned stream re-emits the buffered first batch followed by the rest. pub(super) async fn dispatch_remote_stream( args: RemoteDispatchArgs<'_>, @@ -53,6 +58,8 @@ pub(super) async fn dispatch_remote_stream( linearizable, read_groups, } = args; + // The statement's deadline, fixed before any wait of this attempt. + let deadline = tokio::time::Instant::now() + std::time::Duration::from_millis(deadline_ms); let transport = shared.cluster_transport.as_ref().ok_or(Error::Internal { detail: "gateway: cluster transport not available for remote stream dispatch".into(), })?; @@ -91,6 +98,15 @@ pub(super) async fn dispatch_remote_stream( crate::control::server::exchange::Resolved::Stream(s) => return Ok(s), }; + // A read can be sent again after a lost answer. Any other plan can have + // run on the target, so a lost answer is an unknown outcome. + let send = RemoteSend { + node_id, + vshard_id, + resend_safe: required_permission(&plan) == Permission::Read, + deadline, + }; + let plan_bytes = plan_wire::encode(&plan).map_err(|e| Error::Internal { detail: format!("gateway: plan encode failed: {e}"), })?; @@ -110,6 +126,7 @@ pub(super) async fn dispatch_remote_stream( plan_bytes, tenant_id: tenant_id.as_u64(), database_id: database_id.as_u64(), + // The remote bounds the whole stream with the statement's remainder. deadline_remaining_ms: deadline_ms, trace_id: trace_id.0, descriptor_versions, @@ -125,25 +142,28 @@ pub(super) async fn dispatch_remote_stream( "gateway: dispatching ExecuteStreamRequest to remote node" ); - // Open the stream eagerly. A failure to even open / send the request is a - // pre-row condition: map it like a transport failure in `dispatch_remote` - // so the retry loop routes elsewhere on the next attempt. - let stream = transport - .send_rpc_stream(node_id, req) - .await - .map_err(|e| Error::NotLeader { - vshard_id: VShardId::new((vshard_id % VShardId::COUNT as u64) as u32), - leader_node: 0, - leader_addr: format!("node-{node_id} (stream open error: {e})"), - leader_term: 0, - })?; - // The `async_stream` body is `!Unpin`; pin it on the heap so we can pull - // the eager first frame and then keep the tail around for `.chain`. - let mut stream = Box::pin(stream); - - // Eagerly pull the FIRST frame so a pre-row terminal error is catchable and - // retryable. Any error here is a pre-row error. - let first = match stream.next().await { + // Open the stream and pull the FIRST frame eagerly, within the reply + // wait, so a pre-row error is catchable and retryable. Any error here is + // a pre-row error. + let reply_wait = nodedb_cluster::rpc_codec::reply_wait(deadline_ms); + let opened = tokio::time::timeout(reply_wait, async move { + let stream = transport + .send_rpc_stream(node_id, req) + .await + .map_err(|e| send_error(e, &send))?; + // The `async_stream` body is `!Unpin`; pin it on the heap so the tail + // stays usable for `.chain` after the first frame. + let mut stream = Box::pin(stream); + let first = stream.next().await; + Ok::<_, Error>((stream, first)) + }) + .await; + let (stream, first) = match opened { + Ok(opened) => opened?, + // The wait was the statement's whole budget: no retry has time left. + Err(_) => return Err(reply_wait_expired(&send, reply_wait)), + }; + let first = match first { Some(Ok((payload, lsn))) => RowBatch { payload, watermark_lsn: Lsn::new(lsn), @@ -151,7 +171,9 @@ pub(super) async fn dispatch_remote_stream( // read version is threaded on this remote path. read_versions: crate::types::ReadVersions::new(), }, - Some(Err(e)) => return Err(map_stream_cluster_error(e, vshard_id)), + Some(Err(e)) => { + return Err(map_stream_cluster_error(e, &send)); + } // Clean EOF with zero rows: a valid empty result. Return an empty stream. None => return Ok(Box::pin(futures::stream::empty())), }; @@ -179,11 +201,13 @@ pub(super) async fn dispatch_remote_stream( /// /// A typed error (`StreamTerminal`, `ShardExecution`) maps through the same /// [`map_typed_cluster_error`] used by the one-shot path, so the gateway retry -/// loop handles it identically. A Data-Plane verdict keeps its code. Any other -/// cluster error becomes a transport-style `NotLeader` (leader_node = 0), so -/// the next attempt re-resolves routing rather than re-entrenching an -/// unreachable node. -fn map_stream_cluster_error(err: ClusterError, vshard_id: u64) -> Error { +/// loop handles it identically. A Data-Plane verdict keeps its code. A lost +/// answer goes through [`send_error`]: a read retries, any other plan is an +/// unknown outcome. Any other cluster error becomes a transport-style +/// `NotLeader` (leader_node = 0), so the next attempt re-resolves routing +/// rather than re-entrenching an unreachable node. +fn map_stream_cluster_error(err: ClusterError, send: &RemoteSend) -> Error { + let vshard_id = send.vshard_id; match err { ClusterError::StreamTerminal { error, .. } | ClusterError::ShardExecution { error, .. } => { map_typed_cluster_error(*error, vshard_id) @@ -191,6 +215,8 @@ fn map_stream_cluster_error(err: ClusterError, vshard_id: u64) -> Error { // A verdict from a shard that answered. Retrying it on another route // repeats it, so it keeps its SQLSTATE. ClusterError::DataPlane { code } => Error::DataPlane(code.into()), + // The request can have run on the remote node. + lost if lost.is_lost_answer() => send_error(lost, send), other @ (ClusterError::Raft(_) | ClusterError::VShardNotMapped { .. } | ClusterError::GroupNotFound { .. } @@ -200,8 +226,6 @@ fn map_stream_cluster_error(err: ClusterError, vshard_id: u64) -> Error { | ClusterError::NodeUnreachable { .. } | ClusterError::GhostNotFound { .. } | ClusterError::Transport { .. } - | ClusterError::ShardTimeout { .. } - | ClusterError::Unanswered { .. } | ClusterError::Storage { .. } | ClusterError::Codec { .. } | ClusterError::UnsupportedWireVersion { .. } @@ -227,7 +251,10 @@ fn map_stream_cluster_error(err: ClusterError, vshard_id: u64) -> Error { | ClusterError::Bm25Gather(_) | ClusterError::TsGather(_) | ClusterError::ShufflePush(_) - | ClusterError::RemoteUntyped { .. }) => Error::NotLeader { + | ClusterError::RemoteUntyped { .. } + // The guards above take these. They stay named for exhaustiveness. + | ClusterError::ShardTimeout { .. } + | ClusterError::Unanswered { .. }) => Error::NotLeader { vshard_id: VShardId::new((vshard_id % VShardId::COUNT as u64) as u32), leader_node: 0, leader_addr: format!("stream dispatch error: {other}"), @@ -235,3 +262,63 @@ fn map_stream_cluster_error(err: ClusterError, vshard_id: u64) -> Error { }, } } + +#[cfg(test)] +mod tests { + use super::*; + + fn send(resend_safe: bool) -> RemoteSend { + RemoteSend { + node_id: 2, + vshard_id: 7, + resend_safe, + deadline: tokio::time::Instant::now() + std::time::Duration::from_secs(30), + } + } + + fn unanswered() -> ClusterError { + ClusterError::Unanswered { + node_id: 2, + detail: "stream reset".into(), + } + } + + /// A read whose stream answer was lost retries through the leader lookup. + #[test] + fn a_lost_stream_answer_to_a_read_retries() { + assert!(matches!( + map_stream_cluster_error(unanswered(), &send(true)), + Error::NotLeader { leader_node: 0, .. } + )); + } + + /// Any other plan can have run when its stream answer was lost, so it + /// is never sent again. + #[test] + fn a_lost_stream_answer_to_a_non_read_is_an_unknown_outcome() { + assert!(matches!( + map_stream_cluster_error(unanswered(), &send(false)), + Error::DeadlineExceeded { .. } + )); + let timed_out = ClusterError::ShardTimeout { + vshard_id: 7, + elapsed_ms: 1_000, + }; + assert!(matches!( + map_stream_cluster_error(timed_out, &send(false)), + Error::DeadlineExceeded { .. } + )); + } + + /// A transport error before any row retries, read or not. + #[test] + fn a_pre_row_transport_error_retries() { + let unsent = ClusterError::Transport { + detail: "connect refused".into(), + }; + assert!(matches!( + map_stream_cluster_error(unsent, &send(false)), + Error::NotLeader { leader_node: 0, .. } + )); + } +} diff --git a/nodedb/src/control/gateway/dispatcher.rs b/nodedb/src/control/gateway/dispatcher.rs index 79e396855..5398386de 100644 --- a/nodedb/src/control/gateway/dispatcher.rs +++ b/nodedb/src/control/gateway/dispatcher.rs @@ -227,18 +227,6 @@ pub(crate) async fn dispatch_route_stream( } } -/// Milliseconds left on the running statement, for a remote hop's -/// `ExecuteRequest.deadline_remaining_ms`. -/// -/// The session's `statement_timeout` when one is installed on this connection, -/// else the node's configured default. A forwarded route must stop when the -/// statement that spawned it does, not on a budget of its own. -pub fn statement_deadline_ms(shared: &SharedState) -> u64 { - crate::control::server::shared::session::statement_deadline_ms( - shared.tuning.network.default_deadline_secs, - ) -} - #[cfg(test)] mod tests { use super::*; diff --git a/nodedb/src/control/gateway/error_map/class_parity/error_index.rs b/nodedb/src/control/gateway/error_map/class_parity/error_index.rs index d49285b9f..a28c9cfe5 100644 --- a/nodedb/src/control/gateway/error_map/class_parity/error_index.rs +++ b/nodedb/src/control/gateway/error_map/class_parity/error_index.rs @@ -3,7 +3,7 @@ //! A dense index per `crate::Error` variant. /// The number of `crate::Error` variants [`error_variant_index`] numbers. -pub(super) const ERROR_VARIANT_COUNT: usize = 121; +pub(super) const ERROR_VARIANT_COUNT: usize = 123; /// A dense index per `crate::Error` variant. Exhaustive, so a new variant /// fails to compile here until it gets an index, and @@ -72,6 +72,8 @@ pub(crate) fn error_variant_index(err: &crate::Error) -> usize { E::GroupQuorumUnavailable { .. } => 58, E::GroupMarksUnavailable { .. } => 59, E::MetadataLeaderUnavailable => 60, + E::MetadataGroupGone { .. } => 121, + E::MetadataDdlSuperseded { .. } => 122, E::AuthorizationStateBehind { .. } => 61, E::ExecutionLimitExceeded { .. } => 62, E::LimitExceeded { .. } => 63, diff --git a/nodedb/src/control/gateway/error_map/class_parity/error_samples.rs b/nodedb/src/control/gateway/error_map/class_parity/error_samples.rs index 0280a484d..ffedec6d4 100644 --- a/nodedb/src/control/gateway/error_map/class_parity/error_samples.rs +++ b/nodedb/src/control/gateway/error_map/class_parity/error_samples.rs @@ -248,6 +248,8 @@ pub(crate) fn error_samples() -> Vec { }, E::BackupCaptureMoved { group_id: 1 }, E::MetadataLeaderUnavailable, + E::MetadataGroupGone { log_index: 2 }, + E::MetadataDdlSuperseded { log_index: 2 }, E::AuthorizationStateBehind { detail: text() }, E::LinearizableReadRefused { group_id: 1, diff --git a/nodedb/src/control/gateway/error_map/resp.rs b/nodedb/src/control/gateway/error_map/resp.rs index 0da6aa8e4..e9eeb9fab 100644 --- a/nodedb/src/control/gateway/error_map/resp.rs +++ b/nodedb/src/control/gateway/error_map/resp.rs @@ -107,6 +107,8 @@ impl GatewayErrorMap { | Error::GroupMarksUnavailable { .. } | Error::BackupCaptureMoved { .. } | Error::MetadataLeaderUnavailable + | Error::MetadataGroupGone { .. } + | Error::MetadataDdlSuperseded { .. } | Error::AuthorizationStateBehind { .. } | Error::LinearizableReadRefused { .. } | Error::ExecutionLimitExceeded { .. } diff --git a/nodedb/src/control/gateway/live_leaders.rs b/nodedb/src/control/gateway/live_leaders.rs index 929cb8bec..74bbf543b 100644 --- a/nodedb/src/control/gateway/live_leaders.rs +++ b/nodedb/src/control/gateway/live_leaders.rs @@ -92,6 +92,7 @@ mod tests { last_applied: 0, last_log_index: 0, snapshot_index: 0, + replica_state: nodedb_raft::ReplicaState::Whole, member_count: 2, learner_count: 0, vshard_count: 0, diff --git a/nodedb/src/control/gateway/read_leg.rs b/nodedb/src/control/gateway/read_leg.rs index 94b7f29fe..3dc41a2f1 100644 --- a/nodedb/src/control/gateway/read_leg.rs +++ b/nodedb/src/control/gateway/read_leg.rs @@ -13,7 +13,7 @@ use crate::Error; use nodedb_physical::physical_plan::{PhysicalPlan, plan_contains_cluster_partitioned_leaf}; use crate::control::cluster::linearizable_read::{ - confirm_linearizable_read, groups_of_vshards, linearizable_read_deadline, + StatementDeadline, confirm_linearizable_read, groups_of_vshards, }; use crate::control::server::graph_dispatch::graph_read_groups; use crate::control::server::shared::write_admission::plan_is_write; @@ -52,7 +52,7 @@ pub(crate) async fn confirm_local_read( if read_groups.is_empty() { return Ok(()); } - let deadline = linearizable_read_deadline(Duration::from_millis(deadline_ms)); + let deadline = StatementDeadline::after(Duration::from_millis(deadline_ms)); if plan_contains_cluster_partitioned_leaf(plan) { let groups = graph_read_groups(shared, database_id, plan)?; return confirm_linearizable_read(shared, &groups, deadline).await; diff --git a/nodedb/src/control/gateway/retry.rs b/nodedb/src/control/gateway/retry.rs index 3b5880e91..2adb75ec7 100644 --- a/nodedb/src/control/gateway/retry.rs +++ b/nodedb/src/control/gateway/retry.rs @@ -1,55 +1,119 @@ // SPDX-License-Identifier: BUSL-1.1 -//! Typed `NotLeader` retry with 3-attempt budget + 50/100/200 ms backoff. +//! Typed `NotLeader` retry that runs until the statement deadline. //! //! When a remote dispatch returns `Error::NotLeader`, the retry helper: //! 1. Extracts the hinted new leader and its term from the error. //! 2. Offers them to the routing table entry for the affected group. The //! hint moves only when the redirect's term is above the hint's term, so //! a stale redirect never undoes a newer leader. -//! 3. Sleeps for the appropriate backoff duration. +//! 3. Sleeps for the next backoff of its [`RetryPacing`]. //! 4. Re-invokes the closure. //! //! If the hinted leader is unknown (no hint), we still retry after sleep //! without updating the routing table — a subsequent routing lookup will //! re-read the table from the current routing state. //! -//! After `MAX_RETRIES` attempts the final `NotLeader` error is propagated. +//! A re-election can outlast any fixed attempt count, so retries continue +//! while the statement has time left. Every attempt shares the statement's +//! deadline. When the next backoff would reach it, one last attempt starts +//! at once, and its verdict ends the statement. A lost answer to a read then +//! ends as the deadline the last send reports. No retry starts once the +//! deadline passed. The last refusal then ends the statement: it is +//! definite, so it never becomes `DeadlineExceeded`, which means a lost +//! answer. use std::future::Future; use std::sync::RwLock; -use tokio::time::{Duration, sleep}; +use tokio::time::{Duration, Instant, sleep}; use tracing::debug; use nodedb_cluster::RoutingTable; +use nodedb_types::config::tuning::ClusterTransportTuning; use crate::Error; +use crate::control::server::shared::session::deadline::hop_budget_ms; -/// Maximum number of dispatch attempts (initial + 2 retries = 3 total). -pub const MAX_RETRIES: usize = 3; +/// One dispatch attempt of a statement. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub struct Attempt { + /// The attempt's 0-based index. + pub index: usize, + /// The statement's deadline. Every attempt carries the same instant. + pub deadline: Instant, +} + +impl Attempt { + /// The milliseconds left on the statement now, at least 1. See + /// [`hop_budget_ms`]. + pub fn remaining_ms(&self) -> u64 { + hop_budget_ms(self.deadline.into_std()) + } +} -/// Backoff durations for each retry attempt. -const BACKOFF_MS: [u64; MAX_RETRIES] = [50, 100, 200]; +/// The backoff between leader retries: doubles from `first` up to `cap`. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub struct RetryPacing { + /// The pause before the first retry. + pub first: Duration, + /// The longest pause between two retries. + pub cap: Duration, +} + +impl RetryPacing { + /// Pacing from the cluster's Raft timing. + /// + /// The first pause is one Raft tick. The cap is the minimum election + /// timeout: no new leader can appear faster than that. + pub fn from_tuning(tuning: &ClusterTransportTuning) -> Self { + let first = Duration::from_millis(tuning.raft_tick_interval_ms.max(1)); + let cap = Duration::from_millis(tuning.effective_election_timeout_min_ms()).max(first); + Self { first, cap } + } -/// Execute `f` up to `MAX_RETRIES` times, retrying on `Error::NotLeader`. + /// The pause after `retries` retries already ran. + fn backoff(&self, retries: u32) -> Duration { + self.first + .checked_mul(2u32.saturating_pow(retries)) + .map_or(self.cap, |d| d.min(self.cap)) + } +} + +impl Default for RetryPacing { + fn default() -> Self { + Self::from_tuning(&ClusterTransportTuning::default()) + } +} + +/// Execute `f`, retrying on `Error::NotLeader` and +/// `Error::RetryableLeaderChange` until `deadline`. /// -/// `f` receives the current attempt index (0-based). +/// `f` receives the current [`Attempt`]: its index and the statement's +/// `deadline`, which every attempt shares. /// /// On `NotLeader` with a hinted leader, the routing table is updated before /// the next retry so the caller's routing decision changes. On non-`NotLeader` /// errors the error is propagated immediately without retry. pub async fn retry_not_leader( routing: Option<&RwLock>, + deadline: Instant, + pacing: RetryPacing, f: F, ) -> Result where - F: Fn(usize) -> Fut, + F: Fn(Attempt) -> Fut, Fut: Future>, { - let mut last_err = None; - for (attempt, &backoff_ms) in BACKOFF_MS.iter().enumerate() { - match f(attempt).await { + let mut attempt = 0; + let mut last_call = false; + loop { + let refusal = match f(Attempt { + index: attempt, + deadline, + }) + .await + { Ok(v) => return Ok(v), Err(Error::NotLeader { vshard_id, @@ -89,48 +153,51 @@ where } } - if attempt + 1 < MAX_RETRIES { - sleep(Duration::from_millis(backoff_ms)).await; - } - - last_err = Some(Error::NotLeader { + Error::NotLeader { vshard_id, leader_node, leader_addr: String::new(), leader_term, - }); + } } Err(Error::RetryableLeaderChange { group_id, log_index, }) => { - // The Raft entry the proposer was waiting on was - // overwritten by a new leader's election no-op. Re- - // propose: the next attempt will land at a fresh log - // index under the new leader's term. Same backoff - // schedule as NotLeader — bounded retries so a - // pathological cluster doesn't loop forever. + // A new leader's election no-op overwrote the entry the + // proposer waited on. The next attempt re-proposes under the + // new leader's term. debug!( attempt, group_id, log_index, "gateway: leader change overwrote entry — will re-propose" ); - if attempt + 1 < MAX_RETRIES { - sleep(Duration::from_millis(backoff_ms)).await; - } - - last_err = Some(Error::RetryableLeaderChange { + Error::RetryableLeaderChange { group_id, log_index, - }); + } } Err(other) => return Err(other), + }; + + // Both refusals mean nothing applied. Once the deadline passed, or + // the last call ran, that definite refusal ends the statement. + let now = Instant::now(); + if last_call || now >= deadline { + debug!( + attempt, + "gateway: statement deadline reached — no further retry" + ); + return Err(refusal); } + let pause = pacing.backoff(u32::try_from(attempt).unwrap_or(u32::MAX)); + if now + pause >= deadline { + last_call = true; + } else { + sleep(pause).await; + } + attempt += 1; } - - Err(last_err.unwrap_or(Error::Internal { - detail: "retry_not_leader exhausted all attempts".into(), - })) } #[cfg(test)] @@ -143,9 +210,32 @@ mod tests { use super::*; use crate::types::VShardId; + fn far_deadline() -> Instant { + Instant::now() + Duration::from_secs(30) + } + + fn pacing() -> RetryPacing { + RetryPacing { + first: Duration::from_millis(10), + cap: Duration::from_millis(150), + } + } + + fn not_leader(leader_node: u64, leader_term: u64) -> Error { + Error::NotLeader { + vshard_id: VShardId::new(0), + leader_node, + leader_addr: String::new(), + leader_term, + } + } + #[tokio::test] async fn success_on_first_attempt() { - let result = retry_not_leader(None, |_attempt| async { Ok::(42) }).await; + let result = retry_not_leader(None, far_deadline(), pacing(), |_attempt| async { + Ok::(42) + }) + .await; assert_eq!(result.unwrap(), 42); } @@ -153,17 +243,11 @@ mod tests { async fn success_on_second_attempt() { let call_count = Arc::new(AtomicUsize::new(0)); let count = Arc::clone(&call_count); - let result = retry_not_leader(None, move |_attempt| { + let result = retry_not_leader(None, far_deadline(), pacing(), move |_attempt| { let c = Arc::clone(&count); async move { - let n = c.fetch_add(1, Ordering::SeqCst); - if n == 0 { - Err(Error::NotLeader { - vshard_id: VShardId::new(0), - leader_node: 2, - leader_addr: "10.0.0.2:9400".into(), - leader_term: 1, - }) + if c.fetch_add(1, Ordering::SeqCst) == 0 { + Err(not_leader(2, 1)) } else { Ok::(99) } @@ -174,25 +258,11 @@ mod tests { assert_eq!(call_count.load(Ordering::SeqCst), 2); } - #[tokio::test] - async fn exhausts_retries_returns_not_leader() { - let result = retry_not_leader(None, |_| async { - Err::(Error::NotLeader { - vshard_id: VShardId::new(1), - leader_node: 0, - leader_addr: String::new(), - leader_term: 0, - }) - }) - .await; - assert!(matches!(result, Err(Error::NotLeader { .. }))); - } - #[tokio::test] async fn non_not_leader_error_propagates_immediately() { let call_count = Arc::new(AtomicUsize::new(0)); let count = Arc::clone(&call_count); - let result = retry_not_leader(None, move |_| { + let result = retry_not_leader(None, far_deadline(), pacing(), move |_| { let c = Arc::clone(&count); async move { c.fetch_add(1, Ordering::SeqCst); @@ -206,6 +276,59 @@ mod tests { assert_eq!(call_count.load(Ordering::SeqCst), 1); } + /// A re-election longer than a few fixed attempts is retried until the + /// new leader answers. + #[tokio::test(start_paused = true)] + async fn a_leader_change_past_350_ms_is_retried_until_it_resolves() { + let started = Instant::now(); + let resolves_at = started + Duration::from_millis(1_000); + let calls = Arc::new(AtomicUsize::new(0)); + let count = Arc::clone(&calls); + let result = retry_not_leader(None, far_deadline(), pacing(), move |_| { + let c = Arc::clone(&count); + async move { + c.fetch_add(1, Ordering::SeqCst); + if Instant::now() < resolves_at { + Err(not_leader(0, 0)) + } else { + Ok::(7) + } + } + }) + .await; + assert_eq!(result.unwrap(), 7); + assert!(calls.load(Ordering::SeqCst) > 3, "more than three attempts"); + let waited = started.elapsed(); + assert!( + waited < Duration::from_millis(1_000) + pacing().cap, + "the retry follows the resolution within one capped backoff: {waited:?}" + ); + } + + /// Retries stop when the next backoff reaches the deadline. The last + /// refusal stays the answer, never `DeadlineExceeded`. + #[tokio::test(start_paused = true)] + async fn a_passing_deadline_stops_retries_with_the_last_refusal() { + let started = Instant::now(); + let deadline = started + Duration::from_millis(600); + let result = retry_not_leader(None, deadline, pacing(), |attempt| async move { + Err::<(), Error>(not_leader(7, attempt.index as u64 + 1)) + }) + .await; + match result { + Err(Error::NotLeader { + leader_node: 7, + leader_term, + .. + }) => assert!(leader_term > 1, "the last refusal returns"), + other => panic!("expected the last NotLeader, got {other:?}"), + } + assert!( + Instant::now() <= deadline, + "no retry runs past the deadline" + ); + } + #[tokio::test] async fn routing_table_updated_on_not_leader_hint() { let table = RoutingTable::uniform(1, &[1, 2], 2); @@ -215,17 +338,11 @@ mod tests { let call_count = Arc::new(AtomicUsize::new(0)); let count = Arc::clone(&call_count); - let _ = retry_not_leader(Some(&*rt_clone), move |_| { + let _ = retry_not_leader(Some(&*rt_clone), far_deadline(), pacing(), move |_| { let c = Arc::clone(&count); async move { - let n = c.fetch_add(1, Ordering::SeqCst); - if n == 0 { - Err(Error::NotLeader { - vshard_id: VShardId::new(0), - leader_node: 2, - leader_addr: "addr".into(), - leader_term: 4, - }) + if c.fetch_add(1, Ordering::SeqCst) == 0 { + Err(not_leader(2, 4)) } else { Ok::<(), Error>(()) } @@ -245,14 +362,9 @@ mod tests { assert!(table.observe_leader(group_id, 3, 7)); let rt = RwLock::new(table); - let _ = retry_not_leader(Some(&rt), |attempt| async move { - if attempt == 0 { - Err(Error::NotLeader { - vshard_id: VShardId::new(0), - leader_node: 2, - leader_addr: "addr".into(), - leader_term: 5, - }) + let _ = retry_not_leader(Some(&rt), far_deadline(), pacing(), |attempt| async move { + if attempt.index == 0 { + Err(not_leader(2, 5)) } else { Ok::<(), Error>(()) } @@ -268,14 +380,9 @@ mod tests { #[tokio::test] async fn a_term_less_redirect_never_replaces_a_termed_hint() { async fn redirect_to(rt: &RwLock, leader_node: u64) { - let _ = retry_not_leader(Some(rt), |attempt| async move { - if attempt == 0 { - Err(Error::NotLeader { - vshard_id: VShardId::new(0), - leader_node, - leader_addr: String::new(), - leader_term: 0, - }) + let _ = retry_not_leader(Some(rt), far_deadline(), pacing(), |attempt| async move { + if attempt.index == 0 { + Err(not_leader(leader_node, 0)) } else { Ok::<(), Error>(()) } @@ -298,4 +405,154 @@ mod tests { (3, 7) ); } + + /// Every attempt carries the one statement deadline, so the budget left + /// shrinks across attempts and never restarts. + #[tokio::test(start_paused = true)] + async fn the_statement_deadline_is_shared_across_attempts() { + let deadline = Instant::now() + Duration::from_millis(500); + let seen = Arc::new(std::sync::Mutex::new(Vec::new())); + let record = Arc::clone(&seen); + let result = retry_not_leader(None, deadline, pacing(), move |attempt| { + let record = Arc::clone(&record); + async move { + record.lock().unwrap().push(attempt); + Err::<(), Error>(not_leader(0, 0)) + } + }) + .await; + assert!(matches!(result, Err(Error::NotLeader { .. }))); + + let seen = seen.lock().unwrap(); + assert!(seen.len() > 1); + for (index, attempt) in seen.iter().enumerate() { + assert_eq!(attempt.index, index); + assert_eq!(attempt.deadline, deadline); + } + } + + /// A retry never starts once the statement's deadline has passed. The + /// refusal of the attempt that ran stays the answer: nothing applied. + #[tokio::test] + async fn no_retry_starts_past_the_deadline() { + let calls = Arc::new(AtomicUsize::new(0)); + let count = Arc::clone(&calls); + let result = retry_not_leader(None, Instant::now(), pacing(), move |_| { + let c = Arc::clone(&count); + async move { + c.fetch_add(1, Ordering::SeqCst); + Err::<(), Error>(not_leader(7, 3)) + } + }) + .await; + assert!( + matches!( + result, + Err(Error::NotLeader { + leader_node: 7, + leader_term: 3, + .. + }) + ), + "got {result:?}" + ); + assert_eq!(calls.load(Ordering::SeqCst), 1); + } + + /// A lost answer to a read just before the deadline gets one last call + /// at once. Its verdict, the deadline, ends the statement. + #[tokio::test(start_paused = true)] + async fn the_last_call_before_the_deadline_decides_the_verdict() { + let deadline = Instant::now() + Duration::from_millis(5); + let calls = Arc::new(AtomicUsize::new(0)); + let count = Arc::clone(&calls); + let result = retry_not_leader(None, deadline, pacing(), move |_| { + let c = Arc::clone(&count); + async move { + if c.fetch_add(1, Ordering::SeqCst) == 0 { + Err::<(), Error>(not_leader(0, 0)) + } else { + Err(Error::DeadlineExceeded { + request_id: crate::types::RequestId::new(0), + }) + } + } + }) + .await; + assert!( + matches!(result, Err(Error::DeadlineExceeded { .. })), + "got {result:?}" + ); + assert_eq!(calls.load(Ordering::SeqCst), 2, "one last call, no more"); + } + + /// A refusal from the last call ends the statement: no call follows it. + #[tokio::test(start_paused = true)] + async fn a_refused_last_call_ends_the_retries() { + let deadline = Instant::now() + Duration::from_millis(5); + let calls = Arc::new(AtomicUsize::new(0)); + let count = Arc::clone(&calls); + let result = retry_not_leader(None, deadline, pacing(), move |_| { + let c = Arc::clone(&count); + async move { + c.fetch_add(1, Ordering::SeqCst); + Err::<(), Error>(not_leader(0, 0)) + } + }) + .await; + assert!( + matches!(result, Err(Error::NotLeader { .. })), + "got {result:?}" + ); + assert_eq!(calls.load(Ordering::SeqCst), 2); + } + + /// An overwritten entry was never applied. Past the deadline it stays a + /// definite refusal, never an unknown outcome. + #[tokio::test] + async fn an_overwritten_entry_past_the_deadline_stays_definite() { + let result = retry_not_leader(None, Instant::now(), pacing(), |_| async { + Err::<(), Error>(Error::RetryableLeaderChange { + group_id: 4, + log_index: 9, + }) + }) + .await; + assert!( + matches!( + result, + Err(Error::RetryableLeaderChange { + group_id: 4, + log_index: 9 + }) + ), + "got {result:?}" + ); + } + + /// The backoff doubles from one Raft tick and stops at the minimum + /// election timeout. + #[test] + fn pacing_follows_the_raft_timing() { + let tuning = ClusterTransportTuning { + raft_tick_interval_ms: 10, + election_timeout_min_ms: 150, + ..ClusterTransportTuning::default() + }; + let pacing = RetryPacing::from_tuning(&tuning); + assert_eq!(pacing.backoff(0), Duration::from_millis(10)); + assert_eq!(pacing.backoff(1), Duration::from_millis(20)); + assert_eq!(pacing.backoff(4), Duration::from_millis(150)); + assert_eq!(pacing.backoff(u32::MAX), Duration::from_millis(150)); + } + + /// A spent deadline still sends a 1 ms hop, so the receiver checks it. + #[tokio::test] + async fn remaining_ms_never_reaches_zero() { + let attempt = Attempt { + index: 0, + deadline: Instant::now(), + }; + assert_eq!(attempt.remaining_ms(), 1); + } } diff --git a/nodedb/src/control/gateway/stream.rs b/nodedb/src/control/gateway/stream.rs index ea90ad3cd..1f8ffaaba 100644 --- a/nodedb/src/control/gateway/stream.rs +++ b/nodedb/src/control/gateway/stream.rs @@ -15,10 +15,11 @@ use crate::control::server::shared::clone_write::CloneCheckedTask; use nodedb_physical::physical_plan::PhysicalPlan; use super::core::{Gateway, QueryContext, authorized_plan_for_context}; -use super::dispatcher::{DispatchRouteStreamParams, dispatch_route_stream, statement_deadline_ms}; +use super::dispatcher::{DispatchRouteStreamParams, dispatch_route_stream}; use super::live_leaders::resolve_live_decision; -use super::retry::retry_not_leader; +use super::retry::{RetryPacing, retry_not_leader}; use super::route::TaskRoute; +use crate::control::server::shared::session::StatementDeadline; impl Gateway { /// Streaming sibling of [`execute`](Gateway::execute). @@ -63,7 +64,9 @@ impl Gateway { let routes = self.compute_routes(plan, ctx)?; - let deadline_ms = statement_deadline_ms(&shared); + // One deadline for every route and attempt of the statement. + let deadline = StatementDeadline::of_statement(&shared).tokio_instant(); + let pacing = RetryPacing::from_tuning(&shared.tuning.cluster_transport); let mut per_route: Vec = Vec::with_capacity(routes.len()); for route in routes { @@ -78,8 +81,8 @@ impl Gateway { // attempt re-resolves the routing decision and re-opens the stream; // a pre-row NotLeader / DescriptorMismatch is retryable, anything // after the first frame is terminal (handled inside the stream). - let stream = retry_not_leader(routing_ref, move |attempt| { - if attempt > 0 { + let stream = retry_not_leader(routing_ref, deadline, pacing, move |attempt| { + if attempt.index > 0 { retry_counter.fetch_add(1, Ordering::Relaxed); } let plan = plan_for_retry.clone(); @@ -102,7 +105,7 @@ impl Gateway { tenant_id, database_id, trace_id, - deadline_ms, + deadline_ms: attempt.remaining_ms(), version_set: &version_set, linearizable, }) diff --git a/nodedb/src/control/lease/drain_propose.rs b/nodedb/src/control/lease/drain_propose.rs index 4b9c39b71..808169f1e 100644 --- a/nodedb/src/control/lease/drain_propose.rs +++ b/nodedb/src/control/lease/drain_propose.rs @@ -194,9 +194,6 @@ fn lease_holder_is_member(shared: &SharedState, node_id: u64) -> bool { } } -/// How long a drain variant's propose waits for its local apply. -pub(super) const DRAIN_PROPOSE_TIMEOUT: Duration = Duration::from_secs(5); - /// Encode a drain variant for the metadata group. pub(super) fn encode_drain(entry: &MetadataEntry, operation: &str) -> Result, Error> { encode_entry(entry).map_err(|e| Error::Config { @@ -204,24 +201,6 @@ pub(super) fn encode_drain(entry: &MetadataEntry, operation: &str) -> Result Result<(), Error> { - if outcome.is_reached() { - return Ok(()); - } - Err(Error::Config { - detail: format!( - "descriptor drain {operation} did not apply within {DRAIN_PROPOSE_TIMEOUT:?} \ - (log index {log_index}, current: {current}, outcome: {outcome:?})" - ), - }) -} - #[cfg(test)] mod tests { use std::sync::Arc; diff --git a/nodedb/src/control/lease/drain_propose_async.rs b/nodedb/src/control/lease/drain_propose_async.rs index 3246ccaf3..3d3d8334f 100644 --- a/nodedb/src/control/lease/drain_propose_async.rs +++ b/nodedb/src/control/lease/drain_propose_async.rs @@ -11,14 +11,11 @@ use std::time::{Duration, Instant}; use nodedb_cluster::{DescriptorId, DrainOwner, MetadataEntry}; -use crate::control::metadata_proposer::wait::wait_applied; +use crate::control::metadata_proposer::wait::await_metadata_apply; use crate::control::state::SharedState; use crate::error::Error; -use super::drain_propose::{ - DRAIN_PROPOSE_TIMEOUT, POLL_INTERVAL, drain_applied_or_error, drain_start_entry, - drained_or_timed_out, encode_drain, -}; +use super::drain_propose::{POLL_INTERVAL, drain_start_entry, drained_or_timed_out, encode_drain}; /// Drain every lease on `id` at `version <= up_to_version` for a `Put*` DDL. /// @@ -140,7 +137,8 @@ pub async fn end_drain_async( .await } -/// Propose a drain variant and await its apply on this node. +/// Propose a drain variant and await its apply on this node, until the +/// statement deadline. async fn propose_drain_async( shared: &SharedState, entry: MetadataEntry, @@ -150,12 +148,5 @@ async fn propose_drain_async( let log_index = handle .propose_async(encode_drain(&entry, operation)?) .await?; - let watcher = shared.applied_index_watcher(nodedb_cluster::METADATA_GROUP_ID); - let outcome = wait_applied( - std::sync::Arc::clone(&watcher), - log_index, - DRAIN_PROPOSE_TIMEOUT, - ) - .await?; - drain_applied_or_error(outcome, operation, log_index, watcher.current()) + await_metadata_apply(shared, log_index).await.map(|_| ()) } diff --git a/nodedb/src/control/lease/leader_wait.rs b/nodedb/src/control/lease/leader_wait.rs index 52c7aca70..f3321d75b 100644 --- a/nodedb/src/control/lease/leader_wait.rs +++ b/nodedb/src/control/lease/leader_wait.rs @@ -10,9 +10,6 @@ use nodedb_cluster::{MetadataEntry, encode_entry}; use crate::control::state::SharedState; use crate::error::Error; -/// Same propose-and-wait timeout the catalog DDL path uses. -pub(in crate::control) const PROPOSE_TIMEOUT: Duration = Duration::from_secs(5); - /// Backoff schedule for re-proposing while the metadata group elects a leader. /// /// Reads take a descriptor lease, so a lease proposal issued in the first @@ -21,7 +18,7 @@ pub(in crate::control) const PROPOSE_TIMEOUT: Duration = Duration::from_secs(5); /// failed proposal, so it is waited out here rather than surfaced as a failed /// statement. Bounded at ~1.6s total: long enough for a single-node or healthy /// multi-node election, short enough that a genuinely leaderless group still -/// fails well inside [`PROPOSE_TIMEOUT`]. +/// fails well inside the statement deadline. const LEADER_ELECTION_BACKOFF_MS: [u64; 7] = [10, 25, 50, 100, 200, 400, 800]; /// Propose `raw`, waiting out an in-progress metadata election. @@ -55,10 +52,10 @@ async fn propose_once_leader_is_elected( /// Encode `entry`, propose it through the metadata raft handle, and await /// the local applied watermark until the proposed log index applies or the -/// timeout fires. +/// statement deadline passes. /// /// Shared by the lease grant and release paths. `operation` is a short label -/// for the encode-failure and timeout errors. +/// for the encode-failure error. pub(in crate::control) async fn propose_and_wait( shared: &SharedState, entry: &MetadataEntry, @@ -69,22 +66,5 @@ pub(in crate::control) async fn propose_and_wait( detail: format!("descriptor lease {operation} encode: {e}"), })?; let log_index = propose_once_leader_is_elected(handle.as_ref(), raw, operation).await?; - - let watcher = shared.applied_index_watcher(nodedb_cluster::METADATA_GROUP_ID); - let outcome = crate::control::metadata_proposer::wait::wait_applied( - std::sync::Arc::clone(&watcher), - log_index, - PROPOSE_TIMEOUT, - ) - .await?; - if !outcome.is_reached() { - return Err(Error::Config { - detail: format!( - "descriptor lease {operation} did not apply within {PROPOSE_TIMEOUT:?} \ - (log index {log_index}, current: {}, outcome: {outcome:?})", - watcher.current() - ), - }); - } - Ok(log_index) + crate::control::metadata_proposer::wait::await_metadata_apply(shared, log_index).await } diff --git a/nodedb/src/control/lease/mod.rs b/nodedb/src/control/lease/mod.rs index 4b398ee51..e248e1a86 100644 --- a/nodedb/src/control/lease/mod.rs +++ b/nodedb/src/control/lease/mod.rs @@ -51,7 +51,7 @@ pub use drain::{DescriptorDrainTracker, DrainEntry}; pub use drain_apply::{apply_drain_ends, apply_drain_start}; pub use drain_propose_async::{drain_for_ddl_async, drain_for_owner_async, end_drain_async}; pub use holders::{LeaseHolders, LeaseRevocation, revoke_if_fenced, revoke_on_release}; -pub(super) use leader_wait::{PROPOSE_TIMEOUT, propose_and_wait}; +pub(super) use leader_wait::propose_and_wait; pub use propose::{DEFAULT_LEASE_DURATION, acquire_lease, compute_expires_at, force_refresh_lease}; pub(crate) use propose::{acquire_lease_after_admission, drain_owner_list, ensure_not_draining}; pub use refcount::{LeaseRefCount, QueryLeaseScope}; diff --git a/nodedb/src/control/lease/release.rs b/nodedb/src/control/lease/release.rs index 805e958a4..383691e3b 100644 --- a/nodedb/src/control/lease/release.rs +++ b/nodedb/src/control/lease/release.rs @@ -17,6 +17,9 @@ pub(crate) struct LeaseReleaseHandle { node_id: u64, metadata_raft: Arc, applied_watcher: Arc, + /// `tuning.network.default_deadline_secs`: the apply wait's budget + /// outside a statement. + default_deadline_secs: u64, grant_gate: Arc>, refcounts: Arc, } @@ -29,6 +32,7 @@ impl LeaseReleaseHandle { node_id: shared.node_id, metadata_raft: Arc::clone(shared.metadata_raft_handle()?), applied_watcher: shared.applied_index_watcher(nodedb_cluster::METADATA_GROUP_ID), + default_deadline_secs: shared.tuning.network.default_deadline_secs, grant_gate: Arc::clone(&shared.lease_grant_gate), refcounts: Arc::clone(&shared.lease_refcount), }) @@ -90,22 +94,16 @@ impl LeaseReleaseHandle { detail: format!("descriptor lease release encode: {error}"), })?; let log_index = self.metadata_raft.propose_async(raw).await?; - let outcome = crate::control::metadata_proposer::wait::wait_applied( - Arc::clone(&self.applied_watcher), + let deadline = crate::control::server::shared::session::StatementDeadline::with_default( + self.default_deadline_secs, + ) + .instant(); + crate::control::metadata_proposer::wait::await_applied( + &self.applied_watcher, log_index, - super::PROPOSE_TIMEOUT, + deadline, ) .await?; - if !outcome.is_reached() { - return Err(Error::Config { - detail: format!( - "descriptor lease release did not apply within {:?} \ - (log index {log_index}, current: {}, outcome: {outcome:?})", - super::PROPOSE_TIMEOUT, - self.applied_watcher.current() - ), - }); - } Ok(()) } } diff --git a/nodedb/src/control/lease/self_fence.rs b/nodedb/src/control/lease/self_fence.rs index a4a16eb3c..9ce2f504c 100644 --- a/nodedb/src/control/lease/self_fence.rs +++ b/nodedb/src/control/lease/self_fence.rs @@ -24,7 +24,11 @@ use crate::control::state::SharedState; /// exists to drain or release this node's leases, so the node never fences. pub(crate) fn lease_use_is_fenced(shared: &SharedState) -> bool { match shared.raft_read_gate.get() { - Some(gate) => !gate.within_staleness_bound(METADATA_GROUP_ID, LEASE_SELF_FENCE_WINDOW), + Some(gate) => !gate.within_staleness_bound( + METADATA_GROUP_ID, + LEASE_SELF_FENCE_WINDOW, + shared.applied_index_watcher(METADATA_GROUP_ID).current(), + ), None => false, } } @@ -66,7 +70,12 @@ mod tests { Err(ReadIndexRefusal::NotLeader) } - fn within_staleness_bound(&self, group_id: u64, max_staleness: Duration) -> bool { + fn within_staleness_bound( + &self, + group_id: u64, + max_staleness: Duration, + _applied: u64, + ) -> bool { group_id == METADATA_GROUP_ID && Duration::from_millis(self.contact_age_ms.load(Ordering::SeqCst)) <= max_staleness diff --git a/nodedb/src/control/maintenance/clone_materializer/chained.rs b/nodedb/src/control/maintenance/clone_materializer/chained.rs index 1a3286d82..d125cf24e 100644 --- a/nodedb/src/control/maintenance/clone_materializer/chained.rs +++ b/nodedb/src/control/maintenance/clone_materializer/chained.rs @@ -259,7 +259,8 @@ async fn commit_relink( target: RedoTarget, payload: &TransactionRedoPayload, ) -> crate::Result<()> { - let deadline = crate::control::wal_replication::statement_propose_deadline(state); + let deadline = crate::control::server::shared::session::StatementDeadline::of_statement(state) + .tokio_instant(); propose_transaction_redo(state, target, payload, deadline).await?; Ok(()) } diff --git a/nodedb/src/control/maintenance/clone_materializer/dispatch.rs b/nodedb/src/control/maintenance/clone_materializer/dispatch.rs index 43040674f..757dd3b0c 100644 --- a/nodedb/src/control/maintenance/clone_materializer/dispatch.rs +++ b/nodedb/src/control/maintenance/clone_materializer/dispatch.rs @@ -7,13 +7,13 @@ //! runtime. use std::sync::atomic::Ordering; -use std::time::{Duration, Instant}; use nodedb_types::{DatabaseId, TenantId}; use crate::bridge::envelope::{ErrorCode, Priority, Request, Response}; use crate::control::gateway::core::QueryContext; use crate::control::server::dispatch_utils::{OwnedRead, OwnedReadScope, route_owned_read}; +use crate::control::server::shared::session::StatementDeadline; use crate::control::server::shared::write_admission::plan_is_write; use crate::control::state::SharedState; use crate::types::{ReadConsistency, RequestId, TraceId, TxnId, VShardId}; @@ -157,15 +157,14 @@ pub(crate) async fn dispatch_on_this_node( txn_id: Option, ) -> crate::Result { let req_id = RequestId::new(state.request_id_counter.fetch_add(1, Ordering::Relaxed)); - let deadline_secs = state.tuning.network.default_deadline_secs; - let deadline_dur = Duration::from_secs(deadline_secs); + let deadline = StatementDeadline::of_statement(state); let req = Request { request_id: req_id, tenant_id, vshard_id, database_id, plan, - deadline: Instant::now() + deadline_dur, + deadline: deadline.instant(), priority: Priority::Normal, trace_id: TraceId::ZERO, consistency: ReadConsistency::Strong, @@ -188,7 +187,7 @@ pub(crate) async fn dispatch_on_this_node( Ok(mut d) => d.dispatch(req)?, Err(p) => p.into_inner().dispatch(req)?, } - tokio::time::timeout(deadline_dur, rx.recv()) + tokio::time::timeout_at(deadline.tokio_instant(), rx.recv()) .await .map_err(|_| crate::Error::DeadlineExceeded { request_id: req_id })? .ok_or(crate::Error::Dispatch { diff --git a/nodedb/src/control/metadata_proposer/catalog.rs b/nodedb/src/control/metadata_proposer/catalog.rs index d71f2408c..3657d229e 100644 --- a/nodedb/src/control/metadata_proposer/catalog.rs +++ b/nodedb/src/control/metadata_proposer/catalog.rs @@ -3,9 +3,9 @@ //! Propose a catalog entry and wait until this node applied it. use std::sync::atomic::Ordering; -use std::time::Duration; +use std::time::Instant; -use nodedb_cluster::{METADATA_GROUP_ID, MetadataEntry, WaitOutcome, encode_entry}; +use nodedb_cluster::{METADATA_GROUP_ID, MetadataEntry, encode_entry}; use crate::control::catalog_entry::{self, CatalogEntry}; use crate::control::propose_outcome::ProposeOutcome; @@ -14,8 +14,9 @@ use crate::error::Error; use super::ddl_prepare::{acquire_ddl_prepare_lease_async, lock_ddl_preparation_async}; use super::handle::MetadataRaftHandle; -use super::timeouts::{DEFAULT_DRAIN_TIMEOUT, DEFAULT_PROPOSE_TIMEOUT}; -use super::wait::wait_tracking_progress_async; +use super::timeouts::DEFAULT_DRAIN_TIMEOUT; +use super::wait::await_applied; +use crate::control::server::shared::session::StatementDeadline; /// Propose a `CatalogEntry` and wait until this node applied it, on any /// runtime flavor. @@ -44,6 +45,8 @@ pub async fn propose_catalog_entry_async( group_id: METADATA_GROUP_ID, through: log_index, }], + crate::control::server::shared::session::StatementDeadline::of_statement(shared) + .instant(), ) .await?; } @@ -80,7 +83,7 @@ async fn propose_and_apply_locally( handle.as_ref(), lease.token(), catalog_ddl_entry(&stamped)?, - DEFAULT_PROPOSE_TIMEOUT, + StatementDeadline::of_statement(shared).instant(), ) .await } @@ -205,17 +208,18 @@ pub(super) fn catalog_ddl_entry_with( } /// Propose `entry` under the preparation lease `token` and wait until this -/// node applied it. Returns the log index. Fails when another lease owner -/// superseded `token` before the apply. +/// node applied it, until `deadline`. Returns the log index. /// -/// `timeout` bounds a stall, not the whole wait, as -/// [`super::wait::wait_tracking_progress_async`] describes. +/// - Past `deadline`: `Error::DeadlineExceeded`. The entry was proposed, so +/// its outcome is unknown and no caller resends it. +/// - Another lease owner superseded `token` before the apply: +/// `Error::MetadataDdlSuperseded`. The apply skipped the entry. pub(super) async fn propose_prepared( shared: &SharedState, handle: &dyn MetadataRaftHandle, token: u64, entry: MetadataEntry, - timeout: Duration, + deadline: Instant, ) -> Result { let raw = encode_entry(&MetadataEntry::DdlPrepared { token, @@ -226,79 +230,100 @@ pub(super) async fn propose_prepared( })?; let log_index = handle.propose_async(raw).await?; let watcher = shared.applied_index_watcher(METADATA_GROUP_ID); - let outcome = - wait_tracking_progress_async(std::sync::Arc::clone(&watcher), log_index, timeout, || { - shared.metadata_apply_progress.load(Ordering::Acquire) - }) - .await?; - match outcome { - WaitOutcome::Reached - if shared.metadata_ddl.applied_token.load(Ordering::Acquire) == token => - { - Ok(log_index) - } - WaitOutcome::Reached => Err(Error::Config { - detail: "metadata DDL preparation ownership was superseded before apply".into(), - }), - WaitOutcome::TimedOut => Err(Error::Config { - detail: format!( - "metadata propose timed out after {timeout:?} without apply progress waiting \ - for log index {log_index} (current: {})", - watcher.current() - ), - }), - WaitOutcome::GroupGone => Err(Error::Config { - detail: "metadata group no longer hosted on this node".into(), - }), + await_applied(&watcher, log_index, deadline).await?; + if shared.metadata_ddl.applied_token.load(Ordering::Acquire) == token { + Ok(log_index) + } else { + Err(Error::MetadataDdlSuperseded { log_index }) } } #[cfg(test)] mod tests { use std::sync::Arc; - use std::sync::atomic::AtomicU64; - - use nodedb_cluster::AppliedIndexWatcher; + use std::time::Duration; use super::*; - const WINDOW: Duration = Duration::from_millis(60); + /// A handle that assigns every proposal log index 5. + struct AtFive; + + impl MetadataRaftHandle for AtFive { + fn propose_async<'a>( + &'a self, + _bytes: Vec, + ) -> crate::control::metadata_proposer::ProposeFuture<'a> { + Box::pin(async { Ok::(5) }) + } + } + + fn test_state() -> (Arc, tempfile::TempDir) { + let dir = tempfile::tempdir().expect("tempdir"); + let wal = Arc::new( + crate::wal::WalManager::open_for_testing(&dir.path().join("propose.wal")).expect("wal"), + ); + let (dispatcher, _sides) = crate::bridge::dispatch::Dispatcher::new(1, 64); + let state = SharedState::new(dispatcher, wal).expect("shared state"); + (state, dir) + } - /// Progress that keeps moving for many windows never times out; the wait - /// ends when the entry applies. + fn ddl() -> MetadataEntry { + MetadataEntry::CatalogDdl { + payload: Vec::new(), + } + } + + /// An apply that lands after a 1 s quiet gap, inside a 2 s deadline, + /// answers the log index. #[tokio::test] - async fn steady_progress_never_times_out() { - let watcher = Arc::new(AppliedIndexWatcher::new()); - let progress = Arc::new(AtomicU64::new(0)); - let driver = { - let (watcher, progress) = (Arc::clone(&watcher), Arc::clone(&progress)); - std::thread::spawn(move || { - for _ in 0..20 { - std::thread::sleep(Duration::from_millis(20)); - progress.fetch_add(1, Ordering::Release); - } - watcher.bump(5); + async fn a_slow_apply_inside_the_deadline_succeeds() { + const TOKEN: u64 = 77; + let (state, _dir) = test_state(); + let applier = { + let state = Arc::clone(&state); + tokio::spawn(async move { + tokio::time::sleep(Duration::from_secs(1)).await; + state + .metadata_ddl + .applied_token + .store(TOKEN, Ordering::Release); + state.applied_index_watcher(METADATA_GROUP_ID).bump(5); }) }; - let outcome = wait_tracking_progress_async(Arc::clone(&watcher), 5, WINDOW, || { - progress.load(Ordering::Acquire) - }) - .await - .expect("the wait finishes"); - driver.join().unwrap(); - assert!(outcome.is_reached(), "{outcome:?}"); + let deadline = Instant::now() + Duration::from_secs(2); + let applied = propose_prepared(&state, &AtFive, TOKEN, ddl(), deadline).await; + applier.await.expect("join applier"); + assert_eq!(applied.expect("the entry applied"), 5); } - /// A stall of one window times the wait out. + /// An entry that never applies ends at the deadline as `DeadlineExceeded`, + /// never as a config error: it was proposed, so its outcome is unknown. #[tokio::test] - async fn a_stall_times_out() { - let watcher = Arc::new(AppliedIndexWatcher::new()); - let started = std::time::Instant::now(); - let outcome = wait_tracking_progress_async(watcher, 5, WINDOW, || 7) - .await - .expect("the wait finishes"); - assert!(matches!(outcome, WaitOutcome::TimedOut), "{outcome:?}"); - assert!(started.elapsed() < WINDOW * 4); + async fn an_entry_that_never_applies_ends_at_the_deadline() { + let (state, _dir) = test_state(); + let budget = Duration::from_millis(300); + let started = Instant::now(); + let result = propose_prepared(&state, &AtFive, 1, ddl(), started + budget).await; + assert!( + matches!(result, Err(Error::DeadlineExceeded { .. })), + "{result:?}" + ); + assert!(started.elapsed() >= budget); + } + + /// An entry whose apply skipped it under another lease owner is + /// superseded, not a success. + #[tokio::test] + async fn a_superseded_token_is_a_typed_refusal() { + let (state, _dir) = test_state(); + state.metadata_ddl.applied_token.store(9, Ordering::Release); + state.applied_index_watcher(METADATA_GROUP_ID).bump(5); + let deadline = Instant::now() + Duration::from_secs(2); + let result = propose_prepared(&state, &AtFive, 1, ddl(), deadline).await; + assert!( + matches!(result, Err(Error::MetadataDdlSuperseded { log_index: 5 })), + "{result:?}" + ); } /// A one-node cluster stamps a create, commits it through its metadata diff --git a/nodedb/src/control/metadata_proposer/catalog_batch.rs b/nodedb/src/control/metadata_proposer/catalog_batch.rs index 4e2dc3122..42f551f6a 100644 --- a/nodedb/src/control/metadata_proposer/catalog_batch.rs +++ b/nodedb/src/control/metadata_proposer/catalog_batch.rs @@ -13,7 +13,7 @@ use crate::error::Error; use super::catalog::{catalog_ddl_entry, propose_prepared}; use super::ddl_prepare::{acquire_ddl_prepare_lease_async, lock_ddl_preparation_async}; use super::handle::MetadataRaftHandle; -use super::timeouts::{DEFAULT_DRAIN_TIMEOUT, DEFAULT_PROPOSE_TIMEOUT}; +use super::timeouts::DEFAULT_DRAIN_TIMEOUT; /// What happened to the entries a batch plan built. #[derive(Debug, Clone, Copy, PartialEq, Eq)] @@ -85,6 +85,8 @@ pub async fn propose_catalog_batch_async( group_id: METADATA_GROUP_ID, through: log_index, }], + crate::control::server::shared::session::StatementDeadline::of_statement(shared) + .instant(), ) .await?; } @@ -113,14 +115,14 @@ async fn propose_replicated( for entry in &entries { wrapped.push(catalog_ddl_entry(entry)?); } - // The wait follows apply progress: a batch of slow purges keeps it - // alive, and only a stall of `DEFAULT_PROPOSE_TIMEOUT` ends it. + // A batch of slow purges applies for as long as the statement lets it. let log_index = propose_prepared( shared, handle, lease.token(), MetadataEntry::Batch { entries: wrapped }, - DEFAULT_PROPOSE_TIMEOUT, + crate::control::server::shared::session::StatementDeadline::of_statement(shared) + .instant(), ) .await?; Ok::<_, Error>(Some((log_index, entries))) diff --git a/nodedb/src/control/metadata_proposer/ddl_prepare.rs b/nodedb/src/control/metadata_proposer/ddl_prepare.rs index 2e03fcc77..a9eb2bffe 100644 --- a/nodedb/src/control/metadata_proposer/ddl_prepare.rs +++ b/nodedb/src/control/metadata_proposer/ddl_prepare.rs @@ -3,18 +3,16 @@ //! The DDL preparation lease: the local lock and the replicated lease that //! serialize descriptor preparation across the cluster. -use std::sync::Arc; use std::sync::atomic::Ordering; use std::time::{Duration, Instant, SystemTime, UNIX_EPOCH}; -use nodedb_cluster::{METADATA_GROUP_ID, MetadataEntry, WaitOutcome, encode_entry}; +use nodedb_cluster::{MetadataEntry, encode_entry}; use crate::control::state::SharedState; use crate::error::Error; use super::handle::MetadataRaftHandle; -use super::timeouts::DEFAULT_PROPOSE_TIMEOUT; -use super::wait::wait_applied; +use super::wait::await_metadata_apply; /// How long a proposer waits for the lease. It outlasts the leader's /// stuck-owner fallback, so a reclaim always frees the lease first. @@ -47,37 +45,15 @@ fn encode_metadata(entry: &MetadataEntry) -> Result, Error> { }) } -/// The result of waiting `timeout` for log index `index` to apply. -fn applied_or_error( - outcome: WaitOutcome, - index: u64, - timeout: Duration, - current: u64, -) -> Result { - match outcome { - WaitOutcome::Reached => Ok(index), - WaitOutcome::TimedOut => Err(Error::Config { - detail: format!( - "metadata propose timed out after {timeout:?} waiting for log index {index} \ - (current: {current})" - ), - }), - WaitOutcome::GroupGone => Err(Error::Config { - detail: "metadata group no longer hosted on this node".into(), - }), - } -} - +/// Propose `entry` and wait until this node applied it, until the statement +/// deadline. Returns its log index. pub(super) async fn propose_metadata_and_wait_async( shared: &SharedState, handle: &dyn MetadataRaftHandle, entry: &MetadataEntry, - timeout: Duration, ) -> Result { let index = handle.propose_async(encode_metadata(entry)?).await?; - let watcher = shared.applied_index_watcher(METADATA_GROUP_ID); - let outcome = wait_applied(Arc::clone(&watcher), index, timeout).await?; - applied_or_error(outcome, index, timeout, watcher.current()) + await_metadata_apply(shared, index).await } /// What the acquire loop does next, from the current lease owner. @@ -117,7 +93,6 @@ pub(crate) async fn release_ddl_prepare_token( shared, handle.as_ref(), &MetadataEntry::DdlPrepareRelease { token }, - DEFAULT_PROPOSE_TIMEOUT, ) .await .map(|_| ()) @@ -145,20 +120,29 @@ impl DdlPrepareLease<'_> { self.token } - /// Release the lease and wait until the release applied here. A failed - /// release is logged: the metadata leader reclaims the lease once - /// [`super::ddl_owner::DDL_PREPARE_LEASE`] passed. + /// Release the lease and wait until the release applied here, until the + /// statement deadline. A failed release is logged: the metadata leader + /// reclaims the lease once [`super::ddl_owner::DDL_PREPARE_LEASE`] passed. pub(crate) async fn release(mut self) { self.released = true; - if let Err(error) = propose_metadata_and_wait_async( + match propose_metadata_and_wait_async( self.shared, self.handle, &MetadataEntry::DdlPrepareRelease { token: self.token }, - DEFAULT_PROPOSE_TIMEOUT, ) .await { - tracing::error!(token = self.token, %error, "metadata DDL lease release failed"); + Ok(_) => {} + // Proposed: it applies in log order, ahead of this node's next + // acquire. + Err(error @ Error::DeadlineExceeded { .. }) => tracing::warn!( + token = self.token, + %error, + "metadata DDL lease release did not apply here before the statement deadline" + ), + Err(error) => { + tracing::error!(token = self.token, %error, "metadata DDL lease release failed"); + } } } } @@ -189,7 +173,6 @@ pub(crate) async fn acquire_ddl_prepare_lease_async<'a>( token, node_id: shared.node_id, }, - DEFAULT_PROPOSE_TIMEOUT, ) .await?; @@ -220,6 +203,8 @@ pub(crate) async fn lock_ddl_preparation_async( #[cfg(test)] mod tests { + use std::sync::Arc; + use super::*; fn owner(shared: &SharedState) -> Option { diff --git a/nodedb/src/control/metadata_proposer/ddl_reclaim.rs b/nodedb/src/control/metadata_proposer/ddl_reclaim.rs index 76ed82c13..dd9b07c0c 100644 --- a/nodedb/src/control/metadata_proposer/ddl_reclaim.rs +++ b/nodedb/src/control/metadata_proposer/ddl_reclaim.rs @@ -20,7 +20,6 @@ use crate::error::Error; use super::ddl_owner::{current_owner, reclaim_cause}; use super::ddl_prepare::propose_metadata_and_wait_async; use super::handle::MetadataRaftHandle; -use super::timeouts::DEFAULT_PROPOSE_TIMEOUT; /// How often the leader checks the lease owner. const RECLAIM_POLL: Duration = Duration::from_millis(250); @@ -99,20 +98,10 @@ pub(crate) async fn reclaim_ddl_prepare_lease( token: u64, ) -> Result<(), Error> { if shared.pending_ddl.contains(token) { - propose_metadata_and_wait_async( - shared, - handle, - &MetadataEntry::DdlPendingCancel { token }, - DEFAULT_PROPOSE_TIMEOUT, - ) - .await?; + propose_metadata_and_wait_async(shared, handle, &MetadataEntry::DdlPendingCancel { token }) + .await?; } - propose_metadata_and_wait_async( - shared, - handle, - &MetadataEntry::DdlPrepareRelease { token }, - DEFAULT_PROPOSE_TIMEOUT, - ) - .await - .map(|_| ()) + propose_metadata_and_wait_async(shared, handle, &MetadataEntry::DdlPrepareRelease { token }) + .await + .map(|_| ()) } diff --git a/nodedb/src/control/metadata_proposer/handle.rs b/nodedb/src/control/metadata_proposer/handle.rs index 175799028..810220f6f 100644 --- a/nodedb/src/control/metadata_proposer/handle.rs +++ b/nodedb/src/control/metadata_proposer/handle.rs @@ -8,6 +8,7 @@ use nodedb_cluster::ClusterError; use nodedb_raft::RaftError; use crate::error::Error; +use crate::types::RequestId; /// Type-erased handle for proposing to the metadata raft group. /// @@ -95,31 +96,41 @@ impl MetadataRaftHandle for RaftLoopProposerHandle { } /// The error a metadata proposal returns for a cluster error. +/// +/// A group with no leader to take the proposal proposed nothing, so the +/// caller can re-propose: [`Error::MetadataLeaderUnavailable`]. A forward +/// with no answer can have been proposed, so its outcome is unknown: +/// [`Error::DeadlineExceeded`]. A typed verdict keeps its class. fn metadata_propose_error(error: ClusterError) -> Error { match error { - // An election in progress is transient, not a failure of this - // proposal. Keep it typed rather than flattening it into a generic - // config error, so callers can wait the election out instead of - // failing the statement — a node that has recently restarted answers - // every metadata proposal this way for a moment. - ClusterError::Raft(RaftError::NotLeader { - leader_hint: None, .. - }) => Error::MetadataLeaderUnavailable, + // An election, a transfer in flight, or a leader that stepped down. + // Nothing was proposed. + ClusterError::Raft( + RaftError::NotLeader { .. } | RaftError::LeadershipTransferInProgress, + ) => Error::MetadataLeaderUnavailable, + // A forward that never reached the leader whole proposed nothing. + error if error.is_unsent() => Error::MetadataLeaderUnavailable, + // The forward is not resend-safe and got no answer. The leader can + // have appended the entry, so the outcome is unknown. + error if error.is_lost_answer() => { + tracing::warn!( + %error, + "metadata proposal forward got no answer from the leader" + ); + Error::DeadlineExceeded { + request_id: RequestId::new(0), + } + } // A typed verdict keeps its class. ClusterError::DataPlane { code } => Error::DataPlane(code.into()), ClusterError::ShardExecution { error, .. } | ClusterError::StreamTerminal { error, .. } => { Error::from(*error) } other @ (ClusterError::Raft( - RaftError::NotLeader { - leader_hint: Some(_), - .. - } - | RaftError::LogCompacted { .. } + RaftError::LogCompacted { .. } | RaftError::CompactionAheadOfApplied { .. } | RaftError::ProposalRejected { .. } | RaftError::InvalidTransferTarget { .. } - | RaftError::LeadershipTransferInProgress | RaftError::GroupNotFound { .. } | RaftError::Transport { .. } | RaftError::Storage { .. } @@ -134,9 +145,6 @@ fn metadata_propose_error(error: ClusterError) -> Error { | ClusterError::MigrationPauseBudgetExceeded { .. } | ClusterError::NodeUnreachable { .. } | ClusterError::GhostNotFound { .. } - | ClusterError::Transport { .. } - | ClusterError::ShardTimeout { .. } - | ClusterError::Unanswered { .. } | ClusterError::Storage { .. } | ClusterError::Codec { .. } | ClusterError::UnsupportedWireVersion { .. } @@ -162,8 +170,54 @@ fn metadata_propose_error(error: ClusterError) -> Error { | ClusterError::Bm25Gather(_) | ClusterError::TsGather(_) | ClusterError::ShufflePush(_) - | ClusterError::RemoteUntyped { .. }) => Error::Config { + | ClusterError::RemoteUntyped { .. } + // The guards above take these three. They stay named for exhaustiveness. + | ClusterError::Transport { .. } + | ClusterError::ShardTimeout { .. } + | ClusterError::Unanswered { .. }) => Error::Config { detail: format!("metadata propose: {other}"), }, } } + +#[cfg(test)] +mod tests { + use super::*; + + /// A forward with no answer can have appended the entry. It is an + /// unknown outcome, never a config error and never retried. + #[test] + fn a_lost_metadata_answer_is_an_unknown_outcome() { + let unanswered = metadata_propose_error(ClusterError::Unanswered { + node_id: 2, + detail: "connection lost".into(), + }); + assert!(matches!(unanswered, Error::DeadlineExceeded { .. })); + + let timed_out = metadata_propose_error(ClusterError::ShardTimeout { + vshard_id: 0, + elapsed_ms: 50, + }); + assert!(matches!(timed_out, Error::DeadlineExceeded { .. })); + } + + /// A forward that never went out proposed nothing. The caller waits for + /// a leader and re-proposes. + #[test] + fn an_unsent_metadata_forward_waits_for_a_leader() { + let unsent = metadata_propose_error(ClusterError::Transport { + detail: "connect to node 2 refused".into(), + }); + assert!(matches!(unsent, Error::MetadataLeaderUnavailable)); + + let stale = metadata_propose_error(ClusterError::Raft(RaftError::NotLeader { + leader_hint: Some(2), + term: 3, + })); + assert!(matches!(stale, Error::MetadataLeaderUnavailable)); + + let moving = + metadata_propose_error(ClusterError::Raft(RaftError::LeadershipTransferInProgress)); + assert!(matches!(moving, Error::MetadataLeaderUnavailable)); + } +} diff --git a/nodedb/src/control/metadata_proposer/mod.rs b/nodedb/src/control/metadata_proposer/mod.rs index c06c8a0e2..c6d65e9ff 100644 --- a/nodedb/src/control/metadata_proposer/mod.rs +++ b/nodedb/src/control/metadata_proposer/mod.rs @@ -23,12 +23,14 @@ //! Semantics: //! //! 1. If this node is the metadata-group leader, proposes the entry, waits -//! until its local applied watermark reaches the assigned log index for -//! as long as the apply moves (a 5s stall ends the wait), and returns the -//! log index on success. -//! 2. If this node is NOT the leader, returns -//! `Error::Config { detail: "metadata propose: not leader ..." }`. -//! Gateway-side redirection will make this transparent. +//! until its local applied watermark reaches the assigned log index, and +//! returns the log index. The wait ends at the statement deadline with +//! `Error::DeadlineExceeded`: the entry was proposed, so its outcome is +//! unknown. +//! 2. If this node is NOT the leader, forwards the entry to the leader. +//! With no leader to take it, returns `Error::MetadataLeaderUnavailable`: +//! nothing was proposed. A forward with no answer returns +//! `Error::DeadlineExceeded`: the outcome is unknown. pub mod catalog; pub mod catalog_batch; @@ -53,4 +55,4 @@ pub use replicated_entries::{ propose_restore_point, propose_surrogate_hwm, propose_surrogate_reserve, propose_sync_peer_bind, propose_sync_producer_fence, propose_sync_producer_register, }; -pub use timeouts::{DEFAULT_DRAIN_TIMEOUT, DEFAULT_PROPOSE_TIMEOUT}; +pub use timeouts::DEFAULT_DRAIN_TIMEOUT; diff --git a/nodedb/src/control/metadata_proposer/replicated_entries.rs b/nodedb/src/control/metadata_proposer/replicated_entries.rs index 0e2cce34b..96d2c6b7f 100644 --- a/nodedb/src/control/metadata_proposer/replicated_entries.rs +++ b/nodedb/src/control/metadata_proposer/replicated_entries.rs @@ -8,19 +8,17 @@ //! runtime flavor. Every node runs a metadata group, a one-node cluster //! included. -use std::sync::Arc; - -use nodedb_cluster::{METADATA_GROUP_ID, MetadataEntry, encode_entry}; +use nodedb_cluster::{MetadataEntry, encode_entry}; use crate::control::catalog_entry::CatalogEntry; use crate::control::state::SharedState; use crate::error::Error; use crate::event::cdc::consumer_group::OffsetCommit; -use super::timeouts::DEFAULT_PROPOSE_TIMEOUT; +use super::wait::await_metadata_apply; -/// Propose `entry` and wait until this node reaches its log index. `label` -/// names the entry in the errors. +/// Propose `entry` and wait until this node reaches its log index, until the +/// statement deadline. `label` names the entry in the errors. async fn propose_and_wait( shared: &SharedState, entry: &MetadataEntry, @@ -32,17 +30,7 @@ async fn propose_and_wait( })?; let log_index = handle.propose_async(raw).await?; - - let watcher = shared.applied_index_watcher(METADATA_GROUP_ID); - let outcome = - super::wait::wait_applied(Arc::clone(&watcher), log_index, DEFAULT_PROPOSE_TIMEOUT).await?; - if !outcome.is_reached() { - return Err(Error::Config { - detail: format!("{label} propose timed out waiting for log index {log_index}"), - }); - } - - Ok(log_index) + await_metadata_apply(shared, log_index).await } /// Propose a cluster restore point at watermark `hlc` and wait until this @@ -86,7 +74,7 @@ pub async fn propose_surrogate_hwm(shared: &SharedState, hwm: u32) -> Result, - index: u64, - timeout: Duration, -) -> Result { - tokio::task::spawn_blocking(move || watcher.wait_for(index, timeout)) - .await - .map_err(|error| Error::Config { - detail: format!("metadata apply wait for log index {index} did not finish: {error}"), - }) +/// Wait until `watcher` reaches `log_index`, the index of a metadata entry +/// this node proposed, or `deadline` passes. Returns `log_index`. +pub(crate) async fn await_applied( + watcher: &AppliedIndexWatcher, + log_index: u64, + deadline: Instant, +) -> Result { + let outcome = watcher.wait_for_async(log_index, deadline).await; + applied_or_error(outcome, log_index, watcher.current()) } -/// Wait until `watcher` reaches `log_index`, for as long as the apply moves. -/// Each window parks on the blocking pool. -/// -/// Each window of `stall` that ends with neither the applied index nor -/// `progress()` changed ends the wait with `TimedOut`. Any change inside a -/// window opens another one, so a slow apply that keeps moving never times -/// out, and a stuck one times out after one quiet window. -pub(super) async fn wait_tracking_progress_async( - watcher: Arc, +/// Wait until this node applied the metadata entry it proposed at +/// `log_index`, until the statement deadline. Returns `log_index`. +pub(crate) async fn await_metadata_apply( + shared: &SharedState, log_index: u64, - stall: Duration, - mut progress: impl FnMut() -> u64, -) -> Result { - let mut seen = (watcher.current(), progress()); - loop { - let outcome = wait_applied(Arc::clone(&watcher), log_index, stall).await?; - if !matches!(outcome, WaitOutcome::TimedOut) { - return Ok(outcome); - } - let now = (watcher.current(), progress()); - if now == seen { - return Ok(outcome); +) -> Result { + let watcher = shared.applied_index_watcher(METADATA_GROUP_ID); + let deadline = StatementDeadline::of_statement(shared).instant(); + await_applied(&watcher, log_index, deadline).await +} + +/// The verdict on a proposed metadata entry from its apply wait. `applied` +/// is the watermark when the wait ended. +fn applied_or_error(outcome: WaitOutcome, log_index: u64, applied: u64) -> Result { + match outcome { + WaitOutcome::Reached => Ok(log_index), + WaitOutcome::TimedOut => { + tracing::warn!( + log_index, + applied, + "a proposed metadata entry did not apply here before the statement deadline; \ + its outcome is unknown" + ); + Err(Error::DeadlineExceeded { + request_id: RequestId::new(0), + }) } - seen = now; + WaitOutcome::GroupGone => Err(Error::MetadataGroupGone { log_index }), + } +} + +#[cfg(test)] +mod tests { + use std::time::Duration; + + use super::*; + + /// A metadata group that left this node ends the wait with its own error. + #[tokio::test] + async fn a_gone_group_ends_the_wait_with_a_typed_error() { + let watcher = AppliedIndexWatcher::new(); + watcher.close(); + let result = await_applied(&watcher, 5, Instant::now() + Duration::from_secs(5)).await; + assert!( + matches!(result, Err(Error::MetadataGroupGone { log_index: 5 })), + "{result:?}" + ); } } diff --git a/nodedb/src/control/orchestrated_write.rs b/nodedb/src/control/orchestrated_write.rs index dd4568dbf..55681ef45 100644 --- a/nodedb/src/control/orchestrated_write.rs +++ b/nodedb/src/control/orchestrated_write.rs @@ -49,7 +49,8 @@ pub(crate) async fn apply_orchestrated_write( } })?; let request_id = RequestId::new(state.request_id_counter.fetch_add(1, Ordering::Relaxed)); - let deadline = crate::control::wal_replication::statement_propose_deadline(state); + let deadline = crate::control::server::shared::session::StatementDeadline::of_statement(state) + .tokio_instant(); match propose_replicated_entry(state, proposer, entry, deadline).await { Ok((payload, write_versions)) => { let response = Response { diff --git a/nodedb/src/control/pitr/restore_point/mod.rs b/nodedb/src/control/pitr/restore_point/mod.rs index f9c9ceb5c..f46c283ee 100644 --- a/nodedb/src/control/pitr/restore_point/mod.rs +++ b/nodedb/src/control/pitr/restore_point/mod.rs @@ -2,10 +2,15 @@ pub mod create; pub mod record; +pub mod recorded_cuts; pub mod seed; pub mod task; pub use create::{RestorePointError, create_restore_point, list_restore_points}; pub use record::{record_group_point, sequencer_hook, spawn_node_cut}; -pub use seed::{RecordedCut, load_recorded_cuts, persist_cut_floor, persist_until_durable}; +pub use recorded_cuts::RecordedCuts; +pub use seed::{ + RecordedCut, group_cut_floors, install_cut_floors, load_recorded_cuts, persist_cut_floor, + persist_until_durable, +}; pub use task::spawn_restore_point_task; diff --git a/nodedb/src/control/pitr/restore_point/recorded_cuts.rs b/nodedb/src/control/pitr/restore_point/recorded_cuts.rs new file mode 100644 index 000000000..4a0578217 --- /dev/null +++ b/nodedb/src/control/pitr/restore_point/recorded_cuts.rs @@ -0,0 +1,116 @@ +// SPDX-License-Identifier: BUSL-1.1 + +//! The cut barriers the apply loop raises commit HLCs above, beyond the ones +//! it applies itself: those the system catalog held at boot, and those a +//! data-group snapshot install carried. +//! +//! The apply loop merges every change into its own floors. A change bumps +//! the generation, so the loop reads the shared floors only after one. + +use std::collections::BTreeMap; +use std::sync::Mutex; +use std::sync::atomic::{AtomicU64, Ordering}; + +use super::seed::RecordedCut; + +/// Per group, barrier log index to watermark, and a generation that moves +/// on every change. +#[derive(Debug, Default)] +pub struct RecordedCuts { + floors: Mutex>>, + generation: AtomicU64, +} + +impl RecordedCuts { + pub fn new() -> Self { + Self::default() + } + + /// Add every cut of `cuts`. + pub fn record(&self, cuts: &[RecordedCut]) { + let mut floors = self.floors.lock().unwrap_or_else(|p| p.into_inner()); + for cut in cuts { + let watermark = floors + .entry(cut.group_id) + .or_default() + .entry(cut.barrier_index) + .or_insert(0); + *watermark = (*watermark).max(cut.hlc); + } + self.generation.fetch_add(1, Ordering::Release); + } + + /// Replace `group_id`'s cuts with `cuts`, the group's durable rows after + /// a snapshot install. + pub fn replace_group(&self, group_id: u64, cuts: &[RecordedCut]) { + let mut floors = self.floors.lock().unwrap_or_else(|p| p.into_inner()); + let group = floors.entry(group_id).or_default(); + group.clear(); + for cut in cuts.iter().filter(|cut| cut.group_id == group_id) { + let watermark = group.entry(cut.barrier_index).or_insert(0); + *watermark = (*watermark).max(cut.hlc); + } + self.generation.fetch_add(1, Ordering::Release); + } + + /// Moves on every change. Read it before [`Self::all`]: a change after + /// the read moves it again. + pub fn generation(&self) -> u64 { + self.generation.load(Ordering::Acquire) + } + + /// Every cut, in group and index order. + pub fn all(&self) -> Vec { + self.floors + .lock() + .unwrap_or_else(|p| p.into_inner()) + .iter() + .flat_map(|(group_id, group)| { + group.iter().map(|(barrier_index, hlc)| RecordedCut { + group_id: *group_id, + barrier_index: *barrier_index, + hlc: *hlc, + }) + }) + .collect() + } + + /// The highest watermark of a cut of `group_id` below `log_index`. + pub fn watermark_below(&self, group_id: u64, log_index: u64) -> Option { + self.floors + .lock() + .unwrap_or_else(|p| p.into_inner()) + .get(&group_id)? + .range(..log_index) + .map(|(_, hlc)| *hlc) + .max() + } +} + +#[cfg(test)] +mod tests { + use super::*; + + fn cut(group_id: u64, barrier_index: u64, hlc: u64) -> RecordedCut { + RecordedCut { + group_id, + barrier_index, + hlc, + } + } + + #[test] + fn an_install_replaces_only_its_group() { + let cuts = RecordedCuts::new(); + let start = cuts.generation(); + cuts.record(&[cut(1, 10, 100), cut(2, 5, 50)]); + assert_ne!(cuts.generation(), start); + assert_eq!(cuts.watermark_below(1, 11), Some(100)); + assert_eq!(cuts.watermark_below(1, 10), None, "the barrier's own place"); + + let before = cuts.generation(); + cuts.replace_group(1, &[cut(1, 30, 300), cut(2, 9, 90)]); + assert_ne!(cuts.generation(), before); + assert_eq!(cuts.all(), [cut(1, 30, 300), cut(2, 5, 50)]); + } +} diff --git a/nodedb/src/control/pitr/restore_point/seed.rs b/nodedb/src/control/pitr/restore_point/seed.rs index f10d22eb0..d6ebde921 100644 --- a/nodedb/src/control/pitr/restore_point/seed.rs +++ b/nodedb/src/control/pitr/restore_point/seed.rs @@ -19,6 +19,7 @@ use crate::control::cluster::metadata_applier::{MetadataApplyWedge, WedgeReport, use crate::control::security::catalog::SystemCatalog; use crate::control::security::catalog::cut_floors::StoredBarrier; use crate::control::state::SharedState; +use crate::types::CutFloor; /// One group's cut barrier. #[derive(Debug, Clone, Copy, PartialEq, Eq)] @@ -62,6 +63,59 @@ pub fn persist_cut_floor( .put_cut_floor(group_id, barrier, durable) } +/// The durable cut floors of `group_id` at or below `through`, in index +/// order, for a data-group snapshot to carry. +pub fn group_cut_floors( + state: &SharedState, + group_id: u64, + through: u64, +) -> crate::Result> { + Ok(state + .credentials + .catalog() + .group_cut_floors(group_id)? + .into_iter() + .filter(|barrier| barrier.index <= through) + .map(|barrier| CutFloor { + index: barrier.index, + watermark: barrier.watermark, + }) + .collect()) +} + +/// Durably record the cut floors `floors` a snapshot of `group_id` carried, +/// and hand the group's rows to the apply loop. The receiver never applies +/// the barriers the snapshot covers, yet every later entry records above them. +pub fn install_cut_floors( + state: &SharedState, + group_id: u64, + floors: &[CutFloor], +) -> crate::Result<()> { + if floors.is_empty() { + return Ok(()); + } + let catalog = state.credentials.catalog(); + let barriers: Vec = floors + .iter() + .map(|floor| StoredBarrier { + index: floor.index, + watermark: floor.watermark, + }) + .collect(); + catalog.put_cut_floors(group_id, &barriers, state.pitr.durable_applied(group_id))?; + let rows: Vec = catalog + .group_cut_floors(group_id)? + .into_iter() + .map(|barrier| RecordedCut { + group_id, + barrier_index: barrier.index, + hlc: barrier.watermark, + }) + .collect(); + state.pitr.recorded_cuts().replace_group(group_id, &rows); + Ok(()) +} + /// First wait between two attempts of a floor write. const FIRST_RETRY: Duration = Duration::from_millis(10); diff --git a/nodedb/src/control/pitr/state.rs b/nodedb/src/control/pitr/state.rs index 0a78a6d1b..f500032ef 100644 --- a/nodedb/src/control/pitr/state.rs +++ b/nodedb/src/control/pitr/state.rs @@ -10,7 +10,7 @@ use std::sync::{Arc, Mutex, OnceLock, RwLock}; use super::node_life::NodeLife; use super::pins::ColdPins; -use super::restore_point::RecordedCut; +use super::restore_point::{RecordedCut, RecordedCuts}; use crate::control::metrics::SystemMetrics; use crate::storage::snapshot::SnapshotCatalog; @@ -35,8 +35,9 @@ pub struct PitrState { /// Chunk ids the kept bases and the bases in progress list. chunk_pins: tokio::sync::RwLock, last_failure: Mutex>, - /// The cut barriers the system catalog held at boot. - recorded_cuts: OnceLock>, + /// The cut barriers the system catalog held at boot, and those a + /// snapshot install carried since. + recorded_cuts: Arc, /// Each group's durable applied index, as this life saved it. durable_applied: Mutex>, } @@ -60,15 +61,14 @@ impl PitrState { self.cluster_catalog.get() } - /// Install the cut barriers read from the system catalog at boot. A - /// second install keeps the first. + /// Add the cut barriers read from the system catalog at boot. pub fn install_recorded_cuts(&self, cuts: Vec) { - let _ = self.recorded_cuts.set(cuts); + self.recorded_cuts.record(&cuts); } - /// The cut barriers the system catalog held at boot. - pub fn recorded_cuts(&self) -> &[RecordedCut] { - self.recorded_cuts.get().map_or(&[], Vec::as_slice) + /// The cut barriers the apply loop reads beyond the ones it applies. + pub fn recorded_cuts(&self) -> Arc { + Arc::clone(&self.recorded_cuts) } /// Note that `group_id`'s durable applied index reached `index`. diff --git a/nodedb/src/control/planner/calvin/dependent_recon.rs b/nodedb/src/control/planner/calvin/dependent_recon.rs index 8bbbb277b..df7221560 100644 --- a/nodedb/src/control/planner/calvin/dependent_recon.rs +++ b/nodedb/src/control/planner/calvin/dependent_recon.rs @@ -30,6 +30,7 @@ use crate::control::planner::calvin::{ use crate::control::planner::implicit_edges::{ EdgeUpdateCtx, append_implicit_edge_delete_tasks, append_implicit_edge_update_tasks, }; +use crate::control::server::shared::session::StatementDeadline; use crate::control::state::SharedState; use crate::types::{DatabaseId, TenantId, TraceId}; use nodedb_physical::physical_plan::OllpPredictedEdge; @@ -241,7 +242,8 @@ async fn dispatch_dependent_edge_recon_inner( ) .await?; - let timeout = std::time::Duration::from_secs(state.tuning.network.default_deadline_secs); + // One deadline for every attempt: the statement's. + let deadline = StatementDeadline::of_statement(state).tokio_instant(); let ollp_max_retries = orc.ollp_max_retries() as u32; // `submit`: build the TxClass with the loop-supplied prediction (NOT a @@ -422,7 +424,7 @@ async fn dispatch_dependent_edge_recon_inner( registry, orchestrator: orc, predicate_class_hash: pred_class, - timeout, + deadline, ollp_max_retries, initial_predicted, submit, diff --git a/nodedb/src/control/planner/calvin/dependent_recon_crdt.rs b/nodedb/src/control/planner/calvin/dependent_recon_crdt.rs index 238961314..439930ab2 100644 --- a/nodedb/src/control/planner/calvin/dependent_recon_crdt.rs +++ b/nodedb/src/control/planner/calvin/dependent_recon_crdt.rs @@ -34,6 +34,7 @@ use crate::Error; use crate::control::cluster::calvin::executor::ollp::error::OllpError; use crate::control::security::identity::AuthenticatedIdentity; use crate::control::server::graph_dispatch::{VShardRead, read_on_vshard}; +use crate::control::server::shared::session::StatementDeadline; use crate::control::state::SharedState; use crate::types::VShardId; use crate::types::{DatabaseId, TenantId}; @@ -162,7 +163,7 @@ pub(super) async fn dispatch_crdt_doc_deletes( registry, orchestrator: orc, predicate_class_hash: pred_class, - timeout: std::time::Duration::from_secs(state.tuning.network.default_deadline_secs), + deadline: StatementDeadline::of_statement(state).tokio_instant(), ollp_max_retries: orc.ollp_max_retries() as u32, initial_predicted, submit, diff --git a/nodedb/src/control/planner/calvin/dependent_recon_finish.rs b/nodedb/src/control/planner/calvin/dependent_recon_finish.rs index 5f6b210f2..a71d0f170 100644 --- a/nodedb/src/control/planner/calvin/dependent_recon_finish.rs +++ b/nodedb/src/control/planner/calvin/dependent_recon_finish.rs @@ -28,7 +28,12 @@ pub(super) async fn finish_committed( .any(|collection| sources.is_source_collection(collection)) }); if binds_authorization { - crate::control::security::auth_lease::calvin_write_barrier(state).await?; + crate::control::security::auth_lease::calvin_write_barrier( + state, + crate::control::server::shared::session::StatementDeadline::of_statement(state) + .instant(), + ) + .await?; } // Completion fired: the scheduler deposited the applied Response (with any diff --git a/nodedb/src/control/planner/calvin/mod.rs b/nodedb/src/control/planner/calvin/mod.rs index b0feb9ac1..e1018a3d9 100644 --- a/nodedb/src/control/planner/calvin/mod.rs +++ b/nodedb/src/control/planner/calvin/mod.rs @@ -43,8 +43,8 @@ pub use explain::calvin_explain_preamble; pub use predicate::predicate_class_for_filters; pub use retry_loop::{DependentOutcome, DependentRetryArgs, run_dependent_with_retry}; pub use submit::{ - RoutedAssignment, submit_and_await_calvin, submit_and_await_calvin_with_timeout, - submit_calvin_routed, submit_calvin_routed_assign, submit_calvin_routed_write, + RoutedAssignment, submit_and_await_calvin, submit_and_await_calvin_until, submit_calvin_routed, + submit_calvin_routed_assign, submit_calvin_routed_write, }; pub use tx_class::{ PassiveReads, build_predicted_tx_class, build_read_dependent_tx_class, diff --git a/nodedb/src/control/planner/calvin/read_dependent/item_move.rs b/nodedb/src/control/planner/calvin/read_dependent/item_move.rs index 05ef4a5e0..4bea8b730 100644 --- a/nodedb/src/control/planner/calvin/read_dependent/item_move.rs +++ b/nodedb/src/control/planner/calvin/read_dependent/item_move.rs @@ -40,6 +40,7 @@ use crate::control::security::audit::ArcAuditEmitter; use crate::control::security::identity::AuthenticatedIdentity; use crate::control::security::request_scope::RequestAuthScope; use crate::control::server::shared::authorization::authorize_task_set; +use crate::control::server::shared::session::StatementDeadline; use crate::control::state::SharedState; use crate::types::{DatabaseId, TenantId}; @@ -196,7 +197,7 @@ pub async fn move_item_across_shards( registry, orchestrator, predicate_class_hash: class_hash, - timeout: std::time::Duration::from_secs(state.tuning.network.default_deadline_secs), + deadline: StatementDeadline::of_statement(state).tokio_instant(), ollp_max_retries: u32::from(orchestrator.ollp_max_retries()), initial_predicted: initial_item, submit, diff --git a/nodedb/src/control/planner/calvin/reservation.rs b/nodedb/src/control/planner/calvin/reservation.rs index e37c6b174..799f349e4 100644 --- a/nodedb/src/control/planner/calvin/reservation.rs +++ b/nodedb/src/control/planner/calvin/reservation.rs @@ -17,21 +17,26 @@ //! storage I/O or io_uring directly. use std::collections::BTreeSet; -use std::time::Duration; use nodedb_cluster::calvin::SEQUENCER_GROUP_ID; use nodedb_cluster::calvin::types::{LockKeyWire, ReleaseReason, TxnIdWire}; +use nodedb_cluster::rpc_codec::reply_wait; use nodedb_cluster::{ ClusterError, RaftRpc, ReleaseReservationRequest, ReleaseReservationResponse, - ReserveReadRequest, ReserveReadResponse, + ReserveReadRequest, ReserveReadResponse, TypedClusterError, }; use crate::Error; use crate::control::cluster::warm_peers::register_peers_from_topology; +use crate::control::server::shared::session::StatementDeadline; use crate::control::state::SharedState; /// Submit a reserve-read to THIS node's reservation inbox and await the -/// assigned owner, bounded by `timeout`. +/// assigned owner until `deadline`. +/// +/// A deadline spent before the submit refuses it: nothing was submitted. A +/// wait that reaches the deadline is an unknown outcome: the service can +/// still grant the reservation, and lease GC reaps a lost grant. /// /// PRECONDITION: this node is the sequencer-group leader (its service /// assigns). Callers that are not the leader MUST route via @@ -44,27 +49,29 @@ pub(crate) async fn submit_local_reserve_read( key: LockKeyWire, vshard: u32, owner: Option, - timeout: Duration, + deadline: tokio::time::Instant, ) -> crate::Result { let inbox = state .reservation_inbox .get() .ok_or(Error::SequencerUnavailable)?; + super::submit::budget::budget_ms_until(deadline)?; let rx = inbox .submit_reserve(key, vshard, owner) .map_err(|e| Error::BadRequest { detail: format!("reservation service rejected reserve-read: {e}"), })?; - tokio::time::timeout(timeout, rx) - .await - .map_err(|_| Error::Internal { - detail: "timed out waiting for reservation assignment".to_owned(), - })? - .map_err(|_| Error::Internal { + match tokio::time::timeout_at(deadline, rx).await { + Ok(Ok(assigned)) => Ok(assigned), + Ok(Err(_)) => Err(Error::Internal { detail: "reservation assignment channel closed".to_owned(), - }) + }), + Err(_) => Err(Error::DeadlineExceeded { + request_id: crate::types::RequestId::new(0), + }), + } } /// Submit a reserve-read, routing it to the sequencer-group leader. @@ -111,11 +118,12 @@ pub(crate) async fn submit_reserve_read( }); } + // Every wait of this reserve-read ends at the statement's deadline. + let deadline = StatementDeadline::of_statement(state).tokio_instant(); // Leader is self: submit locally (a self-RPC will be a pointless extra // hop and the local inbox is the one that gets the assignment). if leader == state.node_id { - let local_timeout = super::submit::budget::statement_budget(state)?; - return submit_local_reserve_read(state, key, vshard, owner, local_timeout).await; + return submit_local_reserve_read(state, key, vshard, owner, deadline).await; } // Remote leader: ensure its address is registered before dispatch, then @@ -138,7 +146,7 @@ pub(crate) async fn submit_reserve_read( .transpose()?; // The leader works on the reservation only as long as the statement waits. - let deadline_remaining_ms = super::submit::budget::statement_budget_ms(state)?; + let deadline_remaining_ms = super::submit::budget::budget_ms_until(deadline)?; let req = ReserveReadRequest { lock_key_bytes, vshard, @@ -151,7 +159,7 @@ pub(crate) async fn submit_reserve_read( // assigned (up to `deadline_remaining_ms`). The generic short `rpc_timeout` // will abort the call long before that, so bound the response read by the // forwarded deadline plus a margin for the round-trip itself. - let read_timeout = Duration::from_millis(deadline_remaining_ms.saturating_add(2_000)); + let read_timeout = reply_wait(deadline_remaining_ms); match transport .send_rpc_with_read_timeout(leader, RaftRpc::ReserveReadRequest(req), read_timeout) .await @@ -163,10 +171,17 @@ pub(crate) async fn submit_reserve_read( format: "msgpack".to_owned(), detail: format!("failed to decode TxnIdWire owner from reserve-read reply: {e}"), }), + // An error with no class keeps the leader in its message. + Ok(RaftRpc::ReserveReadResponse(ReserveReadResponse { + error: Some(TypedClusterError::Internal { code: 0, message }), + .. + })) => Err(Error::Internal { + detail: format!("reserve-read failed on sequencer leader node {leader}: {message}"), + }), + // Every other error keeps its class. A leader-side deadline stays an + // unknown outcome, and an expired budget stays a definite refusal. Ok(RaftRpc::ReserveReadResponse(ReserveReadResponse { error: Some(e), .. })) => { - Err(Error::Internal { - detail: format!("reserve-read failed on sequencer leader node {leader}: {e:?}"), - }) + Err(Error::from(e)) } Ok(other) => Err(Error::Internal { detail: format!("reserve-read: unexpected reply from node {leader}: {other:?}"), @@ -298,7 +313,7 @@ pub(crate) async fn release_reservation( deadline_remaining_ms, trace_id: [0u8; 16], }; - let read_timeout = Duration::from_millis(deadline_remaining_ms.saturating_add(2_000)); + let read_timeout = reply_wait(deadline_remaining_ms); match transport .send_rpc_with_read_timeout( diff --git a/nodedb/src/control/planner/calvin/retry_loop.rs b/nodedb/src/control/planner/calvin/retry_loop.rs index fbe58d25c..63ef39a8c 100644 --- a/nodedb/src/control/planner/calvin/retry_loop.rs +++ b/nodedb/src/control/planner/calvin/retry_loop.rs @@ -15,6 +15,8 @@ use crate::control::cluster::calvin::executor::ollp::error::OllpError; use crate::control::cluster::calvin::executor::ollp::orchestrator::OllpOrchestrator; use crate::control::planner::calvin::abort_error::calvin_abort_error; use crate::control::planner::calvin::submit::RoutedAssignment; +use crate::control::planner::calvin::submit::budget::budget_ms_until; +use crate::control::planner::calvin::submit::wait::await_completion; use crate::{Error, OllpExhaustedCause}; // ── run_dependent_with_retry ────────────────────────────────────────────────── @@ -64,11 +66,16 @@ pub enum DependentOutcome { /// `submit` and `rescan` are injected so this loop is unit-testable WITHOUT a /// live server/executor: a fake scheduler driving the real /// [`CalvinCompletionRegistry`] suffices. +/// +/// Every attempt shares the statement's `deadline`. A spent deadline before a +/// submit refuses it, since nothing of that attempt was submitted. A +/// completion wait that reaches the deadline is an unknown outcome, and so is +/// a submit that answers [`Error::DeadlineExceeded`]: neither is resubmitted. pub struct DependentRetryArgs<'a, P, SF, RF> { pub registry: &'a CalvinCompletionRegistry, pub orchestrator: &'a OllpOrchestrator, pub predicate_class_hash: u64, - pub timeout: std::time::Duration, + pub deadline: tokio::time::Instant, pub ollp_max_retries: u32, pub initial_predicted: P, pub submit: SF, @@ -88,7 +95,7 @@ where registry, orchestrator, predicate_class_hash, - timeout, + deadline, ollp_max_retries, initial_predicted, mut submit, @@ -97,6 +104,7 @@ where let mut predicted = initial_predicted; let mut retry: u32 = 0; loop { + budget_ms_until(deadline)?; let assignment = match submit(&predicted).await { Ok(Some(assignment)) => assignment, // Nothing to write: the prediction decided an empty write set, so @@ -107,6 +115,12 @@ where // it, so surface the real error instead of retrying and reporting // predicate drift that never happened. Err(OllpError::Terminal(error)) => return Err(*error), + // The submit can have been sequenced. A resubmit can apply it twice. + Err(OllpError::Retryable(error)) + if matches!(*error, Error::DeadlineExceeded { .. }) => + { + return Err(*error); + } Err(ollp_err) => { // PRE-ADMISSION failure (routed submit / circuit / budget). // Nothing executed, so there is no aborted attempt to re-scan @@ -132,15 +146,7 @@ where // registry, which receives the replicated completion ack on every // sequencer-group member. let txn_id = TxnId::new(assignment.epoch, assignment.position); - let completion_rx = registry.register_completion_report(txn_id, assignment.participants); - let report = tokio::time::timeout(timeout, completion_rx) - .await - .map_err(|_| Error::Internal { - detail: "timed out waiting for Calvin completion".into(), - })? - .map_err(|_| Error::Internal { - detail: "Calvin completion channel closed".into(), - })?; + let report = await_completion(registry, txn_id, assignment.participants, deadline).await?; match report.outcome { // Return the completed txn's id so the caller can drain the applied @@ -223,6 +229,11 @@ mod tests { use nodedb_cluster::calvin::VerdictOutcome; use nodedb_cluster::calvin::sequencer::error::SequencerError; + /// A deadline no test reaches. + fn far_deadline() -> tokio::time::Instant { + tokio::time::Instant::now() + std::time::Duration::from_secs(5) + } + /// Build an orchestrator with zero backoff so the retry loop runs instantly. fn zero_backoff_orchestrator() -> OllpOrchestrator { OllpOrchestrator::new(OllpConfig { @@ -296,7 +307,7 @@ mod tests { registry: ®istry, orchestrator: &orchestrator, predicate_class_hash: 0xABCD, - timeout: std::time::Duration::from_secs(5), + deadline: far_deadline(), ollp_max_retries: 5, initial_predicted: vec![1, 2, 3], submit: move |_predicted: &Vec| { @@ -367,7 +378,7 @@ mod tests { registry: ®istry, orchestrator: &orchestrator, predicate_class_hash: 0xABCD, - timeout: std::time::Duration::from_secs(5), + deadline: far_deadline(), ollp_max_retries: 3, initial_predicted: vec![1], submit: move |_predicted: &Vec| { @@ -436,7 +447,7 @@ mod tests { registry: ®istry, orchestrator: &orchestrator, predicate_class_hash: 0xABCD, - timeout: std::time::Duration::from_secs(5), + deadline: far_deadline(), ollp_max_retries: 5, initial_predicted: vec![1], submit: move |_predicted: &Vec| { @@ -503,7 +514,7 @@ mod tests { registry: ®istry, orchestrator: &orchestrator, predicate_class_hash: 0xABCD, - timeout: std::time::Duration::from_secs(5), + deadline: far_deadline(), ollp_max_retries: 5, initial_predicted: Vec::::new(), // A zero-match prediction builds no TxClass, so nothing is @@ -551,7 +562,7 @@ mod tests { registry: ®istry, orchestrator: &orchestrator, predicate_class_hash: 0xABCD, - timeout: std::time::Duration::from_secs(5), + deadline: far_deadline(), ollp_max_retries: 5, initial_predicted: vec![1], submit: move |_predicted: &Vec| { @@ -597,7 +608,7 @@ mod tests { registry: ®istry, orchestrator: &orchestrator, predicate_class_hash: 0xABCD, - timeout: std::time::Duration::from_secs(5), + deadline: far_deadline(), ollp_max_retries: 2, initial_predicted: vec![1], // Retryable every time: the loop exhausts, and the carried cause @@ -668,7 +679,7 @@ mod tests { registry: ®istry, orchestrator: &orchestrator, predicate_class_hash: 0xABCD, - timeout: std::time::Duration::from_secs(5), + deadline: far_deadline(), ollp_max_retries: 5, initial_predicted: vec![1], submit: move |_predicted: &Vec| { @@ -735,4 +746,112 @@ mod tests { assert_eq!(submits, 1); assert_eq!(rescans, 0); } + + /// A submit whose assignment the fake never completes. The completion + /// wait ends at the statement's deadline as an unknown outcome. + #[tokio::test(start_paused = true)] + async fn the_completion_wait_ends_at_the_statement_deadline() { + let registry = CalvinCompletionRegistry::new_detached(); + let orchestrator = zero_backoff_orchestrator(); + let start = tokio::time::Instant::now(); + let deadline = start + std::time::Duration::from_millis(100); + let submit_calls = Arc::new(AtomicU64::new(0)); + + let result = { + let submit_calls = Arc::clone(&submit_calls); + run_dependent_with_retry(DependentRetryArgs { + registry: ®istry, + orchestrator: &orchestrator, + predicate_class_hash: 0xABCD, + deadline, + ollp_max_retries: 5, + initial_predicted: vec![1], + submit: move |_predicted: &Vec| { + let submit_calls = Arc::clone(&submit_calls); + async move { + let n = submit_calls.fetch_add(1, Ordering::SeqCst); + Ok::, OllpError>(Some(fake_assignment(n + 1))) + } + }, + rescan: move || async move { Ok(vec![1]) }, + }) + .await + }; + + assert!( + matches!(result, Err(Error::DeadlineExceeded { .. })), + "expected an unknown outcome, got {result:?}" + ); + let waited = start.elapsed(); + assert!( + waited >= std::time::Duration::from_millis(100) + && waited < std::time::Duration::from_millis(200), + "the wait ends at the statement deadline, not later: {waited:?}" + ); + assert_eq!( + submit_calls.load(Ordering::SeqCst), + 1, + "an unknown outcome is never resubmitted" + ); + } + + /// A drift abort whose re-read ends past the deadline. The next attempt + /// is refused before its submit: nothing of it was submitted. + #[tokio::test(start_paused = true)] + async fn a_retry_after_the_deadline_is_refused_before_submit() { + let registry = CalvinCompletionRegistry::new_detached(); + let orchestrator = zero_backoff_orchestrator(); + let deadline = tokio::time::Instant::now() + std::time::Duration::from_millis(100); + + let (tx, rx) = tokio::sync::mpsc::channel::(16); + let fake = spawn_fake_scheduler(Arc::clone(®istry), rx, u32::MAX); + let seq = Arc::new(AtomicU64::new(1)); + let submit_calls = Arc::new(AtomicU64::new(0)); + + let result = { + let seq = Arc::clone(&seq); + let submit_calls = Arc::clone(&submit_calls); + let tx = tx.clone(); + run_dependent_with_retry(DependentRetryArgs { + registry: ®istry, + orchestrator: &orchestrator, + predicate_class_hash: 0xABCD, + deadline, + ollp_max_retries: 5, + initial_predicted: vec![1], + submit: move |_predicted: &Vec| { + let seq = Arc::clone(&seq); + let submit_calls = Arc::clone(&submit_calls); + let tx = tx.clone(); + async move { + submit_calls.fetch_add(1, Ordering::SeqCst); + let assignment = fake_assignment(seq.fetch_add(1, Ordering::SeqCst)); + let txn = TxnId::new(assignment.epoch, assignment.position); + tx.send(txn).await.expect("fake recv alive"); + Ok::, OllpError>(Some(assignment)) + } + }, + rescan: move || async move { + tokio::time::sleep(std::time::Duration::from_millis(150)).await; + Ok(vec![2]) + }, + }) + .await + }; + + let Err(Error::DataPlane(code)) = &result else { + panic!("expected a definite refusal, got {result:?}"); + }; + assert_eq!( + code, + &crate::bridge::envelope::ErrorCode::ExpiredBeforeExecution + ); + assert_eq!( + submit_calls.load(Ordering::SeqCst), + 1, + "the attempt after the deadline is never submitted" + ); + drop(tx); + let _ = fake.await; + } } diff --git a/nodedb/src/control/planner/calvin/submit/assign.rs b/nodedb/src/control/planner/calvin/submit/assign.rs index f2781ac71..a02a14a83 100644 --- a/nodedb/src/control/planner/calvin/submit/assign.rs +++ b/nodedb/src/control/planner/calvin/submit/assign.rs @@ -6,22 +6,25 @@ //! the OLLP coordinator loop drives the transaction to completion itself. use std::collections::BTreeSet; -use std::time::Duration; use nodedb_cluster::calvin::SEQUENCER_GROUP_ID; use nodedb_cluster::calvin::types::TxClass; -use nodedb_cluster::{ClusterError, RaftRpc, SubmitCalvinInboxRequest, SubmitCalvinInboxResponse}; +use nodedb_cluster::rpc_codec::reply_wait; +use nodedb_cluster::{ + ClusterError, RaftRpc, SubmitCalvinInboxRequest, SubmitCalvinInboxResponse, TypedClusterError, +}; use super::stream::{PartStream, StreamTarget, stream_parts}; use crate::Error; use crate::control::cluster::warm_peers::register_peers_from_topology; +use crate::control::server::shared::session::StatementDeadline; use crate::control::state::SharedState; /// The sequencer ASSIGNMENT for a submitted dependent (OLLP) `TxClass`. /// /// Returned by [`submit_calvin_routed_assign`] AS SOON AS the sequencer assigns /// the transaction — completion is NOT awaited (the OLLP coordinator loop drives -/// the dependent transaction to completion itself in a later unit). +/// the dependent transaction to completion itself). #[derive(Debug, Clone, Copy, PartialEq, Eq)] pub struct RoutedAssignment { pub inbox_seq: u64, @@ -31,38 +34,39 @@ pub struct RoutedAssignment { } /// Submit `tx_class` to THIS node's Calvin sequencer inbox and await only its -/// ASSIGNMENT (NOT completion), bounded by `timeout`. +/// ASSIGNMENT (NOT completion), until `deadline`. /// /// PRECONDITION: this node is the sequencer-group leader (its service assigns). /// Callers that are not the leader MUST route via /// [`submit_calvin_routed_assign`]. The local primitive for the OLLP dependent -/// path — the sibling of [`super::local::submit_and_await_calvin_with_timeout`] that stops at -/// the assignment phase. +/// path — the sibling of [`super::local::submit_and_await_calvin_until`] that +/// stops at the assignment phase. /// /// `pub(crate)` so [`crate::control::server::calvin_submit::inbox_hook`] can /// call it after decoding the wire bytes, mirroring how `hook.rs` delegates to -/// `submit_and_await_calvin_with_timeout`. +/// `submit_and_await_calvin_until`. pub(crate) async fn submit_local_assign( state: &SharedState, mut tx_class: TxClass, - timeout: Duration, + deadline: tokio::time::Instant, ) -> crate::Result { super::local::raise_metadata_floor(state, &mut tx_class); super::local::stamp_incarnations(state, &mut tx_class)?; super::unique_claims::stamp_unique_claims(state, &mut tx_class)?; let stream = super::parts::split_into_parts(state, &mut tx_class)?; - assign_prepared(state, tx_class, stream, timeout).await + assign_prepared(state, tx_class, stream, deadline).await } /// Submit a stamped `tx_class` to this node's sequencer, await its /// assignment, then stream its parts when it carries them as parts. /// -/// PRECONDITION: this node is the sequencer-group leader. +/// PRECONDITION: this node is the sequencer-group leader. A deadline spent +/// before the submit refuses it: nothing was submitted. async fn assign_prepared( state: &SharedState, tx_class: TxClass, stream: Option, - timeout: Duration, + deadline: tokio::time::Instant, ) -> crate::Result { let inbox = state .sequencer_inbox @@ -73,6 +77,7 @@ async fn assign_prepared( .get() .ok_or(Error::SequencerUnavailable)?; + super::budget::budget_ms_until(deadline)?; let (inbox_seq, assignment_rx) = inbox .submit_with(tx_class, registry) @@ -80,11 +85,11 @@ async fn assign_prepared( detail: format!("Calvin sequencer rejected transaction: {e}"), })?; let (epoch, position, participants) = - super::local::await_assignment(registry, inbox_seq, assignment_rx, timeout).await?; + super::wait::await_assignment(registry, inbox_seq, assignment_rx, deadline).await?; // A lost stream aborts the transaction, and the caller's completion // wait reports it. if let Some(stream) = &stream { - stream_parts(state, StreamTarget::Local, stream, true) + stream_parts(state, StreamTarget::Local, stream, true, deadline) .await .into_result()?; } @@ -149,11 +154,12 @@ pub async fn submit_calvin_routed_assign( }); } + // Every wait of this submit ends at the statement's deadline. + let deadline = StatementDeadline::of_statement(state).tokio_instant(); // Leader is self: submit-and-assign locally (a self-RPC will be a pointless // extra hop and the local registry is the one that gets the assignment). if leader == state.node_id { - let local_timeout = super::budget::statement_budget(state)?; - return assign_prepared(state, tx_class, stream, local_timeout).await; + return assign_prepared(state, tx_class, stream, deadline).await; } // Remote leader: ensure its address is registered before dispatch, then send @@ -168,7 +174,7 @@ pub async fn submit_calvin_routed_assign( })?; // The leader works on the transaction only as long as the statement waits. - let deadline_remaining_ms = super::budget::statement_budget_ms(state)?; + let deadline_remaining_ms = super::budget::budget_ms_until(deadline)?; let req = SubmitCalvinInboxRequest { tx_class_bytes, deadline_remaining_ms, @@ -179,7 +185,7 @@ pub async fn submit_calvin_routed_assign( // assigned (up to `deadline_remaining_ms`). The generic short `rpc_timeout` // will abort the call long before that, so bound the response read by the // forwarded deadline plus a margin for the round-trip itself. - let read_timeout = Duration::from_millis(deadline_remaining_ms.saturating_add(2_000)); + let read_timeout = reply_wait(deadline_remaining_ms); let assignment = match transport .send_rpc_with_read_timeout(leader, RaftRpc::SubmitCalvinInboxRequest(req), read_timeout) .await @@ -196,14 +202,21 @@ pub async fn submit_calvin_routed_assign( position, participants: participants as usize, }, + // An error with no class keeps the leader in its message. Ok(RaftRpc::SubmitCalvinInboxResponse(SubmitCalvinInboxResponse { - error: Some(e), + error: Some(TypedClusterError::Internal { code: 0, message }), .. })) => { return Err(Error::Internal { - detail: format!("calvin-inbox failed on sequencer leader node {leader}: {e:?}"), + detail: format!("calvin-inbox failed on sequencer leader node {leader}: {message}"), }); } + // Every other error keeps its class. A leader-side deadline stays an + // unknown outcome, and an expired budget stays a definite refusal. + Ok(RaftRpc::SubmitCalvinInboxResponse(SubmitCalvinInboxResponse { + error: Some(e), + .. + })) => return Err(Error::from(e)), Ok(other) => { return Err(Error::Internal { detail: format!("calvin-inbox: unexpected reply from node {leader}: {other:?}"), @@ -232,7 +245,7 @@ pub async fn submit_calvin_routed_assign( // stream aborts the transaction, and the caller's completion wait // reports it. if let Some(stream) = &stream { - stream_parts(state, StreamTarget::Remote(leader), stream, true) + stream_parts(state, StreamTarget::Remote(leader), stream, true, deadline) .await .into_result()?; } diff --git a/nodedb/src/control/planner/calvin/submit/budget.rs b/nodedb/src/control/planner/calvin/submit/budget.rs index 7d50b8dae..a4336fbe9 100644 --- a/nodedb/src/control/planner/calvin/submit/budget.rs +++ b/nodedb/src/control/planner/calvin/submit/budget.rs @@ -6,11 +6,7 @@ //! leader works on it only as long as the statement waits. The budget is //! what is left of the statement's deadline, never the node default. -use std::time::Duration; - use crate::bridge::envelope::ErrorCode; -use crate::control::state::SharedState; -use crate::control::wal_replication::statement_propose_deadline; /// The milliseconds left to `deadline`, for a request about to go out. /// @@ -23,19 +19,10 @@ pub(crate) fn budget_ms_until(deadline: tokio::time::Instant) -> crate::Result crate::Result { - budget_ms_until(statement_propose_deadline(state)) -} - -/// [`statement_budget_ms`] as a wait bound for the local half of the request. -pub(crate) fn statement_budget(state: &SharedState) -> crate::Result { - statement_budget_ms(state).map(Duration::from_millis) -} - #[cfg(test)] mod tests { + use std::time::Duration; + use super::*; use crate::control::server::shared::session::{conn_scope, deadline, statement_deadline}; diff --git a/nodedb/src/control/planner/calvin/submit/local.rs b/nodedb/src/control/planner/calvin/submit/local.rs index 043dac370..f05c08281 100644 --- a/nodedb/src/control/planner/calvin/submit/local.rs +++ b/nodedb/src/control/planner/calvin/submit/local.rs @@ -7,17 +7,15 @@ //! completion ack. A submit on a non-leader is drained and discarded, so //! non-leader callers must route through [`super::routed`]. -use std::time::Duration; - use nodedb_cluster::calvin::types::TxClass; -use nodedb_cluster::calvin::{ - Assignment, AssignmentReceiver, AttemptOutcome, CalvinCompletionRegistry, TxnId, -}; +use nodedb_cluster::calvin::{AttemptOutcome, TxnId}; use super::stream::{PartStream, StreamTarget, stream_parts}; +use super::wait::{await_assignment, await_completion}; use crate::Error; use crate::bridge::envelope::{PhysicalPlan, Response}; use crate::control::planner::calvin::abort_error::calvin_abort_error; +use crate::control::server::shared::session::StatementDeadline; use crate::control::state::{CalvinApplyResult, SharedState}; /// Build a minimal Control-Plane [`Response`] carrying only the RETURNING @@ -95,78 +93,50 @@ pub(crate) fn stamp_incarnations(state: &SharedState, tx_class: &mut TxClass) -> Ok(()) } -/// Await the sequencer assignment of submission `inbox_seq`, bounded by -/// `timeout`. -/// -/// A closed channel means the sequencer rejected or discarded the submission -/// without sequencing it: a read/write cycle in its epoch, a leadership -/// change, or a halted sequencer. Nothing applied, so the caller gets a -/// retryable refusal. On timeout the registration is dropped, so no sender -/// stays behind. A timed-out submission can still be sequenced later. -pub(crate) async fn await_assignment( - registry: &CalvinCompletionRegistry, - inbox_seq: u64, - assignment_rx: AssignmentReceiver, - timeout: Duration, -) -> crate::Result { - match tokio::time::timeout(timeout, assignment_rx).await { - Ok(Ok(assignment)) => Ok(assignment), - Ok(Err(_)) => Err(Error::RetryableRefusal { - reason: "the Calvin sequencer did not sequence the transaction; nothing was \ - applied" - .to_owned(), - }), - Err(_) => { - registry.drop_assignment(inbox_seq); - Err(Error::Internal { - detail: "timed out waiting for Calvin sequencer assignment".to_owned(), - }) - } - } -} - /// Submit `tx_class` to THIS node's Calvin sequencer inbox and await completion. /// /// PRECONDITION: this node is the sequencer-group leader (its service assigns; /// its registry receives the replicated completion ack). Callers that are not /// the leader MUST route via [`super::routed::submit_calvin_routed`]. /// -/// The assignment + completion waits are bounded by -/// `state.tuning.network.default_deadline_secs`. +/// The assignment and completion waits end at the running statement's +/// deadline. pub async fn submit_and_await_calvin( state: &SharedState, tx_class: TxClass, ) -> crate::Result> { - let timeout = Duration::from_secs(state.tuning.network.default_deadline_secs); - submit_and_await_calvin_with_timeout(state, tx_class, timeout).await + let deadline = StatementDeadline::of_statement(state).tokio_instant(); + submit_and_await_calvin_until(state, tx_class, deadline).await } -/// [`submit_and_await_calvin`] with an explicit deadline budget. +/// [`submit_and_await_calvin`] with an explicit deadline. /// -/// Used by the leader-side RPC handler so the forwarded submit-and-await is -/// bounded by the coordinator's remaining deadline rather than this node's full -/// default deadline. -pub async fn submit_and_await_calvin_with_timeout( +/// The leader-side RPC handler passes the coordinator's remaining deadline, +/// so the forwarded submit-and-await never outlives the statement. +pub async fn submit_and_await_calvin_until( state: &SharedState, mut tx_class: TxClass, - timeout: Duration, + deadline: tokio::time::Instant, ) -> crate::Result> { raise_metadata_floor(state, &mut tx_class); stamp_incarnations(state, &mut tx_class)?; super::unique_claims::stamp_unique_claims(state, &mut tx_class)?; let stream = super::parts::split_into_parts(state, &mut tx_class)?; - submit_prepared_and_await(state, tx_class, stream, timeout).await + submit_prepared_and_await(state, tx_class, stream, deadline).await } /// Submit a stamped `tx_class` to this node's sequencer, stream its parts /// when it carries them as parts, and await its completion. /// /// PRECONDITION: this node is the sequencer-group leader. +/// +/// A deadline spent before the submit refuses it: nothing was submitted. A +/// wait that reaches the deadline after the submit is an unknown outcome. pub(crate) async fn submit_prepared_and_await( state: &SharedState, tx_class: TxClass, stream: Option, - timeout: Duration, + deadline: tokio::time::Instant, ) -> crate::Result> { #[cfg(feature = "failpoints")] crate::control::fail_gate::after_calvin_stamp(state.node_id, &tx_class).await; @@ -196,6 +166,7 @@ pub(crate) async fn submit_prepared_and_await( .is_source_vshard(vshard.as_u32()) }); + super::budget::budget_ms_until(deadline)?; let (inbox_seq, assignment_rx) = inbox .submit_with(tx_class, registry) @@ -203,40 +174,23 @@ pub(crate) async fn submit_prepared_and_await( detail: format!("Calvin sequencer rejected transaction: {e}"), })?; let (epoch, position, participants) = - await_assignment(registry, inbox_seq, assignment_rx, timeout).await?; + await_assignment(registry, inbox_seq, assignment_rx, deadline).await?; // The header holds its locks on every participant until the parts // arrive. A lost stream aborts the transaction, and the completion // below reports it. if let Some(stream) = &stream { - stream_parts(state, StreamTarget::Local, stream, true) + stream_parts(state, StreamTarget::Local, stream, true, deadline) .await .into_result()?; } - let completion_rx = - registry.register_completion_report(TxnId::new(epoch, position), participants); - let report = tokio::time::timeout(timeout, completion_rx) - .await - .map_err(|_| { - let err = Error::Internal { - detail: "timed out waiting for Calvin transaction completion".to_owned(), - }; - // This timeout is the only signal a silently-never-completed - // Calvin write ever produces; file it as a structured report at - // the one site that detects it, since the error alone gives an - // operator no clue which transaction or participant stalled. - crate::diag::calvin_completion_timeout( - &err, - epoch, - position, - participants, - timeout.as_secs(), - ); - err - })? - .map_err(|_| Error::Internal { - detail: "Calvin completion channel closed".to_owned(), - })?; + let report = await_completion( + registry, + TxnId::new(epoch, position), + participants, + deadline, + ) + .await?; // The global cross-shard verdict was ABORT and the writes were dropped. // The verdict's reason picks the error the client sees. match report.outcome { @@ -244,7 +198,12 @@ pub(crate) async fn submit_prepared_and_await( AttemptOutcome::Aborted { reason } => return Err(calvin_abort_error(reason)), } if binds_authorization { - crate::control::security::auth_lease::calvin_write_barrier(state).await?; + crate::control::security::auth_lease::calvin_write_barrier( + state, + crate::control::server::shared::session::StatementDeadline::of_statement(state) + .instant(), + ) + .await?; } // Completion fired: the scheduler deposited the applied Response (with any @@ -546,43 +505,6 @@ mod tests { TsInstallCounts::from_payload(response.payload.as_bytes()).expect("install counts") } - /// The sequencer dropped the assignment of a submission it did not - /// sequence. The caller fails at once with a retryable refusal, long - /// before its timeout. - #[tokio::test] - async fn an_unsequenced_submission_is_a_retryable_refusal() { - let registry = CalvinCompletionRegistry::new_detached(); - let rx = registry.register_submission(7); - registry.drop_assignment(7); - match await_assignment(®istry, 7, rx, Duration::from_secs(3600)).await { - Err(Error::RetryableRefusal { .. }) => {} - other => panic!("expected a retryable refusal, got {other:?}"), - } - } - - #[tokio::test] - async fn an_assigned_submission_returns_its_assignment() { - let registry = CalvinCompletionRegistry::new_detached(); - let rx = registry.register_submission(3); - registry.note_assigned(3, TxnId::new(5, 1), 2); - let assignment = await_assignment(®istry, 3, rx, Duration::from_secs(3600)) - .await - .expect("assigned"); - assert_eq!(assignment, (5, 1, 2)); - } - - /// A wait that times out reports the timeout, not a refusal: the - /// submission can still be sequenced. - #[tokio::test] - async fn a_timed_out_wait_is_not_a_refusal() { - let registry = CalvinCompletionRegistry::new_detached(); - let rx = registry.register_submission(8); - match await_assignment(®istry, 8, rx, Duration::from_millis(1)).await { - Err(Error::Internal { detail }) => assert!(detail.contains("timed out"), "{detail}"), - other => panic!("expected the timeout error, got {other:?}"), - } - } - /// With one replica per vShard, the coordinator's node hosts no replica of /// either participant, so its sidecar holds nothing. The counts both /// participants' acks carried still reach the statement. diff --git a/nodedb/src/control/planner/calvin/submit/mod.rs b/nodedb/src/control/planner/calvin/submit/mod.rs index 20bec2a59..cce77da35 100644 --- a/nodedb/src/control/planner/calvin/submit/mod.rs +++ b/nodedb/src/control/planner/calvin/submit/mod.rs @@ -1,6 +1,6 @@ // SPDX-License-Identifier: BUSL-1.1 -//! Calvin submit-and-await primitive and sequencer-leader routing (Cv1). +//! Calvin submit-and-await primitive and sequencer-leader routing. //! //! NodeDB's Calvin cross-shard write path only completes when the transaction is //! submitted on the SEQUENCER-GROUP leader: @@ -38,8 +38,9 @@ pub mod parts; pub mod routed; pub mod stream; pub mod unique_claims; +pub mod wait; pub(crate) use assign::submit_local_assign; pub use assign::{RoutedAssignment, submit_calvin_routed_assign}; -pub use local::{submit_and_await_calvin, submit_and_await_calvin_with_timeout}; +pub use local::{submit_and_await_calvin, submit_and_await_calvin_until}; pub use routed::{submit_calvin_routed, submit_calvin_routed_write}; diff --git a/nodedb/src/control/planner/calvin/submit/routed.rs b/nodedb/src/control/planner/calvin/submit/routed.rs index 7e71b6e80..91e89a3f9 100644 --- a/nodedb/src/control/planner/calvin/submit/routed.rs +++ b/nodedb/src/control/planner/calvin/submit/routed.rs @@ -7,10 +7,10 @@ //! leader over one `SubmitCalvinTxn` RPC. use std::collections::BTreeSet; -use std::time::Duration; use nodedb_cluster::calvin::SEQUENCER_GROUP_ID; use nodedb_cluster::calvin::types::TxClass; +use nodedb_cluster::rpc_codec::reply_wait; use nodedb_cluster::{ ClusterError, RaftRpc, SubmitCalvinTxnRequest, SubmitCalvinTxnResponse, TypedClusterError, }; @@ -18,6 +18,7 @@ use nodedb_cluster::{ use crate::Error; use crate::bridge::envelope::Response; use crate::control::cluster::warm_peers::register_peers_from_topology; +use crate::control::server::shared::session::StatementDeadline; use crate::control::state::SharedState; use super::local::{submit_prepared_and_await, synthetic_returning_response}; @@ -119,11 +120,12 @@ pub async fn submit_calvin_routed( }); } + // Every wait of this submit ends at the statement's deadline. + let deadline = StatementDeadline::of_statement(state).tokio_instant(); // Leader is self: submit-and-await locally (a self-RPC will be a pointless // extra hop and the local registry is the one that completes). if leader == state.node_id { - let timeout = super::budget::statement_budget(state)?; - return submit_prepared_and_await(state, tx_class, stream, timeout).await; + return submit_prepared_and_await(state, tx_class, stream, deadline).await; } // Remote leader: ensure its address is registered before dispatch, then send @@ -138,7 +140,7 @@ pub async fn submit_calvin_routed( })?; // The leader works on the transaction only as long as the statement waits. - let deadline_remaining_ms = super::budget::statement_budget_ms(state)?; + let deadline_remaining_ms = super::budget::budget_ms_until(deadline)?; let req = SubmitCalvinTxnRequest { tx_class_bytes, deadline_remaining_ms, @@ -150,7 +152,7 @@ pub async fn submit_calvin_routed( // short `rpc_timeout` (a normal request/response round-trip budget) will // abort the call long before that, so bound the response read by the // forwarded deadline plus a margin for the round-trip itself. - let read_timeout = Duration::from_millis(deadline_remaining_ms.saturating_add(2_000)); + let read_timeout = reply_wait(deadline_remaining_ms); let header = transport.send_rpc_with_read_timeout( leader, RaftRpc::SubmitCalvinTxnRequest(req), @@ -163,7 +165,8 @@ pub async fn submit_calvin_routed( None => header.await, Some(stream) => { tokio::pin!(header); - let streaming = stream_parts(state, StreamTarget::Remote(leader), stream, false); + let streaming = + stream_parts(state, StreamTarget::Remote(leader), stream, false, deadline); tokio::pin!(streaming); tokio::select! { reply = &mut header => reply, diff --git a/nodedb/src/control/planner/calvin/submit/stream.rs b/nodedb/src/control/planner/calvin/submit/stream.rs index d000a2b68..64132b11b 100644 --- a/nodedb/src/control/planner/calvin/submit/stream.rs +++ b/nodedb/src/control/planner/calvin/submit/stream.rs @@ -19,7 +19,8 @@ //! //! A `Full` answer comes from a live leader that holds the stream, so it //! keeps the stream alive. A stream with no answer from such a leader for -//! the network deadline stops as lost. +//! the stall window stops as lost. A stream also stops as lost at the +//! statement's deadline: the statement no longer waits for its outcome. use std::collections::BTreeSet; use std::time::{Duration, Instant}; @@ -80,24 +81,37 @@ impl StreamEnd { } } -/// Stream `stream` to `target`. `assigned` says the caller holds the -/// header's assignment, so the leader has opened the stream. +/// Stream `stream` to `target` until the statement's `deadline`. +/// `assigned` says the caller holds the header's assignment, so the leader +/// has opened the stream. pub(crate) async fn stream_parts( state: &SharedState, target: StreamTarget, stream: &PartStream, assigned: bool, + deadline: tokio::time::Instant, ) -> StreamEnd { if stream.parts.is_empty() { return StreamEnd::Done; } - let deadline = Duration::from_secs(state.tuning.network.default_deadline_secs.max(1)); + let stall_window = stall_window(state); // no-determinism: coordinator-side liveness timer, never in the log. - let mut progress = StreamProgress::new(stream.parts.len(), assigned, deadline, Instant::now()); + let mut progress = StreamProgress::new( + stream.parts.len(), + assigned, + stall_window, + deadline.into_std(), + Instant::now(), + ); let mut backoff = FIRST_BACKOFF; loop { + let left = deadline.saturating_duration_since(tokio::time::Instant::now()); + if left.is_zero() { + return StreamEnd::Lost; + } let batch = batch_from(&stream.parts, progress.next); - let reply = match offer(state, target, stream.id, batch).await { + let read_timeout = stall_window.min(left); + let reply = match offer(state, target, stream.id, batch, read_timeout).await { Ok(reply) => reply, Err(error) => { warn!(%error, "calvin part stream: offer failed; retrying"); @@ -136,6 +150,11 @@ enum Step { End(StreamEnd), } +/// The longest a stream waits with no answer from a leader that holds it. +fn stall_window(state: &SharedState) -> Duration { + Duration::from_secs(state.tuning.network.default_deadline_secs.max(1)) +} + /// Where a stream stands, and when the leader last showed it alive. #[derive(Debug)] struct StreamProgress { @@ -146,19 +165,28 @@ struct StreamProgress { taken_any: bool, /// Whether the caller holds the header's assignment. assigned: bool, - deadline: Duration, + stall_window: Duration, + /// The statement's deadline. The stream stops there. + ends_at: Instant, /// The last answer from a leader that holds the stream. last_alive: Instant, } impl StreamProgress { - fn new(total: usize, assigned: bool, deadline: Duration, now: Instant) -> Self { + fn new( + total: usize, + assigned: bool, + stall_window: Duration, + ends_at: Instant, + now: Instant, + ) -> Self { Self { total, next: 0, taken_any: false, assigned, - deadline, + stall_window, + ends_at, last_alive: now, } } @@ -168,8 +196,9 @@ impl StreamProgress { /// An `Accepted` or `Full` answer comes from a leader that holds the /// stream, so it counts as liveness: a full queue drains as the leader /// proposes. Only an `Unknown` answer, or no answer, runs down the - /// deadline. A leader that stops proposing closes the stalled stream, - /// and its next answer is `Unknown`. + /// stall window. A leader that stops proposing closes the stalled + /// stream, and its next answer is `Unknown`. The statement's deadline + /// ends any stream that still owes parts. fn on_reply(&mut self, reply: &PartsOfferReply, now: Instant) -> Step { let owed = usize::try_from(reply.next_index).unwrap_or(usize::MAX); match reply.status { @@ -196,7 +225,8 @@ impl StreamProgress { if reply.status == PartsOfferStatus::Accepted { return Step::Offer; } - if now.saturating_duration_since(self.last_alive) > self.deadline { + if now >= self.ends_at || now.saturating_duration_since(self.last_alive) > self.stall_window + { return Step::End(StreamEnd::Lost); } Step::Wait @@ -231,6 +261,7 @@ async fn offer( target: StreamTarget, stream: PartStreamId, batch: Vec, + read_timeout: Duration, ) -> crate::Result { match target { StreamTarget::Local => { @@ -246,7 +277,9 @@ async fn offer( transport_error: false, }) } - StreamTarget::Remote(leader) => offer_remote(state, leader, stream, batch).await, + StreamTarget::Remote(leader) => { + offer_remote(state, leader, stream, batch, read_timeout).await + } } } @@ -255,6 +288,7 @@ async fn offer_remote( leader: u64, stream: PartStreamId, batch: Vec, + read_timeout: Duration, ) -> crate::Result { let transport = state .cluster_transport @@ -270,7 +304,6 @@ async fn offer_remote( stream_seq: stream.seq, parts_bytes, }); - let read_timeout = Duration::from_secs(state.tuning.network.default_deadline_secs.max(1)); let reply = transport .send_rpc_with_read_timeout(leader, request, read_timeout) .await @@ -338,15 +371,16 @@ mod tests { } } - /// A leader that holds the stream and answers `Full` past the deadline - /// keeps it alive: its queue drains as it proposes. Only no answer runs - /// the deadline down. + /// A leader that holds the stream and answers `Full` past the stall + /// window keeps it alive: its queue drains as it proposes. Only no answer + /// runs the stall window down. #[test] - fn full_answers_keep_a_stream_alive_past_the_deadline() { - let deadline = Duration::from_secs(1); + fn full_answers_keep_a_stream_alive_past_the_stall_window() { + let stall_window = Duration::from_secs(1); // no-determinism: test-only synthetic timeline. let start = Instant::now(); - let mut progress = StreamProgress::new(4, true, deadline, start); + let ends_at = start + Duration::from_secs(3600); + let mut progress = StreamProgress::new(4, true, stall_window, ends_at, start); assert_eq!( progress.on_reply(&reply(PartsOfferStatus::Accepted, 1, false), start), Step::Offer @@ -366,7 +400,7 @@ mod tests { alive + Duration::from_millis(500) ), Step::Wait, - "one missed answer within the deadline waits" + "one missed answer within the stall window waits" ); assert_eq!( progress.on_reply( @@ -374,7 +408,29 @@ mod tests { alive + Duration::from_secs(2) ), Step::End(StreamEnd::Lost), - "no answer for the deadline loses the stream" + "no answer for the stall window loses the stream" + ); + } + + /// A leader that keeps answering `Full` still loses the stream at the + /// statement's deadline: nobody waits for the outcome past it. + #[test] + fn the_statement_deadline_ends_a_live_stream() { + let stall_window = Duration::from_secs(10); + // no-determinism: test-only synthetic timeline. + let start = Instant::now(); + let ends_at = start + Duration::from_secs(2); + let mut progress = StreamProgress::new(4, true, stall_window, ends_at, start); + assert_eq!( + progress.on_reply( + &reply(PartsOfferStatus::Full, 0, false), + start + Duration::from_secs(1) + ), + Step::Wait + ); + assert_eq!( + progress.on_reply(&reply(PartsOfferStatus::Full, 0, false), ends_at), + Step::End(StreamEnd::Lost) ); } } diff --git a/nodedb/src/control/planner/calvin/submit/wait.rs b/nodedb/src/control/planner/calvin/submit/wait.rs new file mode 100644 index 000000000..d04468117 --- /dev/null +++ b/nodedb/src/control/planner/calvin/submit/wait.rs @@ -0,0 +1,138 @@ +// SPDX-License-Identifier: BUSL-1.1 + +//! The two waits of a submitted Calvin transaction: its sequencer assignment +//! and its completion. +//! +//! Both end at the statement's deadline. A submission that reached the +//! sequencer inbox can still be sequenced after its wait ends, so an elapsed +//! wait is an unknown outcome, never a definite refusal. + +use nodedb_cluster::calvin::{ + Assignment, AssignmentReceiver, CalvinCompletionRegistry, CompletionReport, TxnId, +}; + +use crate::Error; +use crate::types::RequestId; + +/// The error of a wait that ended at the statement's deadline: the outcome +/// is unknown. +fn unknown_outcome() -> Error { + Error::DeadlineExceeded { + request_id: RequestId::new(0), + } +} + +/// Await the sequencer assignment of submission `inbox_seq` until `deadline`. +/// +/// A closed channel means the sequencer rejected or discarded the submission +/// without sequencing it: a read/write cycle in its epoch, a leadership +/// change, or a halted sequencer. Nothing applied, so the caller gets a +/// retryable refusal. On the deadline the registration is dropped, so no +/// sender stays behind. +pub(crate) async fn await_assignment( + registry: &CalvinCompletionRegistry, + inbox_seq: u64, + assignment_rx: AssignmentReceiver, + deadline: tokio::time::Instant, +) -> crate::Result { + match tokio::time::timeout_at(deadline, assignment_rx).await { + Ok(Ok(assignment)) => Ok(assignment), + Ok(Err(_)) => Err(Error::RetryableRefusal { + reason: "the Calvin sequencer did not sequence the transaction; nothing was \ + applied" + .to_owned(), + }), + Err(_) => { + registry.drop_assignment(inbox_seq); + tracing::warn!( + inbox_seq, + "Calvin submission reached the statement deadline before its assignment" + ); + Err(unknown_outcome()) + } + } +} + +/// Await the completion report of `txn_id` until `deadline`. +/// +/// An elapsed wait files a `calvin_completion_timeout` report: it is the only +/// signal a never-completed Calvin write produces. +pub(crate) async fn await_completion( + registry: &CalvinCompletionRegistry, + txn_id: TxnId, + participants: usize, + deadline: tokio::time::Instant, +) -> crate::Result { + let waiting_since = tokio::time::Instant::now(); + let completion_rx = registry.register_completion_report(txn_id, participants); + match tokio::time::timeout_at(deadline, completion_rx).await { + Ok(Ok(report)) => Ok(report), + Ok(Err(_)) => Err(Error::Internal { + detail: "Calvin completion channel closed".to_owned(), + }), + Err(_) => { + let error = unknown_outcome(); + let waited_ms = u64::try_from( + deadline + .saturating_duration_since(waiting_since) + .as_millis(), + ) + .unwrap_or(u64::MAX); + crate::diag::calvin_completion_timeout( + &error, + txn_id.epoch, + txn_id.position, + participants, + waited_ms, + ); + Err(error) + } + } +} + +#[cfg(test)] +mod tests { + use std::time::Duration; + + use super::*; + + /// The sequencer dropped the assignment of a submission it did not + /// sequence. The caller fails at once with a retryable refusal, long + /// before its deadline. + #[tokio::test] + async fn an_unsequenced_submission_is_a_retryable_refusal() { + let registry = CalvinCompletionRegistry::new_detached(); + let rx = registry.register_submission(7); + registry.drop_assignment(7); + let deadline = tokio::time::Instant::now() + Duration::from_secs(3600); + match await_assignment(®istry, 7, rx, deadline).await { + Err(Error::RetryableRefusal { .. }) => {} + other => panic!("expected a retryable refusal, got {other:?}"), + } + } + + #[tokio::test] + async fn an_assigned_submission_returns_its_assignment() { + let registry = CalvinCompletionRegistry::new_detached(); + let rx = registry.register_submission(3); + registry.note_assigned(3, TxnId::new(5, 1), 2); + let deadline = tokio::time::Instant::now() + Duration::from_secs(3600); + let assignment = await_assignment(®istry, 3, rx, deadline) + .await + .expect("assigned"); + assert_eq!(assignment, (5, 1, 2)); + } + + /// A wait that reaches the deadline is an unknown outcome, never a + /// refusal: the submission can still be sequenced. + #[tokio::test] + async fn a_timed_out_wait_is_not_a_refusal() { + let registry = CalvinCompletionRegistry::new_detached(); + let rx = registry.register_submission(8); + let deadline = tokio::time::Instant::now() + Duration::from_millis(1); + match await_assignment(®istry, 8, rx, deadline).await { + Err(Error::DeadlineExceeded { .. }) => {} + other => panic!("expected an unknown outcome, got {other:?}"), + } + } +} diff --git a/nodedb/src/control/planner/calvin/tx_class/static_builder.rs b/nodedb/src/control/planner/calvin/tx_class/static_builder.rs index 00178eb3f..920175898 100644 --- a/nodedb/src/control/planner/calvin/tx_class/static_builder.rs +++ b/nodedb/src/control/planner/calvin/tx_class/static_builder.rs @@ -379,7 +379,8 @@ mod tests { /// A predicate read locks its whole collection `Shared`. A later row /// writer of that collection takes `Intent` on it, so it waits until the - /// reader releases. A point read locks only its row. + /// reader releases. A point read locks its row `Shared` and its + /// collection `IntentShared`. #[test] fn a_predicate_read_blocks_a_later_writer_until_released() { use crate::control::cluster::calvin::scheduler::LockMode; @@ -414,11 +415,12 @@ mod tests { }), Some(&LockMode::Shared) ); - assert!( - !reader_locks.contains_key(&LockKey::Collection { + assert_eq!( + reader_locks.get(&LockKey::Collection { collection: std::sync::Arc::from("read_col"), }), - "a point read locks nothing collection-wide" + Some(&LockMode::IntentShared), + "a point read announces itself on its collection" ); let writer = build_single_vshard_tx_class( diff --git a/nodedb/src/control/planner/sql_plan_convert/array_convert/dml.rs b/nodedb/src/control/planner/sql_plan_convert/array_convert/dml.rs index 6e1538742..557824317 100644 --- a/nodedb/src/control/planner/sql_plan_convert/array_convert/dml.rs +++ b/nodedb/src/control/planner/sql_plan_convert/array_convert/dml.rs @@ -86,9 +86,9 @@ pub(in super::super) fn convert_insert_array( let surrogate = ctx.surrogate_for_pk(ctx.collection_key(name), &pk_bytes)?; // A cell routes by its tile's key, the key a slice's shard // fan-out and each shard's tile filter use. - let hilbert = + let tile_key = cell_tile_prefix(&schema, &coord).map_err(|e| crate::Error::PlanError { - detail: format!("INSERT INTO ARRAY {name}: Hilbert prefix: {e}"), + detail: format!("INSERT INTO ARRAY {name}: tile key: {e}"), })?; let cell = ArrayPutCell { coord, @@ -103,7 +103,7 @@ pub(in super::super) fn convert_insert_array( format: "msgpack".into(), detail: format!("array put cell encode: {e}"), })?; - partitioned.push((hilbert, cell_bytes)); + partitioned.push((tile_key, cell_bytes)); } let array_id_msgpack = zerompk::to_msgpack_vec(&aid).map_err(|e| crate::Error::Serialization { @@ -212,9 +212,9 @@ pub(in super::super) fn convert_delete_array( let mut partitioned: Vec<(u64, Vec)> = Vec::with_capacity(coords.len()); for row in coords { let typed = coerce_coords(row, &schema)?; - let hilbert = + let tile_key = cell_tile_prefix(&schema, &typed).map_err(|e| crate::Error::PlanError { - detail: format!("DELETE FROM ARRAY {name}: Hilbert prefix: {e}"), + detail: format!("DELETE FROM ARRAY {name}: tile key: {e}"), })?; let cell = ArrayDeleteCell { coord: typed, @@ -226,7 +226,7 @@ pub(in super::super) fn convert_delete_array( format: "msgpack".into(), detail: format!("array delete cell encode: {e}"), })?; - partitioned.push((hilbert, cell_bytes)); + partitioned.push((tile_key, cell_bytes)); } let array_id_msgpack = zerompk::to_msgpack_vec(&aid).map_err(|e| crate::Error::Serialization { diff --git a/nodedb/src/control/planner/sql_plan_convert/array_fn_convert/aggregate.rs b/nodedb/src/control/planner/sql_plan_convert/array_fn_convert/aggregate.rs index fea8d2ab4..6f51fcda3 100644 --- a/nodedb/src/control/planner/sql_plan_convert/array_fn_convert/aggregate.rs +++ b/nodedb/src/control/planner/sql_plan_convert/array_fn_convert/aggregate.rs @@ -69,7 +69,7 @@ pub(crate) fn convert_agg( attr_idx, reducer_msgpack, group_by_dim: group_by_dim_idx, - slice_hilbert_ranges: vec![], + slice_tile_keys: vec![], prefix_bits: entry.prefix_bits, system_as_of, valid_at_ms, @@ -82,7 +82,7 @@ pub(crate) fn convert_agg( group_by_dim: group_by_dim_idx, cell_filter: None, return_partial: false, - hilbert_range: None, + shard_bucket: None, system_as_of, valid_at_ms, }) diff --git a/nodedb/src/control/planner/sql_plan_convert/array_fn_convert/slice.rs b/nodedb/src/control/planner/sql_plan_convert/array_fn_convert/slice.rs index 453669ffc..5ae48d1d2 100644 --- a/nodedb/src/control/planner/sql_plan_convert/array_fn_convert/slice.rs +++ b/nodedb/src/control/planner/sql_plan_convert/array_fn_convert/slice.rs @@ -10,6 +10,7 @@ use nodedb_array::types::ArrayId; use nodedb_array::types::domain::DomainBound; use nodedb_sql::temporal::TemporalScope; use nodedb_sql::types_array::ArraySliceAst; +use nodedb_types::array_route::prefix_bucket; use crate::bridge::envelope::PhysicalPlan; use crate::types::TenantId; @@ -69,14 +70,14 @@ pub(crate) fn convert_slice( // // The coordinator fans out only to the shards that own a tile the // slice covers. An empty list fans out to every shard. - let slice_hilbert_ranges = - compute_slice_hilbert_ranges(&schema, &slice.dim_ranges, entry.prefix_bits); + let slice_tile_keys = + compute_slice_tile_keys(&schema, &slice.dim_ranges, entry.prefix_bits); PhysicalPlan::ClusterArray(ClusterArrayOp::Slice { array_id: aid, slice_msgpack, attr_projection: attr_indices, limit, - slice_hilbert_ranges, + slice_tile_keys, prefix_bits: entry.prefix_bits, system_time, valid_at_ms, @@ -88,7 +89,7 @@ pub(crate) fn convert_slice( attr_projection: attr_indices, limit, cell_filter: None, - hilbert_range: None, + shard_bucket: None, system_time, valid_at_ms, }) @@ -108,8 +109,8 @@ pub(crate) fn convert_slice( /// tiles fans out to every shard. const MAX_FAN_OUT_TILES: u128 = 4096; -/// The Hilbert ranges a slice's shard fan-out covers: one `(key, key)` pair -/// per routing bucket, each holding the key of one tile in that bucket. +/// The tile keys a slice's shard fan-out covers: one key per routing bucket +/// (`prefix_bucket`), each the key of one tile in that bucket. /// /// Every tile whose cells can match the slice is enumerated, so every shard /// that holds such a cell, or will hold one a later write lands there, is @@ -120,11 +121,11 @@ const MAX_FAN_OUT_TILES: u128 = 4096; /// Returns an empty vec, which fans out to every shard, when the slice /// covers more than [`MAX_FAN_OUT_TILES`] tiles, when the schema has no /// dimension, or when a tile key fails to encode. -fn compute_slice_hilbert_ranges( +fn compute_slice_tile_keys( schema: &ArraySchema, dim_ranges: &[Option], prefix_bits: u8, -) -> Vec<(u64, u64)> { +) -> Vec { if schema.dims.is_empty() || prefix_bits == 0 || prefix_bits > 16 { return Vec::new(); } @@ -145,19 +146,20 @@ fn compute_slice_hilbert_ranges( return Vec::new(); } - let shift = 64 - u32::from(prefix_bits); - let mut buckets: std::collections::BTreeMap = std::collections::BTreeMap::new(); + let mut buckets: std::collections::BTreeMap = std::collections::BTreeMap::new(); let mut indices: Vec = spans.iter().map(|(lo, _)| *lo).collect(); loop { let Ok(key) = tile_prefix(schema, &indices) else { return Vec::new(); }; - buckets.entry(key >> shift).or_insert(key); + buckets + .entry(prefix_bucket(key, prefix_bits)) + .or_insert(key); // Advance the odometer over the per-dimension tile spans. let mut dim = 0; loop { if dim == indices.len() { - return buckets.into_values().map(|key| (key, key)).collect(); + return buckets.into_values().collect(); } if indices[dim] < spans[dim].1 { indices[dim] += 1; @@ -290,9 +292,9 @@ mod tests { Some(DimRange::new(DomainBound::Int64(0), DomainBound::Int64(2))), Some(DimRange::new(DomainBound::Int64(0), DomainBound::Int64(99))), ]; - let ranges = compute_slice_hilbert_ranges(&schema, &box_ranges, 8); - assert!(!ranges.is_empty(), "a three-tile slice fans out by tile"); - let fanned = nodedb_cluster::distributed_array::array_vshards_for_slice(&ranges, 8, 1024) + let keys = compute_slice_tile_keys(&schema, &box_ranges, 8); + assert!(!keys.is_empty(), "a three-tile slice fans out by tile"); + let fanned = nodedb_cluster::distributed_array::array_vshards_for_slice(&keys, 8, 1024) .expect("fan-out"); let mut owners = std::collections::BTreeSet::new(); for chr in 0..=2 { @@ -324,8 +326,8 @@ mod tests { .tile_extents(vec![1]) .build() .expect("schema build"); - assert!(compute_slice_hilbert_ranges(&wide, &[None], 8).is_empty()); - assert!(!compute_slice_hilbert_ranges(&schema, &[None], 8).is_empty()); + assert!(compute_slice_tile_keys(&wide, &[None], 8).is_empty()); + assert!(!compute_slice_tile_keys(&schema, &[None], 8).is_empty()); } #[test] diff --git a/nodedb/src/control/security/auth_fence/cluster.rs b/nodedb/src/control/security/auth_fence/cluster.rs index a99dfb790..f704a7c24 100644 --- a/nodedb/src/control/security/auth_fence/cluster.rs +++ b/nodedb/src/control/security/auth_fence/cluster.rs @@ -87,14 +87,10 @@ pub(crate) async fn wait_applied( if watcher.current() >= target { return Ok(()); } - // The watcher parks its caller on a condition variable, so the wait runs - // on the blocking pool. - let outcome = tokio::task::spawn_blocking(move || watcher.wait_for(target, timeout)) + match watcher + .wait_for_async(target, Instant::now() + timeout) .await - .map_err(|e| crate::Error::Internal { - detail: format!("applied-index wait for raft group {group_id} did not finish: {e}"), - })?; - match outcome { + { WaitOutcome::Reached => Ok(()), WaitOutcome::TimedOut => Err(behind(format!( "raft group {group_id} was not applied through index {target} in time" diff --git a/nodedb/src/control/security/auth_lease/barrier.rs b/nodedb/src/control/security/auth_lease/barrier.rs index f0cdfae8c..17b2b7bfa 100644 --- a/nodedb/src/control/security/auth_lease/barrier.rs +++ b/nodedb/src/control/security/auth_lease/barrier.rs @@ -8,8 +8,6 @@ //! lease holder too, so its own Event Plane lag closes the same way. A //! single-node cluster runs the same barrier against its one lease. -use std::time::{Duration, Instant}; - use nodedb_cluster::calvin::SEQUENCER_GROUP_ID; use nodedb_cluster::{AuthBarrierOutcome, AuthBarrierRequest, GroupCoverage, RaftRpc}; @@ -22,7 +20,7 @@ use super::leadership::{metadata_leader, send_to_leader}; /// /// `targets` name the change: per group, the index it committed at. The /// change itself is committed; an error means only that the barrier did not -/// release within the request deadline. +/// release by `deadline`, the caller's statement deadline. /// /// Latency: in a cluster, every authorization-bearing write waits about one /// renewal interval (the Raft heartbeat) before it is acknowledged. Each @@ -35,15 +33,16 @@ use super::leadership::{metadata_leader, send_to_leader}; pub async fn authorization_barrier( state: &SharedState, targets: Vec, + deadline: std::time::Instant, ) -> crate::Result<()> { - let deadline_secs = state.tuning.network.default_deadline_secs; - let deadline = Instant::now() + Duration::from_secs(deadline_secs); + let remaining_until = || deadline.saturating_duration_since(std::time::Instant::now()); + let budget_ms = remaining_until().as_millis(); let timing = lease_timing(state)?; loop { - let remaining = deadline.saturating_duration_since(Instant::now()); + let remaining = remaining_until(); if remaining.is_zero() { return Err(committed_but_pending(format!( - "the barrier did not release within {deadline_secs}s" + "the barrier did not release within the statement deadline ({budget_ms} ms)" ))); } let request = AuthBarrierRequest { @@ -86,7 +85,7 @@ pub async fn authorization_barrier( // leader again. A new leader holds the barrier to its own floors. Some(AuthBarrierOutcome::NotLeader { .. }) | Some(AuthBarrierOutcome::Timeout { .. }) - | None => tokio::time::sleep(timing.renew_every).await, + | None => tokio::time::sleep(timing.renew_every.min(remaining_until())).await, } } } @@ -96,8 +95,12 @@ pub async fn authorization_barrier( /// /// Its completion acks sit in the sequencer log, at or below the sequencer /// group's commit index now. A node covers that index only once its own -/// replicas applied every acknowledged transaction below it. -pub async fn calvin_write_barrier(state: &SharedState) -> crate::Result<()> { +/// replicas applied every acknowledged transaction below it. The wait ends +/// at `deadline`, the caller's statement deadline. +pub async fn calvin_write_barrier( + state: &SharedState, + deadline: std::time::Instant, +) -> crate::Result<()> { lease_timing(state)?; let commit_index = state .raft_status_fn @@ -115,6 +118,7 @@ pub async fn calvin_write_barrier(state: &SharedState) -> crate::Result<()> { group_id: SEQUENCER_GROUP_ID, through: commit_index, }], + deadline, ) .await } diff --git a/nodedb/src/control/security/catalog/calvin_barrier_logs.rs b/nodedb/src/control/security/catalog/calvin_barrier_logs.rs index 9be335917..59b343901 100644 --- a/nodedb/src/control/security/catalog/calvin_barrier_logs.rs +++ b/nodedb/src/control/security/catalog/calvin_barrier_logs.rs @@ -9,12 +9,18 @@ //! applied floor, so an entry at or below it is never delivered again. Its //! row is what a barrier of the txn reads after a restart. //! +//! A fold commits without persisting: the row is visible at once, and a +//! crash rolls it back. [`SystemCatalog::persist_calvin_barrier_logs`] +//! persists every such fold in one commit. The apply loop runs it before it +//! saves a floor, so the floor never covers an entry whose row a crash loses. +//! //! Each row holds the encoded barrier log of one txn on one vShard. This //! module stores the bytes and never decodes them. use std::collections::BTreeSet; +use std::sync::atomic::Ordering; -use redb::{ReadableDatabase, ReadableTable, TableDefinition}; +use redb::{Durability, ReadableDatabase, ReadableTable, TableDefinition}; use super::types::{SystemCatalog, catalog_err}; @@ -59,6 +65,9 @@ impl SystemCatalog { /// Replace the row of `(vshard_id, epoch, position)` with what `fold` /// returns for the row it holds, in one transaction. A fold that /// returns `None` keeps the row, and nothing commits. + /// + /// The commit is visible at once and persists with the next + /// [`Self::persist_calvin_barrier_logs`] or any durable commit. pub fn fold_calvin_barrier_log( &self, vshard_id: u32, @@ -66,10 +75,13 @@ impl SystemCatalog { position: u32, fold: impl FnOnce(Option<&[u8]>) -> crate::Result>>, ) -> crate::Result<()> { - let write_txn = self + let mut write_txn = self .db .begin_write() .map_err(|e| catalog_err("fold_calvin_barrier_log txn", e))?; + write_txn + .set_durability(Durability::None) + .map_err(|e| catalog_err("fold_calvin_barrier_log durability", e))?; { let mut table = write_txn .open_table(CALVIN_BARRIER_LOGS) @@ -90,7 +102,50 @@ impl SystemCatalog { } write_txn .commit() - .map_err(|e| catalog_err("commit calvin_barrier_logs", e)) + .map_err(|e| catalog_err("commit calvin_barrier_logs", e))?; + // Set after the commit: a persist that clears the flag first + // persists this commit too. + self.calvin_barrier_logs_pending + .store(true, Ordering::Release); + Ok(()) + } + + /// Persist every barrier log fold committed since the last persist, in + /// one durable commit. With no such fold, nothing commits. + /// + /// On an error the folds stay pending, so the next persist retries. + pub fn persist_calvin_barrier_logs(&self) -> crate::Result<()> { + #[cfg(test)] + if self + .fail_next_calvin_barrier_persist + .swap(false, Ordering::SeqCst) + { + return Err(crate::Error::Storage { + engine: "catalog".into(), + detail: "injected calvin_barrier_logs persist failure".into(), + }); + } + if !self + .calvin_barrier_logs_pending + .swap(false, Ordering::AcqRel) + { + return Ok(()); + } + let persisted = self + .db + .begin_write() + .map_err(|e| catalog_err("persist_calvin_barrier_logs txn", e)) + .and_then(|mut txn| { + txn.set_durability(Durability::Immediate) + .map_err(|e| catalog_err("persist_calvin_barrier_logs durability", e))?; + txn.commit() + .map_err(|e| catalog_err("persist_calvin_barrier_logs commit", e)) + }); + if persisted.is_err() { + self.calvin_barrier_logs_pending + .store(true, Ordering::Release); + } + persisted } /// The row of `(vshard_id, epoch, position)`, when one is stored. @@ -200,6 +255,16 @@ impl SystemCatalog { .map_err(|e| catalog_err("insert calvin_barrier_logs", e))?; } } + #[cfg(test)] + if self + .fail_next_calvin_barrier_replace + .swap(false, Ordering::SeqCst) + { + return Err(crate::Error::Storage { + engine: "catalog".into(), + detail: "injected calvin_barrier_logs replace failure".into(), + }); + } write_txn .commit() .map_err(|e| catalog_err("commit calvin_barrier_logs", e)) @@ -280,6 +345,72 @@ mod tests { assert_eq!(only, vec![row(4, 2, 0, b"y")]); } + /// The rows a crash at this instant leaves: the file as it stands, + /// opened from a copy while the live catalog keeps its own open. + fn rows_after_crash(path: &std::path::Path, copy: &std::path::Path) -> Vec<(u32, u64, u32)> { + std::fs::copy(path, copy).expect("copy the catalog file"); + let db = redb::Database::create(copy).expect("open the copy"); + let txn = db.begin_read().expect("read txn"); + let Ok(table) = txn.open_table(CALVIN_BARRIER_LOGS) else { + return Vec::new(); + }; + table + .iter() + .expect("iter") + .map(|row| row.expect("row").0.value()) + .collect() + } + + /// A fold is visible at once, and a crash rolls it back. The persist + /// after it makes every fold durable, and a second persist commits + /// nothing. + #[test] + fn a_fold_persists_only_with_the_persist() { + let dir = tempfile::tempdir().expect("tempdir"); + let path = dir.path().join("system.redb"); + let catalog = SystemCatalog::open(&path).expect("catalog"); + put(&catalog, &row(3, 7, 1, b"a")); + put(&catalog, &row(4, 2, 0, b"b")); + assert_eq!( + catalog.load_calvin_barrier_log(3, 7, 1).expect("load"), + Some(b"a".to_vec()), + "a fold is visible to reads" + ); + assert!( + rows_after_crash(&path, &dir.path().join("crash-1.redb")).is_empty(), + "a crash rolls the folds back" + ); + + catalog.persist_calvin_barrier_logs().expect("persist"); + assert_eq!( + rows_after_crash(&path, &dir.path().join("crash-2.redb")), + vec![(3, 7, 1), (4, 2, 0)] + ); + assert!( + !catalog.calvin_barrier_logs_pending.load(Ordering::Acquire), + "nothing waits to persist" + ); + catalog.persist_calvin_barrier_logs().expect("persist"); + } + + /// A failed persist keeps the folds pending, and the next persist + /// makes them durable. + #[test] + fn a_failed_persist_keeps_the_folds_pending() { + let dir = tempfile::tempdir().expect("tempdir"); + let path = dir.path().join("system.redb"); + let catalog = SystemCatalog::open(&path).expect("catalog"); + put(&catalog, &row(3, 7, 1, b"a")); + catalog.fail_next_calvin_barrier_persist_for_test(); + assert!(catalog.persist_calvin_barrier_logs().is_err()); + assert!(rows_after_crash(&path, &dir.path().join("crash-1.redb")).is_empty()); + catalog.persist_calvin_barrier_logs().expect("persist"); + assert_eq!( + rows_after_crash(&path, &dir.path().join("crash-2.redb")), + vec![(3, 7, 1)] + ); + } + /// A replace drops every row of the named vShards and keeps the rest. #[test] fn a_replace_swaps_the_named_vshards_rows() { diff --git a/nodedb/src/control/security/catalog/cut_floors.rs b/nodedb/src/control/security/catalog/cut_floors.rs index d9808c9c5..71190d620 100644 --- a/nodedb/src/control/security/catalog/cut_floors.rs +++ b/nodedb/src/control/security/catalog/cut_floors.rs @@ -49,6 +49,18 @@ impl SystemCatalog { group_id: u64, barrier: StoredBarrier, durable_applied: u64, + ) -> crate::Result<()> { + self.put_cut_floors(group_id, &[barrier], durable_applied) + } + + /// Record every barrier of `barriers` group `group_id` holds, in one + /// write. The barriers at or below `durable_applied` fold as in + /// [`Self::put_cut_floor`]. + pub fn put_cut_floors( + &self, + group_id: u64, + barriers: &[StoredBarrier], + durable_applied: u64, ) -> crate::Result<()> { crate::fail_point_err!("cut_floor::before_persist", |detail: String| { crate::Error::Internal { @@ -70,8 +82,10 @@ impl SystemCatalog { Some(bytes) => decode(bytes.value())?, None => StoredFloors::default(), }; - floors.barriers.retain(|held| held.index != barrier.index); - floors.barriers.push(barrier); + for barrier in barriers { + floors.barriers.retain(|held| held.index != barrier.index); + floors.barriers.push(*barrier); + } floors.barriers.sort_by_key(|held| held.index); let settled = floors .barriers @@ -105,6 +119,26 @@ impl SystemCatalog { txn.commit().map_err(|e| catalog_err("cut floor commit", e)) } + /// Every recorded barrier of `group_id`, in index order. + pub fn group_cut_floors(&self, group_id: u64) -> crate::Result> { + let txn = self + .db + .begin_read() + .map_err(|e| catalog_err("cut floor read txn", e))?; + let table = match txn.open_table(CUT_FLOORS) { + Ok(table) => table, + Err(TableError::TableDoesNotExist(_)) => return Ok(Vec::new()), + Err(e) => return Err(catalog_err("open cut floors", e)), + }; + match table + .get(group_id) + .map_err(|e| catalog_err("get cut floors", e))? + { + Some(bytes) => Ok(decode(bytes.value())?.barriers), + None => Ok(Vec::new()), + } + } + /// Every recorded barrier, as `(group_id, barrier)`. pub fn load_cut_floors(&self) -> crate::Result> { let txn = self @@ -170,4 +204,27 @@ mod tests { ] ); } + + #[test] + fn a_group_takes_several_barriers_in_one_write() { + let dir = tempfile::tempdir().unwrap(); + let catalog = SystemCatalog::open(&dir.path().join("system.redb")).unwrap(); + assert!(catalog.group_cut_floors(3).unwrap().is_empty()); + catalog.put_cut_floor(3, barrier(40, 400), 0).unwrap(); + catalog + .put_cut_floors(3, &[barrier(20, 200), barrier(10, 100)], 0) + .unwrap(); + catalog.put_cut_floor(4, barrier(7, 70), 0).unwrap(); + assert_eq!( + catalog.group_cut_floors(3).unwrap(), + [barrier(10, 100), barrier(20, 200), barrier(40, 400)] + ); + + // Durable through 25 folds the two barriers below it. + catalog.put_cut_floors(3, &[barrier(10, 100)], 25).unwrap(); + assert_eq!( + catalog.group_cut_floors(3).unwrap(), + [barrier(20, 200), barrier(40, 400)] + ); + } } diff --git a/nodedb/src/control/security/catalog/redo_snapshot_owed.rs b/nodedb/src/control/security/catalog/redo_snapshot_owed.rs index 6ae86066a..46a537e0f 100644 --- a/nodedb/src/control/security/catalog/redo_snapshot_owed.rs +++ b/nodedb/src/control/security/catalog/redo_snapshot_owed.rs @@ -58,6 +58,16 @@ impl SystemCatalog { .map_err(|e| catalog_err("insert redo_snapshot_owed", e))?; } } + #[cfg(test)] + if self + .fail_next_redo_snapshot_owed_write + .swap(false, std::sync::atomic::Ordering::SeqCst) + { + return Err(crate::Error::Storage { + engine: "catalog".into(), + detail: "injected redo_snapshot_owed write failure".into(), + }); + } txn.commit() .map_err(|e| catalog_err("redo_snapshot_owed commit", e)) } diff --git a/nodedb/src/control/security/catalog/system_catalog.rs b/nodedb/src/control/security/catalog/system_catalog.rs index 80efd3e78..e410a1029 100644 --- a/nodedb/src/control/security/catalog/system_catalog.rs +++ b/nodedb/src/control/security/catalog/system_catalog.rs @@ -24,6 +24,13 @@ pub struct SystemCatalog { /// Committed DEFINE EVENT definitions, for readers that must not read /// redb. pub(super) event_defs: Arc, + /// Whether a barrier log row committed without persisting since the + /// last `persist_calvin_barrier_logs`. + pub(super) calvin_barrier_logs_pending: Arc, + #[cfg(test)] + pub(super) fail_next_calvin_barrier_persist: Arc, + #[cfg(test)] + pub(super) fail_next_calvin_barrier_replace: Arc, #[cfg(test)] pub(super) fail_next_user_counter_write: Arc, #[cfg(test)] @@ -32,6 +39,8 @@ pub struct SystemCatalog { pub(super) fail_next_collection_write: Arc, #[cfg(test)] pub(super) fail_next_surrogate_write: Arc, + #[cfg(test)] + pub(super) fail_next_redo_snapshot_owed_write: Arc, } impl crate::storage::RedbBacked for SystemCatalog { @@ -59,6 +68,11 @@ impl SystemCatalog { db: Arc::new(db), crdt_signing_root: Arc::new(std::sync::RwLock::new(None)), event_defs: Arc::new(super::event_defs_index::EventDefsIndex::new()), + calvin_barrier_logs_pending: Arc::new(std::sync::atomic::AtomicBool::new(false)), + #[cfg(test)] + fail_next_calvin_barrier_persist: Arc::new(std::sync::atomic::AtomicBool::new(false)), + #[cfg(test)] + fail_next_calvin_barrier_replace: Arc::new(std::sync::atomic::AtomicBool::new(false)), #[cfg(test)] fail_next_user_counter_write: Arc::new(std::sync::atomic::AtomicBool::new(false)), #[cfg(test)] @@ -67,6 +81,8 @@ impl SystemCatalog { fail_next_collection_write: Arc::new(std::sync::atomic::AtomicBool::new(false)), #[cfg(test)] fail_next_surrogate_write: Arc::new(std::sync::atomic::AtomicBool::new(false)), + #[cfg(test)] + fail_next_redo_snapshot_owed_write: Arc::new(std::sync::atomic::AtomicBool::new(false)), }; catalog.bootstrap_default_database()?; catalog.reload_event_definitions()?; @@ -85,6 +101,11 @@ impl SystemCatalog { db: Arc::new(db), crdt_signing_root: Arc::new(std::sync::RwLock::new(None)), event_defs: Arc::new(super::event_defs_index::EventDefsIndex::new()), + calvin_barrier_logs_pending: Arc::new(std::sync::atomic::AtomicBool::new(false)), + #[cfg(test)] + fail_next_calvin_barrier_persist: Arc::new(std::sync::atomic::AtomicBool::new(false)), + #[cfg(test)] + fail_next_calvin_barrier_replace: Arc::new(std::sync::atomic::AtomicBool::new(false)), #[cfg(test)] fail_next_user_counter_write: Arc::new(std::sync::atomic::AtomicBool::new(false)), #[cfg(test)] @@ -93,6 +114,8 @@ impl SystemCatalog { fail_next_collection_write: Arc::new(std::sync::atomic::AtomicBool::new(false)), #[cfg(test)] fail_next_surrogate_write: Arc::new(std::sync::atomic::AtomicBool::new(false)), + #[cfg(test)] + fail_next_redo_snapshot_owed_write: Arc::new(std::sync::atomic::AtomicBool::new(false)), }; catalog.bootstrap_default_database()?; catalog.reload_event_definitions()?; @@ -126,6 +149,29 @@ impl SystemCatalog { .store(true, std::sync::atomic::Ordering::SeqCst); } + /// Make the next `save_redo_snapshot_owed` raise, so a held group's debt + /// write fails once. + #[cfg(test)] + pub(crate) fn fail_next_redo_snapshot_owed_write_for_test(&self) { + self.fail_next_redo_snapshot_owed_write + .store(true, std::sync::atomic::Ordering::SeqCst); + } + + /// Make the next `persist_calvin_barrier_logs` raise. The pending rows + /// stay pending. + #[cfg(test)] + pub(crate) fn fail_next_calvin_barrier_persist_for_test(&self) { + self.fail_next_calvin_barrier_persist + .store(true, std::sync::atomic::Ordering::SeqCst); + } + + /// Make the next `replace_calvin_barrier_logs` raise before it commits. + #[cfg(test)] + pub(crate) fn fail_next_calvin_barrier_replace_for_test(&self) { + self.fail_next_calvin_barrier_replace + .store(true, std::sync::atomic::Ordering::SeqCst); + } + /// Bootstrap every `_system.*` table from the canonical registry — /// but only if at least one is actually missing. Probing read-only /// first keeps `open` byte-idempotent on an already-bootstrapped diff --git a/nodedb/src/control/server/calvin_submit/hook.rs b/nodedb/src/control/server/calvin_submit/hook.rs index 57a061d0a..a3ef71c61 100644 --- a/nodedb/src/control/server/calvin_submit/hook.rs +++ b/nodedb/src/control/server/calvin_submit/hook.rs @@ -1,7 +1,7 @@ // SPDX-License-Identifier: BUSL-1.1 //! `RegistryCalvinSubmit` — bridges the cluster `SubmitCalvinTxn` trigger to a -//! node-local Calvin submit-and-await (Cv1). +//! node-local Calvin submit-and-await. //! //! `nodedb-cluster` cannot depend on `nodedb` (circular), so the submit logic //! lives here and is exposed to the transport via the @@ -17,9 +17,10 @@ //! 1. decodes `tx_class_bytes` (msgpack) into a `TxClass` and re-derives its //! cached participating-vshard set via `restore_derived`; //! 2. submits it to this node's Calvin sequencer inbox and awaits assignment + -//! completion through `submit_and_await_calvin_with_timeout`, bounded by the -//! forwarded `deadline_remaining_ms` so it cannot outlive the coordinator's -//! remaining budget; +//! completion through `submit_and_await_calvin_until`, ending at the +//! deadline the forwarded `deadline_remaining_ms` names, so it cannot +//! outlive the coordinator's remaining budget. A spent budget is refused +//! unsubmitted, a definite refusal; //! 3. maps `Ok(())` → `error: None` and `Err` → a typed cluster error that //! keeps the Data-Plane verdict's code — never a silent drop. //! @@ -32,13 +33,13 @@ //! io_uring directly. use std::sync::Arc; -use std::time::Duration; use nodedb_cluster::calvin::types::TxClass; use nodedb_cluster::{SubmitCalvinTxnRequest, SubmitCalvinTxnResponse, TypedClusterError}; use crate::control::cluster::data_plane_error_wire::execution_error_to_typed; -use crate::control::planner::calvin::submit_and_await_calvin_with_timeout; +use crate::control::cluster::forwarded_budget::hook_deadline; +use crate::control::planner::calvin::submit_and_await_calvin_until; use crate::control::state::SharedState; /// `nodedb`-side implementation of [`nodedb_cluster::CalvinSubmit`]. @@ -62,6 +63,17 @@ impl RegistryCalvinSubmit { #[async_trait::async_trait] impl nodedb_cluster::CalvinSubmit for RegistryCalvinSubmit { async fn on_submit_calvin_txn(&self, req: SubmitCalvinTxnRequest) -> SubmitCalvinTxnResponse { + // A spent budget is refused before any work: nothing is submitted. + let deadline = match hook_deadline(req.deadline_remaining_ms) { + Ok(deadline) => deadline, + Err(refusal) => { + return SubmitCalvinTxnResponse { + error: Some(refusal), + payload_bytes: None, + }; + } + }; + let mut tx_class: TxClass = match zerompk::from_msgpack(&req.tx_class_bytes) { Ok(tc) => tc, Err(e) => { @@ -86,8 +98,7 @@ impl nodedb_cluster::CalvinSubmit for RegistryCalvinSubmit { }; } - let timeout = Duration::from_millis(req.deadline_remaining_ms.max(1)); - match submit_and_await_calvin_with_timeout(&self.state, tx_class, timeout).await { + match submit_and_await_calvin_until(&self.state, tx_class, deadline).await { // Forward the applied RETURNING payload (drained from this leader's // local sidecar) back to the remote coordinator over the non-Raft // RPC response; `None` for a plain write with no rows to surface. @@ -105,3 +116,40 @@ impl nodedb_cluster::CalvinSubmit for RegistryCalvinSubmit { } } } + +#[cfg(test)] +mod tests { + use nodedb_cluster::CalvinSubmit; + + use super::*; + use crate::bridge::dispatch::Dispatcher; + use crate::bridge::envelope::ErrorCode; + use crate::wal::WalManager; + + /// A forward with a spent budget is refused unsubmitted. The coordinator + /// sees a definite refusal, never a 1 ms wait that ends unknown. + #[tokio::test] + async fn a_spent_budget_is_refused_before_the_submit() { + let dir = tempfile::tempdir().unwrap(); + let wal = Arc::new(WalManager::open_for_testing(&dir.path().join("test.wal")).unwrap()); + let (dispatcher, _data_sides) = Dispatcher::new(1, 64); + let state = SharedState::new(dispatcher, wal).unwrap(); + + let reply = RegistryCalvinSubmit::new(state) + .on_submit_calvin_txn(SubmitCalvinTxnRequest { + tx_class_bytes: Vec::new(), + deadline_remaining_ms: 0, + trace_id: [0u8; 16], + }) + .await; + let refusal = reply.error.expect("a spent budget is refused"); + assert!( + matches!( + crate::Error::from(refusal), + crate::Error::DataPlane(ErrorCode::ExpiredBeforeExecution) + ), + "a spent budget must come back as ExpiredBeforeExecution" + ); + assert!(reply.payload_bytes.is_none()); + } +} diff --git a/nodedb/src/control/server/calvin_submit/inbox_hook.rs b/nodedb/src/control/server/calvin_submit/inbox_hook.rs index cf4578d59..e903f6b29 100644 --- a/nodedb/src/control/server/calvin_submit/inbox_hook.rs +++ b/nodedb/src/control/server/calvin_submit/inbox_hook.rs @@ -1,7 +1,7 @@ // SPDX-License-Identifier: BUSL-1.1 //! `RegistryCalvinSubmitInbox` — bridges the cluster `SubmitCalvinInbox` trigger -//! to a node-local Calvin submit-and-ASSIGN (Cv1). +//! to a node-local Calvin submit-and-ASSIGN. //! //! OLLP dependent sibling of [`RegistryCalvinSubmit`](super::hook::RegistryCalvinSubmit): //! `nodedb-cluster` cannot depend on `nodedb` (circular), so the submit logic @@ -19,14 +19,15 @@ //! 2. submits it to this node's Calvin sequencer inbox and awaits only the //! ASSIGNMENT (NOT completion) through the local `CalvinCompletionRegistry`, //! bounded by the forwarded `deadline_remaining_ms` so it cannot outlive the -//! coordinator's remaining budget; +//! coordinator's remaining budget. A spent budget is refused unsubmitted; //! 3. maps `Ok(RoutedAssignment { .. })` → a success response carrying -//! the assignment and `Err` → a typed [`TypedClusterError::Internal`] — never -//! a silent drop. +//! the assignment and `Err` → a typed cluster error that keeps the error's +//! class — never a silent drop. A deadline stays an unknown outcome, so +//! the coordinator never resubmits a transaction that can be sequenced. //! //! It must NOT await completion: the OLLP coordinator loop drives the dependent -//! transaction to completion itself in a later unit; this hook returns as soon as -//! the assignment is observed. +//! transaction to completion itself. This hook returns as soon as the +//! assignment is observed. //! //! # Plane discipline //! @@ -37,7 +38,6 @@ //! io_uring directly. use std::sync::Arc; -use std::time::Duration; use nodedb_cluster::calvin::PartsOfferStatus; use nodedb_cluster::calvin::types::{PartStreamId, StreamedPart, TxClass}; @@ -46,6 +46,8 @@ use nodedb_cluster::{ TypedClusterError, }; +use crate::control::cluster::data_plane_error_wire::execution_error_to_typed; +use crate::control::cluster::forwarded_budget::hook_deadline; use crate::control::planner::calvin::submit::submit_local_assign; use crate::control::state::SharedState; @@ -73,6 +75,20 @@ impl nodedb_cluster::CalvinSubmitInbox for RegistryCalvinSubmitInbox { &self, req: SubmitCalvinInboxRequest, ) -> SubmitCalvinInboxResponse { + // A spent budget is refused before any work: nothing is submitted. + let deadline = match hook_deadline(req.deadline_remaining_ms) { + Ok(deadline) => deadline, + Err(refusal) => { + return SubmitCalvinInboxResponse { + inbox_seq: 0, + epoch: 0, + position: 0, + participants: 0, + error: Some(refusal), + }; + } + }; + let mut tx_class: TxClass = match zerompk::from_msgpack(&req.tx_class_bytes) { Ok(tc) => tc, Err(e) => { @@ -103,8 +119,7 @@ impl nodedb_cluster::CalvinSubmitInbox for RegistryCalvinSubmitInbox { }; } - let timeout = Duration::from_millis(req.deadline_remaining_ms.max(1)); - match submit_local_assign(&self.state, tx_class, timeout).await { + match submit_local_assign(&self.state, tx_class, deadline).await { Ok(a) => SubmitCalvinInboxResponse { inbox_seq: a.inbox_seq, epoch: a.epoch, @@ -117,10 +132,7 @@ impl nodedb_cluster::CalvinSubmitInbox for RegistryCalvinSubmitInbox { epoch: 0, position: 0, participants: 0, - error: Some(TypedClusterError::Internal { - code: 0, - message: format!("calvin-inbox local submit-and-assign failed: {e}"), - }), + error: Some(execution_error_to_typed(e)), }, } } @@ -154,3 +166,40 @@ impl nodedb_cluster::CalvinSubmitInbox for RegistryCalvinSubmitInbox { } } } + +#[cfg(test)] +mod tests { + use nodedb_cluster::CalvinSubmitInbox; + + use super::*; + use crate::bridge::dispatch::Dispatcher; + use crate::bridge::envelope::ErrorCode; + use crate::wal::WalManager; + + /// A forward with a spent budget is refused unsubmitted. The coordinator + /// sees a definite refusal, never a 1 ms wait that ends unknown. + #[tokio::test] + async fn a_spent_budget_is_refused_before_the_submit() { + let dir = tempfile::tempdir().unwrap(); + let wal = Arc::new(WalManager::open_for_testing(&dir.path().join("test.wal")).unwrap()); + let (dispatcher, _data_sides) = Dispatcher::new(1, 64); + let state = SharedState::new(dispatcher, wal).unwrap(); + + let reply = RegistryCalvinSubmitInbox::new(state) + .on_submit_calvin_inbox(SubmitCalvinInboxRequest { + tx_class_bytes: Vec::new(), + deadline_remaining_ms: 0, + trace_id: [0u8; 16], + }) + .await; + let refusal = reply.error.expect("a spent budget is refused"); + assert!( + matches!( + crate::Error::from(refusal), + crate::Error::DataPlane(ErrorCode::ExpiredBeforeExecution) + ), + "a spent budget must come back as ExpiredBeforeExecution" + ); + assert_eq!(reply.inbox_seq, 0); + } +} diff --git a/nodedb/src/control/server/calvin_submit/mod.rs b/nodedb/src/control/server/calvin_submit/mod.rs index 149e72b2a..c6f68c919 100644 --- a/nodedb/src/control/server/calvin_submit/mod.rs +++ b/nodedb/src/control/server/calvin_submit/mod.rs @@ -1,6 +1,6 @@ // SPDX-License-Identifier: BUSL-1.1 -//! Routed Calvin-submit (Cv1): the sequencer-leader-side cluster hook that +//! Routed Calvin-submit: the sequencer-leader-side cluster hook that //! submits a forwarded `TxClass` to the local Calvin sequencer inbox and awaits //! its completion. The coordinator-side routing helper lives in //! `crate::control::planner::calvin::submit::submit_calvin_routed`. diff --git a/nodedb/src/control/server/dispatch_utils/change_events/extract.rs b/nodedb/src/control/server/dispatch_utils/change_events/extract.rs index aa7fd992b..ecf0452b3 100644 --- a/nodedb/src/control/server/dispatch_utils/change_events/extract.rs +++ b/nodedb/src/control/server/dispatch_utils/change_events/extract.rs @@ -659,7 +659,7 @@ mod tests { slice_msgpack: Vec::new(), attr_projection: Vec::new(), limit: 0, - slice_hilbert_ranges: Vec::new(), + slice_tile_keys: Vec::new(), prefix_bits: 8, system_time: nodedb_types::SystemTimeScope::Current, valid_at_ms: None, @@ -671,7 +671,7 @@ mod tests { attr_idx: 0, reducer_msgpack: Vec::new(), group_by_dim: -1, - slice_hilbert_ranges: Vec::new(), + slice_tile_keys: Vec::new(), prefix_bits: 8, system_as_of: None, valid_at_ms: None, diff --git a/nodedb/src/control/server/dispatch_utils/durability_barrier.rs b/nodedb/src/control/server/dispatch_utils/durability_barrier.rs index 6f5a94006..ff9958523 100644 --- a/nodedb/src/control/server/dispatch_utils/durability_barrier.rs +++ b/nodedb/src/control/server/dispatch_utils/durability_barrier.rs @@ -26,7 +26,7 @@ use std::sync::atomic::{AtomicU64, Ordering}; use crate::bridge::envelope::PhysicalPlan; use crate::control::server::shared::write_admission::plan_is_write; -use nodedb_physical::physical_plan::{GraphOp, MetaOp}; +use nodedb_physical::physical_plan::{GraphOp, KvOp, MetaOp}; /// Count of writes acknowledged with no durable redo record despite belonging /// to an engine whose every write-class op mints one on this path. @@ -63,7 +63,9 @@ pub fn writes_acked_without_durability() -> u64 { /// * `ClusterArray` — a coordinator-side routing wrapper: each owning shard's /// apply mints the redo for the cells it actually holds; /// * an empty `EdgePutBatch` / `EdgeDeleteBatch`, which has no edge to make -/// durable and appends nothing on purpose. +/// durable and appends nothing on purpose; +/// * a KV `ResolvedWrite` with no mutation, such as a predicate UPDATE / +/// DELETE that matched no row. It appends one record per mutation, so none. /// /// Exchange / gather plans never reach the barrier at all: they are resolved /// into a merged response before `submit_write` is entered. @@ -75,6 +77,7 @@ pub(super) fn funnel_minted_redo_engine(plan: &PhysicalPlan) -> Option<&'static // `PhysicalPlan` variant must be classified here by name rather than // silently inheriting "no durability expected". match plan { + PhysicalPlan::Kv(KvOp::ResolvedWrite { mutations, .. }) if mutations.is_empty() => None, PhysicalPlan::Kv(_) => Some("kv"), PhysicalPlan::Vector(_) => Some("vector"), PhysicalPlan::Columnar(_) => Some("columnar"), @@ -174,6 +177,19 @@ mod tests { assert_eq!(funnel_minted_redo_engine(&plan), Some("transaction")); } + /// A resolved write with no mutation appends no record, so its + /// acknowledgement waits on nothing. A predicate write that matched no + /// row resolves to it. + #[test] + fn an_empty_kv_resolved_write_is_not_held_to_the_barrier() { + let empty = PhysicalPlan::Kv(KvOp::ResolvedWrite { + mutations: Vec::new(), + response_payload: Vec::new(), + rls_write_check: nodedb_types::RlsWriteCheck::decided_earlier_in_request(), + }); + assert_eq!(funnel_minted_redo_engine(&empty), None); + } + /// A read carries no durability obligation at all. #[test] fn read_requires_no_redo() { diff --git a/nodedb/src/control/server/dispatch_utils/durable_write.rs b/nodedb/src/control/server/dispatch_utils/durable_write.rs index 3458f1439..3e3205d17 100644 --- a/nodedb/src/control/server/dispatch_utils/durable_write.rs +++ b/nodedb/src/control/server/dispatch_utils/durable_write.rs @@ -215,7 +215,8 @@ async fn propose_if_replicable( return Ok(None); }; let entry = entry.with_event_source(event_source); - let deadline = crate::control::wal_replication::statement_propose_deadline(shared); + let deadline = crate::control::server::shared::session::StatementDeadline::of_statement(shared) + .tokio_instant(); let (payload, write_versions) = propose_replicated_entry(shared, proposer, entry, deadline).await?; Ok(Some(replicated_write_response( diff --git a/nodedb/src/control/server/dispatch_utils/owner_read.rs b/nodedb/src/control/server/dispatch_utils/owner_read.rs index 5050277c6..4f9880986 100644 --- a/nodedb/src/control/server/dispatch_utils/owner_read.rs +++ b/nodedb/src/control/server/dispatch_utils/owner_read.rs @@ -22,7 +22,7 @@ use futures::future::{BoxFuture, Either, Ready, ready}; use crate::bridge::envelope::{ErrorCode, Payload, PhysicalPlan, Response, Status}; use crate::control::cluster::linearizable_read::{ - confirm_linearizable_read, hosted_groups_of_vshards, statement_read_deadline, + StatementDeadline, confirm_linearizable_read, hosted_groups_of_vshards, }; use crate::control::gateway::RouteDecision; use crate::control::gateway::core::QueryContext; @@ -78,7 +78,7 @@ async fn route_cluster_read( scope: OwnedReadScope, plan: PhysicalPlan, ) -> crate::Result { - let deadline = statement_read_deadline(shared); + let deadline = StatementDeadline::of_statement(shared); if nodedb_physical::physical_plan::plan_contains_cluster_partitioned_leaf(&plan) { if scope.linearizable { let groups = crate::control::server::graph_dispatch::graph_read_groups( @@ -200,7 +200,8 @@ pub(crate) async fn prepare_local_pass( ) -> crate::Result<()> { match read_placement(shared, vshard_id, txn_id)? { ReadPlacement::Here(hosted) => { - confirm_linearizable_read(shared, &hosted, statement_read_deadline(shared)).await + confirm_linearizable_read(shared, &hosted, StatementDeadline::of_statement(shared)) + .await } ReadPlacement::Owner => { let (leader_node, leader_term) = shared diff --git a/nodedb/src/control/server/exchange/all_cores/home_versions.rs b/nodedb/src/control/server/exchange/all_cores/home_versions.rs index 950b81069..3bc7cb3a5 100644 --- a/nodedb/src/control/server/exchange/all_cores/home_versions.rs +++ b/nodedb/src/control/server/exchange/all_cores/home_versions.rs @@ -22,7 +22,7 @@ use nodedb_physical::physical_plan::{ }; use crate::control::cluster::leased_read::{LeaseRefusal, confirm_leased_read}; -use crate::control::cluster::linearizable_read::statement_read_deadline; +use crate::control::cluster::linearizable_read::StatementDeadline; use crate::control::server::exchange::owning_core::dispatch_single_owning_core; use crate::control::state::SharedState; use crate::types::{DatabaseId, Lsn, TenantId, TraceId, VShardId}; @@ -49,7 +49,7 @@ pub(super) async fn fan_home_versions( groups.sort_unstable(); groups.dedup(); let refusals: HashMap = - confirm_leased_read(state, &groups, statement_read_deadline(state)) + confirm_leased_read(state, &groups, StatementDeadline::of_statement(state)) .await? .into_iter() .map(|refusal| (refusal.group_id, refusal)) diff --git a/nodedb/src/control/server/exchange/all_cores/read_cut.rs b/nodedb/src/control/server/exchange/all_cores/read_cut.rs index a66df9898..879d20548 100644 --- a/nodedb/src/control/server/exchange/all_cores/read_cut.rs +++ b/nodedb/src/control/server/exchange/all_cores/read_cut.rs @@ -22,6 +22,7 @@ use std::time::Duration; use nodedb_cluster::calvin::SequencerEntry; use nodedb_physical::physical_plan::{GraphOp, PhysicalPlan}; +use crate::control::server::shared::session::StatementDeadline; use crate::control::state::SharedState; /// How long a node waits for the marker before it proposes it again. A @@ -81,8 +82,9 @@ async fn resolve_read_cut(state: &SharedState, marker: u64) -> crate::Result crate::Result crate::Result<()> { let groups = graph_read_groups(state, database_id, plan)?; - confirm_linearizable_read(state, &groups, statement_read_deadline(state)).await + confirm_linearizable_read(state, &groups, StatementDeadline::of_statement(state)).await } fn keyed_groups<'a>( diff --git a/nodedb/src/control/server/graph_dispatch/whole_graph.rs b/nodedb/src/control/server/graph_dispatch/whole_graph.rs index 75f68861f..0052aad94 100644 --- a/nodedb/src/control/server/graph_dispatch/whole_graph.rs +++ b/nodedb/src/control/server/graph_dispatch/whole_graph.rs @@ -37,9 +37,9 @@ use std::collections::BTreeMap; use futures::future::join_all; use crate::bridge::envelope::{Payload, PhysicalPlan}; -use crate::control::gateway::dispatcher::statement_deadline_ms; use crate::control::gateway::version_set::GatewayVersionSet; use crate::control::server::exchange::execute_plan_all_local_cores; +use crate::control::server::shared::session::StatementDeadline; use crate::control::state::SharedState; use crate::types::{DatabaseId, TenantId, TraceId, VShardId}; use nodedb_graph::{AlgoParams, GraphAlgorithm}; @@ -76,7 +76,7 @@ pub async fn scatter_to_graph_owners( let targets = enumerate_shards(state)?.targets; let shared_arc = gateway_shared(state)?; let version_set = GatewayVersionSet::from_pairs(Vec::new()); - let deadline_ms = statement_deadline_ms(state); + let deadline_ms = StatementDeadline::of_statement(state).hop_budget_ms(); let legs = targets.into_iter().map(|target| { let plan = plan.clone(); let shared_arc = shared_arc.clone(); @@ -123,7 +123,7 @@ pub async fn run_graph_algo( system_as_of_ms: Option, linearizable: bool, ) -> crate::Result { - let deadline_ms = statement_deadline_ms(state); + let deadline_ms = StatementDeadline::of_statement(state).hop_budget_ms(); // A current-state BSP run pins its own read cut from a Calvin cut marker // (`run_cut`). A historical run gathers the edges live at its system time. if state.cluster_routing.is_some() && system_as_of_ms.is_none() { diff --git a/nodedb/src/control/server/http/routes/crdt.rs b/nodedb/src/control/server/http/routes/crdt.rs index 607eb2703..da6b0ce05 100644 --- a/nodedb/src/control/server/http/routes/crdt.rs +++ b/nodedb/src/control/server/http/routes/crdt.rs @@ -175,8 +175,8 @@ pub async fn crdt_apply( crate::control::crdt_admission::AuthorizedCrdtApplyAdmissionRequest { authorized, collection: &collection, - timeout: std::time::Duration::from_secs( - state.shared.tuning.network.default_deadline_secs, + deadline: crate::control::server::shared::session::StatementDeadline::of_statement( + &state.shared, ), event_source: crate::event::EventSource::User, policy: &policy, diff --git a/nodedb/src/control/server/http/routes/health.rs b/nodedb/src/control/server/http/routes/health.rs index 85519e9e8..33998b111 100644 --- a/nodedb/src/control/server/http/routes/health.rs +++ b/nodedb/src/control/server/http/routes/health.rs @@ -247,8 +247,8 @@ fn outcome_floor_stuck_body( /// How long a write window can hold the outcome floor before readiness reports /// it: the longest path from a window's open to its final outcome. /// -/// - A statement write reaches its outcome by the longest statement deadline, -/// plus the wait the node gives a committed entry to apply. +/// - A statement write reaches its outcome by the longest statement deadline. +/// Its waits for a committed entry to apply end at that deadline too. /// - A vector index install waits two core dispatch deadlines. /// /// A window older than both is stuck. @@ -258,8 +258,7 @@ fn outcome_floor_bound(state: &AppState) -> std::time::Duration { network .default_deadline_secs .max(network.copy_deadline_secs), - ) - .saturating_add(crate::control::metadata_proposer::DEFAULT_PROPOSE_TIMEOUT); + ); statement.max(crate::control::catalog_entry::post_apply::vector_install_longest_core_wait()) } @@ -486,8 +485,8 @@ mod tests { assert_ne!(body["reason"], "calvin_apply_halted"); } - /// The bound covers the longest statement deadline plus the apply wait, - /// and the vector install's two core dispatch deadlines. + /// The bound covers the longest statement deadline, and the vector + /// install's two core dispatch deadlines. #[tokio::test] async fn the_outcome_floor_bound_covers_every_path_to_a_final_outcome() { let dir = tempfile::tempdir().expect("tempdir"); @@ -497,7 +496,7 @@ mod tests { network .default_deadline_secs .max(network.copy_deadline_secs), - ) + crate::control::metadata_proposer::DEFAULT_PROPOSE_TIMEOUT; + ); let vector = crate::control::catalog_entry::post_apply::vector_install_longest_core_wait(); let bound = outcome_floor_bound(&state); diff --git a/nodedb/src/control/server/native/dispatch/graph_match.rs b/nodedb/src/control/server/native/dispatch/graph_match.rs index b1ba9fbfd..7e7ad061d 100644 --- a/nodedb/src/control/server/native/dispatch/graph_match.rs +++ b/nodedb/src/control/server/native/dispatch/graph_match.rs @@ -199,7 +199,8 @@ async fn match_across_shards( ctx.tenant_id(), ctx.database_id(), query.clone(), - crate::control::gateway::dispatcher::statement_deadline_ms(ctx.state), + crate::control::server::shared::session::StatementDeadline::of_statement(ctx.state) + .hop_budget_ms(), crate::control::server::graph_dispatch::GraphRead { txn_id, // The native protocol has no read-consistency setting. diff --git a/nodedb/src/control/server/native/dispatch/raw_dispatch.rs b/nodedb/src/control/server/native/dispatch/raw_dispatch.rs index 93a95aab2..87c87348b 100644 --- a/nodedb/src/control/server/native/dispatch/raw_dispatch.rs +++ b/nodedb/src/control/server/native/dispatch/raw_dispatch.rs @@ -164,7 +164,9 @@ async fn dispatch_external_crdt_apply( crate::control::crdt_admission::AuthorizedCrdtApplyAdmissionRequest { authorized, collection: collection.as_str(), - timeout: std::time::Duration::from_secs(ctx.state.tuning.network.default_deadline_secs), + deadline: crate::control::server::shared::session::StatementDeadline::of_statement( + ctx.state, + ), event_source: crate::event::EventSource::User, policy: &policy, }, diff --git a/nodedb/src/control/server/pgwire/handler/dispatch/replicated.rs b/nodedb/src/control/server/pgwire/handler/dispatch/replicated.rs index 2c9237cdd..05192caf7 100644 --- a/nodedb/src/control/server/pgwire/handler/dispatch/replicated.rs +++ b/nodedb/src/control/server/pgwire/handler/dispatch/replicated.rs @@ -34,7 +34,8 @@ impl NodeDbPgHandler { &self.state, proposer, entry, - crate::control::wal_replication::statement_propose_deadline(&self.state), + crate::control::server::shared::session::StatementDeadline::of_statement(&self.state) + .tokio_instant(), ) .await?; diff --git a/nodedb/src/control/server/pgwire/handler/dispatch/routing.rs b/nodedb/src/control/server/pgwire/handler/dispatch/routing.rs index 5cd4c48da..9d7efa4e1 100644 --- a/nodedb/src/control/server/pgwire/handler/dispatch/routing.rs +++ b/nodedb/src/control/server/pgwire/handler/dispatch/routing.rs @@ -6,9 +6,7 @@ use std::sync::Arc; use crate::bridge::envelope::Response; -use crate::control::cluster::linearizable_read::{ - confirm_linearizable_read, statement_read_deadline, -}; +use crate::control::cluster::linearizable_read::{StatementDeadline, confirm_linearizable_read}; use crate::control::security::identity::AuthenticatedIdentity; use crate::control::server::exchange::ReadScope; use crate::control::server::exchange::resolve::{ @@ -228,7 +226,7 @@ impl NodeDbPgHandler { match read_placement(&self.state, checked.vshard_id(), checked.txn_id())? { ReadPlacement::Here(groups) => { if linearizable { - let deadline = statement_read_deadline(&self.state); + let deadline = StatementDeadline::of_statement(&self.state); confirm_linearizable_read(&self.state, &groups, deadline).await?; } } diff --git a/nodedb/src/control/server/pgwire/handler/routing/placement.rs b/nodedb/src/control/server/pgwire/handler/routing/placement.rs index 2ed7cae99..a8c78bd31 100644 --- a/nodedb/src/control/server/pgwire/handler/routing/placement.rs +++ b/nodedb/src/control/server/pgwire/handler/routing/placement.rs @@ -273,15 +273,20 @@ impl NodeDbPgHandler { /// Whether this node's replica of `group_id` meets `consistency`. /// /// `Eventual` asks for no freshness at all, so any replica satisfies it. - /// `BoundedStaleness` asks how far behind the leader this replica is, - /// which only Raft can answer. With no gate installed there is no cluster, - /// and the local copy is the only copy. + /// `BoundedStaleness` asks how far behind the leader this replica is: + /// Raft's last leader contact against this node's applied watermark. With + /// no gate installed there is no cluster, and the local copy is the only + /// copy. fn replica_satisfies(&self, group_id: u64, consistency: ReadConsistency) -> bool { let Some(max_staleness) = consistency.max_staleness() else { return true; }; match self.state.raft_read_gate.get() { - Some(gate) => gate.within_staleness_bound(group_id, max_staleness), + Some(gate) => gate.within_staleness_bound( + group_id, + max_staleness, + self.state.applied_index_watcher(group_id).current(), + ), None => true, } } diff --git a/nodedb/src/control/server/pgwire/types/error_map.rs b/nodedb/src/control/server/pgwire/types/error_map.rs index c393d9e39..994fa5eb3 100644 --- a/nodedb/src/control/server/pgwire/types/error_map.rs +++ b/nodedb/src/control/server/pgwire/types/error_map.rs @@ -411,7 +411,8 @@ pub fn error_to_sqlstate(err: &crate::Error) -> (&'static str, &'static str, Str // Class `XX` is never treated as transient, so no client re-proposes // a write that can have committed. crate::Error::CommittedResultUnavailable { .. } - | crate::Error::ProposalOutcomeUnknown { .. } => { + | crate::Error::ProposalOutcomeUnknown { .. } + | crate::Error::MetadataGroupGone { .. } => { ("ERROR", sqlstate::INTERNAL_ERROR, err.to_string()) } // Server-side faults and system defects. The client can act on none @@ -419,6 +420,7 @@ pub fn error_to_sqlstate(err: &crate::Error) -> (&'static str, &'static str, Str crate::Error::MaterializedSumResolutionMissing { .. } | crate::Error::RetryableLeaderChange { .. } | crate::Error::MetadataLeaderUnavailable + | crate::Error::MetadataDdlSuperseded { .. } | crate::Error::Wal(_) | crate::Error::Dispatch { .. } | crate::Error::Storage { .. } diff --git a/nodedb/src/control/server/reservation/hooks.rs b/nodedb/src/control/server/reservation/hooks.rs index d928b85f4..095124edf 100644 --- a/nodedb/src/control/server/reservation/hooks.rs +++ b/nodedb/src/control/server/reservation/hooks.rs @@ -23,7 +23,6 @@ //! directly. use std::sync::Arc; -use std::time::Duration; use nodedb_cluster::calvin::types::{LockKeyWire, ReleaseReason, TxnIdWire}; use nodedb_cluster::{ @@ -31,6 +30,8 @@ use nodedb_cluster::{ TypedClusterError, }; +use crate::control::cluster::data_plane_error_wire::execution_error_to_typed; +use crate::control::cluster::forwarded_budget::hook_deadline; use crate::control::planner::calvin::reservation::{ submit_local_release, submit_local_reserve_read, }; @@ -57,6 +58,16 @@ impl RegistryReserveRead { #[async_trait::async_trait] impl nodedb_cluster::ReserveRead for RegistryReserveRead { async fn on_reserve_read(&self, req: ReserveReadRequest) -> ReserveReadResponse { + // A spent budget is refused before any work: nothing is reserved. + let deadline = match hook_deadline(req.deadline_remaining_ms) { + Ok(deadline) => deadline, + Err(refusal) => { + return ReserveReadResponse { + owner_bytes: None, + error: Some(refusal), + }; + } + }; let key: LockKeyWire = match zerompk::from_msgpack(&req.lock_key_bytes) { Ok(k) => k, Err(e) => { @@ -85,8 +96,7 @@ impl nodedb_cluster::ReserveRead for RegistryReserveRead { None => None, }; - let timeout = Duration::from_millis(req.deadline_remaining_ms.max(1)); - match submit_local_reserve_read(&self.state, key, req.vshard, owner, timeout).await { + match submit_local_reserve_read(&self.state, key, req.vshard, owner, deadline).await { Ok(assigned) => match zerompk::to_msgpack_vec(&assigned) { Ok(bytes) => ReserveReadResponse { owner_bytes: Some(bytes), @@ -100,12 +110,11 @@ impl nodedb_cluster::ReserveRead for RegistryReserveRead { }), }, }, + // The error keeps its class: a deadline stays an unknown + // outcome, an expired budget a definite refusal. Err(e) => ReserveReadResponse { owner_bytes: None, - error: Some(TypedClusterError::Internal { - code: 0, - message: format!("reserve-read local submit failed: {e}"), - }), + error: Some(execution_error_to_typed(e)), }, } } @@ -172,3 +181,42 @@ impl nodedb_cluster::ReleaseReservation for RegistryReleaseReservation { } } } + +#[cfg(test)] +mod tests { + use nodedb_cluster::ReserveRead; + + use super::*; + use crate::bridge::dispatch::Dispatcher; + use crate::bridge::envelope::ErrorCode; + use crate::wal::WalManager; + + /// A reserve-read with a spent budget is refused unsubmitted, a definite + /// refusal the coordinator keeps. + #[tokio::test] + async fn a_spent_budget_is_refused_before_the_reserve() { + let dir = tempfile::tempdir().unwrap(); + let wal = Arc::new(WalManager::open_for_testing(&dir.path().join("test.wal")).unwrap()); + let (dispatcher, _data_sides) = Dispatcher::new(1, 64); + let state = SharedState::new(dispatcher, wal).unwrap(); + + let reply = RegistryReserveRead::new(state) + .on_reserve_read(ReserveReadRequest { + lock_key_bytes: Vec::new(), + vshard: 0, + owner_bytes: None, + deadline_remaining_ms: 0, + trace_id: [0u8; 16], + }) + .await; + let refusal = reply.error.expect("a spent budget is refused"); + assert!( + matches!( + crate::Error::from(refusal), + crate::Error::DataPlane(ErrorCode::ExpiredBeforeExecution) + ), + "a spent budget must come back as ExpiredBeforeExecution" + ); + assert!(reply.owner_bytes.is_none()); + } +} diff --git a/nodedb/src/control/server/shared/clone_write/probes.rs b/nodedb/src/control/server/shared/clone_write/probes.rs index 341083029..72eb7a4c7 100644 --- a/nodedb/src/control/server/shared/clone_write/probes.rs +++ b/nodedb/src/control/server/shared/clone_write/probes.rs @@ -4,8 +4,6 @@ //! write-interception paths: presence probes on the target collection, and //! source-row/value fetches for copy-up. -use std::time::Duration; - use nodedb_types::{DatabaseId, QualifiedCollection, Surrogate, TenantId}; use crate::bridge::envelope::{Priority, Request, Response, Status}; @@ -269,15 +267,14 @@ pub(super) async fn dispatch_data_plane_raw( .request_id_counter .fetch_add(1, std::sync::atomic::Ordering::Relaxed), ); - let deadline_secs = state.tuning.network.default_deadline_secs; - let deadline_dur = Duration::from_secs(deadline_secs); + let deadline = crate::control::server::shared::session::StatementDeadline::of_statement(state); let req = Request { request_id: req_id, tenant_id, vshard_id, database_id, plan, - deadline: std::time::Instant::now() + deadline_dur, + deadline: deadline.instant(), priority: Priority::Normal, trace_id: TraceId::ZERO, consistency: ReadConsistency::Strong, @@ -300,7 +297,7 @@ pub(super) async fn dispatch_data_plane_raw( Ok(mut d) => d.dispatch(req)?, Err(p) => p.into_inner().dispatch(req)?, } - tokio::time::timeout(deadline_dur, rx.recv()) + tokio::time::timeout_at(deadline.tokio_instant(), rx.recv()) .await .map_err(|_| crate::Error::DeadlineExceeded { request_id: req_id })? .ok_or(crate::Error::Dispatch { diff --git a/nodedb/src/control/server/shared/ddl/engine_apply.rs b/nodedb/src/control/server/shared/ddl/engine_apply.rs index 6aaef0f89..c841a6859 100644 --- a/nodedb/src/control/server/shared/ddl/engine_apply.rs +++ b/nodedb/src/control/server/shared/ddl/engine_apply.rs @@ -7,14 +7,13 @@ //! refused configuration as a successful statement. Index DDL routes its //! engine registration through here so the refusal always reaches the client. -use std::time::Duration; - use crate::bridge::envelope::{ErrorCode, PhysicalPlan}; +use crate::control::server::shared::session::StatementDeadline; use crate::control::state::SharedState; use crate::types::{DatabaseId, TenantId}; use super::result::DdlError; -use super::sync_dispatch::{SystemReason, SystemTask, dispatch_system}; +use super::sync_dispatch::{SystemReason, SystemTask, dispatch_system_by}; /// Dispatch `plan` for `collection` and translate any Data-Plane refusal into /// a [`DdlError`] with `context` before its message. @@ -29,8 +28,7 @@ pub(crate) async fn apply_in_engine( plan: PhysicalPlan, context: &str, ) -> Result<(), DdlError> { - let timeout = Duration::from_secs(state.tuning.network.default_deadline_secs); - dispatch_system( + dispatch_system_by( state, SystemTask::new( SystemReason::DdlApply, @@ -38,7 +36,7 @@ pub(crate) async fn apply_in_engine( nodedb_types::CollectionKey::from_bare(database_id, collection), plan, ), - timeout, + StatementDeadline::of_statement(state), ) .await .map(|_| ()) @@ -63,7 +61,6 @@ pub(crate) async fn refuse_materialized_vector_index( field_name: &str, context: &str, ) -> Result<(), DdlError> { - let timeout = Duration::from_secs(state.tuning.network.default_deadline_secs); let plan = PhysicalPlan::Vector(nodedb_physical::physical_plan::VectorOp::QueryStats { collection: nodedb_types::QualifiedCollection::new(database_id, collection), field_name: field_name.to_string(), @@ -76,7 +73,7 @@ pub(crate) async fn refuse_materialized_vector_index( nodedb_types::CollectionKey::from_bare(database_id, collection), plan, ), - timeout, + StatementDeadline::of_statement(state), crate::event::EventSource::User, ) .await diff --git a/nodedb/src/control/server/shared/ddl/neutral/collection/purge/dispatch.rs b/nodedb/src/control/server/shared/ddl/neutral/collection/purge/dispatch.rs index e07ee1859..e95990d1c 100644 --- a/nodedb/src/control/server/shared/ddl/neutral/collection/purge/dispatch.rs +++ b/nodedb/src/control/server/shared/ddl/neutral/collection/purge/dispatch.rs @@ -102,8 +102,8 @@ pub async fn dispatch_unregister_collection( } } - // Each core's acknowledgement counts as apply progress as it arrives, so - // a proposer waiting on this purge sees a slow reclaim move. + // Every core is awaited, so the first error never cuts a core's reclaim + // short. let mut pending: FuturesUnordered<_> = receivers .into_iter() .map(|(_request_id, core_id, mut receiver)| async move { @@ -134,15 +134,8 @@ pub async fn dispatch_unregister_collection( let mut first_error = None; while let Some(response) = pending.next().await { - match response { - Ok(()) => { - state - .metadata_apply_progress - .fetch_add(1, std::sync::atomic::Ordering::Release); - } - Err(error) => { - first_error.get_or_insert(error); - } + if let Err(error) = response { + first_error.get_or_insert(error); } } first_error.map_or(Ok(()), Err) diff --git a/nodedb/src/control/server/shared/ddl/neutral/convert/driver.rs b/nodedb/src/control/server/shared/ddl/neutral/convert/driver.rs index 59e6c98ac..2ced86de2 100644 --- a/nodedb/src/control/server/shared/ddl/neutral/convert/driver.rs +++ b/nodedb/src/control/server/shared/ddl/neutral/convert/driver.rs @@ -18,6 +18,7 @@ use crate::control::server::pgwire::types::error_to_sqlstate; use crate::control::server::shared::ddl::sync_dispatch::{ SystemReason, SystemTask, dispatch_system_response_with_source, }; +use crate::control::server::shared::session::StatementDeadline; use crate::control::state::SharedState; use nodedb_physical::physical_plan::MetaOp; @@ -114,7 +115,7 @@ pub async fn convert_collection( nodedb_types::CollectionKey::from_bare(database_id, &collection), plan, ), - Duration::from_secs(60), + StatementDeadline::after(Duration::from_secs(60)), event_source, ) .await diff --git a/nodedb/src/control/server/shared/ddl/neutral/crdt_ops.rs b/nodedb/src/control/server/shared/ddl/neutral/crdt_ops.rs index e310bf3e3..fd2a00760 100644 --- a/nodedb/src/control/server/shared/ddl/neutral/crdt_ops.rs +++ b/nodedb/src/control/server/shared/ddl/neutral/crdt_ops.rs @@ -8,8 +8,6 @@ //! Handlers build [`DdlResult`](super::super::result::DdlResult) directly and //! carry no pgwire types. -use std::time::Duration; - use serde_json::{Map, Value as JsonValue}; use crate::bridge::envelope::PhysicalPlan; @@ -20,6 +18,7 @@ use crate::control::security::identity::{AuthenticatedIdentity, Permission}; use crate::control::server::response_shape::types::ShapedRows; use crate::control::server::shared::authorization::{authorize_collection, authorize_task_set}; use crate::control::server::shared::ddl::sql_parse::hex_decode; +use crate::control::server::shared::session::StatementDeadline; use crate::control::state::SharedState; use crate::types::DatabaseId; use nodedb_physical::physical_plan::CrdtOp; @@ -87,7 +86,7 @@ pub async fn crdt_state( database_id, collection, plan, - timeout: Duration::from_secs(state.tuning.network.default_deadline_secs), + deadline: StatementDeadline::of_statement(state), // This handler receives no session or peer information; the // request was admitted at its own transport entry, and // `AlreadyAdmitted` carries no address precisely so there is @@ -228,7 +227,7 @@ pub async fn crdt_apply( crate::control::crdt_admission::AuthorizedCrdtApplyAdmissionRequest { authorized, collection: &qualified_collection, - timeout: Duration::from_secs(state.tuning.network.default_deadline_secs), + deadline: StatementDeadline::of_statement(state), event_source: crate::event::EventSource::User, policy: &policy, }, diff --git a/nodedb/src/control/server/shared/ddl/neutral/dsl/crdt_merge.rs b/nodedb/src/control/server/shared/ddl/neutral/dsl/crdt_merge.rs index 3929ad00a..0f2f1051a 100644 --- a/nodedb/src/control/server/shared/ddl/neutral/dsl/crdt_merge.rs +++ b/nodedb/src/control/server/shared/ddl/neutral/dsl/crdt_merge.rs @@ -2,13 +2,12 @@ //! `CRDT MERGE INTO` DSL handler. -use std::time::Duration; - use crate::bridge::envelope::PhysicalPlan; use crate::control::crdt_post_image_policy::ExternalCrdtPostImagePolicy; use crate::control::security::audit::ArcAuditEmitter; use crate::control::security::identity::{AuthenticatedIdentity, Permission}; use crate::control::server::shared::authorization::{authorize_collection, authorize_task_set}; +use crate::control::server::shared::session::StatementDeadline; use crate::control::state::SharedState; use crate::types::DatabaseId; use nodedb_physical::physical_plan::CrdtOp; @@ -80,7 +79,7 @@ pub async fn crdt_merge( database_id, collection, plan: source_plan, - timeout: Duration::from_secs(state.tuning.network.default_deadline_secs), + deadline: StatementDeadline::of_statement(state), // Admitted at this request's own transport entry; this handler // has no session or peer information of its own. admission: crate::control::server::shared::ddl::user_dispatch::RequestAdmission::AlreadyAdmitted, @@ -165,7 +164,7 @@ pub async fn crdt_merge( crate::control::crdt_admission::AuthorizedCrdtApplyAdmissionRequest { authorized, collection: &qualified_collection, - timeout: Duration::from_secs(state.tuning.network.default_deadline_secs), + deadline: StatementDeadline::of_statement(state), event_source: crate::event::EventSource::User, policy: &policy, }, diff --git a/nodedb/src/control/server/shared/ddl/neutral/graph_ops/rag_fusion.rs b/nodedb/src/control/server/shared/ddl/neutral/graph_ops/rag_fusion.rs index dad1e6fa2..779564f12 100644 --- a/nodedb/src/control/server/shared/ddl/neutral/graph_ops/rag_fusion.rs +++ b/nodedb/src/control/server/shared/ddl/neutral/graph_ops/rag_fusion.rs @@ -7,8 +7,6 @@ //! this single function, so caps and defaults cannot drift between the //! two surfaces. -use std::time::Duration; - use serde_json::{Map, Value as JsonValue}; use nodedb_sql::ddl_ast::FusionParams; @@ -18,6 +16,7 @@ use crate::bridge::envelope::PhysicalPlan; use crate::control::security::identity::AuthenticatedIdentity; use crate::control::server::response_shape::types::ShapedRows; use crate::control::server::shared::ddl::user_dispatch; +use crate::control::server::shared::session::StatementDeadline; use crate::control::state::SharedState; use crate::data::executor::response_codec; use crate::engine::graph::edge_store::Direction; @@ -127,7 +126,7 @@ pub async fn rag_fusion( database_id, collection: &collection, plan, - timeout: Duration::from_secs(state.tuning.network.default_deadline_secs), + deadline: StatementDeadline::of_statement(state), // Admitted at this request's own transport entry; this handler has no // session or peer information of its own. admission: user_dispatch::RequestAdmission::AlreadyAdmitted, diff --git a/nodedb/src/control/server/shared/ddl/neutral/last_value.rs b/nodedb/src/control/server/shared/ddl/neutral/last_value.rs index 4583e3c94..c9416c129 100644 --- a/nodedb/src/control/server/shared/ddl/neutral/last_value.rs +++ b/nodedb/src/control/server/shared/ddl/neutral/last_value.rs @@ -18,6 +18,7 @@ use serde_json::{Map, Value as JsonValue}; use crate::bridge::envelope::PhysicalPlan; use crate::control::security::identity::AuthenticatedIdentity; use crate::control::server::response_shape::types::{DdlColType, ShapedRows}; +use crate::control::server::shared::session::StatementDeadline; use crate::control::state::SharedState; use crate::types::DatabaseId; use nodedb_physical::physical_plan::MetaOp; @@ -44,7 +45,7 @@ pub async fn query_last_values( database_id, collection, plan, - timeout: Duration::from_secs(5), + deadline: StatementDeadline::after(Duration::from_secs(5)), // Admitted at this request's own transport entry; this handler // has no session or peer information of its own. admission: crate::control::server::shared::ddl::user_dispatch::RequestAdmission::AlreadyAdmitted, @@ -116,7 +117,7 @@ pub async fn query_last_value( database_id, collection, plan, - timeout: Duration::from_secs(5), + deadline: StatementDeadline::after(Duration::from_secs(5)), // Admitted at this request's own transport entry; this handler // has no session or peer information of its own. admission: crate::control::server::shared::ddl::user_dispatch::RequestAdmission::AlreadyAdmitted, diff --git a/nodedb/src/control/server/shared/ddl/neutral/match_ops.rs b/nodedb/src/control/server/shared/ddl/neutral/match_ops.rs index ead880675..84fea4c2a 100644 --- a/nodedb/src/control/server/shared/ddl/neutral/match_ops.rs +++ b/nodedb/src/control/server/shared/ddl/neutral/match_ops.rs @@ -200,7 +200,9 @@ pub async fn match_query( // LOCAL scatter/resume leg so this node's cores merge the transaction's // staged edge overlay (read-your-own-writes); remote legs read committed CSR // (multi-node overlay forwarding is a separate unit). - let deadline_ms = crate::control::gateway::dispatcher::statement_deadline_ms(state); + let deadline_ms = + crate::control::server::shared::session::StatementDeadline::of_statement(state) + .hop_budget_ms(); match graph_dispatch::scatter_match( state, tenant_id, diff --git a/nodedb/src/control/server/shared/ddl/neutral/permission_tree.rs b/nodedb/src/control/server/shared/ddl/neutral/permission_tree.rs index c967830fd..2b9f7fa2e 100644 --- a/nodedb/src/control/server/shared/ddl/neutral/permission_tree.rs +++ b/nodedb/src/control/server/shared/ddl/neutral/permission_tree.rs @@ -150,7 +150,12 @@ async fn source_group_barrier(state: &SharedState, sources: &[TreeKey]) -> crate .await?; targets.push(nodedb_cluster::GroupCoverage { group_id, through }); } - crate::control::security::auth_lease::authorization_barrier(state, targets).await + crate::control::security::auth_lease::authorization_barrier( + state, + targets, + crate::control::server::shared::session::StatementDeadline::of_statement(state).instant(), + ) + .await } /// ALTER COLLECTION DROP PERMISSION_TREE diff --git a/nodedb/src/control/server/shared/ddl/neutral/tenant/move_tenant/arrays.rs b/nodedb/src/control/server/shared/ddl/neutral/tenant/move_tenant/arrays.rs index b377ae33a..01c3f64b2 100644 --- a/nodedb/src/control/server/shared/ddl/neutral/tenant/move_tenant/arrays.rs +++ b/nodedb/src/control/server/shared/ddl/neutral/tenant/move_tenant/arrays.rs @@ -2,7 +2,7 @@ //! The arrays of a `MOVE TENANT`. //! -//! A cell routes to its vShard by Hilbert prefix alone, so moving an array to +//! A cell routes to its vShard by tile key alone, so moving an array to //! another database moves no cell to another vShard. Only the catalog key, //! the surrogate bindings, and each core's store directory carry the //! database. The cutover rekeys all three with a `DeleteArray` at the source diff --git a/nodedb/src/control/server/shared/ddl/neutral/version_history/checkpoint.rs b/nodedb/src/control/server/shared/ddl/neutral/version_history/checkpoint.rs index c0abbaf98..0144c5097 100644 --- a/nodedb/src/control/server/shared/ddl/neutral/version_history/checkpoint.rs +++ b/nodedb/src/control/server/shared/ddl/neutral/version_history/checkpoint.rs @@ -7,8 +7,6 @@ //! DROP CHECKPOINT 'launch-ready' ON documents WHERE id = 'doc-123'; //! ``` -use std::time::Duration; - use nodedb_sql::parser::preprocess::lex::find_ascii_case_insensitive; use crate::bridge::envelope::PhysicalPlan; @@ -16,8 +14,9 @@ use crate::control::security::catalog::types::{CheckpointDoc, CheckpointRecord}; use crate::control::security::identity::AuthenticatedIdentity; use crate::control::server::shared::ddl::sql_parse::parse_ident_token; use crate::control::server::shared::ddl::sync_dispatch::{ - SystemReason, SystemTask, dispatch_system, + SystemReason, SystemTask, dispatch_system_by, }; +use crate::control::server::shared::session::StatementDeadline; use crate::control::state::SharedState; use crate::types::DatabaseId; use nodedb_physical::physical_plan::CrdtOp; @@ -42,8 +41,7 @@ pub async fn create_checkpoint( let plan = PhysicalPlan::Crdt(CrdtOp::GetVersionVector { collection: nodedb_types::QualifiedCollection::new(database_id, &collection), }); - let timeout = Duration::from_secs(state.tuning.network.default_deadline_secs); - let vv_bytes = dispatch_system( + let vv_bytes = dispatch_system_by( state, SystemTask::new( SystemReason::CatalogMaintenance, @@ -51,7 +49,7 @@ pub async fn create_checkpoint( nodedb_types::CollectionKey::from_bare(database_id, &collection), plan, ), - timeout, + StatementDeadline::of_statement(state), ) .await .map_err(|e| DdlError::from_error_in_context("dispatch", &e))?; diff --git a/nodedb/src/control/server/shared/ddl/neutral/version_history/dispatch.rs b/nodedb/src/control/server/shared/ddl/neutral/version_history/dispatch.rs index fa6387830..f18d1c361 100644 --- a/nodedb/src/control/server/shared/ddl/neutral/version_history/dispatch.rs +++ b/nodedb/src/control/server/shared/ddl/neutral/version_history/dispatch.rs @@ -10,8 +10,6 @@ //! these, so they mint a capability instead: the plan that reaches storage is //! the plan authorization approved. -use std::time::Duration; - use nodedb_physical::physical_task::{PhysicalTask, PostSetOp}; use crate::bridge::envelope::PhysicalPlan; @@ -21,6 +19,7 @@ use crate::control::server::shared::clone_write::{ }; use crate::control::server::shared::ddl::sync_dispatch::dispatch_authorized; use crate::control::server::shared::response_payload::payload_or_typed_error; +use crate::control::server::shared::session::StatementDeadline; use crate::control::state::SharedState; use crate::types::DatabaseId; @@ -64,7 +63,7 @@ pub(super) async fn dispatch_authorized_read( .await .map_err(gate_error)?; - let timeout = Duration::from_secs(state.tuning.network.default_deadline_secs); + let deadline = StatementDeadline::of_statement(state); match outcome { // The clone-read hook merged the target with its source and produced the // answer itself; the payload is flattened the same way the Data Plane's @@ -76,7 +75,7 @@ pub(super) async fn dispatch_authorized_read( // No session reaches this handler: the read takes the strong // default (see `DmlTxnCtx::linearizable_reads`). let linearizable = true; - dispatch_authorized(state, checked, collection, timeout, linearizable) + dispatch_authorized(state, checked, collection, deadline, linearizable) .await .map_err(|e| DdlError::from_error_in_context("dispatch", &e)) } diff --git a/nodedb/src/control/server/shared/ddl/neutral/version_history/restore.rs b/nodedb/src/control/server/shared/ddl/neutral/version_history/restore.rs index ae7ea0dba..9517806c6 100644 --- a/nodedb/src/control/server/shared/ddl/neutral/version_history/restore.rs +++ b/nodedb/src/control/server/shared/ddl/neutral/version_history/restore.rs @@ -2,8 +2,6 @@ //! RESTORE collection SET VERSION = 'checkpoint' WHERE id = 'doc-id' -use std::time::Duration; - #[cfg(test)] use crate::bridge::envelope::PhysicalPlan; use crate::control::crdt_post_image_policy::ExternalCrdtPostImagePolicy; @@ -11,6 +9,7 @@ use crate::control::security::audit::ArcAuditEmitter; use crate::control::security::identity::{AuthenticatedIdentity, Permission}; use crate::control::server::shared::authorization::authorize_collection; use crate::control::server::shared::ddl::sql_parse::parse_ident_token; +use crate::control::server::shared::session::StatementDeadline; use crate::control::state::SharedState; use crate::types::DatabaseId; #[cfg(test)] @@ -72,7 +71,7 @@ pub async fn restore_version( .await .map_err(|e| DdlError::from_error_in_context("surrogate assign", &e))?; - let timeout = Duration::from_secs(state.tuning.network.default_deadline_secs); + let deadline = StatementDeadline::of_statement(state); // RLS write policies are stored keyed by `db_qualified(database_id, // collection)`, so the policy is handed that same key or it silently // misses a policy on a non-default database. The admission workflow @@ -98,7 +97,7 @@ pub async fn restore_version( target_version_json: &vv_json, surrogate, peer_id: identity.user_id, - timeout, + deadline, event_source: crate::event::EventSource::User, policy: &policy, }, @@ -169,14 +168,14 @@ async fn persist_restore_delta( expected_frontier_digest: None, }); - let outcome = crate::control::crdt_admission::dispatch_crdt_apply_admitted_outcome( + let outcome = crate::control::crdt_admission::apply::dispatch_crdt_apply_admitted_outcome( state, crate::control::crdt_admission::CrdtApplyAdmissionRequest { tenant_id, database_id, collection: qualified.as_str(), plan, - timeout: Duration::from_secs(state.tuning.network.default_deadline_secs), + deadline: StatementDeadline::of_statement(state), event_source: crate::event::EventSource::User, policy: &crate::control::crdt_admission::TrustedInternalCrdtPolicy, }, @@ -431,7 +430,7 @@ mod tests { target_version_json: "{}", surrogate: Surrogate(1), peer_id: 42, - timeout: Duration::from_secs(2), + deadline: StatementDeadline::after(std::time::Duration::from_secs(2)), event_source: crate::event::EventSource::User, policy: &crate::control::crdt_admission::TrustedInternalCrdtPolicy, }, diff --git a/nodedb/src/control/server/shared/ddl/sync_dispatch/dispatch.rs b/nodedb/src/control/server/shared/ddl/sync_dispatch/dispatch.rs index 09cf73d83..a321b3632 100644 --- a/nodedb/src/control/server/shared/ddl/sync_dispatch/dispatch.rs +++ b/nodedb/src/control/server/shared/ddl/sync_dispatch/dispatch.rs @@ -7,7 +7,7 @@ use std::time::{Duration, Instant}; use crate::bridge::envelope::{PhysicalPlan, Priority, Request, Response}; use crate::control::server::shared::clone_write::CloneCheckedTask; use crate::control::server::shared::response_payload::payload_or_typed_error; -use crate::control::server::shared::session::statement_deadline; +use crate::control::server::shared::session::{StatementDeadline, statement_deadline}; use crate::control::server::shared::write_admission::plan_is_write; use crate::control::state::SharedState; use crate::types::{DatabaseId, ReadConsistency, TenantId, TraceId, VShardId}; @@ -23,13 +23,25 @@ use super::system_task::SystemTask; /// own [`crate::bridge::envelope::ErrorCode`]. Each protocol renders that code /// through its SQLSTATE or status table. Only a refusal with no code is /// `crate::Error::Internal`. +/// +/// `timeout` bounds work the server starts on its own. Work inside a statement +/// passes the statement's deadline to [`dispatch_system_by`]. pub(crate) async fn dispatch_system( state: &SharedState, task: SystemTask<'_>, timeout: Duration, +) -> crate::Result> { + dispatch_system_by(state, task, StatementDeadline::after(timeout)).await +} + +/// [`dispatch_system`] bounded by `deadline`. +pub(crate) async fn dispatch_system_by( + state: &SharedState, + task: SystemTask<'_>, + deadline: StatementDeadline, ) -> crate::Result> { let event_source = task.reason.event_source(); - let resp = dispatch_system_response_with_source(state, task, timeout, event_source).await?; + let resp = dispatch_system_response_with_source(state, task, deadline, event_source).await?; // A system task never advances the tenant's observed write-HLC. Its // `SystemReason` states that no client asked for the work, and RESTORE's @@ -49,7 +61,7 @@ pub(crate) async fn dispatch_system( pub(crate) async fn dispatch_system_response_with_source( state: &SharedState, task: SystemTask<'_>, - timeout: Duration, + deadline: StatementDeadline, event_source: crate::event::EventSource, ) -> crate::Result { let vshard_id = task.collection.vshard(); @@ -65,7 +77,7 @@ pub(crate) async fn dispatch_system_response_with_source( database_id: task.collection.database_id(), vshard_id, plan: task.plan, - timeout, + deadline, event_source, }, ) @@ -94,7 +106,7 @@ pub(crate) async fn dispatch_authorized( state: &SharedState, checked: CloneCheckedTask, collection: &str, - timeout: Duration, + deadline: StatementDeadline, linearizable: bool, ) -> crate::Result> { // The write lease lives until the dispatch returns its outcome. @@ -143,7 +155,7 @@ pub(crate) async fn dispatch_authorized( database_id: task.database_id, vshard_id, plan, - timeout, + deadline, event_source: crate::event::EventSource::User, }, ) @@ -158,7 +170,8 @@ struct PlanDispatch { database_id: DatabaseId, vshard_id: VShardId, plan: PhysicalPlan, - timeout: Duration, + /// The caller's own bound, never past the running statement's deadline. + deadline: StatementDeadline, event_source: crate::event::EventSource, } @@ -169,17 +182,17 @@ async fn dispatch_plan(state: &SharedState, dispatch: PlanDispatch) -> crate::Re database_id, vshard_id, plan, - timeout, + deadline: cap, event_source, } = dispatch; let request_id = state.next_request_id(); // Whichever comes first: the running statement's deadline, or the caller's - // own timeout. A DDL statement a session bounded with `statement_timeout` + // own bound. A DDL statement a session bounded with `statement_timeout` // stops with the statement; a background caller with no statement scope - // keeps exactly the timeout it asked for. - let deadline = statement_deadline(state.tuning.network.default_deadline_secs) - .min(Instant::now() + timeout); + // keeps exactly the bound it asked for. + let deadline = + statement_deadline(state.tuning.network.default_deadline_secs).min(cap.instant()); let request = Request { request_id, diff --git a/nodedb/src/control/server/shared/ddl/sync_dispatch/mod.rs b/nodedb/src/control/server/shared/ddl/sync_dispatch/mod.rs index 34c852c8f..2ea05d459 100644 --- a/nodedb/src/control/server/shared/ddl/sync_dispatch/mod.rs +++ b/nodedb/src/control/server/shared/ddl/sync_dispatch/mod.rs @@ -11,6 +11,6 @@ mod dispatch; mod system_task; pub(crate) use dispatch::{ - dispatch_authorized, dispatch_system, dispatch_system_response_with_source, + dispatch_authorized, dispatch_system, dispatch_system_by, dispatch_system_response_with_source, }; pub(crate) use system_task::{SystemReason, SystemTask}; diff --git a/nodedb/src/control/server/shared/ddl/user_dispatch.rs b/nodedb/src/control/server/shared/ddl/user_dispatch.rs index a507c446d..3e9475439 100644 --- a/nodedb/src/control/server/shared/ddl/user_dispatch.rs +++ b/nodedb/src/control/server/shared/ddl/user_dispatch.rs @@ -8,8 +8,6 @@ //! this instead of the system door, which exists only for work no user asked //! for. -use std::time::Duration; - use crate::bridge::envelope::PhysicalPlan; use crate::control::security::audit::ArcAuditEmitter; use crate::control::security::identity::AuthenticatedIdentity; @@ -21,6 +19,7 @@ use crate::control::server::shared::metering::{ PlanMeteringInfo, meter_dispatch, operation_for_plan, }; use crate::control::server::shared::response_payload::payload_or_typed_error; +use crate::control::server::shared::session::StatementDeadline; use crate::control::state::SharedState; use crate::types::DatabaseId; use nodedb_physical::physical_task::{PhysicalTask, PostSetOp}; @@ -62,7 +61,7 @@ pub(crate) enum RequestAdmission<'a> { /// Parameters for [`dispatch_for_identity`]. /// /// Grouped into a struct rather than passed positionally because the -/// argument count (state, identity, database, collection, plan, timeout, +/// argument count (state, identity, database, collection, plan, deadline, /// admission) exceeds what a positional call stays readable at. pub(crate) struct DispatchRequest<'a> { pub state: &'a SharedState, @@ -70,7 +69,8 @@ pub(crate) struct DispatchRequest<'a> { pub database_id: DatabaseId, pub collection: &'a str, pub plan: PhysicalPlan, - pub timeout: Duration, + /// The bound of the dispatch, never past the running statement. + pub deadline: StatementDeadline, /// Whether this request still has to pass the admission gate, and — when /// it does — the real remote address it is admitted against. pub admission: RequestAdmission<'a>, @@ -90,7 +90,7 @@ pub(crate) async fn dispatch_for_identity(req: DispatchRequest<'_>) -> crate::Re database_id, collection, plan, - timeout, + deadline, admission, linearizable, } = req; @@ -115,7 +115,7 @@ pub(crate) async fn dispatch_for_identity(req: DispatchRequest<'_>) -> crate::Re // caller cannot tell the two apart by error shape. CloneCheckedOutcome::Handled(response) => payload_or_typed_error(response), CloneCheckedOutcome::Proceed(checked) => { - dispatch_authorized(state, checked, collection, timeout, linearizable).await + dispatch_authorized(state, checked, collection, deadline, linearizable).await } }; if result.is_ok() { @@ -534,7 +534,7 @@ mod tests { database_id: DatabaseId::DEFAULT, collection: "widgets", plan: trivial_kv_get_plan(), - timeout: Duration::from_secs(5), + deadline: StatementDeadline::after(Duration::from_secs(5)), admission: RequestAdmission::AlreadyAdmitted, linearizable: false, }) @@ -570,7 +570,7 @@ mod tests { database_id: DatabaseId::DEFAULT, collection: "widgets", plan: trivial_kv_get_plan(), - timeout: Duration::from_secs(5), + deadline: StatementDeadline::after(Duration::from_secs(5)), admission: RequestAdmission::NotYetAdmitted { peer_addr: "127.0.0.1:9", }, @@ -605,7 +605,7 @@ mod tests { database_id: DatabaseId::DEFAULT, collection: "widgets", plan: trivial_kv_get_plan(), - timeout: Duration::from_secs(5), + deadline: StatementDeadline::after(Duration::from_secs(5)), admission: RequestAdmission::AlreadyAdmitted, linearizable: false, }) @@ -635,7 +635,7 @@ mod tests { database_id: DatabaseId::DEFAULT, collection: "widgets", plan: trivial_kv_get_plan(), - timeout: Duration::from_secs(5), + deadline: StatementDeadline::after(Duration::from_secs(5)), admission: RequestAdmission::AlreadyAdmitted, linearizable: false, }) diff --git a/nodedb/src/control/server/shared/retry.rs b/nodedb/src/control/server/shared/retry.rs index 9897ce4a0..afe191e0d 100644 --- a/nodedb/src/control/server/shared/retry.rs +++ b/nodedb/src/control/server/shared/retry.rs @@ -134,6 +134,8 @@ impl RetryableSchemaChange for Error { | Error::GroupMarksUnavailable { .. } | Error::BackupCaptureMoved { .. } | Error::MetadataLeaderUnavailable + | Error::MetadataGroupGone { .. } + | Error::MetadataDdlSuperseded { .. } | Error::AuthorizationStateBehind { .. } | Error::LinearizableReadRefused { .. } | Error::ExecutionLimitExceeded { .. } diff --git a/nodedb/src/control/server/shared/session/array_fanout_stage.rs b/nodedb/src/control/server/shared/session/array_fanout_stage.rs index 874d0e713..7fdab3986 100644 --- a/nodedb/src/control/server/shared/session/array_fanout_stage.rs +++ b/nodedb/src/control/server/shared/session/array_fanout_stage.rs @@ -26,6 +26,7 @@ use crate::control::gateway::RouteDecision; use crate::control::security::request_scope::RequestAuthScope; use crate::control::server::shared::metering::{PlanMeteringInfo, meter_staged_write}; use crate::control::server::shared::quota_admission::admit_quota_for_dispatch; +use crate::control::server::shared::session::StatementDeadline; use crate::control::server::shared::sql::staging_predicates::require_affected_count; use crate::control::state::SharedState; use crate::types::VShardId; @@ -122,7 +123,8 @@ where // the INNER write, so the un-wrapped plan rides alongside. remote => { let version_plan = inner.clone(); - forward_to_leader(state, remote, wrap(inner), &version_plan).await + let deadline = StatementDeadline::of_statement(state).tokio_instant(); + forward_to_leader(state, remote, wrap(inner), &version_plan, deadline).await } } } diff --git a/nodedb/src/control/server/shared/session/commit/read_validation.rs b/nodedb/src/control/server/shared/session/commit/read_validation.rs index 6397554f7..008c395b9 100644 --- a/nodedb/src/control/server/shared/session/commit/read_validation.rs +++ b/nodedb/src/control/server/shared/session/commit/read_validation.rs @@ -35,14 +35,13 @@ use std::collections::BTreeMap; use nodedb_physical::physical_plan::{HomeAnswer, HomeVersion, HomeVersionProbe, MetaOp}; use crate::bridge::envelope::PhysicalPlan; -use crate::control::gateway::dispatcher::{ - DispatchRouteParams, dispatch_route, statement_deadline_ms, -}; +use crate::control::gateway::dispatcher::{DispatchRouteParams, dispatch_route}; use crate::control::gateway::live_leaders::resolve_live_decision; use crate::control::gateway::version_set::GatewayVersionSet; use crate::control::gateway::{RouteDecision, TaskRoute}; use crate::control::server::exchange::all_cores::execute_plan_all_local_cores; use crate::control::server::graph_dispatch::cluster_resolve::gateway_shared; +use crate::control::server::shared::session::StatementDeadline; use crate::control::state::SharedState; use crate::types::{DatabaseId, TenantId, TraceId, VShardId}; use nodedb_types::WriteVersion; @@ -294,7 +293,7 @@ async fn current_versions( tenant_id, database_id, trace_id: TraceId::ZERO, - deadline_ms: statement_deadline_ms(shared), + deadline_ms: StatementDeadline::of_statement(shared).hop_budget_ms(), version_set: &GatewayVersionSet::from_pairs(Vec::new()), txn_id: None, // The leader proves its leadership itself: it answers each diff --git a/nodedb/src/control/server/shared/session/commit/run.rs b/nodedb/src/control/server/shared/session/commit/run.rs index b0b861dc2..5470310cc 100644 --- a/nodedb/src/control/server/shared/session/commit/run.rs +++ b/nodedb/src/control/server/shared/session/commit/run.rs @@ -369,9 +369,9 @@ pub async fn run_commit( // longer reach the overlay) cannot un-commit it, so it is // surfaced at ERROR and the remaining vShards are still reaped // rather than aborting a committed transaction. `drop_txn_overlay` - // already retries a transient remote failure a bounded number of - // times internally (see `retry_not_leader`); a drop that still - // fails after that budget strands a bounded, invisible (the + // already retries a transient remote failure until its cleanup + // deadline (see `retry_not_leader`); a drop that still fails + // after that budget strands a bounded, invisible (the // `txn_id` is never reused) overlay on the unreachable former // leader, cleared on that node's restart and visible meanwhile // via `active_txn_overlays`. diff --git a/nodedb/src/control/server/shared/session/commit/single_shard.rs b/nodedb/src/control/server/shared/session/commit/single_shard.rs index e61fe5d36..461652d33 100644 --- a/nodedb/src/control/server/shared/session/commit/single_shard.rs +++ b/nodedb/src/control/server/shared/session/commit/single_shard.rs @@ -207,7 +207,8 @@ async fn commit_redo( // The proposal forwards to the group leader and returns once the redo is // committed and applied on this node by the apply loop. A redo past the // entry limit travels as chunks, then its final entry. - let deadline = crate::control::wal_replication::statement_propose_deadline(state); + let deadline = crate::control::server::shared::session::StatementDeadline::of_statement(state) + .tokio_instant(); match propose_transaction_redo(state, target, payload, deadline).await { Ok((applied, _)) => Ok(super::ts_rejections::applied_rejected_by_collection( &applied, diff --git a/nodedb/src/control/server/shared/session/ddl_authorization.rs b/nodedb/src/control/server/shared/session/ddl_authorization.rs index 4be804b1f..f752cc6d9 100644 --- a/nodedb/src/control/server/shared/session/ddl_authorization.rs +++ b/nodedb/src/control/server/shared/session/ddl_authorization.rs @@ -41,6 +41,7 @@ pub(super) async fn barrier_at(state: &SharedState, log_index: u64) -> crate::Re group_id: METADATA_GROUP_ID, through: log_index, }], + super::StatementDeadline::of_statement(state).instant(), ) .await } diff --git a/nodedb/src/control/server/shared/session/ddl_compensate.rs b/nodedb/src/control/server/shared/session/ddl_compensate.rs index e698bc197..b6b1b0cc8 100644 --- a/nodedb/src/control/server/shared/session/ddl_compensate.rs +++ b/nodedb/src/control/server/shared/session/ddl_compensate.rs @@ -74,10 +74,7 @@ async fn propose_reversals( }; let log_index = propose_and_await(state, handle, &prepared).await?; if state.metadata_ddl.applied_token.load(Ordering::Acquire) != token { - return Err(crate::Error::Config { - detail: "commit compensation: DDL preparation ownership was superseded before apply" - .into(), - }); + return Err(crate::Error::MetadataDdlSuperseded { log_index }); } for entry in &stamped { verify_applied(entry, catalog)?; diff --git a/nodedb/src/control/server/shared/session/ddl_flush.rs b/nodedb/src/control/server/shared/session/ddl_flush.rs index 296a862cf..dcba93250 100644 --- a/nodedb/src/control/server/shared/session/ddl_flush.rs +++ b/nodedb/src/control/server/shared/session/ddl_flush.rs @@ -13,7 +13,7 @@ use std::sync::Arc; -use nodedb_cluster::{METADATA_GROUP_ID, MetadataEntry, PendingDdlObject, encode_entry}; +use nodedb_cluster::{MetadataEntry, PendingDdlObject, encode_entry}; use crate::control::catalog_entry::{self, CatalogEntry}; use crate::control::metadata_proposer::{DdlPrepareLease, MetadataRaftHandle}; @@ -123,7 +123,7 @@ impl PendingDdlHandle<'_> { } /// Propose `entry` to the metadata group and wait until this node's applied -/// watermark reaches its log index. +/// watermark reaches its log index, until the statement deadline. pub(super) async fn propose_and_await( state: &SharedState, handle: &dyn MetadataRaftHandle, @@ -133,22 +133,7 @@ pub(super) async fn propose_and_await( detail: format!("metadata entry encode: {e}"), })?; let log_index = handle.propose_async(raw).await?; - let watcher = state.applied_index_watcher(METADATA_GROUP_ID); - let outcome = crate::control::metadata_proposer::wait::wait_applied( - Arc::clone(&watcher), - log_index, - crate::control::metadata_proposer::DEFAULT_PROPOSE_TIMEOUT, - ) - .await?; - if !outcome.is_reached() { - return Err(crate::Error::Internal { - detail: format!( - "metadata propose timed out waiting for log index {log_index} (current: {})", - watcher.current() - ), - }); - } - Ok(log_index) + crate::control::metadata_proposer::wait::await_metadata_apply(state, log_index).await } /// The committed prior value for `entry`, encoded the same shape its @@ -307,22 +292,11 @@ async fn reserve_buffered( // The propose reserves only while `token` owns the lease. A lease the // metadata leader reclaimed meanwhile left no record. if !state.pending_ddl.contains(token) { - return Err(superseded(token, "reserved")); + return Err(crate::Error::MetadataDdlSuperseded { log_index }); } Ok((log_index, objects)) } -/// The error for a pending DDL whose preparation lease the metadata leader -/// reclaimed before its `step` applied. -fn superseded(token: u64, step: &str) -> crate::Error { - crate::Error::Config { - detail: format!( - "metadata DDL preparation ownership of token {token} was reclaimed before the \ - buffered DDL was {step}; nothing was applied" - ), - } -} - /// Commit the objects `handle` reserved: propose `DdlPendingFinalize`, wait /// for local apply, then release the preparation lease `handle` has held /// since [`begin_commit`] constructed it via `propose_pending_buffered`. @@ -378,7 +352,7 @@ async fn finalize_reserved( .load(std::sync::atomic::Ordering::Acquire) != token { - return Err(superseded(token, "finalized")); + return Err(crate::Error::MetadataDdlSuperseded { log_index }); } Ok((log_index, bears_authorization)) } diff --git a/nodedb/src/control/server/shared/session/deadline.rs b/nodedb/src/control/server/shared/session/deadline.rs index cbe6447db..56e290646 100644 --- a/nodedb/src/control/server/shared/session/deadline.rs +++ b/nodedb/src/control/server/shared/session/deadline.rs @@ -89,15 +89,84 @@ pub fn statement_deadline(default_deadline_secs: u64) -> Instant { } /// Milliseconds left on the current statement, for the remote-hop -/// `ExecuteRequest.deadline_remaining_ms` field. -/// -/// Saturates at 1: a hop dispatched with `0` is refused up front by the -/// receiver's deadline check, which will report a deadline the local half has -/// not actually reached yet. +/// `ExecuteRequest.deadline_remaining_ms` field. See [`hop_budget_ms`]. pub fn statement_deadline_ms(default_deadline_secs: u64) -> u64 { - let remaining = - statement_deadline(default_deadline_secs).saturating_duration_since(Instant::now()); - (remaining.as_millis() as u64).max(1) + hop_budget_ms(statement_deadline(default_deadline_secs)) +} + +/// The budget a remote hop carries for `deadline`: the milliseconds left now, +/// at least 1. +/// +/// A receiver refuses a hop sent with `0` before it checks anything else. It +/// reports a deadline the local half has not reached yet. +pub fn hop_budget_ms(deadline: Instant) -> u64 { + let remaining = deadline.saturating_duration_since(Instant::now()); + u64::try_from(remaining.as_millis()) + .unwrap_or(u64::MAX) + .max(1) +} + +/// The deadline of the statement running on this task, as a value. +/// +/// The one place a caller with `SharedState` reads the statement deadline: +/// the installed deadline, or now plus `tuning.network.default_deadline_secs` +/// outside a statement. Waits, retry loops and proposals that share the +/// statement's budget all take it from here. +#[derive(Debug, Clone, Copy, PartialEq, Eq, PartialOrd, Ord)] +pub struct StatementDeadline(Instant); + +impl StatementDeadline { + /// The running statement's deadline, or the node default outside one. + pub fn of_statement(state: &crate::control::state::SharedState) -> Self { + Self::with_default(state.tuning.network.default_deadline_secs) + } + + /// The running statement's deadline, or `default_deadline_secs` from now + /// outside one. + pub fn with_default(default_deadline_secs: u64) -> Self { + Self(statement_deadline(default_deadline_secs)) + } + + /// A deadline at `instant`. + pub fn at(instant: Instant) -> Self { + Self(instant) + } + + /// A deadline `budget` from now. + pub fn after(budget: Duration) -> Self { + Self(Instant::now() + budget) + } + + /// The deadline as a std instant. + pub fn instant(self) -> Instant { + self.0 + } + + /// The deadline as a Tokio instant, for `tokio::time` waits. + pub fn tokio_instant(self) -> tokio::time::Instant { + tokio::time::Instant::from_std(self.0) + } + + /// The milliseconds a remote hop carries for this deadline. See + /// [`hop_budget_ms`]. + pub fn hop_budget_ms(self) -> u64 { + hop_budget_ms(self.0) + } + + /// Time left until the deadline, zero once it passed. + pub fn remaining(self) -> Duration { + self.0.saturating_duration_since(Instant::now()) + } +} + +/// Install a fresh node-default deadline for cleanup work, such as an overlay +/// drop. +/// +/// Cleanup often runs after its statement ran out of time. Under that dead +/// deadline every request it sends is refused before it runs. Dropping the +/// guard restores the statement's deadline. +pub fn enter_cleanup(default_deadline_secs: u64) -> StatementScope { + enter(None, default_deadline_secs) } fn store(deadline: Option) { @@ -179,6 +248,33 @@ mod tests { .await; } + #[test] + fn a_spent_deadline_still_sends_a_one_millisecond_hop() { + assert_eq!(hop_budget_ms(Instant::now()), 1); + let live = hop_budget_ms(Instant::now() + Duration::from_secs(10)); + assert!(live > 9_000 && live <= 10_000, "budget {live} ms"); + } + + /// Cleanup after a timed-out statement runs on the node default, and the + /// statement's deadline returns once cleanup ends. + #[tokio::test] + async fn cleanup_gets_a_fresh_deadline_after_its_statement_timed_out() { + conn_scope::scoped(async { + let _statement = enter(Some(Duration::from_millis(1)), 30); + let dead = current().expect("the statement installs a deadline"); + tokio::time::sleep(Duration::from_millis(5)).await; + assert!(dead <= Instant::now()); + { + let _cleanup = enter_cleanup(30); + let fresh = statement_deadline(30); + assert!(fresh >= Instant::now() + Duration::from_secs(29)); + assert!(statement_deadline_ms(30) > 29_000); + } + assert_eq!(current(), Some(dead)); + }) + .await; + } + #[tokio::test] async fn outside_a_connection_scope_the_node_default_applies() { assert!(current().is_none()); diff --git a/nodedb/src/control/server/shared/session/leader_forward.rs b/nodedb/src/control/server/shared/session/leader_forward.rs index 843a3c180..3a522e335 100644 --- a/nodedb/src/control/server/shared/session/leader_forward.rs +++ b/nodedb/src/control/server/shared/session/leader_forward.rs @@ -34,12 +34,11 @@ //! on a non-leader. use crate::bridge::envelope::{Payload, PhysicalPlan, Response}; -use crate::control::gateway::dispatcher::{ - DispatchRouteParams, dispatch_route, statement_deadline_ms, -}; +use crate::control::gateway::dispatcher::{DispatchRouteParams, dispatch_route}; use crate::control::gateway::live_leaders::resolve_live_decision; use crate::control::gateway::{GatewayVersionSet, RouteDecision, TaskRoute}; use crate::control::server::graph_dispatch::cluster_resolve::gateway_shared; +use crate::control::server::shared::session::deadline::hop_budget_ms; use crate::control::server::shared::write_admission::bare_ok_response; use crate::control::state::SharedState; use crate::types::{DatabaseId, RequestId, TenantId, TraceId}; @@ -68,11 +67,15 @@ pub(crate) fn resolve_leader(task: &PhysicalTask, state: &SharedState) -> RouteD /// write, so the forwarded stage carries the same descriptor versions a normal /// remote write does and is not spuriously rejected; a `DropTxnOverlay` touches /// no user collection, so its version set is empty. +/// +/// `deadline` is the statement's deadline. The forward carries what is left +/// of it. pub(crate) async fn forward_to_leader( state: &SharedState, decision: RouteDecision, forward_task: PhysicalTask, version_plan: &PhysicalPlan, + deadline: tokio::time::Instant, ) -> crate::Result { // The remote dispatch primitive needs an owned `Arc`; upgrade // the gateway's back-reference (loud typed error if the gateway is not wired @@ -99,7 +102,7 @@ pub(crate) async fn forward_to_leader( tenant_id: forward_task.tenant_id, database_id: forward_task.database_id, trace_id: TraceId::ZERO, - deadline_ms: statement_deadline_ms(&shared), + deadline_ms: hop_budget_ms(deadline.into_std()), version_set: &version_set, // The leader's session-less staging handler reads THIS to key the // per-transaction overlay; forwarding it is the whole point. diff --git a/nodedb/src/control/server/shared/session/mod.rs b/nodedb/src/control/server/shared/session/mod.rs index 45b8465a4..9881f09ab 100644 --- a/nodedb/src/control/server/shared/session/mod.rs +++ b/nodedb/src/control/server/shared/session/mod.rs @@ -54,7 +54,9 @@ pub use self::connection::{ ConnectionId, ConnectionIdError, ConnectionMetadata, ConnectionRegistrationError, SessionId, }; pub use self::cross_shard_mode::{CrossShardTxnMode, parse_value as parse_cross_shard_value}; -pub use self::deadline::{statement_budget, statement_deadline, statement_deadline_ms}; +pub use self::deadline::{ + StatementDeadline, statement_budget, statement_deadline, statement_deadline_ms, +}; pub use self::outcome::{AbortReason, CommitOutcome, TxnDataPlane}; pub use self::params::{parse_set_command, parse_show_command}; pub use self::read_set::{ diff --git a/nodedb/src/control/server/shared/session/overlay_drop.rs b/nodedb/src/control/server/shared/session/overlay_drop.rs index b65914c27..9427a77e6 100644 --- a/nodedb/src/control/server/shared/session/overlay_drop.rs +++ b/nodedb/src/control/server/shared/session/overlay_drop.rs @@ -7,14 +7,18 @@ use nodedb_physical::physical_plan::meta::MetaOp; use nodedb_physical::physical_task::{PhysicalTask, PostSetOp}; use crate::control::gateway::RouteDecision; -use crate::control::gateway::retry::retry_not_leader; +use crate::control::gateway::retry::{RetryPacing, retry_not_leader}; use crate::control::state::SharedState; +use super::deadline::{self, StatementDeadline}; use super::leader_forward::{forward_to_leader, resolve_leader}; use super::outcome::TxnDataPlane; use crate::control::server::graph_dispatch::cluster_resolve::gateway_shared; /// Release a transaction's staging overlay on the vShard's current leader. +/// +/// The drop runs on its own node-default deadline. Its statement can have +/// timed out, and that dead deadline refuses every send before it runs. pub(super) async fn drop_txn_overlay( state: &SharedState, dp: &impl TxnDataPlane, @@ -22,6 +26,7 @@ pub(super) async fn drop_txn_overlay( vshard_id: crate::types::VShardId, txn_id: crate::types::TxnId, ) -> crate::Result<()> { + let _cleanup = deadline::enter_cleanup(state.tuning.network.default_deadline_secs); let drop_plan = PhysicalPlan::Meta(MetaOp::DropTxnOverlay { txn_id }); let task = PhysicalTask { tenant_id, @@ -39,13 +44,19 @@ pub(super) async fn drop_txn_overlay( _ => { let shared = gateway_shared(state)?; let routing_ref = shared.cluster_routing.as_deref(); - retry_not_leader(routing_ref, |_attempt| { + // Reads the cleanup deadline installed above. + let cleanup_deadline = StatementDeadline::of_statement(&shared).tokio_instant(); + let pacing = RetryPacing::from_tuning(&shared.tuning.cluster_transport); + retry_not_leader(routing_ref, cleanup_deadline, pacing, |attempt| { let task = task.clone(); let drop_plan = drop_plan.clone(); async move { match resolve_leader(&task, state) { RouteDecision::Local => dp.dispatch_no_wal(task).await, - remote => forward_to_leader(state, remote, task, &drop_plan).await, + remote => { + forward_to_leader(state, remote, task, &drop_plan, attempt.deadline) + .await + } } } }) @@ -54,3 +65,91 @@ pub(super) async fn drop_txn_overlay( } } } + +#[cfg(test)] +mod tests { + use std::future::Future; + use std::pin::Pin; + use std::sync::Mutex; + use std::time::{Duration, Instant}; + + use super::*; + use crate::bridge::dispatch::Dispatcher; + use crate::bridge::envelope::{Payload, Response, Status}; + use crate::control::server::shared::session::conn_scope; + use crate::types::{Lsn, RequestId, TenantId, TxnId, VShardId}; + use crate::wal::WalManager; + + /// Records the statement deadline each dispatch runs under. + #[derive(Default)] + struct DeadlineDp { + seen: Mutex>>, + } + + impl TxnDataPlane for DeadlineDp { + fn dispatch_no_wal<'a>( + &'a self, + _task: PhysicalTask, + ) -> Pin> + Send + 'a>> { + self.seen.lock().unwrap().push(deadline::current()); + Box::pin(async move { + Ok(Response { + request_id: RequestId::new(1), + status: Status::Ok, + attempt: 1, + partial: false, + payload: Payload::empty(), + watermark_lsn: Lsn::ZERO, + error_code: None, + stage_vote: None, + read_versions: crate::types::ReadVersions::new(), + write_set: Vec::new(), + }) + }) + } + + fn event_source(&self) -> crate::event::EventSource { + crate::event::EventSource::User + } + } + + /// A drop after its statement timed out runs on a fresh node-default + /// deadline, and the statement's deadline returns afterwards. + #[tokio::test] + async fn an_overlay_drop_outlives_its_timed_out_statement() { + let dir = tempfile::tempdir().unwrap(); + let wal = std::sync::Arc::new( + WalManager::open_for_testing(&dir.path().join("test.wal")).unwrap(), + ); + let (dispatcher, _data_sides) = Dispatcher::new(1, 64); + let state = SharedState::new(dispatcher, wal).unwrap(); + let default_secs = state.tuning.network.default_deadline_secs; + let dp = DeadlineDp::default(); + + conn_scope::scoped(async { + let _statement = deadline::enter(Some(Duration::from_millis(1)), default_secs); + let dead = deadline::current().expect("the statement installs a deadline"); + tokio::time::sleep(Duration::from_millis(5)).await; + + drop_txn_overlay( + &state, + &dp, + TenantId::new(1), + VShardId::new(3), + TxnId::new(7), + ) + .await + .unwrap(); + + let seen = dp.seen.lock().unwrap(); + assert_eq!(seen.len(), 1); + let cleanup = seen[0].expect("the drop runs under a deadline"); + assert!( + cleanup > Instant::now() + Duration::from_secs(default_secs.saturating_sub(1)), + "the drop ran under the statement's dead deadline" + ); + assert_eq!(deadline::current(), Some(dead)); + }) + .await; + } +} diff --git a/nodedb/src/control/server/shared/session/staging_gate.rs b/nodedb/src/control/server/shared/session/staging_gate.rs index 301f01784..70a8a0005 100644 --- a/nodedb/src/control/server/shared/session/staging_gate.rs +++ b/nodedb/src/control/server/shared/session/staging_gate.rs @@ -28,6 +28,7 @@ use crate::control::gateway::RouteDecision; use crate::control::security::request_scope::RequestAuthScope; use crate::control::server::shared::metering::{PlanMeteringInfo, meter_staged_write}; use crate::control::server::shared::quota_admission::admit_quota_for_dispatch; +use crate::control::server::shared::session::StatementDeadline; use crate::control::server::shared::sql::staging_predicates::{ is_stageable_write, require_affected_count, stageable_write_shape, stages_returning, }; @@ -371,9 +372,15 @@ where RouteDecision::Local => dispatch(stage_task) .await .map_err(StagingGateError::Dispatch)?, - remote => forward_to_leader(state, remote, stage_task, &task.plan) - .await - .map_err(StagingGateError::Dispatch)?, + remote => forward_to_leader( + state, + remote, + stage_task, + &task.plan, + StatementDeadline::of_statement(state).tokio_instant(), + ) + .await + .map_err(StagingGateError::Dispatch)?, }; if resp.status == Status::Error { diff --git a/nodedb/src/control/server/shared/write_admission/admission_keys.rs b/nodedb/src/control/server/shared/write_admission/admission_keys.rs index 469be6e1f..d48c7e299 100644 --- a/nodedb/src/control/server/shared/write_admission/admission_keys.rs +++ b/nodedb/src/control/server/shared/write_admission/admission_keys.rs @@ -20,9 +20,8 @@ //! alike on every replica. //! - Every Calvin writer of a columnar or timeseries collection locks it //! `Exclusive`. So an append still orders against each Calvin writer. -//! - A contended append waits and never routes. A routed write applies -//! outside the data-group log. Each replica then installs it against its -//! own schema at its own point. +//! - A contended append waits and never routes. The data-group log already +//! orders appends, so routing one to Calvin gains nothing. //! //! A write no Calvin transaction sequences still takes keys: //! @@ -354,6 +353,27 @@ mod tests { } } + /// A held TRUNCATE locks its collection `Exclusive`. A fast-path point + /// write announces itself `Intent` on that collection, so it cannot take + /// its keys while the TRUNCATE holds. + #[test] + fn a_held_truncate_fences_a_point_write() { + let mut table = LockManager::new(); + let truncate = PhysicalPlan::Document(DocumentOp::Truncate { + collection: qualified("docs"), + restart_identity: false, + resolved_sum_targets: Vec::new(), + declared_primary_key: None, + }); + let (truncate_keys, _) = keys_of(&truncate); + assert!(table.try_acquire(TxnId::new(1, 0), truncate_keys)); + let (point_keys, _) = keys_of(&doc_insert("x", 17)); + assert!( + !table.try_acquire(TxnId::new(TxnId::AUTOCOMMIT_EPOCH, 0), point_keys), + "the point write must wait for the truncate" + ); + } + /// A resolved write no Calvin transaction sequences still locks the rows /// it names, and waits rather than routes when contended. #[test] diff --git a/nodedb/src/control/server/shared/write_admission/leader_gate.rs b/nodedb/src/control/server/shared/write_admission/leader_gate.rs index 27a8c5c3a..a90dd4ea3 100644 --- a/nodedb/src/control/server/shared/write_admission/leader_gate.rs +++ b/nodedb/src/control/server/shared/write_admission/leader_gate.rs @@ -280,8 +280,6 @@ fn entry_scope(write: &ReplicatedWrite, database_id: crate::types::DatabaseId) - | ReplicatedWrite::ColumnarTruncate { .. } | ReplicatedWrite::TimeseriesTruncate { .. } | ReplicatedWrite::KvResolvedWrite { .. } - | ReplicatedWrite::KvPredicateUpdate { .. } - | ReplicatedWrite::KvPredicateDelete { .. } | ReplicatedWrite::DocumentResolvedWrite { .. } | ReplicatedWrite::VectorDirectDelete { .. } | ReplicatedWrite::VectorDirectTruncate { .. } diff --git a/nodedb/src/control/server/shared/write_admission/predicate/txn_buffering/classify.rs b/nodedb/src/control/server/shared/write_admission/predicate/txn_buffering/classify.rs index 2a878ca9e..0476aefbf 100644 --- a/nodedb/src/control/server/shared/write_admission/predicate/txn_buffering/classify.rs +++ b/nodedb/src/control/server/shared/write_admission/predicate/txn_buffering/classify.rs @@ -466,11 +466,42 @@ mod tests { ) } + /// A write whose rows a predicate names. It resolves to rows before it is + /// logged, so the log encoder refuses its raw form. + fn resolves_before_log(plan: &PhysicalPlan) -> bool { + matches!( + plan, + PhysicalPlan::Kv(KvOp::PredicateUpdate { .. } | KvOp::PredicateDelete { .. }) + | PhysicalPlan::Vector( + VectorOp::DirectDelete { + targets: nodedb_physical::physical_plan::VectorWriteTargets::Predicate(_), + .. + } | VectorOp::DirectUpdate { + targets: nodedb_physical::physical_plan::VectorWriteTargets::Predicate(_), + .. + } + ) + ) + } + + /// The classifier agrees with the encoder. A predicate write is refused + /// at the log boundary with a typed error and still buffers in a + /// transaction, where staging resolves it. fn assert_matches_oracle(plan: &PhysicalPlan) { - let expected = to_replicated_entry(tenant(), db(), vshard(), plan) + let encoded = to_replicated_entry(tenant(), db(), vshard(), plan); + let actual = plan_requires_txn_buffering(plan); + if resolves_before_log(plan) { + assert!( + matches!(encoded, Err(crate::Error::Internal { .. })), + "expected {plan:?} to be refused at the log boundary, got {:?}", + encoded.as_ref().err() + ); + assert!(actual, "expected {plan:?} to require txn buffering"); + return; + } + let expected = encoded .expect("encode must not error for this fixture") .is_some(); - let actual = plan_requires_txn_buffering(plan); assert_eq!( actual, expected, "plan_requires_txn_buffering disagrees with to_replicated_entry for {plan:?}" @@ -2001,7 +2032,7 @@ mod tests { attr_projection: Vec::new(), limit: 0, cell_filter: None, - hilbert_range: None, + shard_bucket: None, system_time: SystemTimeScope::Current, valid_at_ms: None, }), @@ -2016,7 +2047,7 @@ mod tests { group_by_dim: -1, cell_filter: None, return_partial: false, - hilbert_range: None, + shard_bucket: None, system_as_of: None, valid_at_ms: None, }), @@ -2063,7 +2094,7 @@ mod tests { slice_msgpack: Vec::new(), attr_projection: Vec::new(), limit: 0, - slice_hilbert_ranges: Vec::new(), + slice_tile_keys: Vec::new(), prefix_bits: 0, system_time: SystemTimeScope::Current, valid_at_ms: None, @@ -2073,7 +2104,7 @@ mod tests { attr_idx: 0, reducer_msgpack: Vec::new(), group_by_dim: -1, - slice_hilbert_ranges: Vec::new(), + slice_tile_keys: Vec::new(), prefix_bits: 0, system_as_of: None, valid_at_ms: None, diff --git a/nodedb/src/control/server/surrogate_exchange/authority.rs b/nodedb/src/control/server/surrogate_exchange/authority.rs index fc0420d10..d162eddbe 100644 --- a/nodedb/src/control/server/surrogate_exchange/authority.rs +++ b/nodedb/src/control/server/surrogate_exchange/authority.rs @@ -78,7 +78,9 @@ pub(crate) async fn assign_many_at_home( home.as_u32(), ReplicatedWrite::SurrogateBind { identities }, ); - let deadline = crate::control::wal_replication::statement_propose_deadline(state); + let deadline = + crate::control::server::shared::session::StatementDeadline::of_statement(state) + .tokio_instant(); propose_replicated_entry(state, proposer, entry, deadline).await?; } pks.iter() diff --git a/nodedb/src/control/server/sync/async_dispatch/delta/apply.rs b/nodedb/src/control/server/sync/async_dispatch/delta/apply.rs index aa88a8616..f39d15561 100644 --- a/nodedb/src/control/server/sync/async_dispatch/delta/apply.rs +++ b/nodedb/src/control/server/sync/async_dispatch/delta/apply.rs @@ -3,8 +3,6 @@ //! Dispatch a CRDT delta to the Data Plane and hand its outcome to the frame //! builder. -use std::time::Duration; - use tracing::warn; use nodedb_types::sync::wire::{EngineKind, SyncProvenance, stream_id_for}; @@ -12,6 +10,7 @@ use nodedb_types::sync::wire::{EngineKind, SyncProvenance, stream_id_for}; use crate::control::security::audit::ArcAuditEmitter; use crate::control::security::identity::AuthenticatedIdentity; use crate::control::server::shared::metering::{PlanMeteringInfo, meter_dispatch}; +use crate::control::server::shared::session::StatementDeadline; use crate::control::state::SharedState; use crate::types::DatabaseId; @@ -81,6 +80,8 @@ pub(crate) async fn apply_delta_and_finalize( epoch: session_epoch, peer_addr, } = session; + // Pinned once: preview, propose and apply all end at this instant. + let deadline = StatementDeadline::of_statement(shared); use crate::bridge::envelope::PhysicalPlan; use nodedb_physical::physical_plan::CrdtOp; @@ -354,7 +355,7 @@ pub(crate) async fn apply_delta_and_finalize( shared, &delta_msg.collection, authorized, - Duration::from_secs(10), + deadline, crate::event::EventSource::CrdtSync, &policy, ) diff --git a/nodedb/src/control/server/sync/async_dispatch/delta/compensation.rs b/nodedb/src/control/server/sync/async_dispatch/delta/compensation.rs index 6029411a7..4ac641779 100644 --- a/nodedb/src/control/server/sync/async_dispatch/delta/compensation.rs +++ b/nodedb/src/control/server/sync/async_dispatch/delta/compensation.rs @@ -54,6 +54,7 @@ pub(super) fn compensation_hint_for_dispatch_error(e: &crate::Error) -> Compensa | crate::Error::GroupMarksUnavailable { .. } | crate::Error::BackupCaptureMoved { .. } | crate::Error::MetadataLeaderUnavailable + | crate::Error::MetadataDdlSuperseded { .. } | crate::Error::AuthorizationStateBehind { .. } | crate::Error::LinearizableReadRefused { .. } | crate::Error::DispatchCapacity { .. } @@ -69,7 +70,8 @@ pub(super) fn compensation_hint_for_dispatch_error(e: &crate::Error) -> Compensa }, // The write committed or was overwritten, and nothing here says which. // Never re-pushed: a committed write will apply twice. - other @ crate::Error::ProposalOutcomeUnknown { .. } => CompensationHint::Custom { + other @ (crate::Error::ProposalOutcomeUnknown { .. } + | crate::Error::MetadataGroupGone { .. }) => CompensationHint::Custom { constraint: "proposal_outcome_unknown".into(), detail: other.to_string(), }, diff --git a/nodedb/src/control/server/sync/async_dispatch/shape/snapshot.rs b/nodedb/src/control/server/sync/async_dispatch/shape/snapshot.rs index e47060626..d594d6b32 100644 --- a/nodedb/src/control/server/sync/async_dispatch/shape/snapshot.rs +++ b/nodedb/src/control/server/sync/async_dispatch/shape/snapshot.rs @@ -2,8 +2,6 @@ //! Shape snapshot production: plan construction, RLS, and predicate filtering. -use std::time::Duration; - use tracing::{info, warn}; use nodedb_types::sync::shape::{ShapeDefinition, ShapeType}; @@ -185,7 +183,9 @@ async fn document_snapshot(req: DocumentSnapshot<'_>) -> Option crate::Result> { let RecordOwner { tenant_id, @@ -63,7 +65,13 @@ pub(crate) async fn propose_sync_plan( ), }, )?; - propose_sync_write(state, entry.with_event_source(event_source), proposer).await + propose_sync_write( + state, + entry.with_event_source(event_source), + proposer, + deadline, + ) + .await } /// Propose a `ReplicatedEntry` through Raft and block until the entry is @@ -84,6 +92,7 @@ pub(crate) async fn propose_sync_write( state: &SharedState, mut entry: ReplicatedEntry, proposer: &Arc, + deadline: StatementDeadline, ) -> crate::Result> { if let Some(response) = crate::control::planner::calvin::sequence_replicated_edge_write(state, &entry).await? @@ -100,8 +109,7 @@ pub(crate) async fn propose_sync_write( let idempotency_key = entry.idempotency_key; let data = entry.encode()?; let vshard_id = entry.vshard_id; - let deadline = tokio::time::Instant::now() - + Duration::from_secs(state.tuning.network.default_deadline_secs); + let deadline = deadline.tokio_instant(); const BACKOFF_MS: [u64; 5] = [10, 25, 50, 100, 200]; let mut payload: Option> = None; diff --git a/nodedb/src/control/server/sync/raft_dispatch/response.rs b/nodedb/src/control/server/sync/raft_dispatch/response.rs index 8be27caf8..1bb90b5d7 100644 --- a/nodedb/src/control/server/sync/raft_dispatch/response.rs +++ b/nodedb/src/control/server/sync/raft_dispatch/response.rs @@ -8,6 +8,7 @@ use crate::bridge::envelope::{PhysicalPlan, Response, Status}; use crate::control::server::dispatch_utils::RecordOwner; use crate::control::server::shared::authorization::AuthorizedTask; +use crate::control::server::shared::session::StatementDeadline; use crate::control::state::SharedState; use crate::event::EventSource; use crate::types::{DatabaseId, Lsn, TenantId, VShardId}; @@ -58,7 +59,14 @@ async fn sync_write( event_source: EventSource, ) -> crate::Result> { reject_unadmitted_crdt_apply(plan)?; - propose_sync_plan(state, owner, plan, event_source).await + propose_sync_plan( + state, + owner, + plan, + event_source, + StatementDeadline::of_statement(state), + ) + .await } /// The applied payload wrapped in `Status::Ok`. A non-`Ok` status means a diff --git a/nodedb/src/control/server/sync/raft_dispatch/write.rs b/nodedb/src/control/server/sync/raft_dispatch/write.rs index 000b91b5a..f410ca4da 100644 --- a/nodedb/src/control/server/sync/raft_dispatch/write.rs +++ b/nodedb/src/control/server/sync/raft_dispatch/write.rs @@ -2,11 +2,10 @@ //! Sync dispatch that returns raw payload bytes, used by the CRDT delta path. -use std::time::Duration; - use crate::bridge::envelope::PhysicalPlan; use crate::control::server::dispatch_utils::RecordOwner; use crate::control::server::shared::authorization::AuthorizedTask; +use crate::control::server::shared::session::StatementDeadline; use crate::control::state::SharedState; use crate::event::EventSource; @@ -21,7 +20,7 @@ pub async fn dispatch_sync_bytes( state: &SharedState, collection: &str, authorized: AuthorizedTask, - timeout: Duration, + deadline: StatementDeadline, event_source: EventSource, policy: &dyn crate::control::crdt_admission::CrdtPostImagePolicy, ) -> crate::Result { @@ -39,7 +38,7 @@ pub async fn dispatch_sync_bytes( crate::control::crdt_admission::AuthorizedCrdtApplyAdmissionRequest { authorized, collection, - timeout, + deadline, event_source, policy, }, @@ -51,7 +50,7 @@ pub async fn dispatch_sync_bytes( }); } // The Raft entry's apply owns durability. - dispatch_write_replicated(state, collection, authorized, event_source) + dispatch_write_replicated(state, collection, authorized, deadline, event_source) .await .map(SyncDispatchOutcome::untrimmed) } @@ -63,6 +62,7 @@ pub(crate) async fn dispatch_write_replicated( state: &SharedState, collection: &str, authorized: AuthorizedTask, + deadline: StatementDeadline, event_source: EventSource, ) -> crate::Result> { let task = authorized.into_physical_task(); @@ -91,7 +91,7 @@ pub(crate) async fn dispatch_write_replicated( ) .await?; // The apply's write funnel records the write's mark on every replica. - propose_sync_plan(state, owner, &plan, event_source).await + propose_sync_plan(state, owner, &plan, event_source, deadline).await } #[cfg(test)] @@ -100,6 +100,7 @@ mod tests { COLLECTION, applying_proposer, authorized_write, fixture, }; use super::dispatch_write_replicated; + use crate::control::server::shared::session::StatementDeadline; use crate::event::EventSource; /// A proposed write answers with the applied entry's payload. @@ -113,10 +114,15 @@ mod tests { .expect("install proposer"); let authorized = authorized_write(&state); - let payload = - dispatch_write_replicated(&state, COLLECTION, authorized, EventSource::CrdtSync) - .await - .expect("the proposal applies"); + let payload = dispatch_write_replicated( + &state, + COLLECTION, + authorized, + StatementDeadline::of_statement(&state), + EventSource::CrdtSync, + ) + .await + .expect("the proposal applies"); assert_eq!(payload, b"applied".to_vec()); } @@ -128,8 +134,14 @@ mod tests { let (state, _side, _directory) = fixture(); let authorized = authorized_write(&state); - let result = - dispatch_write_replicated(&state, COLLECTION, authorized, EventSource::CrdtSync).await; + let result = dispatch_write_replicated( + &state, + COLLECTION, + authorized, + StatementDeadline::of_statement(&state), + EventSource::CrdtSync, + ) + .await; assert!( matches!(result, Err(crate::Error::Internal { .. })), diff --git a/nodedb/src/control/server/sync/refusal.rs b/nodedb/src/control/server/sync/refusal.rs index e97ccf6d8..593148db5 100644 --- a/nodedb/src/control/server/sync/refusal.rs +++ b/nodedb/src/control/server/sync/refusal.rs @@ -146,6 +146,8 @@ pub(super) fn retryable_refusal_reason(error: &crate::Error) -> Option<&str> { | crate::Error::GroupMarksUnavailable { .. } | crate::Error::BackupCaptureMoved { .. } | crate::Error::MetadataLeaderUnavailable + | crate::Error::MetadataGroupGone { .. } + | crate::Error::MetadataDdlSuperseded { .. } | crate::Error::AuthorizationStateBehind { .. } | crate::Error::LinearizableReadRefused { .. } | crate::Error::ExecutionLimitExceeded { .. } @@ -234,6 +236,7 @@ fn is_indeterminate(error: &crate::Error) -> bool { | crate::Error::GroupMarksUnavailable { .. } | crate::Error::BackupCaptureMoved { .. } | crate::Error::MetadataLeaderUnavailable + | crate::Error::MetadataDdlSuperseded { .. } | crate::Error::AuthorizationStateBehind { .. } | crate::Error::LinearizableReadRefused { .. } // Shed by a resource or rate gate: the class `53`. @@ -248,7 +251,8 @@ fn is_indeterminate(error: &crate::Error) -> bool { }, // The write committed: pushing the same bytes again applies it twice. crate::Error::CommittedResultUnavailable { .. } - | crate::Error::ProposalOutcomeUnknown { .. } => false, + | crate::Error::ProposalOutcomeUnknown { .. } + | crate::Error::MetadataGroupGone { .. } => false, // Refused on its merits, or a fault the same bytes reproduce. crate::Error::RejectedConstraint { .. } | crate::Error::TxnOverlayMemoryExceeded { .. } diff --git a/nodedb/src/control/server/sync/timeseries_handler.rs b/nodedb/src/control/server/sync/timeseries_handler.rs index 5f0feccdc..58db02a8e 100644 --- a/nodedb/src/control/server/sync/timeseries_handler.rs +++ b/nodedb/src/control/server/sync/timeseries_handler.rs @@ -138,6 +138,7 @@ impl<'a> TimeseriesDispatcher for SharedStateTimeseriesDispatcher<'a> { self.shared, &collection, authorized, + crate::control::server::shared::session::StatementDeadline::of_statement(self.shared), crate::event::EventSource::CrdtSync, ) .await?; diff --git a/nodedb/src/control/server/wal_dispatch/mod.rs b/nodedb/src/control/server/wal_dispatch/mod.rs index fb8b498e8..e57251fc3 100644 --- a/nodedb/src/control/server/wal_dispatch/mod.rs +++ b/nodedb/src/control/server/wal_dispatch/mod.rs @@ -37,6 +37,8 @@ pub(crate) use text::encode_text_op_record; pub(crate) use timeseries::{ TimeseriesIngestRecord, encode_columnar_truncate_payload, encode_timeseries_ingest_payload, }; +#[cfg(test)] +pub(crate) use vector::encode_vector_direct_delete_payload; pub(crate) use vector::{ VectorDirectDeleteRecord, VectorDirectTruncateRecord, VectorDirectUpdateRecord, VectorDirectUpsertRecord, VectorResolvedDirectWritePayload, VectorResolvedDirectWriteRecord, @@ -46,6 +48,7 @@ pub(crate) use vector::{ encode_vector_delete_payload, encode_vector_direct_truncate_payload, encode_vector_put_payload, encode_vector_resolved_direct_write_payload, }; +pub(crate) use vector::{check_logged_patch, logged_rows}; pub(crate) use super::wal_dispatch_fts_spatial::{ encode_spatial_delete_payload, encode_spatial_put_payload, diff --git a/nodedb/src/control/server/wal_dispatch/vector/append.rs b/nodedb/src/control/server/wal_dispatch/vector/append.rs index 46487dcfd..304d76059 100644 --- a/nodedb/src/control/server/wal_dispatch/vector/append.rs +++ b/nodedb/src/control/server/wal_dispatch/vector/append.rs @@ -231,7 +231,8 @@ pub(crate) fn wal_append_vector_op( rls_filters: _, rls_write_check: _, } => { - let entry = encode_vector_direct_delete_payload(collection.as_str(), field, targets)?; + let rows = super::logged::logged_rows(collection.as_str(), targets)?; + let entry = encode_vector_direct_delete_payload(collection.as_str(), field, rows)?; Some(wal.append_vector_direct_delete(tenant_id, vshard_id, database_id, &entry)?) } // `restart_identity` is applied by the Control Plane after commit, @@ -258,10 +259,11 @@ pub(crate) fn wal_append_vector_op( rls_filters: _, rls_write_check: _, } => { + let rows = super::logged::logged_rows(collection.as_str(), targets)?; let entry = encode_vector_direct_update_payload(VectorDirectUpdatePayload { collection: collection.as_str(), field, - targets, + rows, new_vector: new_vector.as_deref(), payload_patch, quantization: *quantization, diff --git a/nodedb/src/control/server/wal_dispatch/vector/encode.rs b/nodedb/src/control/server/wal_dispatch/vector/encode.rs index 3e413f6d9..586717517 100644 --- a/nodedb/src/control/server/wal_dispatch/vector/encode.rs +++ b/nodedb/src/control/server/wal_dispatch/vector/encode.rs @@ -152,6 +152,7 @@ pub(crate) fn encode_vector_direct_upsert_payload( intent, on_conflict_updates, } = args; + super::logged::check_logged_patch(collection, on_conflict_updates)?; zerompk::to_msgpack_vec(&( collection, field, @@ -173,28 +174,22 @@ pub(crate) fn encode_vector_direct_upsert_payload( /// Encode the payload of a `VectorDirectDelete` WAL record. /// -/// Produces the 3-element shape `(collection, field, targets)`. Replay -/// resolves `targets` against the state it rebuilt; a surrogate with no node -/// is skipped, so re-applying over a restored checkpoint is idempotent. +/// Produces the 3-element shape `(collection, field, rows)`: the rows the +/// delete removed, never a predicate. Replay removes each row whose node it +/// rebuilt, so re-applying over a restored checkpoint is idempotent. pub(crate) fn encode_vector_direct_delete_payload( collection: &str, field: &str, - targets: &nodedb_physical::physical_plan::VectorWriteTargets, + rows: &[nodedb_types::Surrogate], ) -> crate::Result> { - zerompk::to_msgpack_vec(&(collection, field, targets)).map_err(|e| { - crate::Error::Serialization { - format: "msgpack".into(), - detail: format!("wal vector direct delete: {e}"), - } + zerompk::to_msgpack_vec(&(collection, field, rows)).map_err(|e| crate::Error::Serialization { + format: "msgpack".into(), + detail: format!("wal vector direct delete: {e}"), }) } /// The decoded form of a `VectorDirectDelete` record. -pub(crate) type VectorDirectDeleteRecord = ( - String, - String, - nodedb_physical::physical_plan::VectorWriteTargets, -); +pub(crate) type VectorDirectDeleteRecord = (String, String, Vec); /// Encode a `VectorDirectTruncate` record: `(collection, field)`. The /// `restart_identity` flag is a Control-Plane sequence concern and never @@ -216,7 +211,8 @@ pub(crate) type VectorDirectTruncateRecord = (String, String); pub(crate) struct VectorDirectUpdatePayload<'a> { pub collection: &'a str, pub field: &'a str, - pub targets: &'a nodedb_physical::physical_plan::VectorWriteTargets, + /// The rows the update rewrote, never a predicate. + pub rows: &'a [nodedb_types::Surrogate], pub new_vector: Option<&'a [f32]>, pub payload_patch: &'a [(String, nodedb_physical::physical_plan::UpdateValue)], pub quantization: nodedb_types::VectorQuantization, @@ -229,7 +225,7 @@ pub(crate) struct VectorDirectUpdatePayload<'a> { pub(crate) type VectorDirectUpdateRecord = ( String, String, - nodedb_physical::physical_plan::VectorWriteTargets, + Vec, Option>, Vec<(String, nodedb_physical::physical_plan::UpdateValue)>, nodedb_types::VectorQuantization, @@ -239,27 +235,28 @@ pub(crate) type VectorDirectUpdateRecord = ( /// Encode the payload of a `VectorDirectUpdate` WAL record. /// -/// Produces the 8-element shape `(collection, field, targets, new_vector, -/// payload_patch, quantization, storage_dtype, payload_indexes)`. Replay -/// re-runs the update handler, so a row the update already reached before -/// the crash is patched to the same image again. +/// Produces the 8-element shape `(collection, field, rows, new_vector, +/// payload_patch, quantization, storage_dtype, payload_indexes)`. The patch +/// holds literals only, so a row the update already reached before the +/// crash is patched to the same image again. pub(crate) fn encode_vector_direct_update_payload( args: VectorDirectUpdatePayload<'_>, ) -> crate::Result> { let VectorDirectUpdatePayload { collection, field, - targets, + rows, new_vector, payload_patch, quantization, storage_dtype, payload_indexes, } = args; + super::logged::check_logged_patch(collection, payload_patch)?; zerompk::to_msgpack_vec(&( collection, field, - targets, + rows, new_vector, payload_patch, quantization, diff --git a/nodedb/src/control/server/wal_dispatch/vector/logged.rs b/nodedb/src/control/server/wal_dispatch/vector/logged.rs new file mode 100644 index 000000000..2506964b4 --- /dev/null +++ b/nodedb/src/control/server/wal_dispatch/vector/logged.rs @@ -0,0 +1,74 @@ +// SPDX-License-Identifier: BUSL-1.1 + +//! What a vector-primary direct write may put in a log record. +//! +//! The sidecar store is written through, while the vector index restores +//! from its checkpoint. Replay therefore runs against sidecars newer than +//! the index. A record names its rows and their values, never a row +//! predicate or a patch expression replay evaluates again. Such a write +//! resolves to rows first (`control::write_resolve`). + +use nodedb_physical::physical_plan::{UpdateValue, VectorWriteTargets}; +use nodedb_types::Surrogate; + +/// The rows `targets` names. Refuses a row predicate. +pub(crate) fn logged_rows<'a>( + collection: &str, + targets: &'a VectorWriteTargets, +) -> crate::Result<&'a [Surrogate]> { + match targets { + VectorWriteTargets::Surrogates(rows) => Ok(rows), + VectorWriteTargets::Predicate(_) => Err(crate::Error::Internal { + detail: format!( + "internal invariant break: a vector-primary write on '{collection}' reached \ + the log with a row predicate; write_resolve resolves it to rows first" + ), + }), + } +} + +/// Refuse a `patch` that holds an expression. +pub(crate) fn check_logged_patch( + collection: &str, + patch: &[(String, UpdateValue)], +) -> crate::Result<()> { + for (column, value) in patch { + match value { + UpdateValue::Literal(_) => {} + UpdateValue::Expr(_) => { + return Err(crate::Error::Internal { + detail: format!( + "internal invariant break: a vector-primary write on '{collection}' \ + reached the log with an expression for '{column}'; write_resolve \ + resolves it to a value first" + ), + }); + } + } + } + Ok(()) +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn a_row_predicate_is_refused_and_rows_pass() { + let rows = VectorWriteTargets::Surrogates(vec![Surrogate::new(3)]); + assert_eq!(logged_rows("v", &rows).expect("rows"), &[Surrogate::new(3)]); + let predicate = VectorWriteTargets::Predicate(Vec::new()); + assert!(logged_rows("v", &predicate).is_err()); + } + + #[test] + fn an_expression_is_refused_and_a_literal_passes() { + let literal = [("n".to_string(), UpdateValue::Literal(vec![0x01]))]; + assert!(check_logged_patch("v", &literal).is_ok()); + let expression = [( + "n".to_string(), + UpdateValue::Expr(nodedb_query::expr::SqlExpr::Column("n".into())), + )]; + assert!(check_logged_patch("v", &expression).is_err()); + } +} diff --git a/nodedb/src/control/server/wal_dispatch/vector/mod.rs b/nodedb/src/control/server/wal_dispatch/vector/mod.rs index dcd13641b..49572b1be 100644 --- a/nodedb/src/control/server/wal_dispatch/vector/mod.rs +++ b/nodedb/src/control/server/wal_dispatch/vector/mod.rs @@ -5,8 +5,11 @@ pub mod append; pub mod encode; +pub mod logged; pub(crate) use append::wal_append_vector_op; +#[cfg(test)] +pub(crate) use encode::encode_vector_direct_delete_payload; pub(crate) use encode::{ VectorDirectDeleteRecord, VectorDirectTruncateRecord, VectorDirectUpdateRecord, VectorDirectUpsertRecord, encode_multi_vector_delete_payload, encode_multi_vector_put_payload, @@ -18,3 +21,4 @@ pub(crate) use encode::{ VectorResolvedDirectWritePayload, VectorResolvedDirectWriteRecord, encode_vector_resolved_direct_write_payload, }; +pub(crate) use logged::{check_logged_patch, logged_rows}; diff --git a/nodedb/src/control/server/wal_dispatch_kv/append.rs b/nodedb/src/control/server/wal_dispatch_kv/append.rs index 39d89117c..47c289bdf 100644 --- a/nodedb/src/control/server/wal_dispatch_kv/append.rs +++ b/nodedb/src/control/server/wal_dispatch_kv/append.rs @@ -10,10 +10,9 @@ use super::encode::{ KvIncrRecord, KvRegisterSortedIndexFields, KvTransferFields, encode_kv_batch_put, encode_kv_cas, encode_kv_delete, encode_kv_drop_index, encode_kv_drop_sorted_index, encode_kv_expire, encode_kv_field_set, encode_kv_getset, encode_kv_incr, encode_kv_incr_float, - encode_kv_insert_on_conflict_update, encode_kv_persist, encode_kv_predicate_delete, - encode_kv_predicate_update, encode_kv_put, encode_kv_register_index, - encode_kv_register_sorted_index, encode_kv_rewrite, encode_kv_transfer, - encode_kv_transfer_item, encode_kv_truncate, + encode_kv_insert_on_conflict_update, encode_kv_persist, encode_kv_put, + encode_kv_register_index, encode_kv_register_sorted_index, encode_kv_rewrite, + encode_kv_transfer, encode_kv_transfer_item, encode_kv_truncate, }; /// Outcome of [`wal_append_kv_op`]: the allocated WAL LSN and, for a TTL-bearing @@ -314,29 +313,16 @@ pub fn wal_append_kv_op( let entry = encode_kv_truncate(collection.as_str())?; Some(wal.append_delete(tenant_id, vshard_id, database_id, &entry)?) } - // The predicate is the durable record; replay re-executes it via the live handler. - KvOp::PredicateUpdate { - collection, - filters, - updates, - // Per-request authorization input, not part of the durable image. - rls_write_check: _, - // Per-request reply projection, not part of the durable image. - returning: _, - rls_filters: _, - } => { - let entry = encode_kv_predicate_update(collection.as_str(), filters, updates)?; - Some(wal.append_put(tenant_id, vshard_id, database_id, &entry)?) - } - KvOp::PredicateDelete { - collection, - filters, - rls_write_check: _, - returning: _, - rls_filters: _, - } => { - let entry = encode_kv_predicate_delete(collection.as_str(), filters)?; - Some(wal.append_delete(tenant_id, vshard_id, database_id, &entry)?) + // A row set replay scans for itself depends on its clock (TTL expiry), + // so a predicate write resolves to a mutation list first. + KvOp::PredicateUpdate { collection, .. } | KvOp::PredicateDelete { collection, .. } => { + return Err(crate::Error::Internal { + detail: format!( + "internal invariant break: a kv predicate UPDATE/DELETE on '{}' reached \ + the WAL unresolved; write_resolve resolves it to a mutation list first", + collection.as_str() + ), + }); } KvOp::Transfer { collection, diff --git a/nodedb/src/control/server/wal_dispatch_kv/encode.rs b/nodedb/src/control/server/wal_dispatch_kv/encode.rs index be1d77d30..9bf958fcc 100644 --- a/nodedb/src/control/server/wal_dispatch_kv/encode.rs +++ b/nodedb/src/control/server/wal_dispatch_kv/encode.rs @@ -370,33 +370,6 @@ pub(crate) fn encode_kv_drop_sorted_index(index_name: &str) -> crate::Result)], -) -> crate::Result> { - encode( - "predicate update", - &("kv_predicate_update", collection, filters, updates), - ) -} - -/// Encode a `kv_predicate_delete` WAL payload: -/// `("kv_predicate_delete", collection, filters)`. See -/// [`encode_kv_predicate_update`] for why the predicate travels. -pub(crate) fn encode_kv_predicate_delete( - collection: &str, - filters: &[u8], -) -> crate::Result> { - encode( - "predicate delete", - &("kv_predicate_delete", collection, filters), - ) -} - /// Encode a `kv_truncate` WAL payload: `("kv_truncate", collection)`. pub(crate) fn encode_kv_truncate(collection: &str) -> crate::Result> { encode("truncate", &("kv_truncate", collection)) diff --git a/nodedb/src/control/state/calvin_bases.rs b/nodedb/src/control/state/calvin_bases.rs index 3c1fe7269..d03939e0b 100644 --- a/nodedb/src/control/state/calvin_bases.rs +++ b/nodedb/src/control/state/calvin_bases.rs @@ -133,6 +133,11 @@ impl CalvinBases { reaches } + /// Every vShard that holds a base here. + pub fn vshards(&self) -> BTreeSet { + self.state().bases.keys().copied().collect() + } + /// Stop waiting for every vShard not in `mounted`, the vShards of the /// data groups this node mounts: an unmounted group starts no scheduler. pub fn retain_waiting(&self, mounted: &BTreeSet) { diff --git a/nodedb/src/control/state/calvin_local.rs b/nodedb/src/control/state/calvin_local.rs index c612b64e3..0914cd96c 100644 --- a/nodedb/src/control/state/calvin_local.rs +++ b/nodedb/src/control/state/calvin_local.rs @@ -3,7 +3,7 @@ //! This node's in-process Calvin state, shared between the per-vShard //! schedulers and the Control-Plane paths that read or fence against them. -use std::collections::BTreeMap; +use std::collections::{BTreeMap, BTreeSet}; use std::sync::atomic::{AtomicU32, AtomicU64}; use std::sync::{Arc, Mutex, OnceLock}; @@ -102,6 +102,9 @@ pub struct CalvinLocalState { /// scheduler: read results and timeouts the data-group apply loop /// applied before the scheduler took them. pub read_results: CalvinReadResults, + /// vShards whose stored Calvin state waits for a forget. Each scheduler + /// reconcile forgets those this node does not host and drops the rest. + pub forget_pending: Mutex>, } impl CalvinLocalState { @@ -131,6 +134,7 @@ impl CalvinLocalState { caught_up: CaughtUpRegistry::default(), inboxes: CalvinInboxes::default(), read_results: CalvinReadResults::default(), + forget_pending: Mutex::new(BTreeSet::new()), } } } diff --git a/nodedb/src/control/state/fields.rs b/nodedb/src/control/state/fields.rs index 29891b91a..5b4622932 100644 --- a/nodedb/src/control/state/fields.rs +++ b/nodedb/src/control/state/fields.rs @@ -119,10 +119,6 @@ pub struct SharedState { pub group_watchers: Arc, /// This node's state of the replicated descriptor preparation lease. pub metadata_ddl: super::metadata_ddl::MetadataDdlState, - /// Apply-progress steps this node made on metadata entries: each batch - /// sub-entry and each purge-reclaim step (a core acknowledgement, a closed - /// scan). A proposer on a long apply reads it to tell slow from stalled. - pub metadata_apply_progress: AtomicU64, /// Node-local table of in-flight `DdlPendingPropose` records. Persisted in /// `SystemCatalog` and seeded at boot. pub pending_ddl: crate::control::pending_ddl::PendingDdlTable, @@ -172,6 +168,10 @@ pub struct SharedState { /// Never call it while holding a `cluster_routing` guard. pub raft_status_fn: std::sync::OnceLock Vec + Send + Sync>>, + /// Set once boot recovery saw every mounted data group apply its + /// committed log here. From then on the routing table names every group + /// this node replicates. + pub data_groups_replayed: std::sync::atomic::AtomicBool, /// Cluster observability handle. Set once by `start_raft`. pub cluster_observer: OnceLock>, /// Registry of standardized per-loop metrics. Populated by `start_raft`. diff --git a/nodedb/src/control/state/init.rs b/nodedb/src/control/state/init.rs index f1fbf5f35..a44590b75 100644 --- a/nodedb/src/control/state/init.rs +++ b/nodedb/src/control/state/init.rs @@ -139,7 +139,6 @@ impl SharedState { .0, group_watchers: Arc::new(nodedb_cluster::GroupAppliedWatchers::new()), metadata_ddl: super::MetadataDdlState::new(), - metadata_apply_progress: std::sync::atomic::AtomicU64::new(0), pending_ddl: crate::control::pending_ddl::PendingDdlTable::new(), metadata_apply_wedge: std::sync::Arc::default(), sequencer_halt: std::sync::Arc::default(), @@ -158,6 +157,7 @@ impl SharedState { multi_raft: std::sync::OnceLock::new(), cluster_epoch: std::sync::OnceLock::new(), raft_status_fn: std::sync::OnceLock::new(), + data_groups_replayed: std::sync::atomic::AtomicBool::new(false), cluster_observer: std::sync::OnceLock::new(), loop_metrics_registry: nodedb_cluster::LoopMetricsRegistry::new(), per_vshard_metrics: crate::control::metrics::PerVShardMetricsRegistry::new(), diff --git a/nodedb/src/control/state/init_prod/open.rs b/nodedb/src/control/state/init_prod/open.rs index 256000b9c..a4ab66d8f 100644 --- a/nodedb/src/control/state/init_prod/open.rs +++ b/nodedb/src/control/state/init_prod/open.rs @@ -210,7 +210,6 @@ impl SharedState { .0, group_watchers: Arc::new(nodedb_cluster::GroupAppliedWatchers::new()), metadata_ddl: crate::control::state::MetadataDdlState::new(), - metadata_apply_progress: std::sync::atomic::AtomicU64::new(0), pending_ddl: crate::control::pending_ddl::PendingDdlTable::new(), metadata_apply_wedge: std::sync::Arc::default(), sequencer_halt: std::sync::Arc::default(), @@ -229,6 +228,7 @@ impl SharedState { multi_raft: std::sync::OnceLock::new(), cluster_epoch: std::sync::OnceLock::new(), raft_status_fn: std::sync::OnceLock::new(), + data_groups_replayed: std::sync::atomic::AtomicBool::new(false), cluster_observer: std::sync::OnceLock::new(), loop_metrics_registry: nodedb_cluster::LoopMetricsRegistry::new(), per_vshard_metrics: crate::control::metrics::PerVShardMetricsRegistry::new(), diff --git a/nodedb/src/control/surrogate/assign/core/flush.rs b/nodedb/src/control/surrogate/assign/core/flush.rs index d4c9d52d6..f0476a7dd 100644 --- a/nodedb/src/control/surrogate/assign/core/flush.rs +++ b/nodedb/src/control/surrogate/assign/core/flush.rs @@ -76,10 +76,10 @@ impl SurrogateHwmPersist for CombinedPersist<'_> { // view of the watermark. // // Awaiting it inline stalls the writer. `propose_surrogate_hwm` - // blocks for `DEFAULT_PROPOSE_TIMEOUT` (5s) waiting for the entry - // to commit. Surrogate assignment runs on the Raft apply loop as - // well as on the coordinator. A flush from the apply path parks - // the very loop that must commit the entry until the timeout fires. + // waits up to the statement deadline for the entry to apply. + // Surrogate assignment runs on the Raft apply loop as well as on the + // coordinator. A flush from the apply path parks the very loop that + // must commit the entry until the deadline passes. // // Out-of-order or duplicate delivery is safe: `apply_surrogate_alloc` // advances the watermark through `restore_hwm`, which is idempotent diff --git a/nodedb/src/control/vshard_admission.rs b/nodedb/src/control/vshard_admission.rs index 6092b2fbd..27750af0a 100644 --- a/nodedb/src/control/vshard_admission.rs +++ b/nodedb/src/control/vshard_admission.rs @@ -296,7 +296,7 @@ pub(crate) async fn await_admitted_applies( state, at.group_id, at.log_index, - deadline, + crate::control::cluster::linearizable_read::StatementDeadline::at(deadline), ) .await?; } diff --git a/nodedb/src/control/wal_replication/decode/entry.rs b/nodedb/src/control/wal_replication/decode/entry.rs index 7bfdbb723..a107f2239 100644 --- a/nodedb/src/control/wal_replication/decode/entry.rs +++ b/nodedb/src/control/wal_replication/decode/entry.rs @@ -172,9 +172,7 @@ fn to_physical_plan( | ReplicatedWrite::KvFieldSet { .. } | ReplicatedWrite::KvTransfer { .. } | ReplicatedWrite::KvTransferItem { .. } - | ReplicatedWrite::KvResolvedWrite { .. } - | ReplicatedWrite::KvPredicateUpdate { .. } - | ReplicatedWrite::KvPredicateDelete { .. } => entry_kv::decode_arm(write), + | ReplicatedWrite::KvResolvedWrite { .. } => entry_kv::decode_arm(write), // A timeseries ingest carries the proposer's instant for its untimed // rows, installed on every replica as `resolved_now_ms`. ReplicatedWrite::TimeseriesIngest { diff --git a/nodedb/src/control/wal_replication/decode/entry_kv.rs b/nodedb/src/control/wal_replication/decode/entry_kv.rs index f777e5468..f803b07fc 100644 --- a/nodedb/src/control/wal_replication/decode/entry_kv.rs +++ b/nodedb/src/control/wal_replication/decode/entry_kv.rs @@ -264,34 +264,6 @@ pub(super) fn decode_arm(write: &ReplicatedWrite) -> crate::Result<(PhysicalPlan mutations, response_payload, } => kv_resolved::resolved_write(mutations, response_payload)?, - ReplicatedWrite::KvPredicateUpdate { - collection, - filters, - updates, - returning, - rls_filters, - } => kv::predicate_update( - collection, - filters, - updates, - ReturningFields { - returning: decode_returning(returning)?, - rls_filters, - }, - ), - ReplicatedWrite::KvPredicateDelete { - collection, - filters, - returning, - rls_filters, - } => kv::predicate_delete( - collection, - filters, - ReturningFields { - returning: decode_returning(returning)?, - rls_filters, - }, - ), ReplicatedWrite::KvTransferItem { source_collection, dest_collection, diff --git a/nodedb/src/control/wal_replication/decode/kv.rs b/nodedb/src/control/wal_replication/decode/kv.rs index 50ef00a6f..7c26fd7db 100644 --- a/nodedb/src/control/wal_replication/decode/kv.rs +++ b/nodedb/src/control/wal_replication/decode/kv.rs @@ -73,40 +73,6 @@ pub(super) fn delete( }) } -/// Reconstruct a KV predicate `UPDATE` plan. The predicate is re-scanned -/// against this node's own committed state — see `delete` above for why the -/// check slot is stamped rather than re-derived. -pub(super) fn predicate_update( - collection: &str, - filters: &[u8], - updates: &[(String, Vec)], - returning: ReturningFields<'_>, -) -> PhysicalPlan { - PhysicalPlan::Kv(KvOp::PredicateUpdate { - collection: nodedb_types::QualifiedCollection::from_stored(collection.to_owned()), - filters: filters.to_vec(), - updates: updates.to_vec(), - rls_write_check: RlsWriteCheck::already_decided_elsewhere(), - returning: returning.returning, - rls_filters: returning.rls_filters.to_vec(), - }) -} - -/// Reconstruct a KV predicate `DELETE` plan — see [`predicate_update`]. -pub(super) fn predicate_delete( - collection: &str, - filters: &[u8], - returning: ReturningFields<'_>, -) -> PhysicalPlan { - PhysicalPlan::Kv(KvOp::PredicateDelete { - collection: nodedb_types::QualifiedCollection::from_stored(collection.to_owned()), - filters: filters.to_vec(), - rls_write_check: RlsWriteCheck::already_decided_elsewhere(), - returning: returning.returning, - rls_filters: returning.rls_filters.to_vec(), - }) -} - pub(super) fn insert( collection: &str, key: &[u8], diff --git a/nodedb/src/control/wal_replication/decode/vector.rs b/nodedb/src/control/wal_replication/decode/vector.rs index bcc69acb9..170e26c9e 100644 --- a/nodedb/src/control/wal_replication/decode/vector.rs +++ b/nodedb/src/control/wal_replication/decode/vector.rs @@ -157,13 +157,13 @@ pub(super) fn decode_arm(write: &ReplicatedWrite) -> crate::Result ReplicatedWrite::VectorDirectDelete { collection, field, - targets, + rows, returning, rls_filters, } => Ok(super::vector_direct::direct_delete( collection, field, - targets, + rows, decode_sync_engines::decode_returning(returning)?, rls_filters, )), @@ -179,7 +179,7 @@ pub(super) fn decode_arm(write: &ReplicatedWrite) -> crate::Result ReplicatedWrite::VectorDirectUpdate { collection, field, - targets, + rows, new_vector, payload_patch, quantization, @@ -191,7 +191,7 @@ pub(super) fn decode_arm(write: &ReplicatedWrite) -> crate::Result super::vector_direct::DirectUpdateFields { collection, field, - targets, + rows, new_vector, payload_patch, quantization: *quantization, diff --git a/nodedb/src/control/wal_replication/decode/vector_direct.rs b/nodedb/src/control/wal_replication/decode/vector_direct.rs index ac491bacb..5808037d9 100644 --- a/nodedb/src/control/wal_replication/decode/vector_direct.rs +++ b/nodedb/src/control/wal_replication/decode/vector_direct.rs @@ -4,11 +4,9 @@ //! `PhysicalPlan::Vector`. //! //! Every surrogate is rebuilt verbatim from the record; `entry.rs` binds the -//! insert family to its primary key afterwards. `DELETE` / `UPDATE` carry -//! their targets verbatim: a point-targeted write names the leader's -//! surrogates, which every replica bound when the rows were inserted; a -//! predicate-targeted write re-resolves the predicate against the replica's -//! own sidecar rows, exactly as `KvPredicateDelete` does. +//! insert family to its primary key afterwards. `DELETE` / `UPDATE` name the +//! leader's surrogates, which every replica bound when the rows were +//! inserted. A row predicate resolves on the proposer, never here. use nodedb_physical::physical_plan::{ ReturningSpec, UpdateValue, VectorDirectWriteIntent, VectorOp, VectorResolvedMutation, @@ -22,14 +20,14 @@ use crate::bridge::envelope::PhysicalPlan; pub(super) fn direct_delete( collection: &str, field: &str, - targets: &VectorWriteTargets, + rows: &[Surrogate], returning: Option, rls_filters: &[u8], ) -> PhysicalPlan { PhysicalPlan::Vector(VectorOp::DirectDelete { collection: nodedb_types::QualifiedCollection::from_stored(collection.to_owned()), field: field.to_owned(), - targets: targets.clone(), + targets: VectorWriteTargets::Surrogates(rows.to_vec()), returning, rls_filters: rls_filters.to_vec(), // A write-policy collection never reaches this shape: the leader @@ -56,7 +54,7 @@ pub(super) fn direct_truncate( pub(super) struct DirectUpdateFields<'a> { pub(super) collection: &'a str, pub(super) field: &'a str, - pub(super) targets: &'a VectorWriteTargets, + pub(super) rows: &'a [Surrogate], pub(super) new_vector: &'a Option>, pub(super) payload_patch: &'a [(String, UpdateValue)], pub(super) quantization: nodedb_types::VectorQuantization, @@ -71,7 +69,7 @@ pub(super) fn direct_update(f: DirectUpdateFields<'_>) -> PhysicalPlan { PhysicalPlan::Vector(VectorOp::DirectUpdate { collection: nodedb_types::QualifiedCollection::from_stored(f.collection.to_owned()), field: f.field.to_owned(), - targets: f.targets.clone(), + targets: VectorWriteTargets::Surrogates(f.rows.to_vec()), new_vector: f.new_vector.clone(), payload_patch: f.payload_patch.to_vec(), quantization: f.quantization, diff --git a/nodedb/src/control/wal_replication/encode/entry.rs b/nodedb/src/control/wal_replication/encode/entry.rs index a9498560e..69fb9f78c 100644 --- a/nodedb/src/control/wal_replication/encode/entry.rs +++ b/nodedb/src/control/wal_replication/encode/entry.rs @@ -52,8 +52,9 @@ pub fn to_replicated_entry( PhysicalPlan::Document(op) => entry_document::document_write(op), // Fallible: a governed predicate DML refuses rather than encode it bare. PhysicalPlan::Kv(op) => entry_kv::kv_write(op)?, + // Fallible: a row predicate or a patch expression refuses the write. + PhysicalPlan::Vector(op) => vector::encode(op)?, // Exhaustive over their op enums — see their module docs. - PhysicalPlan::Vector(op) => vector::encode(op), PhysicalPlan::Crdt(op) => crdt::encode(op), PhysicalPlan::Graph(op) => entry_graph::graph_write(op), // Fallible for the same reason as the Kv arm above. diff --git a/nodedb/src/control/wal_replication/encode/entry_kv.rs b/nodedb/src/control/wal_replication/encode/entry_kv.rs index 28f42808e..8010a0904 100644 --- a/nodedb/src/control/wal_replication/encode/entry_kv.rs +++ b/nodedb/src/control/wal_replication/encode/entry_kv.rs @@ -13,8 +13,8 @@ use super::kv::{WirePut, WireReturning}; use nodedb_physical::physical_plan::KvOp; /// Encode a `KvOp` write variant, `Ok(None)` when not a single-shard replicated -/// write, or an error when it can't replicate safely. `PredicateUpdate`/ -/// `PredicateDelete` refuse on `rls_write_check.has_predicate()`. +/// write, or an error when it can't replicate safely. `PredicateUpdate` / +/// `PredicateDelete` always refuse: they resolve to a mutation list first. pub(super) fn kv_write(op: &KvOp) -> crate::Result> { Ok(Some(match op { KvOp::Put { @@ -304,43 +304,10 @@ pub(super) fn kv_write(op: &KvOp) -> crate::Result> { rls_write_check: _, } => kv::resolved_write(mutations, response_payload), - // With no write policy, each replica re-scans the predicate — deterministic - // replay. A restricting policy refuses; see `refuse_governed_predicate_dml`. - KvOp::PredicateUpdate { - collection, - filters, - updates, - rls_write_check, - returning, - rls_filters, - } => { - refuse_governed_predicate_dml(collection.as_str(), rls_write_check)?; - kv::predicate_update( - collection.as_str(), - filters, - updates, - WireReturning { - returning, - rls_filters, - }, - ) - } - KvOp::PredicateDelete { - collection, - filters, - rls_write_check, - returning, - rls_filters, - } => { - refuse_governed_predicate_dml(collection.as_str(), rls_write_check)?; - kv::predicate_delete( - collection.as_str(), - filters, - WireReturning { - returning, - rls_filters, - }, - ) + // A row set a replica scans for itself depends on its clock (TTL + // expiry), so a predicate write resolves to rows first. + KvOp::PredicateUpdate { collection, .. } | KvOp::PredicateDelete { collection, .. } => { + return Err(refuse_predicate_dml(collection.as_str())); } // Not a write — reads/scans/sorted-index queries. `ResolveWrite` mutates @@ -364,19 +331,12 @@ pub(super) fn kv_write(op: &KvOp) -> crate::Result> { /// Refuse a KV predicate UPDATE/DELETE on a governed collection: a follower has /// no writing identity to evaluate the predicate. Must resolve to /// `KvOp::ResolvedWrite` before proposing. -fn refuse_governed_predicate_dml( - collection: &str, - rls_write_check: &nodedb_types::RlsWriteCheck, -) -> crate::Result<()> { - if rls_write_check.has_predicate() { - return Err(crate::Error::PlanError { - detail: format!( - "kv predicate UPDATE/DELETE on '{collection}' cannot be replicated as a \ - predicate because it carries an RLS write policy: a follower has no writing \ - identity to evaluate the predicate against. It must be resolved to a concrete \ - mutation list before it is proposed." - ), - }); +fn refuse_predicate_dml(collection: &str) -> crate::Error { + crate::Error::Internal { + detail: format!( + "internal invariant break: a kv predicate UPDATE/DELETE on '{collection}' reached \ + the Raft propose path unresolved; write_resolve resolves it to a mutation list \ + first" + ), } - Ok(()) } diff --git a/nodedb/src/control/wal_replication/encode/kv.rs b/nodedb/src/control/wal_replication/encode/kv.rs index ab5991c96..608281f8a 100644 --- a/nodedb/src/control/wal_replication/encode/kv.rs +++ b/nodedb/src/control/wal_replication/encode/kv.rs @@ -61,37 +61,6 @@ pub(super) fn put( } } -/// Encode a KV predicate `UPDATE`: the predicate itself, never a row set. Only -/// reached for a collection with no write policy. -pub(super) fn predicate_update( - collection: &str, - filters: &[u8], - updates: &[(String, Vec)], - returning: WireReturning<'_>, -) -> ReplicatedWrite { - ReplicatedWrite::KvPredicateUpdate { - collection: collection.to_owned(), - filters: filters.to_vec(), - updates: updates.to_vec(), - returning: encode_returning(returning.returning), - rls_filters: returning.rls_filters.to_vec(), - } -} - -/// Encode a KV predicate `DELETE` — see [`predicate_update`]. -pub(super) fn predicate_delete( - collection: &str, - filters: &[u8], - returning: WireReturning<'_>, -) -> ReplicatedWrite { - ReplicatedWrite::KvPredicateDelete { - collection: collection.to_owned(), - filters: filters.to_vec(), - returning: encode_returning(returning.returning), - rls_filters: returning.rls_filters.to_vec(), - } -} - pub(super) fn delete( collection: &str, keys: &[Vec], diff --git a/nodedb/src/control/wal_replication/encode/vector.rs b/nodedb/src/control/wal_replication/encode/vector.rs index ab9f40c97..cf1958607 100644 --- a/nodedb/src/control/wal_replication/encode/vector.rs +++ b/nodedb/src/control/wal_replication/encode/vector.rs @@ -8,6 +8,8 @@ use nodedb_physical::physical_plan::{ }; use nodedb_types::Surrogate; +use crate::control::server::wal_dispatch::{check_logged_patch, logged_rows}; + /// Encode a `VectorOp` write variant into its `ReplicatedWrite` wire shape. /// /// Returns `None` for the read / DDL-Alter variants (`Search`, @@ -15,8 +17,11 @@ use nodedb_types::Surrogate; /// `SparseSearch`, `MultiVectorScoreSearch`) — none of those are replicated. /// Exhaustive over `VectorOp` (not a catch-all): a new variant forces an /// explicit decision here instead of silently falling through. -pub(super) fn encode(op: &VectorOp) -> Option { - Some(match op { +/// +/// A direct write names its rows and values. A row predicate or a patch +/// expression is refused: `write_resolve` resolves it first. +pub(super) fn encode(op: &VectorOp) -> crate::Result> { + Ok(Some(match op { VectorOp::Insert { collection, vector, @@ -158,7 +163,7 @@ pub(super) fn encode(op: &VectorOp) -> Option { on_conflict_updates, returning, rls_filters, - }), + })?, VectorOp::DirectInsert { collection, field, @@ -185,7 +190,7 @@ pub(super) fn encode(op: &VectorOp) -> Option { on_conflict_updates: &[], returning, rls_filters, - }), + })?, VectorOp::DirectInsertIfAbsent { collection, field, @@ -212,7 +217,7 @@ pub(super) fn encode(op: &VectorOp) -> Option { on_conflict_updates: &[], returning, rls_filters, - }), + })?, VectorOp::DirectDelete { collection, field, @@ -223,7 +228,7 @@ pub(super) fn encode(op: &VectorOp) -> Option { } => ReplicatedWrite::VectorDirectDelete { collection: collection.as_str().to_owned(), field: field.to_owned(), - targets: targets.clone(), + rows: logged_rows(collection.as_str(), targets)?.to_vec(), returning: super::entry::encode_returning(returning), rls_filters: rls_filters.clone(), }, @@ -248,18 +253,21 @@ pub(super) fn encode(op: &VectorOp) -> Option { returning, rls_filters, rls_write_check: _, - } => ReplicatedWrite::VectorDirectUpdate { - collection: collection.as_str().to_owned(), - field: field.to_owned(), - targets: targets.clone(), - new_vector: new_vector.clone(), - payload_patch: payload_patch.clone(), - quantization: *quantization, - storage_dtype: *storage_dtype, - payload_indexes: payload_indexes.clone(), - returning: super::entry::encode_returning(returning), - rls_filters: rls_filters.clone(), - }, + } => { + check_logged_patch(collection.as_str(), payload_patch)?; + ReplicatedWrite::VectorDirectUpdate { + collection: collection.as_str().to_owned(), + field: field.to_owned(), + rows: logged_rows(collection.as_str(), targets)?.to_vec(), + new_vector: new_vector.clone(), + payload_patch: payload_patch.clone(), + quantization: *quantization, + storage_dtype: *storage_dtype, + payload_indexes: payload_indexes.clone(), + returning: super::entry::encode_returning(returning), + rls_filters: rls_filters.clone(), + } + } VectorOp::DropIndex { collection, field_name, @@ -296,8 +304,8 @@ pub(super) fn encode(op: &VectorOp) -> Option { | VectorOp::CompactIndex { .. } | VectorOp::Rebuild { .. } | VectorOp::SparseSearch { .. } - | VectorOp::MultiVectorScoreSearch { .. } => return None, - }) + | VectorOp::MultiVectorScoreSearch { .. } => return Ok(None), + })) } /// Fields shared by the three insert-family ops, bundled so [`direct_write`] @@ -318,8 +326,9 @@ struct DirectWriteFields<'a> { rls_filters: &'a [u8], } -fn direct_write(f: DirectWriteFields<'_>) -> ReplicatedWrite { - ReplicatedWrite::DirectUpsert { +fn direct_write(f: DirectWriteFields<'_>) -> crate::Result { + check_logged_patch(f.collection, f.on_conflict_updates)?; + Ok(ReplicatedWrite::DirectUpsert { collection: f.collection.to_owned(), field: f.field.to_owned(), surrogate: f.surrogate.as_u32(), @@ -333,7 +342,7 @@ fn direct_write(f: DirectWriteFields<'_>) -> ReplicatedWrite { on_conflict_updates: f.on_conflict_updates.to_vec(), returning: super::entry::encode_returning(f.returning), rls_filters: f.rls_filters.to_vec(), - } + }) } pub(super) fn insert( @@ -560,6 +569,41 @@ mod tests { assert!(!rls_write_check.has_predicate()); } + /// A row predicate never reaches the wire, even with no write policy: + /// replay would evaluate it again. A delete naming rows replicates them. + #[test] + fn an_ungoverned_predicate_delete_is_refused_and_named_rows_replicate() { + let tenant = TenantId::new(1); + let vshard = VShardId::new(0); + let by_predicate = predicate_delete(nodedb_types::RlsWriteCheck::NoPolicyApplies); + assert!(to_replicated_entry(tenant, DatabaseId::DEFAULT, vshard, &by_predicate).is_err()); + + let by_rows = PhysicalPlan::Vector(VectorOp::DirectDelete { + collection: QualifiedCollection::new(DatabaseId::DEFAULT, "vecs"), + field: "emb".into(), + targets: nodedb_physical::physical_plan::VectorWriteTargets::Surrogates(vec![ + Surrogate::new(4), + ]), + returning: None, + rls_filters: Vec::new(), + rls_write_check: nodedb_types::RlsWriteCheck::NoPolicyApplies, + }); + let entry = to_replicated_entry(tenant, DatabaseId::DEFAULT, vshard, &by_rows) + .expect("encode must not error") + .expect("a delete naming rows must replicate"); + let (_, _, decoded, _) = + crate::control::wal_replication::decode_replicated_entry(&entry.to_bytes()) + .expect("decode") + .expect("a replicated entry"); + let PhysicalPlan::Vector(VectorOp::DirectDelete { targets, .. }) = decoded else { + panic!("decoded to the wrong shape"); + }; + assert_eq!( + targets, + nodedb_physical::physical_plan::VectorWriteTargets::Surrogates(vec![Surrogate::new(4)]) + ); + } + /// A vector-primary truncate replicates with its `restart_identity` /// flag, so every applying node resets the same sequences. #[test] diff --git a/nodedb/src/control/wal_replication/mod.rs b/nodedb/src/control/wal_replication/mod.rs index bf0e1cd61..46d9e8d7e 100644 --- a/nodedb/src/control/wal_replication/mod.rs +++ b/nodedb/src/control/wal_replication/mod.rs @@ -19,7 +19,6 @@ pub use decode::{decode_replicated_entry, from_replicated_entry}; pub use encode::to_replicated_entry; pub(crate) use propose::{ propose_replicated_entry, stamp_collection_incarnations, stamp_metadata_floor, - statement_propose_deadline, }; pub use replicable_write::ReplicableWrite; pub use types::{ diff --git a/nodedb/src/control/wal_replication/propose.rs b/nodedb/src/control/wal_replication/propose.rs index 6f85d6503..d7185ab3c 100644 --- a/nodedb/src/control/wal_replication/propose.rs +++ b/nodedb/src/control/wal_replication/propose.rs @@ -19,14 +19,6 @@ const FIRST_BACKOFF: std::time::Duration = std::time::Duration::from_millis(10); /// Longest wait between two re-proposals. const MAX_BACKOFF: std::time::Duration = std::time::Duration::from_millis(200); -/// The deadline a proposal made now carries: the running statement's -/// deadline, or the node default outside a statement. -pub(crate) fn statement_propose_deadline(state: &SharedState) -> tokio::time::Instant { - tokio::time::Instant::from_std(crate::control::server::shared::session::statement_deadline( - state.tuning.network.default_deadline_secs, - )) -} - /// Stamp this node's metadata floor on `entry`: the catalog the write was /// planned against, including the batch being applied now (see /// `AppliedIndexWatcher::floor`). Every replica applies the entry only once diff --git a/nodedb/src/control/wal_replication/transaction_redo/chunks/owed.rs b/nodedb/src/control/wal_replication/transaction_redo/chunks/owed.rs index a3ee4317c..15abb36c5 100644 --- a/nodedb/src/control/wal_replication/transaction_redo/chunks/owed.rs +++ b/nodedb/src/control/wal_replication/transaction_redo/chunks/owed.rs @@ -7,13 +7,14 @@ //! hold a later chunk or the final entry of a dropped stream. Here it //! refuses, while every other replica installs the redo. So the drop marks //! the group owed, durably, before the streams release their WAL floor -//! holds: +//! holds. The apply loop also marks a group owed when its apply holds an +//! entry this replica cannot apply as its peers do. //! //! - The group's snapshot requirement holds on this node while it is owed. -//! Its replica refuses log entries, and its leader sends a snapshot. The -//! snapshot carries the group's open streams. +//! Its replica refuses log entries, never leads, and its leader sends a +//! snapshot. The snapshot carries the group's open streams. //! - A refusal of a chunk or final entry of an owed group holds the group's -//! durable floor instead of passing the entry. +//! apply at the entry instead of passing it. //! - A snapshot install of the group settles the debt. //! //! The set is a leaf lock: no other lock is taken while it is held. diff --git a/nodedb/src/control/wal_replication/types/replicated_write.rs b/nodedb/src/control/wal_replication/types/replicated_write.rs index b63fb766b..f0a1bbd55 100644 --- a/nodedb/src/control/wal_replication/types/replicated_write.rs +++ b/nodedb/src/control/wal_replication/types/replicated_write.rs @@ -15,9 +15,9 @@ use crate::wal::RedoStreamId; use nodedb_physical::physical_plan::document::MergeClauseOp; use nodedb_physical::physical_plan::{ ColumnarInsertIntent, CrdtWriteVerb, KvCounterShape, UpdateValue, VectorDirectWriteIntent, - VectorResolvedMutation, VectorWriteTargets, + VectorResolvedMutation, }; -use nodedb_types::{PayloadIndexKind, VectorQuantization, VectorStorageDtype}; +use nodedb_types::{PayloadIndexKind, Surrogate, VectorQuantization, VectorStorageDtype}; #[derive( Debug, @@ -874,36 +874,6 @@ pub enum ReplicatedWrite { response_payload: Vec, }, - /// KV predicate `UPDATE` on a collection with NO write policy — the - /// predicate travels and every replica re-scans it against its own state. - /// A write-policy collection never reaches this shape. - KvPredicateUpdate { - collection: String, - /// Serialized `Vec`. Empty matches every row. - filters: Vec, - updates: Vec<(String, Vec)>, - /// See `ReplicatedWrite::PointPut::returning`. - #[serde(default)] - returning: Option>, - /// See `ReplicatedWrite::PointPut::rls_filters`. - #[serde(default)] - rls_filters: Vec, - }, - - /// KV predicate `DELETE` on a collection with NO write policy — see - /// [`ReplicatedWrite::KvPredicateUpdate`]. - KvPredicateDelete { - collection: String, - /// Serialized `Vec`. Empty matches every row. - filters: Vec, - /// See `ReplicatedWrite::PointPut::returning`. - #[serde(default)] - returning: Option>, - /// See `ReplicatedWrite::PointPut::rls_filters`. - #[serde(default)] - rls_filters: Vec, - }, - /// Resolved form of a deferred document write (`PointUpdate`, /// `PointDelete`, `Upsert`, `BulkUpdate`, `BulkDelete`) on a write-policy /// collection: mutations and reply, already decided against the live @@ -915,12 +885,12 @@ pub enum ReplicatedWrite { response_payload: Vec, }, - /// Vector-primary `DELETE` on a collection with NO write policy — the - /// targets travel and every replica resolves them against its own state. + /// Vector-primary `DELETE` on a collection with NO write policy. It + /// names the rows it removes: a row predicate resolves on the proposer. VectorDirectDelete { collection: String, field: String, - targets: VectorWriteTargets, + rows: Vec, /// See `ReplicatedWrite::PointPut::returning`. #[serde(default)] returning: Option>, @@ -940,11 +910,11 @@ pub enum ReplicatedWrite { }, /// Vector-primary `UPDATE` on a collection with NO write policy — see - /// [`ReplicatedWrite::VectorDirectDelete`]. + /// [`ReplicatedWrite::VectorDirectDelete`]. The patch holds literals only. VectorDirectUpdate { collection: String, field: String, - targets: VectorWriteTargets, + rows: Vec, new_vector: Option>, payload_patch: Vec<(String, UpdateValue)>, quantization: VectorQuantization, diff --git a/nodedb/src/control/write_resolve/kv.rs b/nodedb/src/control/write_resolve/kv.rs index eff276603..2748d90e2 100644 --- a/nodedb/src/control/write_resolve/kv.rs +++ b/nodedb/src/control/write_resolve/kv.rs @@ -3,7 +3,8 @@ //! KV implementation of [`EngineWriteResolver`]. //! //! Resolves a governed state-dependent KV write (increment, CAS, field -//! merge, TTL, transfer, predicate UPDATE/DELETE) into `KvOp::ResolvedWrite` +//! merge, TTL, transfer), and every predicate UPDATE/DELETE, into +//! `KvOp::ResolvedWrite` //! — a mutation list plus response payload, since e.g. a `CAS` miss owes a //! reply while writing nothing. @@ -80,23 +81,17 @@ pub(super) fn resolver_for_kv_op(op: &KvOp) -> Option { if !rls_write_check.has_predicate() { return None; } collection } + // A predicate write always resolves: the row set it scans depends on + // the clock (TTL expiry), so a log record names the rows instead. + KvOp::PredicateUpdate { collection, .. } | KvOp::PredicateDelete { collection, .. } => { + collection + } // Both collections checked: an identity may give a row up but not // receive it — `sole_rls_write_check` would wave the write through. KvOp::TransferItem { diff --git a/nodedb/src/control/write_resolve/propose.rs b/nodedb/src/control/write_resolve/propose.rs index 16e7b697a..79522cd93 100644 --- a/nodedb/src/control/write_resolve/propose.rs +++ b/nodedb/src/control/write_resolve/propose.rs @@ -42,7 +42,8 @@ pub(super) async fn propose_resolved( ), })?; - let deadline = crate::control::wal_replication::statement_propose_deadline(state); + let deadline = crate::control::server::shared::session::StatementDeadline::of_statement(state) + .tokio_instant(); match propose_replicated_entry(state, proposer, entry, deadline).await { Ok((payload, write_versions)) => { let request_id = diff --git a/nodedb/src/control/write_resolve/vector.rs b/nodedb/src/control/write_resolve/vector.rs index ab1a6735b..9e8c095e6 100644 --- a/nodedb/src/control/write_resolve/vector.rs +++ b/nodedb/src/control/write_resolve/vector.rs @@ -2,11 +2,15 @@ //! Vector-primary implementation of [`EngineWriteResolver`]. //! -//! Resolves a governed vector-primary write — `DirectDelete`, `DirectUpdate`, -//! or a `DirectUpsert` whose conflict patch is decided on the Data Plane — -//! into `VectorOp::ResolvedDirectWrite`: a row-mutation list plus the -//! response payload, each mutation carrying the sidecar it read so the apply -//! refuses a resolution that drifted. +//! Resolves a vector-primary write into `VectorOp::ResolvedDirectWrite`: a +//! row-mutation list plus the response payload, each mutation carrying the +//! sidecar it read so the apply refuses a resolution that drifted. +//! +//! A write resolves when it carries a live write predicate, a row predicate, +//! or a patch expression. The sidecar store is written through while the +//! index restores from its checkpoint, so replay runs against newer sidecars. +//! A log record therefore names rows and their values, never something +//! replay evaluates again. use async_trait::async_trait; use nodedb_types::{QualifiedCollection, RlsWriteCheck}; @@ -14,7 +18,9 @@ use nodedb_types::{QualifiedCollection, RlsWriteCheck}; use crate::bridge::envelope::{PhysicalPlan, Status}; use crate::control::maintenance::clone_materializer::dispatch_local; use crate::control::state::SharedState; -use nodedb_physical::physical_plan::{VectorOp, VectorResolveOutcome}; +use nodedb_physical::physical_plan::{ + UpdateValue, VectorOp, VectorResolveOutcome, VectorWriteTargets, +}; use super::resolved_rows::ResolvedRows; use super::resolver::{EngineWriteResolver, WriteResolveContext}; @@ -41,42 +47,78 @@ pub struct VectorWriteResolver { op: VectorOp, } -/// The resolver for `op`, or `None` when it carries no live write predicate. -/// Exhaustive over `VectorOp` — a new op fails to compile here. +/// Whether `targets` names rows by a predicate replay would evaluate again. +fn by_predicate(targets: &VectorWriteTargets) -> bool { + match targets { + VectorWriteTargets::Predicate(_) => true, + VectorWriteTargets::Surrogates(_) => false, + } +} + +/// Whether `patch` holds an expression replay would evaluate again. +fn has_expression(patch: &[(String, UpdateValue)]) -> bool { + patch.iter().any(|(_, value)| match value { + UpdateValue::Expr(_) => true, + UpdateValue::Literal(_) => false, + }) +} + +/// The resolver for `op`, or `None` when the op logs as it is. Exhaustive +/// over `VectorOp` — a new op fails to compile here. pub(super) fn resolver_for_vector_op(op: &VectorOp) -> Option> { let (collection, field, settings) = match op { VectorOp::DirectDelete { collection, field, + targets, rls_write_check, .. } => { - if !rls_write_check.has_predicate() { + if !rls_write_check.has_predicate() && !by_predicate(targets) { return None; } (collection, field, ResolvedIndexSettings::default()) } - // Only a conflict patch leaves an upsert's predicate live: a - // whole-row upsert is decided Control-Plane-side against its payload. VectorOp::DirectUpdate { collection, field, + targets, + payload_patch, quantization, storage_dtype, payload_indexes, rls_write_check, .. + } => { + if !rls_write_check.has_predicate() + && !by_predicate(targets) + && !has_expression(payload_patch) + { + return None; + } + ( + collection, + field, + ResolvedIndexSettings { + quantization: *quantization, + storage_dtype: *storage_dtype, + payload_indexes: payload_indexes.clone(), + }, + ) } - | VectorOp::DirectUpsert { + // A whole-row upsert is decided Control-Plane-side against its + // payload. A conflict patch is decided against the stored row. + VectorOp::DirectUpsert { collection, field, + on_conflict_updates, quantization, storage_dtype, payload_indexes, rls_write_check, .. } => { - if !rls_write_check.has_predicate() { + if !rls_write_check.has_predicate() && !has_expression(on_conflict_updates) { return None; } ( @@ -204,45 +246,81 @@ impl EngineWriteResolver for VectorWriteResolver { #[cfg(test)] mod tests { - use nodedb_physical::physical_plan::{VectorOp, VectorWriteTargets}; - use nodedb_types::{DatabaseId, QualifiedCollection, RlsWriteCheck}; + use nodedb_physical::physical_plan::{UpdateValue, VectorOp, VectorWriteTargets}; + use nodedb_types::{DatabaseId, QualifiedCollection, RlsWriteCheck, Surrogate}; use super::resolver_for_vector_op; use crate::bridge::envelope::PhysicalPlan; - fn predicate_delete(check: RlsWriteCheck) -> VectorOp { + fn delete(targets: VectorWriteTargets, check: RlsWriteCheck) -> VectorOp { VectorOp::DirectDelete { collection: QualifiedCollection::new(DatabaseId::DEFAULT, "vecs"), field: "emb".into(), - targets: VectorWriteTargets::Predicate(Vec::new()), + targets, returning: None, rls_filters: Vec::new(), rls_write_check: check, } } - /// Only a live predicate needs resolving; a decided or absent policy - /// replicates the write as it is. + fn update(targets: VectorWriteTargets, value: UpdateValue) -> VectorOp { + VectorOp::DirectUpdate { + collection: QualifiedCollection::new(DatabaseId::DEFAULT, "vecs"), + field: "emb".into(), + targets, + new_vector: None, + payload_patch: vec![("n".into(), value)], + quantization: nodedb_types::VectorQuantization::None, + storage_dtype: nodedb_types::VectorStorageDtype::F32, + payload_indexes: Vec::new(), + returning: None, + rls_filters: Vec::new(), + rls_write_check: RlsWriteCheck::NoPolicyApplies, + } + } + + fn rows() -> VectorWriteTargets { + VectorWriteTargets::Surrogates(vec![Surrogate::new(4)]) + } + + /// A write that names its rows and values logs as it is. #[test] - fn only_a_live_predicate_selects_the_resolver() { - assert!( - resolver_for_vector_op(&predicate_delete(RlsWriteCheck::NoPolicyApplies)).is_none() - ); - assert!( - resolver_for_vector_op(&predicate_delete( - RlsWriteCheck::decided_earlier_in_request() - )) - .is_none() + fn a_write_naming_rows_and_values_logs_as_it_is() { + for check in [ + RlsWriteCheck::NoPolicyApplies, + RlsWriteCheck::decided_earlier_in_request(), + ] { + assert!(resolver_for_vector_op(&delete(rows(), check)).is_none()); + } + let literal = UpdateValue::Literal(vec![0x01]); + assert!(resolver_for_vector_op(&update(rows(), literal)).is_none()); + } + + /// A live write predicate, a row predicate, or a patch expression is + /// resolved to rows and values before it is logged. + #[test] + fn anything_replay_evaluates_again_selects_the_resolver() { + let governed = delete(rows(), RlsWriteCheck::from_injected(vec![1, 2, 3])); + let by_predicate = delete( + VectorWriteTargets::Predicate(Vec::new()), + RlsWriteCheck::NoPolicyApplies, ); - let resolver = - resolver_for_vector_op(&predicate_delete(RlsWriteCheck::from_injected(vec![ - 1, 2, 3, - ]))) - .expect("a live predicate selects the resolver"); - assert_eq!(resolver.collection(), "vecs"); - assert!(matches!( - resolver.build_resolve_op(), - PhysicalPlan::Vector(VectorOp::ResolveDirectWrite(_)) - )); + let literal = UpdateValue::Literal(vec![0x01]); + let update_by_predicate = update(VectorWriteTargets::Predicate(Vec::new()), literal); + let expression = UpdateValue::Expr(nodedb_query::expr::SqlExpr::Column("n".into())); + let update_by_expression = update(rows(), expression); + for op in [ + governed, + by_predicate, + update_by_predicate, + update_by_expression, + ] { + let resolver = resolver_for_vector_op(&op).expect("the op resolves before logging"); + assert_eq!(resolver.collection(), "vecs"); + assert!(matches!( + resolver.build_resolve_op(), + PhysicalPlan::Vector(VectorOp::ResolveDirectWrite(_)) + )); + } } } diff --git a/nodedb/src/data/executor/array_checkpoint.rs b/nodedb/src/data/executor/array_checkpoint.rs index ff234e7bf..80bd20115 100644 --- a/nodedb/src/data/executor/array_checkpoint.rs +++ b/nodedb/src/data/executor/array_checkpoint.rs @@ -337,7 +337,7 @@ mod tests { attr_projection: vec![], limit: 0, cell_filter: None, - hilbert_range: None, + shard_bucket: None, system_time: nodedb_types::SystemTimeScope::Current, valid_at_ms: None, }); diff --git a/nodedb/src/data/executor/core_loop/write_index/index.rs b/nodedb/src/data/executor/core_loop/write_index/index.rs index 7fd5d7136..743dfe800 100644 --- a/nodedb/src/data/executor/core_loop/write_index/index.rs +++ b/nodedb/src/data/executor/core_loop/write_index/index.rs @@ -50,6 +50,10 @@ pub struct WriteVersionIndex { /// floor: a predicate reader validates against it). Never GC'd: bounded /// by the live collections of the core's vShards. coll_write: HashMap, + /// Last write that removed every row of a collection, such as a TRUNCATE + /// that records no per-row version. A row read validates against it. + /// Never GC'd: bounded like `coll_write`. + coll_wipe: HashMap, /// Per-secondary-index-dimension write-VALUE versions, the finer-grained /// sibling of `coll_write` an index-range read validates against. pub(in crate::data::executor) index_values: IndexValueVersionIndex, @@ -103,6 +107,28 @@ impl WriteVersionIndex { self.raise_latest(vshard, version); } + /// Record a committed write at `version` on `vshard` that removed every + /// row of `collection`. Advances the collection's version and its wipe + /// version, so every row read older than `version` fails validation. + pub fn note_wipe( + &mut self, + db: DatabaseId, + tenant: TenantId, + vshard: VShardId, + collection: &str, + version: WriteVersion, + ) { + self.note_write(db, tenant, vshard, collection, None, version); + let coll_key = CollKey { + vshard, + db, + tenant, + collection: Box::from(collection), + }; + let slot = self.coll_wipe.entry(coll_key).or_default(); + *slot = (*slot).max(version); + } + /// Raise `vshard`'s latest version to `version` when it is higher. pub(in crate::data::executor) fn raise_latest( &mut self, @@ -120,6 +146,7 @@ impl WriteVersionIndex { pub fn install(&mut self, vshard: VShardId, version: WriteVersion) { self.last_write.retain(|key, _| key.vshard != vshard); self.coll_write.retain(|key, _| key.vshard != vshard); + self.coll_wipe.retain(|key, _| key.vshard != vshard); self.index_values.drop_vshard(vshard); let bounds = self.vshards.entry(vshard).or_default(); bounds.installed = bounds.installed.max(version); @@ -203,6 +230,24 @@ impl WriteVersionIndex { .unwrap_or_else(|| self.bounds(vshard).installed) } + /// The version of the last write that removed every row of + /// `collection` on `vshard`, `ZERO` when none is recorded. + fn collection_wipe( + &self, + db: DatabaseId, + tenant: TenantId, + vshard: VShardId, + collection: &str, + ) -> WriteVersion { + let key = CollKey { + vshard, + db, + tenant, + collection: Box::from(collection), + }; + self.coll_wipe.get(&key).copied().unwrap_or_default() + } + fn bounds(&self, vshard: VShardId) -> VShardBounds { self.vshards.get(&vshard).copied().unwrap_or_default() } @@ -218,6 +263,9 @@ impl WriteVersionIndex { /// A key or value with no recorded version was not written since the /// vShard was installed, or GC evicted it. Its version is then at most /// the installed or evicted bound, and never above the collection's. + /// + /// A write that removed every row of the collection also removed the + /// key or value read, so its version bounds every point and index read. pub(crate) fn read_is_valid( &self, db: DatabaseId, @@ -230,6 +278,7 @@ impl WriteVersionIndex { let collection_current = self.collection_current(db, tenant, vshard, collection); let bounds = self.bounds(vshard); let unrecorded = collection_current.min(bounds.installed.max(bounds.evicted)); + let wipe = self.collection_wipe(db, tenant, vshard, collection); match key { ReadKeyIdent::Point(repr) => { let write_key = WriteKey { @@ -243,6 +292,7 @@ impl WriteVersionIndex { .get(&write_key) .copied() .unwrap_or(unrecorded) + .max(wipe) <= read } ReadKeyIdent::Predicate => collection_current <= read, @@ -256,7 +306,9 @@ impl WriteVersionIndex { }; match self.index_values.eq_version(dim, value) { IndexVersion::Untracked => collection_current <= read, - IndexVersion::Tracked(recorded) => recorded.unwrap_or(unrecorded) <= read, + IndexVersion::Tracked(recorded) => { + recorded.unwrap_or(unrecorded).max(wipe) <= read + } } } ReadKeyIdent::IndexRange { field, lo, hi } => { @@ -275,7 +327,7 @@ impl WriteVersionIndex { // A value GC evicted from the range is unknown, so the // range takes the unrecorded bound as well. IndexVersion::Tracked(recorded) => { - recorded.unwrap_or_default().max(unrecorded) <= read + recorded.unwrap_or_default().max(unrecorded).max(wipe) <= read } } } @@ -288,7 +340,7 @@ impl WriteVersionIndex { /// latest version in the same epoch, then, if more than /// [`MAX_KEY_ENTRIES`] per-key entries remain, drops the lowest versions /// until back within bound. Each vShard's evicted bound rises to the - /// highest version it lost. The per-collection map is left untouched. + /// highest version it lost. The per-collection and wipe maps are left untouched. pub fn gc(&mut self) { let latest: HashMap = self.latest_by_vshard().collect(); let mut evicted: HashMap = HashMap::new(); @@ -417,6 +469,55 @@ mod tests { assert!(!valid(&index, &ReadKeyIdent::Predicate, 10)); } + #[test] + fn a_collection_write_leaves_a_point_read_valid() { + let mut index = WriteVersionIndex::new(); + write(&mut index, Some(KeyRepr::Surrogate(7)), 5); + write(&mut index, None, 20); + assert!(valid(&index, &point(7), 10)); + } + + #[test] + fn a_wipe_after_the_read_invalidates_a_point_read() { + let mut index = WriteVersionIndex::new(); + write(&mut index, Some(KeyRepr::Surrogate(7)), 5); + index.note_wipe(db(), tenant(), vshard(), "orders", at(20)); + assert!(!valid(&index, &point(7), 10), "the wipe removed row 7"); + assert!( + !valid(&index, &point(8), 10), + "the wipe bounds a row with no recorded version" + ); + assert!(!valid(&index, &ReadKeyIdent::Predicate, 10)); + assert!(valid(&index, &point(7), 20)); + assert!(valid(&index, &point(8), 20)); + } + + #[test] + fn a_wipe_invalidates_a_tracked_index_read() { + let mut index = WriteVersionIndex::new(); + record_value(&mut index, "email", "a@b.c", 5); + index.note_wipe(db(), tenant(), vshard(), "orders", at(20)); + let eq = ReadKeyIdent::IndexEq { + field: "email".to_string(), + value: "a@b.c".to_string(), + }; + let range = ReadKeyIdent::IndexRange { + field: "email".to_string(), + lo: Some("a".to_string()), + hi: Some("b".to_string()), + }; + assert!(!valid(&index, &eq, 10)); + assert!(!valid(&index, &range, 10)); + assert!(valid(&index, &eq, 20)); + } + + #[test] + fn a_wipe_on_another_collection_leaves_a_point_read_valid() { + let mut index = WriteVersionIndex::new(); + index.note_wipe(db(), tenant(), vshard(), "items", at(20)); + assert!(valid(&index, &point(7), 10)); + } + #[test] fn a_write_on_another_vshard_never_moves_a_read() { let mut index = WriteVersionIndex::new(); diff --git a/nodedb/src/data/executor/core_loop/write_index/record.rs b/nodedb/src/data/executor/core_loop/write_index/record.rs index 6b54575a7..c1a1c859f 100644 --- a/nodedb/src/data/executor/core_loop/write_index/record.rs +++ b/nodedb/src/data/executor/core_loop/write_index/record.rs @@ -94,6 +94,61 @@ impl CoreLoop { } } + /// Record a committed write that removed every row of `collection`: a + /// TRUNCATE. Every point and index read of the collection older than the + /// write then fails validation. + /// + /// In the install pass of a committed-redo apply the version waits in the + /// apply scope, exactly like [`Self::note_write`]. + pub(in crate::data::executor) fn note_collection_wipe( + &mut self, + db: DatabaseId, + tenant: TenantId, + collection: &str, + stamp: WriteStamp, + ) { + if let Some(scope) = self.redo_apply.scope.as_mut() { + scope.defer_collection_wipe(db, tenant, collection, stamp); + return; + } + self.publish_collection_wipe(db, tenant, collection, stamp); + } + + /// Record a wipe version and advance the core watermark monotonically. + pub(in crate::data::executor) fn publish_collection_wipe( + &mut self, + db: DatabaseId, + tenant: TenantId, + collection: &str, + stamp: WriteStamp, + ) { + self.floors.applied_prefix.note_applied(stamp.lsn); + let version = self.write_version_of(stamp); + self.write_index + .note_wipe(db, tenant, stamp.vshard, collection, version); + if stamp.lsn > self.watermark { + self.watermark = stamp.lsn; + } + } + + /// Record a committed whole-collection removal, if `task` carries a WAL + /// record. See [`Self::note_collection_wipe`]. + pub(in crate::data::executor) fn note_task_collection_wipe( + &mut self, + task: &ExecutionTask, + tid: u64, + collection: &str, + ) { + if let Some(stamp) = self.task_write_stamp(task) { + self.note_collection_wipe( + task.request.database_id, + TenantId::new(tid), + collection, + stamp, + ); + } + } + /// Record a committed write's collection version only (no per-key /// entry), if `task` carries a WAL record. Shared by the columnar-family /// write handlers (columnar / timeseries / array / spatial / FTS) whose @@ -155,11 +210,39 @@ impl CoreLoop { key: Option, record_lsn: u64, ) { - if record_lsn == 0 { - return; + let db = DatabaseId::new(database_id); + if let Some(stamp) = self.replay_stamp(db, collection, record_lsn) { + self.note_write(db, TenantId::new(tenant_id), collection, key, stamp); } + } + + /// Record a whole-collection removal a replay arm applied from the WAL + /// record at `record_lsn`, stamped like [`Self::note_replay_write`]. + pub(in crate::data::executor) fn note_replay_wipe( + &mut self, + database_id: u64, + tenant_id: u64, + collection: &str, + record_lsn: u64, + ) { let db = DatabaseId::new(database_id); - let stamp = match self.write_index.record_stamps.get(record_lsn) { + if let Some(stamp) = self.replay_stamp(db, collection, record_lsn) { + self.note_collection_wipe(db, TenantId::new(tenant_id), collection, stamp); + } + } + + /// The stamp of a write a replay arm applied from the WAL record at + /// `record_lsn`. `None` when `record_lsn == 0` (no durable record). + fn replay_stamp( + &self, + db: DatabaseId, + collection: &str, + record_lsn: u64, + ) -> Option { + if record_lsn == 0 { + return None; + } + Some(match self.write_index.record_stamps.get(record_lsn) { Some(stamp) => stamp, None => WriteStamp { // A name without its database's qualifier is a bare name. @@ -169,8 +252,7 @@ impl CoreLoop { lsn: Lsn::new(record_lsn), entry: None, }, - }; - self.note_write(db, TenantId::new(tenant_id), collection, key, stamp); + }) } /// Run horizon GC on the per-core version index. Invoked from the periodic @@ -411,6 +493,103 @@ mod tests { assert_eq!(core.watermark, Lsn::new(42)); } + #[test] + fn a_kv_truncate_invalidates_a_point_read_of_a_removed_key() { + let (mut core, _, _, _dir) = make_core(); + let resp = core.execute_kv_put( + &wal_task(42), + KvWriteParams { + did: DatabaseId::DEFAULT.as_u64(), + tid: 1, + collection: "kv", + key: b"k1".as_slice(), + value: b"v1".as_slice(), + ttl_ms: 0, + surrogate: Surrogate::new(3), + returning: None, + rls_filters: &[], + }, + ); + assert_eq!(resp.status, Status::Ok); + let read = core.write_index.latest(vshard0()); + let point = + nodedb_types::calvin::ReadKeyIdent::Point(KeyRepr::KvKey(Box::from(b"k1".as_slice()))); + let current = |core: &CoreLoop| { + core.write_index.read_is_valid( + DatabaseId::DEFAULT, + TenantId::new(1), + vshard0(), + "kv", + &point, + read, + ) + }; + assert!(current(&core), "the read saw the put"); + + let resp = core.execute_kv_truncate(&wal_task(50), DatabaseId::DEFAULT.as_u64(), 1, "kv"); + assert_eq!(resp.status, Status::Ok); + assert!( + !current(&core), + "the truncate removed k1 after the read observed it" + ); + assert_eq!(core.watermark, Lsn::new(50)); + } + + #[test] + fn a_document_truncate_invalidates_an_index_read_of_a_removed_row() { + let (mut core, _, _, _dir) = make_core(); + let put = wal_task(42); + let resp = point_put(&mut core, &put, "docs", "d1", 3, "open"); + assert_eq!(resp.status, Status::Ok, "{:?}", resp.error_code); + // The index value the put of an indexed collection records. + let stamp = core.task_write_stamp(&put).expect("a WAL task has a stamp"); + core.note_index_write_values( + DatabaseId::DEFAULT, + TenantId::new(1), + "docs", + &[("a".to_string(), "open".to_string())], + stamp, + ); + let read = core.write_index.latest(vshard0()); + let index_eq = nodedb_types::calvin::ReadKeyIdent::IndexEq { + field: "a".to_string(), + value: "open".to_string(), + }; + let point = nodedb_types::calvin::ReadKeyIdent::Point(KeyRepr::Surrogate(3)); + let current = |core: &CoreLoop, key: &nodedb_types::calvin::ReadKeyIdent| { + core.write_index.read_is_valid( + DatabaseId::DEFAULT, + TenantId::new(1), + vshard0(), + "docs", + key, + read, + ) + }; + assert!(current(&core, &index_eq), "the index read saw the put"); + assert!(current(&core, &point), "the point read saw the put"); + + let resp = core.execute_truncate( + &wal_task(50), + 1, + crate::data::executor::handlers::truncate::TruncateParams { + collection: "docs", + resolved_sum_targets: &[], + declared_primary_key: None, + }, + ); + assert_eq!(resp.status, Status::Ok, "{:?}", resp.error_code); + assert!( + !current(&core, &index_eq), + "the truncate removed the indexed row after the read observed it" + ); + assert!( + !current(&core, &point), + "the truncate removed d1 after the read observed it" + ); + assert_eq!(core.watermark, Lsn::new(50)); + } + #[test] fn an_autocommit_apply_is_named_by_the_next_replay_stamp() { let (mut core, _, _, _dir) = make_core(); diff --git a/nodedb/src/data/executor/core_loop/write_index/validate.rs b/nodedb/src/data/executor/core_loop/write_index/validate.rs index 51991a501..be11d6da6 100644 --- a/nodedb/src/data/executor/core_loop/write_index/validate.rs +++ b/nodedb/src/data/executor/core_loop/write_index/validate.rs @@ -255,7 +255,7 @@ mod tests { attr_projection: Vec::new(), limit: 0, cell_filter: None, - hilbert_range: None, + shard_bucket: None, system_time: nodedb_types::SystemTimeScope::Current, valid_at_ms: None, }, diff --git a/nodedb/src/data/executor/dispatch/array/aggregate.rs b/nodedb/src/data/executor/dispatch/array/aggregate.rs index bd0a03648..3e1582e5a 100644 --- a/nodedb/src/data/executor/dispatch/array/aggregate.rs +++ b/nodedb/src/data/executor/dispatch/array/aggregate.rs @@ -15,7 +15,8 @@ use nodedb_array::segment::TilePayload; use nodedb_array::types::ArrayId; use nodedb_array::types::coord::value::CoordValue; use nodedb_cluster::distributed_array::merge::ArrayAggPartial; -use nodedb_types::SurrogateBitmap; +use nodedb_types::array_route::shard_bucket_admits; +use nodedb_types::{ShardBucket, SurrogateBitmap}; use crate::bridge::envelope::{ErrorCode, Response}; use crate::data::executor::core_loop::CoreLoop; @@ -37,8 +38,8 @@ pub(in crate::data::executor) struct AggParams<'a> { pub group_by_dim_idx: i32, pub cell_filter: Option<&'a SurrogateBitmap>, pub return_partial: bool, - /// Optional Hilbert-prefix range `[lo, hi]` for shard-level partitioning. - pub hilbert_range: Option<(u64, u64)>, + /// Optional shard bucket. Only tiles whose key routes to it contribute. + pub shard_bucket: Option, /// Bitemporal system-time cutoff. `None` = live read. pub system_as_of: Option, /// Bitemporal valid-time point. `None` = no valid-time filter. @@ -49,7 +50,7 @@ pub(in crate::data::executor) struct AggParams<'a> { struct AggTileScan<'a> { array_id: &'a ArrayId, schema: &'a ArraySchema, - hilbert_range: Option<(u64, u64)>, + shard_bucket: Option, system_as_of: Option, valid_at_ms: Option, /// The transaction whose staged cells merge into the scan, or `None` @@ -78,13 +79,6 @@ struct AggEmit<'a> { emit_horizon: bool, } -fn hilbert_prefix_in_range(hp: u64, range: Option<(u64, u64)>) -> bool { - match range { - Some((lo, hi)) => hp >= lo && hp <= hi, - None => true, - } -} - impl CoreLoop { pub(in crate::data::executor) fn dispatch_array_aggregate( &mut self, @@ -98,7 +92,7 @@ impl CoreLoop { group_by_dim_idx, cell_filter, return_partial, - hilbert_range, + shard_bucket, system_as_of, valid_at_ms, } = p; @@ -126,7 +120,7 @@ impl CoreLoop { // shard's result, for the finalizing single-node read and for the // distributed partial alike. Each shard's overlay holds only the // cells partitioned to that shard, and `collect_agg_tiles` applies - // the shard's `hilbert_range` to overlay tiles as to base tiles, so + // the shard's `shard_bucket` to overlay tiles as to base tiles, so // folding them into the shard's partial counts every staged cell // exactly once at the coordinator's merge. `AS OF SYSTEM TIME` is a // snapshot of committed history and skips the overlay. @@ -138,7 +132,7 @@ impl CoreLoop { let (all_tiles, truncated_before_horizon) = match self.collect_agg_tiles(AggTileScan { array_id, schema: &schema, - hilbert_range, + shard_bucket, system_as_of, valid_at_ms, overlay_txn, @@ -172,7 +166,7 @@ impl CoreLoop { /// double-count any cell overwritten across segments in a bitemporal array /// (e.g. v1 in one sealed segment, v2 in another). `system_as_of`/ /// `valid_at_ms` narrow the cutoff for point-in-time queries; the same - /// `hilbert_range` shard filter applies in all cases. + /// `shard_bucket` filter applies in all cases. /// /// Returns an error *detail* string (wrapped into `ErrorCode::Internal` by /// the caller) rather than a `Response`, so it can borrow `&self` cleanly. @@ -180,7 +174,7 @@ impl CoreLoop { let AggTileScan { array_id, schema, - hilbert_range, + shard_bucket, system_as_of, valid_at_ms, overlay_txn, @@ -194,7 +188,7 @@ impl CoreLoop { .scan_tiles_at(cutoff, valid_at_ms) .map_err(|e| format!("array aggregate scan: {e}"))?; // Staged cells join the SAME tile set the base cells reduce through, - // ahead of the shard-range filter, so they fold into the same partial + // ahead of the shard bucket filter, so they fold into the same partial // accumulator under the same surrogate filter. self.merge_array_overlay_tiles( ArrayOverlayMergeParams { @@ -208,7 +202,7 @@ impl CoreLoop { .map_err(|e| format!("array aggregate overlay merge: {e}"))?; let tiles = resolved_tiles .into_iter() - .filter(|(hp, _)| hilbert_prefix_in_range(*hp, hilbert_range)) + .filter(|(hp, _)| shard_bucket_admits(shard_bucket, *hp)) .map(|(_, tile)| TilePayload::Sparse(tile)) .collect(); Ok((tiles, truncated_before_horizon)) @@ -553,7 +547,7 @@ mod tests { group_by_dim: -1, cell_filter: None, return_partial: false, - hilbert_range: None, + shard_bucket: None, system_as_of: None, valid_at_ms: None, }); @@ -587,7 +581,7 @@ mod tests { group_by_dim: -1, cell_filter: None, return_partial: true, - hilbert_range: None, + shard_bucket: None, system_as_of: None, valid_at_ms: None, }); @@ -640,7 +634,7 @@ mod tests { group_by_dim: -1, cell_filter: None, return_partial: false, - hilbert_range: None, + shard_bucket: None, system_as_of: Some(150), valid_at_ms: None, }); @@ -672,7 +666,7 @@ mod tests { group_by_dim: -1, cell_filter: None, return_partial: false, - hilbert_range: None, + shard_bucket: None, system_as_of: Some(50), valid_at_ms: None, }); @@ -707,7 +701,7 @@ mod tests { group_by_dim: 0, cell_filter: None, return_partial: false, - hilbert_range: None, + shard_bucket: None, system_as_of: None, valid_at_ms: None, }); @@ -761,7 +755,7 @@ mod tests { group_by_dim: -1, cell_filter: Some(bm), return_partial: false, - hilbert_range: None, + shard_bucket: None, system_as_of: None, valid_at_ms: None, }); diff --git a/nodedb/src/data/executor/dispatch/array/entry.rs b/nodedb/src/data/executor/dispatch/array/entry.rs index bd2314daa..06bc8e6a1 100644 --- a/nodedb/src/data/executor/dispatch/array/entry.rs +++ b/nodedb/src/data/executor/dispatch/array/entry.rs @@ -88,7 +88,7 @@ impl CoreLoop { attr_projection, limit, cell_filter, - hilbert_range, + shard_bucket, system_time, valid_at_ms, } => self.dispatch_array_slice( @@ -99,7 +99,7 @@ impl CoreLoop { attr_projection, limit: *limit, cell_filter: cell_filter.as_ref(), - hilbert_range: *hilbert_range, + shard_bucket: *shard_bucket, system_time: *system_time, valid_at_ms: *valid_at_ms, }, @@ -119,7 +119,7 @@ impl CoreLoop { group_by_dim, cell_filter, return_partial, - hilbert_range, + shard_bucket, system_as_of, valid_at_ms, } => self.dispatch_array_aggregate( @@ -131,7 +131,7 @@ impl CoreLoop { group_by_dim_idx: *group_by_dim, cell_filter: cell_filter.as_ref(), return_partial: *return_partial, - hilbert_range: *hilbert_range, + shard_bucket: *shard_bucket, system_as_of: *system_as_of, valid_at_ms: *valid_at_ms, }, diff --git a/nodedb/src/data/executor/dispatch/array/read.rs b/nodedb/src/data/executor/dispatch/array/read.rs index 387469017..d622eb0be 100644 --- a/nodedb/src/data/executor/dispatch/array/read.rs +++ b/nodedb/src/data/executor/dispatch/array/read.rs @@ -14,7 +14,8 @@ use nodedb_array::query::slice::{Slice, slice_sparse, tile_overlaps_slice}; use nodedb_array::segment::{MbrQueryPredicate, TilePayload}; use nodedb_array::tile::sparse_tile::SparseTile; use nodedb_array::types::ArrayId; -use nodedb_types::{ArrayCell, SurrogateBitmap, SystemTimeScope, Value}; +use nodedb_types::array_route::shard_bucket_admits; +use nodedb_types::{ArrayCell, ShardBucket, SurrogateBitmap, SystemTimeScope, Value}; /// Slice parameters bundled to avoid exceeding the 7-argument limit. pub(in crate::data::executor) struct SliceParams<'a> { @@ -23,11 +24,10 @@ pub(in crate::data::executor) struct SliceParams<'a> { pub attr_projection: &'a [u32], pub limit: u32, pub cell_filter: Option<&'a SurrogateBitmap>, - /// Optional Hilbert-prefix range `[lo, hi]` for shard-level partitioning. - /// When set, only tiles whose Hilbert prefix falls within this range are - /// included. Used by the distributed shard handler to prevent duplicate - /// rows when all vShards share a single Data Plane. - pub hilbert_range: Option<(u64, u64)>, + /// Optional shard bucket. When set, only tiles whose key routes to this + /// bucket are included, so vShards that share one Data Plane return no + /// duplicate rows. + pub shard_bucket: Option, /// Bitemporal system-time scope. /// /// - `Current`: live read (effective cutoff `i64::MAX`). @@ -59,7 +59,7 @@ impl CoreLoop { attr_projection, limit, cell_filter, - hilbert_range, + shard_bucket, system_time, valid_at_ms, } = p; @@ -126,11 +126,9 @@ impl CoreLoop { } }; - // Apply Hilbert-range and slice coord filters. + // Apply the shard bucket and slice coord filters. versions.retain(|(hp, coord, _sys_ms, _payload)| { - if let Some((lo, hi)) = hilbert_range - && (*hp < lo || *hp > hi) - { + if !shard_bucket_admits(shard_bucket, *hp) { return false; } // Check coord against slice predicate dim-by-dim. @@ -235,9 +233,7 @@ impl CoreLoop { let mut rows: Vec = Vec::new(); 'outer: for (hp, sparse) in resolved_tiles { - if let Some((lo, hi)) = hilbert_range - && (hp < lo || hp > hi) - { + if !shard_bucket_admits(shard_bucket, hp) { continue; } if !tile_overlaps_slice(&sparse.mbr.dim_mins, &sparse.mbr.dim_maxs, &slice) { @@ -742,7 +738,7 @@ mod tests { attr_projection: vec![], limit: 0, cell_filter: None, - hilbert_range: None, + shard_bucket: None, system_time: nodedb_types::SystemTimeScope::Current, valid_at_ms: None, }); @@ -794,7 +790,7 @@ mod tests { attr_projection: vec![], limit: 0, cell_filter: None, - hilbert_range: None, + shard_bucket: None, system_time: nodedb_types::SystemTimeScope::AllVersions, valid_at_ms: None, }); @@ -867,7 +863,7 @@ mod tests { attr_projection: vec![], limit: 0, cell_filter: Some(bm), - hilbert_range: None, + shard_bucket: None, system_time: nodedb_types::SystemTimeScope::Current, valid_at_ms: None, }); @@ -887,4 +883,122 @@ mod tests { // 10.0 + 30.0 = 40.0; 20.0 must have been excluded. assert!((total - 40.0).abs() < 1e-9, "total got {total}"); } + + fn schema_full_range_2d(name: &str) -> ArraySchema { + let full = || Domain::new(DomainBound::Int64(i64::MIN), DomainBound::Int64(i64::MAX)); + ArraySchemaBuilder::new(name) + .dim(DimSpec::new("x", DimType::Int64, full())) + .dim(DimSpec::new("y", DimType::Int64, full())) + .attr(AttrSpec::new("v", AttrType::Float64, true)) + .tile_extents(vec![1, 1]) + .build() + .unwrap() + } + + fn slice_in_bucket(h: &mut Harness, aid: &ArrayId, bucket: Option) -> Vec { + let slice = ArraySlice::new(vec![None, None]); + let r = h.send(ArrayOp::Slice { + array_id: aid.clone(), + slice_msgpack: zerompk::to_msgpack_vec(&slice).unwrap(), + attr_projection: vec![], + limit: 0, + cell_filter: None, + shard_bucket: bucket, + system_time: nodedb_types::SystemTimeScope::Current, + valid_at_ms: None, + }); + assert_eq!(r.status, Status::Ok, "slice failed: {r:?}"); + decode_slice_rows(r.payload.as_ref()) + } + + fn row_coords(row: &Value) -> Vec { + match row { + Value::ArrayCell(ArrayCell { coords, .. }) => coords.clone(), + other => panic!("row not ArrayCell: {other:?}"), + } + } + + /// Over every bucket, the shard filter returns each cell exactly once. + #[test] + fn shard_buckets_partition_the_cells_of_a_full_range_array() { + let mut h = Harness::new(); + let s = schema_full_range_2d("t6_buckets"); + let aid = ArrayId::new(TenantId::new(1), "t6_buckets"); + h.open(&aid, &s, 0xB1); + let cells: Vec = (0..8) + .flat_map(|x| (0..8).map(move |y| cell(x, y, (x * 8 + y) as f64))) + .collect(); + h.put(&aid, cells, 1); + h.flush(&aid); + + let prefix_bits = 8u8; + let mut seen: Vec = Vec::new(); + let mut non_empty = 0usize; + for bucket in 0..(1u32 << prefix_bits) { + let rows = slice_in_bucket( + &mut h, + &aid, + Some(ShardBucket { + bucket, + prefix_bits, + }), + ); + if !rows.is_empty() { + non_empty += 1; + } + seen.extend(rows.iter().map(|r| format!("{:?}", row_coords(r)))); + } + assert_eq!( + seen.len(), + 64, + "every cell is returned by exactly one bucket" + ); + seen.sort(); + seen.dedup(); + assert_eq!(seen.len(), 64, "no cell is returned by two buckets"); + assert!(non_empty >= 32, "64 tiles land in only {non_empty} buckets"); + assert_eq!(slice_in_bucket(&mut h, &aid, None).len(), 64); + } + + /// Two tiles `2^bits` apart share one key, so they share one bucket, and + /// a read of that bucket returns both cells. + #[test] + fn tiles_that_share_a_key_route_together_and_both_cells_read_back() { + let mut h = Harness::new(); + let s = schema_full_range_2d("t6_collide"); + let aid = ArrayId::new(TenantId::new(1), "t6_collide"); + h.open(&aid, &s, 0xB2); + let far = 1i64 << nodedb_array::coord::bits_per_dim(2); + let near_coord = [CoordValue::Int64(3), CoordValue::Int64(5)]; + let far_coord = [CoordValue::Int64(3 + far), CoordValue::Int64(5)]; + let key = nodedb_array::tile::cell_tile_prefix(&s, &near_coord).unwrap(); + assert_eq!( + nodedb_array::tile::cell_tile_prefix(&s, &far_coord).unwrap(), + key, + "the fixture's two tiles share one key" + ); + h.put( + &aid, + vec![cell_sur(3, 5, 1.0, 7), cell_sur(3 + far, 5, 2.0, 8)], + 1, + ); + h.flush(&aid); + + let prefix_bits = 8u8; + let owner = ShardBucket { + bucket: nodedb_types::array_route::prefix_bucket(key, prefix_bits), + prefix_bits, + }; + let rows = slice_in_bucket(&mut h, &aid, Some(owner)); + assert_eq!( + rows.len(), + 2, + "both cells of the shared key read back: {rows:?}" + ); + let other = ShardBucket { + bucket: (owner.bucket + 1) % (1u32 << prefix_bits), + prefix_bits, + }; + assert!(slice_in_bucket(&mut h, &aid, Some(other)).is_empty()); + } } diff --git a/nodedb/src/data/executor/handlers/columnar_truncate.rs b/nodedb/src/data/executor/handlers/columnar_truncate.rs index 51c4bc833..1077aa7d6 100644 --- a/nodedb/src/data/executor/handlers/columnar_truncate.rs +++ b/nodedb/src/data/executor/handlers/columnar_truncate.rs @@ -78,6 +78,8 @@ impl CoreLoop { self.checkpoint_coordinator .mark_dirty("columnar", truncated); + // No wipe: columnar and spatial reads record only `Predicate` keys, + // which the collection version bounds. self.note_collection_write(task, collection); self.invalidate_aggregate_cache_for_collection(db.as_u64(), tid.as_u64(), collection); diff --git a/nodedb/src/data/executor/handlers/control/calvin_txn_id.rs b/nodedb/src/data/executor/handlers/control/calvin_txn_id.rs index a104cb2d8..d0e5c9a0b 100644 --- a/nodedb/src/data/executor/handlers/control/calvin_txn_id.rs +++ b/nodedb/src/data/executor/handlers/control/calvin_txn_id.rs @@ -27,13 +27,16 @@ //! distinct `(epoch, position, vshard)` triples collide on the same //! synthetic `TxnId` and corrupt each other's staged overlay. -use crate::types::TxnId; +use crate::types::{TxnId, VShardId}; -const VSHARD_BITS: u32 = 10; +const VSHARD_BITS: u32 = VShardId::BITS as u32; const POSITION_BITS: u32 = 20; const EPOCH_BITS: u32 = 33; -const VSHARD_MAX: u32 = 1 << VSHARD_BITS; +// The marker bit and the three fields fill the id exactly. +const _: () = assert!(1 + EPOCH_BITS + POSITION_BITS + VSHARD_BITS == 64); + +const VSHARD_MAX: u32 = VShardId::COUNT; const POSITION_MAX: u32 = 1 << POSITION_BITS; const EPOCH_MAX: u64 = 1 << EPOCH_BITS; diff --git a/nodedb/src/data/executor/handlers/kv/crud/delete.rs b/nodedb/src/data/executor/handlers/kv/crud/delete.rs index de25fd2e9..a6bd0885d 100644 --- a/nodedb/src/data/executor/handlers/kv/crud/delete.rs +++ b/nodedb/src/data/executor/handlers/kv/crud/delete.rs @@ -107,6 +107,8 @@ impl CoreLoop { ) -> Response { debug!(core = self.core_id, %collection, "kv truncate"); let count = self.kv_engine.truncate(did, tid, collection); + // Every key leaves, and no per-key version names them. + self.note_task_collection_wipe(task, tid, collection); match response_codec::encode_count("deleted", count) { Ok(payload) => self.response_with_payload(task, payload), Err(e) => self.response_error(task, ErrorCode::from(e)), diff --git a/nodedb/src/data/executor/handlers/kv/dispatch.rs b/nodedb/src/data/executor/handlers/kv/dispatch.rs index f4775d5a5..fc917cfc4 100644 --- a/nodedb/src/data/executor/handlers/kv/dispatch.rs +++ b/nodedb/src/data/executor/handlers/kv/dispatch.rs @@ -2,7 +2,7 @@ //! KV operation dispatch: routes `KvOp` variants to their handler methods. -use crate::bridge::envelope::Response; +use crate::bridge::envelope::{ErrorCode, Response}; use crate::data::executor::core_loop::CoreLoop; use crate::data::executor::task::ExecutionTask; use nodedb_physical::physical_plan::KvOp; @@ -441,44 +441,20 @@ impl CoreLoop { cursor, *count, ), - KvOp::PredicateUpdate { - collection, - filters, - updates, - rls_write_check, - returning, - rls_filters, - } => self.execute_kv_predicate_update( - task, - super::predicate::KvPredicateCtx { - did, - tid, - collection: collection.as_str(), - filters, - rls_write_check, - returning: returning.as_ref(), - rls_filters, - }, - updates, - ), - KvOp::PredicateDelete { - collection, - filters, - rls_write_check, - returning, - rls_filters, - } => self.execute_kv_predicate_delete( - task, - super::predicate::KvPredicateCtx { - did, - tid, - collection: collection.as_str(), - filters, - rls_write_check, - returning: returning.as_ref(), - rls_filters, - }, - ), + // A predicate write resolves to a mutation list before it is + // logged, and a transaction stages it. It never applies directly. + KvOp::PredicateUpdate { collection, .. } | KvOp::PredicateDelete { collection, .. } => { + self.response_error( + task, + ErrorCode::Internal { + detail: format!( + "kv predicate UPDATE/DELETE on '{}' reached the apply path \ + unresolved; it resolves to a mutation list first", + collection.as_str() + ), + }, + ) + } KvOp::ResolveWrite(inner) => self.execute_kv_resolve_write(task, did, tid, inner), KvOp::ResolvedWrite { mutations, diff --git a/nodedb/src/data/executor/handlers/kv/predicate/apply.rs b/nodedb/src/data/executor/handlers/kv/predicate/apply.rs deleted file mode 100644 index a623d6718..000000000 --- a/nodedb/src/data/executor/handlers/kv/predicate/apply.rs +++ /dev/null @@ -1,166 +0,0 @@ -// SPDX-License-Identifier: BUSL-1.1 - -//! Live handlers for `KvOp::PredicateUpdate` / `KvOp::PredicateDelete`. Both -//! resolve their row set via [`CoreLoop::kv_predicate_matches`] and reuse the -//! keyed path verbatim — re-deriving either here is how a predicate write -//! would drift from the keyed one. - -use nodedb_physical::physical_plan::ReturningSpec; -use nodedb_types::RlsWriteCheck; -use tracing::debug; - -use crate::bridge::envelope::{ErrorCode, Response}; -use crate::data::executor::core_loop::CoreLoop; -use crate::data::executor::handlers::kv::crud::KvDeleteParams; -use crate::data::executor::handlers::kv::field_compute::merge_field_updates; -use crate::data::executor::handlers::kv::rls::admit_kv_row; -use crate::data::executor::handlers::returning_rows::KvStoredRow; -use crate::data::executor::response_codec; -use crate::data::executor::task::ExecutionTask; -use crate::engine::kv::entry::NO_EXPIRY; -use crate::engine::kv::{KvRewriteParams, current_ms}; - -/// Routing and policy inputs shared by both predicate handlers. -pub(in crate::data::executor) struct KvPredicateCtx<'a> { - pub did: u64, - pub tid: u64, - pub collection: &'a str, - pub filters: &'a [u8], - pub rls_write_check: &'a RlsWriteCheck, - /// When `Some`, project one STORED row per matched key per spec instead - /// of reporting a bare count: the post-image for an update, the - /// pre-image for a delete. - pub returning: Option<&'a ReturningSpec>, - /// Compiled read policy bounding which of those rows may be shown back. - pub rls_filters: &'a [u8], -} - -impl CoreLoop { - /// Merge `updates` into every row the predicate matches. Every post-image - /// is decided against the write policy before the first row is written, - /// so one rejected row leaves the whole statement without effect. - pub(in crate::data::executor) fn execute_kv_predicate_update( - &mut self, - task: &ExecutionTask, - ctx: KvPredicateCtx<'_>, - updates: &[(String, Vec)], - ) -> Response { - let KvPredicateCtx { - did, - tid, - collection, - filters, - rls_write_check, - returning, - rls_filters, - } = ctx; - debug!(core = self.core_id, %collection, "kv predicate update"); - let now_ms = current_ms(); - - let matched = match self.kv_predicate_matches(did, tid, collection, filters, now_ms) { - Ok(rows) => rows, - Err(e) => return self.response_error(task, e), - }; - - let declared = self.declared_columns_of(did, tid, collection); - let mut writes: Vec<(Vec, Vec, Vec)> = Vec::with_capacity(matched.len()); - for (key, body) in matched { - let computed = - match merge_field_updates(collection, Some(body.as_slice()), updates, declared) { - Ok(c) => c, - Err(e) => return self.response_error(task, e), - }; - if let Err(e) = - admit_kv_row(rls_write_check, &computed.new_value, &key, tid, collection) - { - return self.response_error(task, e); - } - writes.push((key, body, computed.new_value)); - } - - for (key, old_body, new_value) in &writes { - // The matched row exists and keeps its bound identity. - self.kv_engine.rewrite_with_absolute_expiry( - KvRewriteParams { - database_id: did, - tenant_id: tid, - collection, - key, - value: new_value, - // Mirrors `execute_kv_field_set`, whose keyed merge this is - // the predicate form of: the merge writes no TTL of its own. - ttl_ms: 0, - now_ms, - }, - NO_EXPIRY, - ); - if let Some(ref m) = self.metrics { - m.record_kv_put(); - } - self.emit_kv_write_event( - task, - collection, - crate::event::WriteOp::Update, - key, - Some(new_value), - Some(old_body), - ); - self.note_kv_write(task, did, tid, collection, key); - } - - if let Some(spec) = returning { - // Each `new_value` IS the stored body: the merge is persisted - // verbatim, so projecting it is projecting the post-image. Zero - // matches ships an EMPTY row set, never a count. - let rows: Vec> = writes - .iter() - .map(|(key, _old_body, new_value)| (key.as_slice(), new_value.as_slice())) - .collect(); - return self.kv_stored_returning_response(task, spec, rls_filters, &rows); - } - - match response_codec::encode_count("affected", writes.len()) { - Ok(payload) => self.response_with_payload(task, payload), - Err(e) => self.response_error(task, ErrorCode::from(e)), - } - } - - /// Delete every row the predicate matches. Resolved keys go through - /// `execute_kv_delete`, so the gate, metrics, and reply are the keyed - /// delete's, not a second copy. - pub(in crate::data::executor) fn execute_kv_predicate_delete( - &mut self, - task: &ExecutionTask, - ctx: KvPredicateCtx<'_>, - ) -> Response { - let KvPredicateCtx { - did, - tid, - collection, - filters, - rls_write_check, - returning, - rls_filters, - } = ctx; - debug!(core = self.core_id, %collection, "kv predicate delete"); - let now_ms = current_ms(); - - let matched = match self.kv_predicate_matches(did, tid, collection, filters, now_ms) { - Ok(rows) => rows, - Err(e) => return self.response_error(task, e), - }; - let keys: Vec> = matched.into_iter().map(|(key, _body)| key).collect(); - self.execute_kv_delete( - task, - KvDeleteParams { - did, - tid, - collection, - keys: &keys, - rls_write_check, - returning, - rls_filters, - }, - ) - } -} diff --git a/nodedb/src/data/executor/handlers/kv/predicate/ctx.rs b/nodedb/src/data/executor/handlers/kv/predicate/ctx.rs new file mode 100644 index 000000000..db4345d58 --- /dev/null +++ b/nodedb/src/data/executor/handlers/kv/predicate/ctx.rs @@ -0,0 +1,21 @@ +// SPDX-License-Identifier: BUSL-1.1 + +//! Inputs of a KV predicate `UPDATE` / `DELETE` resolve. + +use nodedb_physical::physical_plan::ReturningSpec; +use nodedb_types::RlsWriteCheck; + +/// Routing and policy inputs shared by both predicate resolves. +pub(in crate::data::executor) struct KvPredicateCtx<'a> { + pub did: u64, + pub tid: u64, + pub collection: &'a str, + pub filters: &'a [u8], + pub rls_write_check: &'a RlsWriteCheck, + /// When `Some`, project one STORED row per matched key per spec instead + /// of reporting a bare count: the post-image for an update, the + /// pre-image for a delete. + pub returning: Option<&'a ReturningSpec>, + /// Compiled read policy bounding which of those rows may be shown back. + pub rls_filters: &'a [u8], +} diff --git a/nodedb/src/data/executor/handlers/kv/predicate/mod.rs b/nodedb/src/data/executor/handlers/kv/predicate/mod.rs index 84d9919fb..f654b1d8a 100644 --- a/nodedb/src/data/executor/handlers/kv/predicate/mod.rs +++ b/nodedb/src/data/executor/handlers/kv/predicate/mod.rs @@ -1,8 +1,10 @@ // SPDX-License-Identifier: BUSL-1.1 //! KV predicate DML: `UPDATE` / `DELETE` whose `WHERE` names no primary key. +//! The Control Plane resolves both to a mutation list before they are +//! logged, so the Data Plane reads the predicate only to resolve or stage it. -pub(in crate::data::executor) mod apply; +mod ctx; mod matches; -pub(in crate::data::executor) use apply::KvPredicateCtx; +pub(in crate::data::executor) use ctx::KvPredicateCtx; diff --git a/nodedb/src/data/executor/handlers/kv/resolve/predicate_ops.rs b/nodedb/src/data/executor/handlers/kv/resolve/predicate_ops.rs index a86e3445f..d847fc33f 100644 --- a/nodedb/src/data/executor/handlers/kv/resolve/predicate_ops.rs +++ b/nodedb/src/data/executor/handlers/kv/resolve/predicate_ops.rs @@ -63,8 +63,8 @@ impl CoreLoop { collection, key: &key, value: new_value, - // `execute_kv_predicate_update` writes with `ttl_ms: 0`, the - // keyed field merge's behaviour. Preserved verbatim. + // The keyed field merge writes with `ttl_ms: 0`, so the + // resolved rewrite does too. ttl_ms: 0, expire_at_ms: 0, precondition: body, diff --git a/nodedb/src/data/executor/handlers/mod.rs b/nodedb/src/data/executor/handlers/mod.rs index a30ee2c1c..b0f9c5926 100644 --- a/nodedb/src/data/executor/handlers/mod.rs +++ b/nodedb/src/data/executor/handlers/mod.rs @@ -99,6 +99,8 @@ pub mod vector_direct_delete; pub mod vector_direct_resolve; pub mod vector_direct_row; pub mod vector_direct_targets; +#[cfg(test)] +pub mod vector_direct_test_rows; pub mod vector_direct_truncate; pub mod vector_direct_update; pub mod vector_index_drop; diff --git a/nodedb/src/data/executor/handlers/snapshot/capture_arrays.rs b/nodedb/src/data/executor/handlers/snapshot/capture_arrays.rs index ab14cad02..c216c1d66 100644 --- a/nodedb/src/data/executor/handlers/snapshot/capture_arrays.rs +++ b/nodedb/src/data/executor/handlers/snapshot/capture_arrays.rs @@ -12,7 +12,7 @@ use std::sync::Arc; use nodedb_array::schema::ArraySchema; use nodedb_array::tile::cell_tile_prefix; -use nodedb_cluster::distributed_array::array_vshard_for_tile; +use nodedb_types::array_route::array_vshard_for_tile; use crate::data::executor::core_loop::CoreLoop; use crate::engine::array::export::ArrayCellVersion; @@ -81,18 +81,18 @@ fn by_vshard( ) -> crate::Result>> { let mut grouped: BTreeMap> = BTreeMap::new(); for version in versions { - let prefix = match cell_tile_prefix(schema, &version.coord) { - Ok(prefix) => prefix, + let tile_key = match cell_tile_prefix(schema, &version.coord) { + Ok(tile_key) => tile_key, Err(_) if version.payload.is_none() => continue, Err(e) => { return Err(capture_error( array, - format!("Hilbert prefix of a live cell: {e}"), + format!("tile key of a live cell: {e}"), )); } }; - let vshard = array_vshard_for_tile(prefix, prefix_bits) - .map_err(|e| capture_error(array, format!("vShard of Hilbert prefix {prefix}: {e}")))?; + let vshard = array_vshard_for_tile(tile_key, prefix_bits) + .map_err(|e| capture_error(array, format!("vShard of tile key {tile_key}: {e}")))?; grouped.entry(vshard).or_default().push(version); } Ok(grouped) @@ -152,12 +152,71 @@ mod tests { ); for (vshard, cells) in &grouped { for cell in cells { - let prefix = cell_tile_prefix(&s, &cell.coord).expect("prefix"); - assert_eq!(array_vshard_for_tile(prefix, 8).expect("route"), *vshard); + let tile_key = cell_tile_prefix(&s, &cell.coord).expect("tile key"); + assert_eq!(array_vshard_for_tile(tile_key, 8).expect("route"), *vshard); } } } + fn full_range_2d() -> ArraySchema { + let full = || Domain::new(DomainBound::Int64(i64::MIN), DomainBound::Int64(i64::MAX)); + ArraySchemaBuilder::new("open") + .dim(DimSpec::new("x", DimType::Int64, full())) + .dim(DimSpec::new("y", DimType::Int64, full())) + .attr(AttrSpec::new("v", AttrType::Int64, true)) + .tile_extents(vec![1, 1]) + .build() + .expect("schema") + } + + fn live_2d(x: i64, y: i64) -> ArrayCellVersion { + ArrayCellVersion { + coord: vec![CoordValue::Int64(x), CoordValue::Int64(y)], + ..live(x) + } + } + + /// Cells near the origin of a full-range 2-D array spread over vShards, + /// and a snapshot captured for a group of vShards installs into exactly + /// the cells that group's install replaces. + #[test] + fn a_full_range_array_captures_and_installs_by_the_same_route() { + let s = full_range_2d(); + let versions: Vec = (0..16) + .flat_map(|x| (0..16).map(move |y| live_2d(x, y))) + .collect(); + let grouped = by_vshard("open", &s, 8, versions.clone()).expect("group"); + assert!( + grouped.len() >= 64, + "256 tiles near the origin reach only {} vShards", + grouped.len() + ); + assert_eq!( + grouped.values().map(Vec::len).sum::(), + versions.len(), + "every version is captured once" + ); + + // A Raft group owning every other captured vShard. + let group: std::collections::HashSet = grouped.keys().copied().step_by(2).collect(); + let captured: Vec<&ArrayCellVersion> = grouped + .iter() + .filter(|(vshard, _)| group.contains(vshard)) + .flat_map(|(_, cells)| cells.iter()) + .collect(); + assert!(!captured.is_empty()); + for version in &versions { + let in_snapshot = captured.iter().any(|c| c.coord == version.coord); + let replaced = + super::super::install_arrays::routes_into_group(&s, 8, &group, &version.coord); + assert_eq!( + replaced, in_snapshot, + "coord {:?}: install replaces it iff the snapshot carries it", + version.coord + ); + } + } + #[test] fn an_unroutable_tombstone_is_left_out_and_a_live_cell_fails() { let s = schema(); diff --git a/nodedb/src/data/executor/handlers/snapshot/install_arrays.rs b/nodedb/src/data/executor/handlers/snapshot/install_arrays.rs index 63b16c4e0..4f8db8549 100644 --- a/nodedb/src/data/executor/handlers/snapshot/install_arrays.rs +++ b/nodedb/src/data/executor/handlers/snapshot/install_arrays.rs @@ -13,7 +13,7 @@ use std::sync::Arc; use nodedb_array::schema::ArraySchema; use nodedb_array::tile::cell_tile_prefix; use nodedb_array::types::coord::value::CoordValue; -use nodedb_cluster::distributed_array::array_vshard_for_tile; +use nodedb_types::array_route::array_vshard_for_tile; use crate::data::executor::core_loop::CoreLoop; use crate::engine::array::export::ArrayCellVersion; @@ -28,6 +28,21 @@ fn install_error(detail: String) -> crate::Error { /// `(database, tenant, array)`. type ArrayKey = (u64, u64, String); +/// Whether a cell write at `coord` routes to a vShard in `group`. The install +/// replaces exactly the cells this admits. A coordinate outside the schema's +/// domain has no route and is never replaced. +pub(super) fn routes_into_group( + schema: &ArraySchema, + prefix_bits: u8, + group: &HashSet, + coord: &[CoordValue], +) -> bool { + cell_tile_prefix(schema, coord) + .ok() + .and_then(|tile_key| array_vshard_for_tile(tile_key, prefix_bits).ok()) + .is_some_and(|vshard| group.contains(&vshard)) +} + impl CoreLoop { /// Replace the array cells of `vshards` on this core with `blobs`. /// Returns the versions installed. @@ -92,12 +107,8 @@ impl CoreLoop { let schema = Arc::new(schema); let routes = Arc::clone(&schema); let prefix_bits = entry.prefix_bits; - let replaced = |coord: &[CoordValue]| { - cell_tile_prefix(&routes, coord) - .ok() - .and_then(|prefix| array_vshard_for_tile(prefix, prefix_bits).ok()) - .is_some_and(|vshard| group.contains(&vshard)) - }; + let replaced = + |coord: &[CoordValue]| routes_into_group(&routes, prefix_bits, &group, coord); self.array_engine .replace_cells( &entry.array_id, diff --git a/nodedb/src/data/executor/handlers/timeseries/truncate.rs b/nodedb/src/data/executor/handlers/timeseries/truncate.rs index 05aa0eb8f..92c610aab 100644 --- a/nodedb/src/data/executor/handlers/timeseries/truncate.rs +++ b/nodedb/src/data/executor/handlers/timeseries/truncate.rs @@ -165,6 +165,8 @@ impl CoreLoop { self.checkpoint_coordinator .mark_dirty("timeseries", truncated as usize); + // No wipe: timeseries reads record only `Predicate` keys, which the + // collection version bounds. self.note_collection_write(task, collection); self.invalidate_aggregate_cache_for_collection(db.as_u64(), tid.as_u64(), collection); diff --git a/nodedb/src/data/executor/handlers/transaction/overlay/array_merge.rs b/nodedb/src/data/executor/handlers/transaction/overlay/array_merge.rs index c7bbc056a..3dfb7da53 100644 --- a/nodedb/src/data/executor/handlers/transaction/overlay/array_merge.rs +++ b/nodedb/src/data/executor/handlers/transaction/overlay/array_merge.rs @@ -9,8 +9,8 @@ //! //! - base rows whose coordinate this transaction tombstoned OR re-put are //! dropped (a staged put shadows the base cell, last-writer-wins); -//! - staged puts are appended as synthetic tiles, one per Hilbert prefix, so -//! a shard `hilbert_range` filter, the slice window, the attribute +//! - staged puts are appended as synthetic tiles, one per tile key, so +//! a `shard_bucket` filter, the slice window, the attribute //! projection, the surrogate `cell_filter`, and the row cap all apply to //! staged cells exactly as they apply to base cells. //! @@ -130,8 +130,8 @@ fn merge_overlay( tiles.retain(|(_, tile)| tile.row_count() > 0); } - // Staged puts, grouped by the Hilbert prefix each would land in so a - // shard-range filter treats them like base tiles. + // Staged puts, grouped by the tile key each would land in so a shard + // bucket filter treats them like base tiles. let mut by_prefix: HashMap> = HashMap::new(); for (coord, put) in overlay.staged_cells(array_id) { if let Some(vt) = valid_at_ms diff --git a/nodedb/src/data/executor/handlers/transaction/redo_apply/entry.rs b/nodedb/src/data/executor/handlers/transaction/redo_apply/entry.rs index edd5d42f2..bb5333dd8 100644 --- a/nodedb/src/data/executor/handlers/transaction/redo_apply/entry.rs +++ b/nodedb/src/data/executor/handlers/transaction/redo_apply/entry.rs @@ -45,7 +45,7 @@ use crate::types::TenantId; use crate::wal::RedoRecord; use super::passes::{RedoTarget, final_refusal}; -use super::state::{RedoApplyPass, RedoApplyScope, TsInstalled}; +use super::state::{DeferredWriteKind, RedoApplyPass, RedoApplyScope, TsInstalled}; use super::sub_ops::{document_ops, kv_ops, label_ops}; use crate::engine::timeseries::install_counts::TsInstallCounts; @@ -198,13 +198,21 @@ impl CoreLoop { // Write versions and the watermark move only once the record is // settled. for version in std::mem::take(&mut scope.write_versions) { - self.publish_write_version( - version.db, - version.tenant, - &version.collection, - version.key, - version.stamp, - ); + match version.kind { + DeferredWriteKind::Wipe => self.publish_collection_wipe( + version.db, + version.tenant, + &version.collection, + version.stamp, + ), + DeferredWriteKind::Row(key) => self.publish_write_version( + version.db, + version.tenant, + &version.collection, + key, + version.stamp, + ), + } } // Events leave only once the record is settled. Every // one names the record's LSN: the install held the watermark back. diff --git a/nodedb/src/data/executor/handlers/transaction/redo_apply/state.rs b/nodedb/src/data/executor/handlers/transaction/redo_apply/state.rs index 6de4a417d..5d4a80e5f 100644 --- a/nodedb/src/data/executor/handlers/transaction/redo_apply/state.rs +++ b/nodedb/src/data/executor/handlers/transaction/redo_apply/state.rs @@ -121,8 +121,16 @@ pub(in crate::data::executor) struct DeferredWriteVersion { pub db: crate::types::DatabaseId, pub tenant: crate::types::TenantId, pub collection: String, - pub key: Option, pub stamp: WriteStamp, + pub kind: DeferredWriteKind, +} + +/// What a held-back write version records. +pub(in crate::data::executor) enum DeferredWriteKind { + /// A write to one key (`Some`), or to the collection only (`None`). + Row(Option), + /// A write that removed every row of the collection. + Wipe, } /// `(database, tenant, collection)`. @@ -179,8 +187,28 @@ impl RedoApplyScope { db, tenant, collection: collection.to_string(), - key, stamp, + kind: DeferredWriteKind::Row(key), + }); + } + } + + /// Hold back one whole-collection removal until the record settles. The + /// validate pass writes nothing, so it holds nothing. + pub(in crate::data::executor) fn defer_collection_wipe( + &mut self, + db: crate::types::DatabaseId, + tenant: crate::types::TenantId, + collection: &str, + stamp: WriteStamp, + ) { + if self.pass == RedoApplyPass::Install { + self.write_versions.push(DeferredWriteVersion { + db, + tenant, + collection: collection.to_string(), + stamp, + kind: DeferredWriteKind::Wipe, }); } } diff --git a/nodedb/src/data/executor/handlers/transaction/stage_write/stage_kv_predicate.rs b/nodedb/src/data/executor/handlers/transaction/stage_write/stage_kv_predicate.rs index c8c0ffa58..7c5251563 100644 --- a/nodedb/src/data/executor/handlers/transaction/stage_write/stage_kv_predicate.rs +++ b/nodedb/src/data/executor/handlers/transaction/stage_write/stage_kv_predicate.rs @@ -66,7 +66,7 @@ impl CoreLoop { /// Stage a KV predicate `UPDATE` at statement time: resolve the current /// BASE ∪ OVERLAY matching set, merge the SET-list into each match, and /// record the post-image as a staged `Put`. Replies `{"affected": N}`, - /// the same shape `execute_kv_predicate_update` reports in autocommit. + /// the same shape the autocommit resolve reports. pub(in crate::data::executor) fn stage_kv_predicate_update( &mut self, params: StageKvPredicateUpdateParams<'_>, @@ -204,6 +204,8 @@ impl CoreLoop { let filters: Vec = decode_scan_filters(filter_bytes, "kv predicate dml filters") .map_err(|e| self.response_error(task, e))?; + // Deterministic: Calvin staging pins the epoch anchor every replica + // shares, and session staging runs on the leader alone. let mut rows = self .kv_predicate_matches(did, tid, collection, filter_bytes, self.kv_read_now_ms()) .map_err(|e| self.response_error(task, e))?; diff --git a/nodedb/src/data/executor/handlers/transaction/write_version_kv.rs b/nodedb/src/data/executor/handlers/transaction/write_version_kv.rs index d2d9ef4b8..add93bf97 100644 --- a/nodedb/src/data/executor/handlers/transaction/write_version_kv.rs +++ b/nodedb/src/data/executor/handlers/transaction/write_version_kv.rs @@ -88,11 +88,13 @@ impl CoreLoop { ); } } - // Whole-collection mutations: key set is every row or predicate- - // resolved at apply time, so only the collection floor applies. - KvOp::Truncate { collection, .. } - | KvOp::PredicateUpdate { collection, .. } - | KvOp::PredicateDelete { collection, .. } => { + // Every row leaves: a wipe bounds every row read of the collection. + KvOp::Truncate { collection, .. } => { + self.note_collection_wipe(db, tenant, collection.as_str(), stamp); + } + // Predicate-resolved at apply time, so only the collection floor + // applies. + KvOp::PredicateUpdate { collection, .. } | KvOp::PredicateDelete { collection, .. } => { self.note_write(db, tenant, collection.as_str(), None, stamp); } // Read-only: nothing written, no version to record. diff --git a/nodedb/src/data/executor/handlers/truncate.rs b/nodedb/src/data/executor/handlers/truncate.rs index 84d3afaf8..f0a408bc1 100644 --- a/nodedb/src/data/executor/handlers/truncate.rs +++ b/nodedb/src/data/executor/handlers/truncate.rs @@ -229,6 +229,11 @@ impl CoreLoop { declared_primary_key, *storage_key, ); + // The removed rows' index values record no version, so the + // first removal records a wipe that bounds every index read. + if truncated == 0 { + self.note_task_collection_wipe(task, tid, collection); + } // Record the removal's version against the row's surrogate // and collection, as a point delete does. self.note_surrogate_write(task, tid, collection, surrogate.as_u32()); diff --git a/nodedb/src/data/executor/handlers/vector_direct_delete.rs b/nodedb/src/data/executor/handlers/vector_direct_delete.rs index 9aaaa1183..2209612cd 100644 --- a/nodedb/src/data/executor/handlers/vector_direct_delete.rs +++ b/nodedb/src/data/executor/handlers/vector_direct_delete.rs @@ -98,11 +98,16 @@ impl CoreLoop { } let mut removed: Vec<(StorageKey, Vec)> = Vec::with_capacity(surrogates.len()); - for surrogate in &surrogates { + for (at, surrogate) in surrogates.iter().enumerate() { match self.remove_vector_direct_row(&index_key, tid, collection, *surrogate) { Ok(Some(row)) => removed.push((StorageKey::for_surrogate(*surrogate), row.bytes)), Ok(None) => {} - Err(e) => return self.response_error(task, e), + Err(e) => { + // The failed row stays whole. The rows before it are removed. + let before = &surrogates[..at]; + self.finish_vector_direct_write(task, &index_key, tid, collection, before); + return self.response_error(task, e); + } } } if !removed.is_empty() { diff --git a/nodedb/src/data/executor/handlers/vector_direct_row.rs b/nodedb/src/data/executor/handlers/vector_direct_row.rs index 6e79cdd26..acc8a4496 100644 --- a/nodedb/src/data/executor/handlers/vector_direct_row.rs +++ b/nodedb/src/data/executor/handlers/vector_direct_row.rs @@ -190,6 +190,10 @@ impl CoreLoop { /// Remove every trace of `surrogate`'s row: bitmap entries, HNSW node, /// sidecar row, and the cached sidecar. Returns the sidecar that was /// stored, or `None` when no node was bound to the surrogate. + /// + /// The sidecar delete is the only fallible step, so it runs first. A + /// failure leaves the row fully present. A crash after it leaves a node + /// with no sidecar, which replay of the delete record removes. pub(in crate::data::executor) fn remove_vector_direct_row( &mut self, index_key: &VectorIndexKey, @@ -202,19 +206,23 @@ impl CoreLoop { return Ok(None); }; let sidecar = self.vector_sidecar_row(database_id, tid, collection, surrogate)?; - let Some(coll) = self.vector_collections.get_mut(index_key) else { - return Ok(None); - }; - if let Some(row) = &sidecar { - coll.payload.delete_row(node_id, &row.fields); - } - coll.delete(node_id); let key = StorageKey::for_surrogate(surrogate); + crate::fail_point_err!( + self.fail_scope, + "vector_direct::sidecar_delete", + |detail: String| ErrorCode::Internal { detail } + ); self.sparse .delete(database_id, tid, collection, &key) .map_err(ErrorCode::from)?; self.doc_cache .invalidate(database_id, tid, collection, &key); + if let Some(coll) = self.vector_collections.get_mut(index_key) { + if let Some(row) = &sidecar { + coll.payload.delete_row(node_id, &row.fields); + } + coll.delete(node_id); + } Ok(Some(sidecar.unwrap_or_else(|| VectorSidecarRow { bytes: Vec::new(), fields: HashMap::new(), @@ -286,3 +294,76 @@ impl CoreLoop { } } } + +#[cfg(all(test, feature = "failpoints"))] +mod tests { + use nodedb_physical::physical_plan::VectorWriteTargets; + use nodedb_types::fail_point::FailGuard; + use nodedb_types::{RlsWriteCheck, Surrogate}; + + use crate::bridge::envelope::Status; + use crate::data::executor::core_loop::CoreLoop; + use crate::data::executor::core_loop::tests::{make_core_with_dir, make_default_task}; + use crate::data::executor::handlers::vector_direct_delete::VectorDirectDeleteParams; + use crate::data::executor::handlers::vector_direct_test_rows::{ + COLLECTION, FIELD, TID, has_node, has_sidecar, seed, + }; + + const SIDECAR_DELETE: &str = "vector_direct::sidecar_delete"; + + fn delete(core: &mut CoreLoop, surrogate: Surrogate) -> Status { + let task = make_default_task(); + let targets = VectorWriteTargets::Surrogates(vec![surrogate]); + core.execute_vector_direct_delete(VectorDirectDeleteParams { + task: &task, + tid: TID, + collection: COLLECTION, + field: FIELD, + targets: &targets, + returning: None, + rls_filters: &[], + rls_write_check: &RlsWriteCheck::decided_earlier_in_request(), + }) + .status + } + + #[test] + fn a_failed_sidecar_delete_leaves_the_row_whole() { + let dir = tempfile::tempdir().expect("tempdir"); + let (mut core, _req, _resp) = make_core_with_dir(dir.path()); + let row = Surrogate::new(5); + seed(&mut core, row); + + { + let _fail = FailGuard::fail(SIDECAR_DELETE, "disk full"); + assert_ne!(delete(&mut core, row), Status::Ok); + assert!(has_node(&core, row), "a refused delete keeps the node"); + assert!( + has_sidecar(&core, row), + "a refused delete keeps the sidecar" + ); + } + + assert_eq!(delete(&mut core, row), Status::Ok); + assert!(!has_node(&core, row)); + assert!(!has_sidecar(&core, row)); + } + + #[test] + fn a_failed_sidecar_delete_leaves_a_truncated_row_whole() { + let dir = tempfile::tempdir().expect("tempdir"); + let (mut core, _req, _resp) = make_core_with_dir(dir.path()); + let row = Surrogate::new(9); + seed(&mut core, row); + + let _fail = FailGuard::fail(SIDECAR_DELETE, "disk full"); + let task = make_default_task(); + let resp = core.execute_vector_direct_truncate(&task, TID, COLLECTION, FIELD); + assert_ne!(resp.status, Status::Ok); + assert!(has_node(&core, row), "a refused truncate keeps the node"); + assert!( + has_sidecar(&core, row), + "a refused truncate keeps the sidecar" + ); + } +} diff --git a/nodedb/src/data/executor/handlers/vector_direct_test_rows.rs b/nodedb/src/data/executor/handlers/vector_direct_test_rows.rs new file mode 100644 index 000000000..b8e285de5 --- /dev/null +++ b/nodedb/src/data/executor/handlers/vector_direct_test_rows.rs @@ -0,0 +1,58 @@ +// SPDX-License-Identifier: BUSL-1.1 + +//! Vector-primary row fixtures shared by the direct-write and replay tests. + +use std::collections::HashMap; + +use nodedb_physical::physical_plan::VectorDirectWriteIntent; +use nodedb_types::{RlsWriteCheck, Surrogate, Value}; + +use crate::bridge::envelope::Status; +use crate::data::executor::core_loop::CoreLoop; +use crate::data::executor::core_loop::tests::make_default_task; +use crate::data::executor::handlers::vector_upsert::VectorDirectUpsertParams; +use crate::types::DatabaseId; + +pub const TID: u64 = 1; +pub const COLLECTION: &str = "vp_rows"; +pub const FIELD: &str = "emb"; + +/// Store one row under `surrogate`. +pub fn seed(core: &mut CoreLoop, surrogate: Surrogate) { + let mut fields: HashMap = HashMap::new(); + fields.insert("owner".into(), Value::String("a".into())); + let payload = zerompk::to_msgpack_vec(&fields).expect("encode payload"); + let task = make_default_task(); + let resp = core.execute_vector_direct_upsert(VectorDirectUpsertParams { + task: &task, + tid: TID, + collection: COLLECTION, + field: FIELD, + surrogate, + vector: &[1.0, 0.0, 0.0], + payload: &payload, + quantization: nodedb_types::VectorQuantization::None, + storage_dtype: nodedb_types::VectorStorageDtype::F32, + payload_indexes: &[], + intent: VectorDirectWriteIntent::Insert, + on_conflict_updates: &[], + rls_write_check: &RlsWriteCheck::decided_earlier_in_request(), + returning: None, + rls_filters: &[], + }); + assert_eq!(resp.status, Status::Ok, "seed: {:?}", resp.error_code); +} + +/// Whether the index holds a node for `surrogate`. +pub fn has_node(core: &CoreLoop, surrogate: Surrogate) -> bool { + let index_key = + CoreLoop::vector_index_key(DatabaseId::DEFAULT.as_u64(), TID, COLLECTION, FIELD); + core.vector_direct_node(&index_key, surrogate).is_some() +} + +/// Whether the sidecar store holds `surrogate`'s row. +pub fn has_sidecar(core: &CoreLoop, surrogate: Surrogate) -> bool { + core.vector_sidecar_row(DatabaseId::DEFAULT.as_u64(), TID, COLLECTION, surrogate) + .expect("read sidecar") + .is_some() +} diff --git a/nodedb/src/data/executor/handlers/vector_direct_truncate.rs b/nodedb/src/data/executor/handlers/vector_direct_truncate.rs index 20df17085..3e4ff09b4 100644 --- a/nodedb/src/data/executor/handlers/vector_direct_truncate.rs +++ b/nodedb/src/data/executor/handlers/vector_direct_truncate.rs @@ -41,22 +41,32 @@ impl CoreLoop { Err(e) => return self.response_error(task, e), }; - let mut truncated = 0usize; - for surrogate in &surrogates { - match self.remove_vector_direct_row(&index_key, tid, collection, *surrogate) { - Ok(Some(_)) => truncated += 1, - // A sidecar row with no node bound to it: drop the row so the - // collection reads back empty. - Ok(None) => { - let key = StorageKey::for_surrogate(*surrogate); - if let Err(e) = self.sparse.delete(database_id, tid, collection, &key) { - return self.response_error(task, ErrorCode::from(e)); + for (at, surrogate) in surrogates.iter().enumerate() { + let outcome = + match self.remove_vector_direct_row(&index_key, tid, collection, *surrogate) { + Ok(Some(_)) => Ok(()), + // A sidecar row with no node bound to it: drop the row so the + // collection reads back empty. + Ok(None) => { + let key = StorageKey::for_surrogate(*surrogate); + let deleted = self + .sparse + .delete(database_id, tid, collection, &key) + .map(|_| ()) + .map_err(ErrorCode::from); + if deleted.is_ok() { + self.doc_cache + .invalidate(database_id, tid, collection, &key); + } + deleted } - self.doc_cache - .invalidate(database_id, tid, collection, &key); - truncated += 1; - } - Err(e) => return self.response_error(task, e), + Err(e) => Err(e), + }; + if let Err(e) = outcome { + // The failed row stays whole. The rows before it are removed. + let removed = &surrogates[..at]; + self.finish_vector_direct_write(task, &index_key, tid, collection, removed); + return self.response_error(task, e); } } @@ -68,6 +78,7 @@ impl CoreLoop { self.finish_vector_direct_write(task, &index_key, tid, collection, &surrogates); self.invalidate_aggregate_cache_for_collection(database_id, tid, collection); + let truncated = surrogates.len(); debug!(core = self.core_id, %collection, truncated, "vector direct truncate complete"); self.truncate_response(task, truncated) } diff --git a/nodedb/src/data/executor/mod.rs b/nodedb/src/data/executor/mod.rs index 648850612..ae25b41f6 100644 --- a/nodedb/src/data/executor/mod.rs +++ b/nodedb/src/data/executor/mod.rs @@ -53,7 +53,6 @@ mod wal_replay_kv_field; mod wal_replay_kv_incr; mod wal_replay_kv_index; mod wal_replay_kv_insert_conflict; -mod wal_replay_kv_predicate; mod wal_replay_kv_sorted_index; mod wal_replay_kv_transfer; mod wal_replay_kv_ttl; diff --git a/nodedb/src/data/executor/wal_replay/kv.rs b/nodedb/src/data/executor/wal_replay/kv.rs index d6bf3ab68..702532d78 100644 --- a/nodedb/src/data/executor/wal_replay/kv.rs +++ b/nodedb/src/data/executor/wal_replay/kv.rs @@ -238,20 +238,6 @@ impl CoreLoop { continue; } - // kv_predicate_update (delta): re-resolves the row set against - // current state and re-runs the same merge. - if let Some(applied) = self.try_replay_kv_predicate_update( - &record.payload, - tenant_id, - database_id, - now_ms, - record_lsn, - tombstones, - ) { - puts += applied; - continue; - } - // A KV put-family record of no current shape: refused, never // replayed without its surrogate. Any other `Put` payload // belongs to another engine's replay pass. @@ -336,7 +322,7 @@ impl CoreLoop { }]); } self.kv_engine.truncate(database_id, tenant_id, &collection); - self.note_replay_write(database_id, tenant_id, &collection, None, record_lsn); + self.note_replay_wipe(database_id, tenant_id, &collection, record_lsn); deletes += 1; continue; } @@ -344,20 +330,6 @@ impl CoreLoop { continue; } - // kv_predicate_delete (delta): re-resolves the row set - // against current state. - if let Some(removed) = self.try_replay_kv_predicate_delete( - &record.payload, - tenant_id, - database_id, - now_ms, - record_lsn, - tombstones, - ) { - deletes += removed; - continue; - } - // No tombstone gate: the record carries only `index_name`, // no collection to gate on. if let Some(applied) = diff --git a/nodedb/src/data/executor/wal_replay_kv_predicate.rs b/nodedb/src/data/executor/wal_replay_kv_predicate.rs deleted file mode 100644 index b8a672a26..000000000 --- a/nodedb/src/data/executor/wal_replay_kv_predicate.rs +++ /dev/null @@ -1,144 +0,0 @@ -// SPDX-License-Identifier: BUSL-1.1 - -//! WAL replay for the KV predicate DML records (`kv_predicate_update`, -//! `kv_predicate_delete`). The log carries only the predicate and -//! assignments, never a row set, so replay re-resolves the row set with the -//! same scan and merge the live handlers use. - -use tracing::warn; - -use super::core_loop::CoreLoop; -use super::handlers::kv::field_compute::merge_field_updates; - -impl CoreLoop { - /// Decode + tombstone-gate + replay one `kv_predicate_update` record. - /// - /// Returns `None` when `payload` is not that record shape (the caller - /// tries the next candidate arm), otherwise the number of rows written. - pub(super) fn try_replay_kv_predicate_update( - &mut self, - payload: &[u8], - tenant_id: u64, - database_id: u64, - now_ms: u64, - record_lsn: u64, - tombstones: &nodedb_wal::TombstoneSet, - ) -> Option { - let (disc, collection, filters, updates) = - zerompk::from_msgpack::<(&str, String, Vec, Vec<(String, Vec)>)>(payload) - .ok()?; - if disc != "kv_predicate_update" { - return None; - } - let tombstones = &tombstones.for_database(database_id); - if self.skip_kv_replay_record(tombstones, tenant_id, &collection, record_lsn) { - return Some(0); - } - - let matched = match self.kv_predicate_matches( - database_id, - tenant_id, - &collection, - &filters, - now_ms, - ) { - Ok(rows) => rows, - Err(e) => { - warn!( - core = self.core_id, - %collection, - ?e, - "WAL kv_predicate_update replay: predicate scan failed, skipping record" - ); - return Some(0); - } - }; - - let mut written = 0usize; - for (key, body) in matched { - let computed = match merge_field_updates( - &collection, - Some(body.as_slice()), - &updates, - self.declared_columns_of(database_id, tenant_id, &collection), - ) { - Ok(c) => c, - Err(e) => { - warn!( - core = self.core_id, - %collection, - key = %String::from_utf8_lossy(&key), - ?e, - "WAL kv_predicate_update replay: field merge failed, skipping row" - ); - continue; - } - }; - // The row exists and keeps its bound identity, the same surrogate - // the live merge preserved. - self.kv_engine.rewrite_with_absolute_expiry( - crate::engine::kv::KvRewriteParams { - database_id, - tenant_id, - collection: &collection, - key: &key, - value: &computed.new_value, - ttl_ms: 0, - now_ms, - }, - crate::engine::kv::entry::NO_EXPIRY, - ); - written += 1; - } - // The key set is resolved at replay time rather than carried, so the - // collection floor is the version this can record. - self.note_replay_write(database_id, tenant_id, &collection, None, record_lsn); - Some(written) - } - - /// Decode + tombstone-gate + replay one `kv_predicate_delete` record. - pub(super) fn try_replay_kv_predicate_delete( - &mut self, - payload: &[u8], - tenant_id: u64, - database_id: u64, - now_ms: u64, - record_lsn: u64, - tombstones: &nodedb_wal::TombstoneSet, - ) -> Option { - let (disc, collection, filters) = - zerompk::from_msgpack::<(&str, String, Vec)>(payload).ok()?; - if disc != "kv_predicate_delete" { - return None; - } - let tombstones = &tombstones.for_database(database_id); - if self.skip_kv_replay_record(tombstones, tenant_id, &collection, record_lsn) { - return Some(0); - } - - let matched = match self.kv_predicate_matches( - database_id, - tenant_id, - &collection, - &filters, - now_ms, - ) { - Ok(rows) => rows, - Err(e) => { - warn!( - core = self.core_id, - %collection, - ?e, - "WAL kv_predicate_delete replay: predicate scan failed, skipping record" - ); - return Some(0); - } - }; - let keys: Vec> = matched.into_iter().map(|(key, _body)| key).collect(); - let removed = self - .kv_engine - .delete(database_id, tenant_id, &collection, &keys, now_ms); - self.note_replay_write(database_id, tenant_id, &collection, None, record_lsn); - Some(removed) - } -} diff --git a/nodedb/src/data/executor/wal_replay_vector_direct.rs b/nodedb/src/data/executor/wal_replay_vector_direct.rs index a89d79d8d..a4bf46f77 100644 --- a/nodedb/src/data/executor/wal_replay_vector_direct.rs +++ b/nodedb/src/data/executor/wal_replay_vector_direct.rs @@ -8,7 +8,7 @@ //! gated by the per-collection checkpoint watermark like the direct upsert: //! a restored checkpoint already holds every write at or below it. -use nodedb_physical::physical_plan::VectorOp; +use nodedb_physical::physical_plan::{VectorOp, VectorWriteTargets}; use crate::bridge::envelope::{PhysicalPlan, Status}; use crate::control::server::wal_dispatch::{ @@ -31,7 +31,7 @@ impl CoreLoop { tombstones: &nodedb_wal::TombstoneSet, ) -> bool { let tombstones = tombstones.for_database(database_id); - let Ok((collection, field, targets)) = + let Ok((collection, field, rows)) = zerompk::from_msgpack::(payload) else { self.replay_record_unapplied( @@ -45,6 +45,9 @@ impl CoreLoop { if tombstones.is_tombstoned(tenant_id, &collection, record_lsn) { return false; } + // The record names the rows the delete removed. A row whose sidecar + // delete was durable before a crash still has its restored node. + let targets = VectorWriteTargets::Surrogates(rows); let index_key = CoreLoop::vector_index_key(database_id, tenant_id, &collection, &field); if self.vector_replay_skips(record_lsn) { return false; @@ -192,7 +195,7 @@ impl CoreLoop { let Ok(( collection, field, - targets, + rows, new_vector, payload_patch, quantization, @@ -211,6 +214,7 @@ impl CoreLoop { if tombstones.is_tombstoned(tenant_id, &collection, record_lsn) { return false; } + let targets = VectorWriteTargets::Surrogates(rows); let index_key = CoreLoop::vector_index_key(database_id, tenant_id, &collection, &field); if self.vector_replay_skips(record_lsn) { return false; @@ -283,3 +287,37 @@ impl CoreLoop { true } } + +#[cfg(test)] +mod tests { + use nodedb_types::{StorageKey, Surrogate}; + + use crate::control::server::wal_dispatch::encode_vector_direct_delete_payload; + use crate::data::executor::core_loop::tests::make_core_with_dir; + use crate::data::executor::handlers::vector_direct_test_rows::{ + COLLECTION, FIELD, TID, has_node, has_sidecar, seed, + }; + use crate::types::DatabaseId; + + /// A crash after the sidecar delete was durable leaves the restored node + /// with no sidecar. Replay of the delete record still removes the node. + #[test] + fn replay_removes_a_node_whose_sidecar_delete_was_durable() { + let dir = tempfile::tempdir().expect("tempdir"); + let (mut core, _req, _resp) = make_core_with_dir(dir.path()); + let row = Surrogate::new(7); + seed(&mut core, row); + let db = DatabaseId::DEFAULT.as_u64(); + core.sparse + .delete(db, TID, COLLECTION, &StorageKey::for_surrogate(row)) + .expect("durable sidecar delete"); + assert!(has_node(&core, row) && !has_sidecar(&core, row)); + + let payload = + encode_vector_direct_delete_payload(COLLECTION, FIELD, &[row]).expect("encode"); + let applied = + core.replay_direct_delete(&payload, TID, db, 100, &nodedb_wal::TombstoneSet::new()); + assert!(applied, "the replayed delete applies"); + assert!(!has_node(&core, row), "replay removes the node"); + } +} diff --git a/nodedb/src/diag/context/data_plane.rs b/nodedb/src/diag/context/data_plane.rs index f1859a5ee..2d64dc04d 100644 --- a/nodedb/src/diag/context/data_plane.rs +++ b/nodedb/src/diag/context/data_plane.rs @@ -69,7 +69,8 @@ pub(in crate::diag) struct CalvinCompletionTimeout { pub epoch: u64, pub position: u32, pub participants: usize, - pub timeout_secs: u64, + /// How long the completion wait ran before the statement deadline. + pub waited_ms: u64, } impl DomainContext for CalvinCompletionTimeout { @@ -88,10 +89,10 @@ impl DomainContext for CalvinCompletionTimeout { "epoch": self.epoch, "position": self.position, "participants": self.participants, - "timeout_secs": self.timeout_secs, + "waited_ms": self.waited_ms, "why_fatal": "this timeout is the only signal a Calvin-routed write ever \ produces for a completion ack that never arrived; the caller \ - sees a generic internal error with no indication of which \ + sees a deadline error with no indication of which \ participant or stage stalled, and the write's outcome is \ unknown to the client", "operator_action": "check the sequencer-group leader and the listed \ diff --git a/nodedb/src/diag/context/held_entry.rs b/nodedb/src/diag/context/held_entry.rs new file mode 100644 index 000000000..77c294acd --- /dev/null +++ b/nodedb/src/diag/context/held_entry.rs @@ -0,0 +1,113 @@ +// SPDX-License-Identifier: BUSL-1.1 + +//! Forensic payloads for a committed entry a replica holds: its redo bytes +//! do not decode, its apply panicked, or its group's snapshot debt did not +//! persist. + +use faultbox::DomainContext; +use faultbox::serde_json::{Value, json}; + +/// A committed redo entry whose own bytes do not decode as the redo they +/// name on this replica. +pub(in crate::diag) struct RedoEntryMalformed { + pub group_id: u64, + pub log_index: u64, + /// The decode error. + pub error: String, +} + +impl DomainContext for RedoEntryMalformed { + fn domain_kind(&self) -> &'static str { + "nodedb.redo_entry_malformed" + } + + fn grouping_key(&self) -> String { + // One bug class: a committed redo that does not decode. Group and + // index are the occurrence. + "redo_entry_malformed".to_string() + } + + fn to_json(&self) -> Value { + json!({ + "group_id": self.group_id, + "log_index": self.log_index, + "error": self.error, + "impact": "the group's apply on this replica holds at the entry. Its applied \ + index stays below it and reads wait. The replica refuses log \ + entries, steps down from leadership, and never campaigns until a \ + snapshot of the group installs", + "operator_action": "the group's leader sends a snapshot. When every replica \ + holds the same entry, no replica can lead the group: the \ + proposer encoded a redo no replica decodes, and the error \ + names the field", + }) + } +} + +/// The snapshot debt of a group whose apply holds an entry did not persist. +pub(in crate::diag) struct HeldEntryDebtNotRecorded { + pub group_id: u64, + pub log_index: u64, + /// The catalog error. + pub error: String, +} + +impl DomainContext for HeldEntryDebtNotRecorded { + fn domain_kind(&self) -> &'static str { + "nodedb.held_entry_debt_not_recorded" + } + + fn grouping_key(&self) -> String { + // The apply loop retries on each Raft tick while the write fails: one + // report per root cause. + "held_entry_debt_not_recorded".to_string() + } + + fn to_json(&self) -> Value { + json!({ + "group_id": self.group_id, + "log_index": self.log_index, + "error": self.error, + "impact": "the replica refuses log entries until a snapshot installs, but a \ + sequencer snapshot install can lift that before the held entry is \ + covered. The apply loop writes the debt again on each Raft tick \ + and holds the requirement up until a write persists", + "operator_action": "the error names why the system catalog refused the write", + }) + } +} + +/// A committed entry whose enqueue or apply panicked on this replica. +pub(in crate::diag) struct ApplyPanicked { + pub group_id: u64, + pub log_index: u64, + /// The panic payload. + pub detail: String, +} + +impl DomainContext for ApplyPanicked { + fn domain_kind(&self) -> &'static str { + "nodedb.apply_panicked" + } + + fn grouping_key(&self) -> String { + // One bug class: an apply that panicked. Group and index are the + // occurrence. + "apply_panicked".to_string() + } + + fn to_json(&self) -> Value { + json!({ + "group_id": self.group_id, + "log_index": self.log_index, + "detail": self.detail, + "impact": "the entry's effect on this replica is unknown. Its proposer \ + receives an error. The group's apply holds at the entry, its \ + applied index stays below it, and the replica refuses log entries \ + until a snapshot of the group installs. Other groups keep applying", + "operator_action": "the panic names the code that failed. The group's leader \ + sends a snapshot. When every replica panics on the entry, \ + no replica can lead the group", + }) + } +} diff --git a/nodedb/src/diag/context/mod.rs b/nodedb/src/diag/context/mod.rs index 695e302c4..db4048e4d 100644 --- a/nodedb/src/diag/context/mod.rs +++ b/nodedb/src/diag/context/mod.rs @@ -14,6 +14,7 @@ mod crdt_dead_letter_store; mod cut_barrier; mod data_plane; mod event_image; +mod held_entry; mod index_rebuild; mod ingest; mod lease; @@ -43,6 +44,7 @@ pub(in crate::diag) use data_plane::{ CalvinApplyHalted, CalvinCompletionTimeout, CoreFailStopped, DataPlaneResponseLost, }; pub(in crate::diag) use event_image::{StrictRowImageUnrendered, TimeseriesRowImageUndecodable}; +pub(in crate::diag) use held_entry::{ApplyPanicked, HeldEntryDebtNotRecorded, RedoEntryMalformed}; pub(in crate::diag) use index_rebuild::IndexRebuildNotInstalled; pub(in crate::diag) use ingest::IlpAcceptedLinesDropped; pub use ingest::IlpFlushOutcome; @@ -53,7 +55,9 @@ pub(in crate::diag) use quota::{ QuotaRowNotInstalled, QuotaRowWriteFailed, QuotaScopePurgeIncomplete, QuotaScopeReplayAborted, ScopeQuotaNotInstalled, }; -pub(in crate::diag) use raft_apply::{CalvinBarrierLogStoreFailed, RaftEntryReapplied}; +pub(in crate::diag) use raft_apply::{ + CalvinBarrierLogStoreFailed, CalvinLeftStateNotForgotten, RaftEntryReapplied, +}; pub(in crate::diag) use recovery::{ReplayRecordUnapplied, WalArchivalFailedTruncationHeld}; pub(in crate::diag) use redo_stream::{ RedoAbandonGivenUp, RedoSnapshotDebtNotRecorded, RedoStreamLostHere, diff --git a/nodedb/src/diag/context/raft_apply.rs b/nodedb/src/diag/context/raft_apply.rs index 13178ca52..25b8088b9 100644 --- a/nodedb/src/diag/context/raft_apply.rs +++ b/nodedb/src/diag/context/raft_apply.rs @@ -41,14 +41,15 @@ impl DomainContext for RaftEntryReapplied { } } -/// The catalog refused a read or write of a vShard's stored dependent-read -/// barrier log. +/// The catalog refused a read or write of the stored dependent-read +/// barrier logs. pub(in crate::diag) struct CalvinBarrierLogStoreFailed<'a> { - pub vshard_id: u32, - /// The txn's `(epoch, position)`. A removal of every row of the vShard + /// The vShard. A persist of every vShard's rows names none. + pub vshard_id: Option, + /// The txn's `(epoch, position)`. An operation on every row of a vShard /// names none. pub txn: Option<(u64, u32)>, - /// What the store did: `save`, `load` or `remove`. + /// What the store did: `save`, `load`, `remove`, `forget` or `persist`. pub op: &'static str, pub error_class: &'a str, } @@ -59,11 +60,12 @@ impl DomainContext for CalvinBarrierLogStoreFailed<'_> { } fn grouping_key(&self) -> String { - // One report per vShard and operation: the txn changes per event. - format!( - "calvin_barrier_log_store_failed:{}:{}", - self.op, self.vshard_id - ) + // One report per vShard and operation: the txn changes per event, + // and a retry loop repeats the same pair. + match self.vshard_id { + Some(vshard_id) => format!("calvin_barrier_log_store_failed:{}:{vshard_id}", self.op), + None => format!("calvin_barrier_log_store_failed:{}:all", self.op), + } } fn to_json(&self) -> Value { @@ -77,6 +79,15 @@ impl DomainContext for CalvinBarrierLogStoreFailed<'_> { "the scheduler keeps the txn's barrier open and reads the row \ again on its next pass" } + "forget" => { + "the rows of a vShard this node left stay stored. Every scheduler \ + reconcile pass removes them again until one succeeds" + } + "persist" => { + "the saved barrier rows are not durable yet. No applied floor moves \ + this pass, so a restart delivers their entries again. The next \ + settle pass persists again" + } _ => { "the row of a finished txn stays stored. The scheduler's next \ stall-tick sweep removes it" @@ -93,3 +104,34 @@ impl DomainContext for CalvinBarrierLogStoreFailed<'_> { }) } } + +/// The catalog refused the forget of the stored Calvin state of vShards +/// this node left: their applied positions and bases. +pub(in crate::diag) struct CalvinLeftStateNotForgotten<'a> { + pub vshards: &'a [u32], + pub error_class: &'a str, +} + +impl DomainContext for CalvinLeftStateNotForgotten<'_> { + fn domain_kind(&self) -> &'static str { + "nodedb.calvin_left_state_not_forgotten" + } + + fn grouping_key(&self) -> String { + // One root cause: the catalog refuses the write. The vShards are the + // occurrence, and a retry loop repeats them. + "calvin_left_state_not_forgotten".to_string() + } + + fn to_json(&self) -> Value { + json!({ + "vshards": self.vshards, + "error_class": self.error_class, + "effect": "the applied positions and base of a vShard this node left stay \ + stored. Every scheduler reconcile pass forgets them again until one \ + succeeds", + "operator_action": "check the system catalog's disk: free space, I/O errors, \ + and the redb file's permissions", + }) + } +} diff --git a/nodedb/src/diag/context/redo_stream.rs b/nodedb/src/diag/context/redo_stream.rs index c3ae03284..db7752d7d 100644 --- a/nodedb/src/diag/context/redo_stream.rs +++ b/nodedb/src/diag/context/redo_stream.rs @@ -89,13 +89,13 @@ impl DomainContext for RedoStreamLostHere { "entry": self.entry, "refusal": self.refusal, "impact": "the other replicas can install the stream's redo, which this replica \ - lacks. Its durable floor holds at the entry, so a restart applies the \ - entry again. Later entries of the group apply here on the state \ - without the redo until a snapshot of the group installs", - "operator_action": "the node's membership pass makes the group's replica here \ - refuse log entries, and moves a leadership it holds. The \ - group's leader then sends a snapshot. When the group has no \ - other replica, no snapshot comes: restore the group from a \ + lacks. The group's apply here holds at the entry: no later entry \ + applies, its applied index stays below the entry, and reads wait \ + until a snapshot of the group installs", + "operator_action": "the group's replica here refuses log entries, steps down \ + from leadership, and never campaigns. The group's leader \ + then sends a snapshot. When the group has no other \ + replica, no snapshot comes: restore the group from a \ backup", }) } diff --git a/nodedb/src/diag/mod.rs b/nodedb/src/diag/mod.rs index 82573cd7b..7cb9e4d4c 100644 --- a/nodedb/src/diag/mod.rs +++ b/nodedb/src/diag/mod.rs @@ -10,17 +10,18 @@ mod recording; pub use context::{DATABASE_SCOPE, IlpFlushOutcome, LostResponseWrite, TENANT_SCOPE}; pub use recording::{ - IndexRebuildTarget, VectorBuildTarget, batch_insert_without_surrogates, calvin_apply_halted, - calvin_barrier_log_store_failed, calvin_completion_timeout, catalog_apply_orphan_row, - collection_purge_row_missing, columnar_segment_corrupt, consumer_group_offsets_retained, - continuous_aggregate_not_applied, crdt_dead_letter_not_enqueued, crdt_dead_letter_not_restored, - crdt_dead_letter_not_stored, cut_barrier_not_placed, data_plane_core_fail_stopped, - data_plane_response_lost, data_plane_responses_lost, descriptor_lease_not_renewed, entry_kind, - fts_index_update_failed, history_compaction_not_applied, ilp_invalid_utf8_drop, + IndexRebuildTarget, VectorBuildTarget, apply_panicked, batch_insert_without_surrogates, + calvin_apply_halted, calvin_barrier_log_store_failed, calvin_completion_timeout, + calvin_left_state_not_forgotten, catalog_apply_orphan_row, collection_purge_row_missing, + columnar_segment_corrupt, consumer_group_offsets_retained, continuous_aggregate_not_applied, + crdt_dead_letter_not_enqueued, crdt_dead_letter_not_restored, crdt_dead_letter_not_stored, + cut_barrier_not_placed, data_plane_core_fail_stopped, data_plane_response_lost, + data_plane_responses_lost, descriptor_lease_not_renewed, entry_kind, fts_index_update_failed, + held_entry_debt_not_recorded, history_compaction_not_applied, ilp_invalid_utf8_drop, ilp_line_read_drop, index_rebuild_not_installed, metadata_apply_wedged, orphaned_index_entry_after_delete, quota_row_invalid, quota_row_undecodable, quota_row_write_failed, quota_scope_purge_incomplete, quota_scope_replay_aborted, - raft_entries_reapplied, raft_entry_reapplied, redo_abandon_given_up, + raft_entries_reapplied, raft_entry_reapplied, redo_abandon_given_up, redo_entry_malformed, redo_snapshot_debt_not_recorded, redo_stream_lost_here, replay_record_unapplied, retention_autowire_orphaned, scope_quota_not_installed, strict_row_image_unrendered, strict_row_undecodable, synonym_group_not_applied, timeseries_partition_unreadable, diff --git a/nodedb/src/diag/recording/data_plane.rs b/nodedb/src/diag/recording/data_plane.rs index d7c128c82..aad0d0552 100644 --- a/nodedb/src/diag/recording/data_plane.rs +++ b/nodedb/src/diag/recording/data_plane.rs @@ -34,22 +34,22 @@ pub fn data_plane_response_lost(core_id: usize, write: context::LostResponseWrit .emit(); } -/// Report a Calvin cross-shard transaction whose completion wait timed out. -/// Called from the completion-timeout arm of -/// `submit_and_await_calvin_with_timeout`; the sibling "channel closed" arm -/// is a different root cause and is not reported here. +/// Report a Calvin cross-shard transaction whose completion wait reached the +/// statement deadline. Called from the deadline arm of the coordinator's +/// completion wait. The "channel closed" arm is a different root cause and +/// is not reported here. pub fn calvin_completion_timeout( err: &crate::Error, epoch: u64, position: u32, participants: usize, - timeout_secs: u64, + waited_ms: u64, ) { let ctx = context::CalvinCompletionTimeout { epoch, position, participants, - timeout_secs, + waited_ms, }; let _ = Capture::new( EventKind::Error, diff --git a/nodedb/src/diag/recording/held_entry.rs b/nodedb/src/diag/recording/held_entry.rs new file mode 100644 index 000000000..4d855fa8f --- /dev/null +++ b/nodedb/src/diag/recording/held_entry.rs @@ -0,0 +1,62 @@ +// SPDX-License-Identifier: BUSL-1.1 + +//! Capture sites for a committed entry the data-group apply loop holds. + +use faultbox::{Capture, EventKind, error_chain_of}; + +use crate::diag::context; + +/// Report a committed redo entry of `group_id` at `log_index` whose bytes do +/// not decode on this replica. Called from the apply loop, the one site that +/// holds the entry. +pub fn redo_entry_malformed(group_id: u64, log_index: u64, error: &crate::Error) { + let ctx = context::RedoEntryMalformed { + group_id, + log_index, + error: error.to_string(), + }; + let _ = Capture::new( + EventKind::InvariantViolation, + "committed redo entry does not decode: the group's apply holds until a snapshot \ + installs", + ) + .domain(&ctx) + .error_chain(error_chain_of(error)) + .emit(); +} + +/// Report that the snapshot debt of `group_id`, whose apply holds the entry +/// at `log_index`, did not persist. Called from the apply loop, the one site +/// that records the debt of a held entry, once per failure run. +pub fn held_entry_debt_not_recorded(group_id: u64, log_index: u64, error: &crate::Error) { + let ctx = context::HeldEntryDebtNotRecorded { + group_id, + log_index, + error: error.to_string(), + }; + let _ = Capture::new( + EventKind::Error, + "snapshot debt of a group whose apply holds an entry did not persist", + ) + .domain(&ctx) + .error_chain(error_chain_of(error)) + .emit(); +} + +/// Report that the enqueue or apply of the committed entry of `group_id` at +/// `log_index` panicked on this replica. Called from the apply loop, the one +/// site that collects the entry's apply. +pub fn apply_panicked(group_id: u64, log_index: u64, detail: &str) { + let ctx = context::ApplyPanicked { + group_id, + log_index, + detail: detail.to_string(), + }; + let _ = Capture::new( + EventKind::InvariantViolation, + "a committed entry's apply panicked: the group's apply holds until a snapshot \ + installs", + ) + .domain(&ctx) + .emit(); +} diff --git a/nodedb/src/diag/recording/mod.rs b/nodedb/src/diag/recording/mod.rs index 4a9e24f17..b3c64193b 100644 --- a/nodedb/src/diag/recording/mod.rs +++ b/nodedb/src/diag/recording/mod.rs @@ -15,6 +15,7 @@ mod crdt; mod cut_barrier; mod data_plane; mod event_image; +mod held_entry; mod index_rebuild; mod ingest; mod lease; @@ -44,6 +45,7 @@ pub use data_plane::{ data_plane_response_lost, data_plane_responses_lost, }; pub use event_image::{strict_row_image_unrendered, timeseries_row_image_undecodable}; +pub use held_entry::{apply_panicked, held_entry_debt_not_recorded, redo_entry_malformed}; pub use index_rebuild::{IndexRebuildTarget, index_rebuild_not_installed}; pub use ingest::{ilp_invalid_utf8_drop, ilp_line_read_drop}; pub use lease::descriptor_lease_not_renewed; @@ -53,7 +55,8 @@ pub use quota::{ quota_scope_replay_aborted, scope_quota_not_installed, }; pub use raft_apply::{ - calvin_barrier_log_store_failed, raft_entries_reapplied, raft_entry_reapplied, + calvin_barrier_log_store_failed, calvin_left_state_not_forgotten, raft_entries_reapplied, + raft_entry_reapplied, }; pub use recovery::{ batch_insert_without_surrogates, fts_index_update_failed, orphaned_index_entry_after_delete, diff --git a/nodedb/src/diag/recording/raft_apply.rs b/nodedb/src/diag/recording/raft_apply.rs index 8799ddacf..1b7b8510c 100644 --- a/nodedb/src/diag/recording/raft_apply.rs +++ b/nodedb/src/diag/recording/raft_apply.rs @@ -37,10 +37,11 @@ pub fn raft_entry_reapplied(group_id: u64, log_index: u64, highest_applied: u64) .emit(); } -/// Report a catalog error on a vShard's stored dependent-read barrier log. -/// Called from the sites that read and write the rows. `op` names the operation: `save`, `load` or `remove`. +/// Report a catalog error on the stored dependent-read barrier logs. +/// Called from the sites that read and write the rows. `op` names the +/// operation: `save`, `load`, `remove`, `forget` or `persist`. pub fn calvin_barrier_log_store_failed( - vshard_id: u32, + vshard_id: Option, txn: Option<(u64, u32)>, op: &'static str, err: &crate::Error, @@ -60,3 +61,20 @@ pub fn calvin_barrier_log_store_failed( .domain(&ctx) .emit(); } + +/// Report a catalog error that left the stored Calvin state of vShards this +/// node left. Called from `calvin_snapshot::forget_left`. +pub fn calvin_left_state_not_forgotten(vshards: &[u32], err: &crate::Error) { + let class = error_class(err); + let ctx = context::CalvinLeftStateNotForgotten { + vshards, + error_class: &class, + }; + let _ = Capture::new( + EventKind::Error, + "calvin state of a left vShard was not forgotten", + ) + .error_chain(error_chain_of(err)) + .domain(&ctx) + .emit(); +} diff --git a/nodedb/src/error/types.rs b/nodedb/src/error/types.rs index b023ac085..1e8eeb3cc 100644 --- a/nodedb/src/error/types.rs +++ b/nodedb/src/error/types.rs @@ -470,11 +470,30 @@ pub enum Error { )] BackupCaptureMoved { group_id: u64 }, - /// No leader elected on the metadata group yet. Transient — an election - /// is in progress — so callers can wait it out instead of failing. - #[error("metadata raft group has no elected leader yet; retry needed")] + /// The metadata group has no leader to take a proposal right now. Nothing + /// was proposed. An election, a leadership transfer, a stale leader or an + /// unreachable leader causes it. Callers can wait it out and re-propose. + #[error("metadata raft group has no reachable leader; retry needed")] MetadataLeaderUnavailable, + /// The metadata group left this node before it applied the entry this + /// node proposed at `log_index`. The entry's outcome is unknown, so no + /// caller re-proposes it blindly. + #[error( + "metadata entry at log index {log_index}: the metadata raft group left this node before \ + it applied the entry; its outcome is unknown. Check the catalog before any retry" + )] + MetadataGroupGone { log_index: u64 }, + + /// Another owner took the DDL preparation lease before the DDL this node + /// proposed at `log_index` applied. Its apply skipped the DDL, so + /// nothing changed and the client can retry the statement. + #[error( + "metadata DDL at log index {log_index} was superseded: the DDL preparation lease moved to \ + another owner before it applied; nothing changed. Retry the statement" + )] + MetadataDdlSuperseded { log_index: u64 }, + /// A statement cannot be planned yet: this node's roles, grants, policies /// or permission trees may be behind a change already acknowledged, and /// did not catch up before the deadline. Nothing ran; the client retries. diff --git a/nodedb/src/error_classify/public.rs b/nodedb/src/error_classify/public.rs index a3b0753e5..59907b08c 100644 --- a/nodedb/src/error_classify/public.rs +++ b/nodedb/src/error_classify/public.rs @@ -219,12 +219,13 @@ pub(crate) fn classify(e: &Error) -> NodeDbError { )), // Internal class `XX`: no driver or pool treats it as transient, so // nothing re-proposes a write that already committed. - Error::CommittedResultUnavailable { .. } | Error::ProposalOutcomeUnknown { .. } => { - NodeDbError::internal(e.to_string()) - } + Error::CommittedResultUnavailable { .. } + | Error::ProposalOutcomeUnknown { .. } + | Error::MetadataGroupGone { .. } => NodeDbError::internal(e.to_string()), Error::MetadataLeaderUnavailable => NodeDbError::dispatch( - "metadata raft group has no elected leader yet; retry exhausted".to_string(), + "metadata raft group has no reachable leader; retry exhausted".to_string(), ), + Error::MetadataDdlSuperseded { .. } => NodeDbError::dispatch(e.to_string()), // The code renders `55P03`, the SQLSTATE pgwire gives the variant, so // the class survives a node hop as a bare numeric code. Error::AuthorizationStateBehind { .. } | Error::LinearizableReadRefused { .. } => { diff --git a/nodedb/src/error_classify/unclassified.rs b/nodedb/src/error_classify/unclassified.rs index 65ecd6bfd..fa36d7d6e 100644 --- a/nodedb/src/error_classify/unclassified.rs +++ b/nodedb/src/error_classify/unclassified.rs @@ -105,6 +105,8 @@ pub(crate) fn is_unclassified_failure(e: &Error) -> bool { | Error::GroupMarksUnavailable { .. } | Error::BackupCaptureMoved { .. } | Error::MetadataLeaderUnavailable + | Error::MetadataGroupGone { .. } + | Error::MetadataDdlSuperseded { .. } | Error::AuthorizationStateBehind { .. } | Error::LinearizableReadRefused { .. } | Error::ExecutionLimitExceeded { .. } diff --git a/nodedb/src/error_from.rs b/nodedb/src/error_from.rs index 2cc32a820..e68e35546 100644 --- a/nodedb/src/error_from.rs +++ b/nodedb/src/error_from.rs @@ -435,6 +435,8 @@ impl From for nodedb_cluster::rpc_codec::TypedClusterError { | Error::GroupMarksUnavailable { .. } | Error::BackupCaptureMoved { .. } | Error::MetadataLeaderUnavailable + | Error::MetadataGroupGone { .. } + | Error::MetadataDdlSuperseded { .. } | Error::AuthorizationStateBehind { .. } | Error::LinearizableReadRefused { .. } | Error::ExecutionLimitExceeded { .. } diff --git a/nodedb/src/event/topic/publish.rs b/nodedb/src/event/topic/publish.rs index 04d18d751..a128c80b2 100644 --- a/nodedb/src/event/topic/publish.rs +++ b/nodedb/src/event/topic/publish.rs @@ -104,7 +104,8 @@ async fn publish_replicated( state, proposer, entry, - crate::control::wal_replication::statement_propose_deadline(state), + crate::control::server::shared::session::StatementDeadline::of_statement(state) + .tokio_instant(), ) .await .map_err(|error| match error { diff --git a/nodedb/src/types/mod.rs b/nodedb/src/types/mod.rs index 22ba096bb..8bfadda76 100644 --- a/nodedb/src/types/mod.rs +++ b/nodedb/src/types/mod.rs @@ -22,4 +22,6 @@ pub use snapshot::{ ArrayCellsBlob, SurrogateBindEntry, TenantDataSnapshot, TsFlushedCollectionBlob, TsFlushedPartitionBlob, }; -pub use snapshot_calvin::{CutBarrierLog, GroupCalvinCut, VShardCalvinState}; +pub use snapshot_calvin::{ + CoveredCutBarrier, CutBarrierLog, CutFloor, GroupCalvinCut, VShardCalvinState, +}; diff --git a/nodedb/src/types/record_home.rs b/nodedb/src/types/record_home.rs index c971a775e..777c7638b 100644 --- a/nodedb/src/types/record_home.rs +++ b/nodedb/src/types/record_home.rs @@ -13,7 +13,7 @@ //! that holds edges it also lives on the key vShard of its key: a live edge //! write carries each endpoint's surrogate from the collection home, and //! every replica of each endpoint home binds it on apply. -//! - An array cell lives on the vShard its Hilbert prefix routes to. +//! - An array cell lives on the vShard its tile key routes to. //! //! Every collection is collection-homed. Key-homed data is graph data, whose //! partition key is the node id. @@ -38,7 +38,7 @@ pub enum HomedRecord<'a> { key: &'a [u8], holds_edges: bool, }, - /// Array cells whose Hilbert prefixes route to `vshard`. + /// Array cells whose tile keys route to `vshard`. ArrayCells { vshard: VShardId }, } diff --git a/nodedb/src/types/snapshot.rs b/nodedb/src/types/snapshot.rs index 6220622a8..8792557f0 100644 --- a/nodedb/src/types/snapshot.rs +++ b/nodedb/src/types/snapshot.rs @@ -335,7 +335,7 @@ pub struct GroupEventLane { pub publish_cursors: Vec, } -/// One array's cell versions whose Hilbert prefixes route to one vShard. +/// One array's cell versions whose tile keys route to one vShard. #[derive( Debug, Clone, diff --git a/nodedb/src/types/snapshot_calvin.rs b/nodedb/src/types/snapshot_calvin.rs index 585278355..0548753ec 100644 --- a/nodedb/src/types/snapshot_calvin.rs +++ b/nodedb/src/types/snapshot_calvin.rs @@ -37,6 +37,56 @@ pub struct GroupCalvinCut { /// group vShard. It holds every barrier entry of the group's log at or /// below the snapshot's index, which the receiver never applies. pub barrier_logs: Vec, + /// Every backup cut barrier the builder applied in the group at or below + /// the snapshot's index whose cut window is open, in cut key order. The + /// receiver never applies them, and a cut waiting there needs them. + pub cut_barriers: Vec, + /// The group's durable cut floors at or below the snapshot's index, in + /// index order. Every later entry records a commit HLC above them. + pub cut_floors: Vec, +} + +/// A backup cut's barrier a [`GroupCalvinCut`] covers. +#[derive( + Debug, + Clone, + PartialEq, + Eq, + serde::Serialize, + serde::Deserialize, + zerompk::ToMessagePack, + zerompk::FromMessagePack, +)] +#[msgpack(map)] +pub struct CoveredCutBarrier { + /// The cut's watermark: HLC wall time in nanoseconds. + pub hlc: u64, + /// The cluster restore point the cut takes, `0` for a backup's cut. + pub restore_point: u64, + /// A database backup's capture request. + pub capture: Option, + /// The log index of the cut's first barrier in the group. + pub log_index: u64, +} + +/// One durable cut floor of a group in a [`GroupCalvinCut`]. +#[derive( + Debug, + Clone, + Copy, + PartialEq, + Eq, + serde::Serialize, + serde::Deserialize, + zerompk::ToMessagePack, + zerompk::FromMessagePack, +)] +#[msgpack(map)] +pub struct CutFloor { + /// The barrier's log index. + pub index: u64, + /// The barrier's watermark: HLC wall time in nanoseconds. + pub watermark: u64, } /// One vShard's applied Calvin positions in a [`GroupCalvinCut`]. diff --git a/nodedb/tests/crash_harness/vshards.rs b/nodedb/tests/crash_harness/vshards.rs index f9187388f..8f8776799 100644 --- a/nodedb/tests/crash_harness/vshards.rs +++ b/nodedb/tests/crash_harness/vshards.rs @@ -41,8 +41,8 @@ pub fn single_node_data_group(name: &str) -> u32 { /// `INT64` dimension `[lo..hi]` array in. /// /// An array cell write routes by its tile, never by the array's name. The -/// tile's key picks a bucket from its top `prefix_bits` bits, and the bucket -/// owns vShard `bucket * (1024 >> prefix_bits)`. At the default of 8 bits +/// tile's key picks a bucket from the top `prefix_bits` bits of +/// `fmix64(key)`, and the bucket owns vShard `bucket * (1024 >> prefix_bits)`. At the default of 8 bits /// every array vShard is a multiple of 4, so every cell of every array /// applies in data group 1. Only `prefix_bits` of 9 or more spreads cells /// over more than one group. `tile_extent` must match the array's diff --git a/nodedb/tests/crash_redo_chunk_stream.rs b/nodedb/tests/crash_redo_chunk_stream.rs index 2eb760a61..fe487aa18 100644 --- a/nodedb/tests/crash_redo_chunk_stream.rs +++ b/nodedb/tests/crash_redo_chunk_stream.rs @@ -9,7 +9,8 @@ //! //! - after every chunk is durable and before the final entry applies: boot //! rebuilds the stream from its WAL records, and the re-delivered final -//! entry installs every row once; +//! entry installs every row once. The restart slows that install, and the +//! node reports ready only after it, so the first read sees every row; //! - after the first chunk is durable: no final entry exists, so no row //! installs, and the stream drops at the group's next term. A later //! chunked commit installs as usual. @@ -37,6 +38,9 @@ const STATEMENTS: usize = 6; /// Bounded wait for the injected abort. const CRASH_TIMEOUT: Duration = Duration::from_secs(60); +/// How long the restart holds the re-delivered final entry's install. +const SLOW_INSTALL_MS: u64 = 3_000; + /// A harness whose server reads a config file with the lowered entry limit. fn harness() -> CrashHarness { let h = CrashHarness::new(); @@ -91,8 +95,12 @@ async fn commit_large_transaction(h: &CrashHarness, coll: &str, prefix: &str) -> } /// Arm `fail_point`, crash the server inside a chunked commit, and restart it -/// without the fail point. -async fn crash_inside_a_chunked_commit(h: &mut CrashHarness, fail_point: &str) { +/// with `restart_failpoints` armed, or with none. +async fn crash_inside_a_chunked_commit( + h: &mut CrashHarness, + fail_point: &str, + restart_failpoints: Option<&str>, +) { h.set_env("NODEDB_FAILPOINTS", &format!("{fail_point}=abort")); h.spawn(); h.wait_ready(); @@ -115,19 +123,30 @@ async fn crash_inside_a_chunked_commit(h: &mut CrashHarness, fail_point: &str) { h.keep_data_dir_note(), diagnostics::log_tail_section(&log) ); - h.clear_env("NODEDB_FAILPOINTS"); + match restart_failpoints { + Some(spec) => h.set_env("NODEDB_FAILPOINTS", spec), + None => h.clear_env("NODEDB_FAILPOINTS"), + } h.reopen(); } #[tokio::test(flavor = "multi_thread")] async fn a_crash_before_the_final_entry_applies_installs_the_whole_redo_once() { let mut h = harness(); - crash_inside_a_chunked_commit(&mut h, "redo_chunk::before_final_install").await; + let slow_install = format!("redo_chunk::before_final_install=sleep({SLOW_INSTALL_MS})"); + crash_inside_a_chunked_commit( + &mut h, + "redo_chunk::before_final_install", + Some(&slow_install), + ) + .await; + // The first read after ready: readiness waits for the slowed install. let expected = STATEMENTS * ROWS_PER_STATEMENT; assert_eq!( count(&h, "chunk_crash").await, expected, - "the committed final entry must install every row of its rebuilt stream once\n{}", + "the node reported ready before the re-delivered final entry installed every row of \ + its rebuilt stream once\n{}", diagnostics::log_tail_section(&h.server_log()) ); } @@ -135,7 +154,7 @@ async fn a_crash_before_the_final_entry_applies_installs_the_whole_redo_once() { #[tokio::test(flavor = "multi_thread")] async fn a_crash_between_chunk_entries_installs_nothing_and_frees_the_stream() { let mut h = harness(); - crash_inside_a_chunked_commit(&mut h, "redo_chunk::after_chunk_durable").await; + crash_inside_a_chunked_commit(&mut h, "redo_chunk::after_chunk_durable", None).await; assert_eq!( count(&h, "chunk_crash").await, 0, diff --git a/nodedb/tests/crash_replay_stamp_calvin.rs b/nodedb/tests/crash_replay_stamp_calvin.rs index d15ef09bf..a8d7def09 100644 --- a/nodedb/tests/crash_replay_stamp_calvin.rs +++ b/nodedb/tests/crash_replay_stamp_calvin.rs @@ -47,9 +47,10 @@ const CRASH_TIMEOUT: Duration = Duration::from_secs(60); /// How long the Calvin sequencer may take to elect its leader after a boot. const CALVIN_READY_TIMEOUT: Duration = Duration::from_secs(30); -/// The error a COMMIT returns once its completion deadline passes with no -/// acknowledgement. The redo record stays appended and the install stays held. -const COMMIT_DEADLINE: &str = "timed out waiting for Calvin transaction completion"; +/// The error a COMMIT returns once its statement deadline passes with no +/// acknowledgement: an unknown outcome. The redo record stays appended and +/// the install stays held. +const COMMIT_DEADLINE: &str = "57014: request req:0 exceeded deadline"; const LOG_DIRECTIVES: &str = "warn,nodedb::data::executor::kv_checkpoint=info"; diff --git a/nodedb/tests/inproc/cases/surrogate_round_trip.rs b/nodedb/tests/inproc/cases/surrogate_round_trip.rs index fe0b72849..b9d4cedee 100644 --- a/nodedb/tests/inproc/cases/surrogate_round_trip.rs +++ b/nodedb/tests/inproc/cases/surrogate_round_trip.rs @@ -681,7 +681,7 @@ fn surrogate_round_trip_all_engines() { attr_projection: vec![], limit: 100, cell_filter: None, - hilbert_range: None, + shard_bucket: None, system_time: nodedb_types::SystemTimeScope::Current, valid_at_ms: None, }), @@ -703,7 +703,7 @@ fn surrogate_round_trip_all_engines() { attr_projection: vec![], limit: 100, cell_filter: Some(intersection.clone()), - hilbert_range: None, + shard_bucket: None, system_time: nodedb_types::SystemTimeScope::Current, valid_at_ms: None, }), diff --git a/nodedb/tests/inproc/cases/system_task_call_sites.rs b/nodedb/tests/inproc/cases/system_task_call_sites.rs index d7ff0042a..7e21a4895 100644 --- a/nodedb/tests/inproc/cases/system_task_call_sites.rs +++ b/nodedb/tests/inproc/cases/system_task_call_sites.rs @@ -59,7 +59,7 @@ const ALLOWED: &[&str] = &[ // Event Plane rules dispatched back through the Control Plane. "event/alert/executor.rs", // Legs of a request whose capability was consumed at the entry point. - "control/crdt_admission.rs", + "control/crdt_admission/legs.rs", // `sync/raft_dispatch/write.rs` is absent: a sync write proposes its // plan through Raft, and the entry's apply dispatches it on each replica. ]; diff --git a/nodedb/tests/wire/cases/mod.rs b/nodedb/tests/wire/cases/mod.rs index 96e339a60..8173f040f 100644 --- a/nodedb/tests/wire/cases/mod.rs +++ b/nodedb/tests/wire/cases/mod.rs @@ -191,6 +191,7 @@ mod session_handle_security; mod session_plan_cache_permission_tree_revoke; mod session_plan_cache_rls_revoke; mod session_statement_timeout; +mod slow_catalog_ddl_apply; mod sorted_index_authorization; mod spatial_cp_dp_query; mod sql_aggregate_functions; diff --git a/nodedb/tests/wire/cases/slow_catalog_ddl_apply.rs b/nodedb/tests/wire/cases/slow_catalog_ddl_apply.rs new file mode 100644 index 000000000..ff131bb0d --- /dev/null +++ b/nodedb/tests/wire/cases/slow_catalog_ddl_apply.rs @@ -0,0 +1,47 @@ +// SPDX-License-Identifier: BUSL-1.1 + +//! Compiled only with `--features failpoints`. +//! +//! A catalog DDL whose apply runs longer than a few seconds still succeeds. +//! The proposer waits for its apply until the statement deadline, so a slow +//! apply is never reported as an error while the deadline holds. + +#[cfg(feature = "failpoints")] +use crate::harness::TestServer; + +/// How long the armed fail point holds the collection's apply. +#[cfg(feature = "failpoints")] +const APPLY_PAUSE_MS: u64 = 6_000; + +#[cfg(feature = "failpoints")] +#[tokio::test(flavor = "multi_thread", worker_threads = 4)] +async fn a_create_collection_with_a_six_second_apply_succeeds() { + let server = TestServer::start_with_failpoints(&format!( + "catalog_ddl::before_apply::slow_apply=sleep({APPLY_PAUSE_MS})" + )) + .await; + + let started = std::time::Instant::now(); + server + .exec( + "CREATE COLLECTION slow_apply (id TEXT PRIMARY KEY, val INT) \ + WITH (engine='document_strict')", + ) + .await + .expect("CREATE COLLECTION must succeed while its apply runs inside the deadline"); + assert!( + started.elapsed() >= std::time::Duration::from_millis(APPLY_PAUSE_MS), + "the fail point did not hold the apply, so this test proves nothing: {:?}", + started.elapsed() + ); + + server + .exec("INSERT INTO slow_apply (id, val) VALUES ('a', 1)") + .await + .expect("the created collection takes writes"); + let rows = server + .query_rows("SELECT val FROM slow_apply WHERE id = 'a'") + .await + .expect("the created collection serves reads"); + assert_eq!(rows, vec![vec!["1".to_string()]]); +}