From 9b99099b764cf0fcb6545d6ec283f38447da6a77 Mon Sep 17 00:00:00 2001 From: Farhan Syah Date: Sun, 11 Oct 2026 13:53:27 +0800 Subject: [PATCH 1/5] feat(array): route tiles by a mixed hash of the tile key Tile keys of a full-range array share their top bits, so routing by the raw top bits sent every tile to the same few buckets. Routing now takes the top bits of a mixed hash of the key, shared by the planner, the cluster coordinator and the Data Plane. --- nodedb-array/src/tile/layout.rs | 76 ++++- nodedb-cluster/src/array_routing.rs | 14 +- .../src/distributed_array/coordinator/read.rs | 208 ++++++++----- .../distributed_array/coordinator/write.rs | 74 +++-- .../src/distributed_array/handler.rs | 107 +++---- .../src/distributed_array/local_executor.rs | 11 +- .../src/distributed_array/partition.rs | 123 ++++---- .../src/distributed_array/routing.rs | 293 +++++------------- .../src/distributed_array/scatter.rs | 109 ++++++- nodedb-cluster/src/distributed_array/wire.rs | 54 ++-- nodedb-physical/src/physical_plan/array.rs | 30 +- .../src/physical_plan/cluster_array.rs | 30 +- nodedb-types/src/array_route.rs | 241 ++++++++++++++ nodedb-types/src/id/vshard.rs | 5 + nodedb-types/src/lib.rs | 2 + .../control/backup/restore/array_reissue.rs | 3 +- .../post_apply/async_dispatch/array.rs | 2 +- .../cluster/array_cluster_exec/dispatch.rs | 141 +++++++-- .../cluster/array_cluster_exec/executor.rs | 42 ++- .../control/cluster/array_cluster_helpers.rs | 69 ++++- .../cluster/array_executor/executor.rs | 12 +- .../control/cluster/array_executor/read.rs | 4 +- .../control/cluster/array_executor/refusal.rs | 2 + .../control/cluster/array_executor/write.rs | 5 +- .../sql_plan_convert/array_convert/dml.rs | 12 +- .../array_fn_convert/aggregate.rs | 4 +- .../array_fn_convert/slice.rs | 36 ++- .../ddl/neutral/tenant/move_tenant/arrays.rs | 2 +- .../shared/session/array_fanout_stage.rs | 6 +- nodedb/src/data/executor/array_checkpoint.rs | 2 +- .../data/executor/dispatch/array/aggregate.rs | 42 ++- .../src/data/executor/dispatch/array/entry.rs | 8 +- .../src/data/executor/dispatch/array/read.rs | 148 ++++++++- .../handlers/snapshot/capture_arrays.rs | 75 ++++- .../handlers/snapshot/install_arrays.rs | 25 +- .../transaction/overlay/array_merge.rs | 8 +- 36 files changed, 1323 insertions(+), 702 deletions(-) create mode 100644 nodedb-types/src/array_route.rs diff --git a/nodedb-array/src/tile/layout.rs b/nodedb-array/src/tile/layout.rs index cb61577d3..d6c22f102 100644 --- a/nodedb-array/src/tile/layout.rs +++ b/nodedb-array/src/tile/layout.rs @@ -83,19 +83,21 @@ pub fn tile_id_for_cell( /// The space-filling-curve key of the tile at `tile_indices`. /// /// The memtable, the segment index, the transaction overlay and retention -/// group cells by this key, and a cluster routes a tile by its top bits. -/// Every cell inside a group stays addressed by its full coordinate: a -/// lookup matches the coordinate within the group +/// group cells by this key. A cluster routes a tile by the top bits of +/// `fmix64(key)` (`nodedb_types::array_route::prefix_bucket`), not by the +/// key's own top bits. Every cell inside a group stays addressed by its +/// full coordinate: a lookup matches the coordinate within the group /// (`TileBuffer::get_cell_bytes`, `extract_cell_bytes`), and a tile's MBR /// is built from the cells it holds. Two tiles that share a key therefore -/// share one group and lose no cell. +/// share one group and one route, and lose no cell. /// /// A dimension whose tile count fits its share of the key bits scales each /// index over that count to the full bit width. Its tiles keep distinct, -/// ordered keys that spread over the top bits. A dimension with more tiles -/// than its bits can number, such as one with the default full-range -/// domain, keeps the low bits of the index instead. Nearby tiles then keep -/// distinct keys, and tiles `2^bits` apart share one. +/// ordered keys. A dimension with more tiles than its bits can number, such +/// as one with the default full-range domain, keeps the low bits of the +/// index instead. Nearby tiles then keep distinct keys, and tiles `2^bits` +/// apart share one. Such keys can share their top bits, and the routing mix +/// still spreads them over buckets. pub fn tile_prefix(schema: &ArraySchema, tile_indices: &[u64]) -> ArrayResult { if tile_indices.len() != schema.arity() { return Err(ArrayError::CoordArityMismatch { @@ -178,6 +180,7 @@ mod tests { use crate::schema::attr_spec::{AttrSpec, AttrType}; use crate::schema::dim_spec::DimSpec; use crate::types::domain::Domain; + use nodedb_types::array_route::prefix_bucket; fn schema() -> ArraySchema { ArraySchemaBuilder::new("g") @@ -221,20 +224,63 @@ mod tests { } #[test] - fn tile_keys_spread_over_the_top_bits() { + fn tile_keys_spread_over_routing_buckets() { let s = schema(); - let tops: std::collections::BTreeSet = (0..10) + let buckets: std::collections::BTreeSet = (0..10) .flat_map(|x| (0..10).map(move |y| (x * 10, y * 10))) .map(|(x, y)| { - tile_id_for_cell(&s, &[CoordValue::Int64(x), CoordValue::Int64(y)], 0) + let key = tile_id_for_cell(&s, &[CoordValue::Int64(x), CoordValue::Int64(y)], 0) .unwrap() - .hilbert_prefix - >> 56 + .hilbert_prefix; + prefix_bucket(key, 8) + }) + .collect(); + // 100 tiles over 256 buckets: well over half of them distinct. + assert!( + buckets.len() >= 60, + "the tiles of one array route to {} buckets", + buckets.len() + ); + } + + fn full_range_2d() -> ArraySchema { + ArraySchemaBuilder::new("open") + .dim(DimSpec::new( + "x", + DimType::Int64, + Domain::new(DomainBound::Int64(i64::MIN), DomainBound::Int64(i64::MAX)), + )) + .dim(DimSpec::new( + "y", + DimType::Int64, + Domain::new(DomainBound::Int64(i64::MIN), DomainBound::Int64(i64::MAX)), + )) + .attr(AttrSpec::new("v", AttrType::Int64, false)) + .tile_extents(vec![1, 1]) + .build() + .unwrap() + } + + /// The tiles of coordinates 0..64 x 0..64 in a full-range 2-D array + /// share their key's top bits, yet spread over the routing buckets. + #[test] + fn full_range_tiles_near_zero_spread_over_routing_buckets() { + let s = full_range_2d(); + let keys: Vec = (0..64) + .flat_map(|x| (0..64).map(move |y| (x, y))) + .map(|(x, y)| { + cell_tile_prefix(&s, &[CoordValue::Int64(x), CoordValue::Int64(y)]).unwrap() }) .collect(); + let distinct: std::collections::BTreeSet = keys.iter().copied().collect(); + assert_eq!(distinct.len(), 4096, "every tile keeps a distinct key"); + let buckets: std::collections::BTreeSet = + keys.iter().map(|&k| prefix_bucket(k, 8)).collect(); + // 4096 distinct keys over 256 buckets reach at least 7/8 of them. assert!( - tops.len() > 1, - "the tiles of one array route to more than one bucket: {tops:?}" + buckets.len() >= 224, + "4096 tiles route to only {} of 256 buckets", + buckets.len() ); } diff --git a/nodedb-cluster/src/array_routing.rs b/nodedb-cluster/src/array_routing.rs index 0085835e2..5610aea9a 100644 --- a/nodedb-cluster/src/array_routing.rs +++ b/nodedb-cluster/src/array_routing.rs @@ -16,10 +16,8 @@ //! //! # Returned type //! -//! Functions here return a raw `u16` vShard ID (0..1023). Callers in the -//! `nodedb` crate wrap this in `VShardId::new(…)`. `VShardId` lives in -//! `nodedb::types` which `nodedb-cluster` cannot depend on without creating -//! a circular dependency. +//! Functions here return a raw `u32` vShard ID below `VShardId::COUNT`. +//! Callers wrap it in `VShardId::new(…)`. //! //! # Fallbacks //! @@ -27,8 +25,8 @@ //! - Zero tile extent in any dimension: same fallback (no division by zero). //! - `coord.len() != tile_extents.len()`: fallback. -/// Number of virtual shards — must match `VShardId::COUNT` in `nodedb::types`. -pub const VSHARD_COUNT: u32 = 1024; +/// Number of virtual shards. +pub const VSHARD_COUNT: u32 = nodedb_types::id::VShardId::COUNT; // ─── Collection-level fallback hash ────────────────────────────────────────── @@ -120,7 +118,7 @@ pub fn tile_id_of_coord(coord: &[u64], tile_extents: &[u64]) -> Option { /// - `coord` / `tile_extents` are empty or mismatched. /// - Any `tile_extents[i]` is zero. /// -/// Returns a raw `u16` in `0..1023`. Callers wrap it in `VShardId::new(…)`. +/// Returns a raw `u32` below `VSHARD_COUNT`. Callers wrap it in `VShardId::new(…)`. pub fn vshard_for_array_coord(array_name: &str, coord: &[u64], tile_extents: &[u64]) -> u32 { match tile_id_of_coord(coord, tile_extents) { Some(tile_id) => vshard_for_array_tile(array_name, tile_id), @@ -133,7 +131,7 @@ pub fn vshard_for_array_coord(array_name: &str, coord: &[u64], tile_extents: &[u /// Use this when the tile_id is already known (e.g. from the op-log key or /// snapshot metadata) and coord re-derivation is unnecessary. /// -/// Returns a raw `u16` in `0..1023`. Callers wrap it in `VShardId::new(…)`. +/// Returns a raw `u32` below `VSHARD_COUNT`. Callers wrap it in `VShardId::new(…)`. pub fn vshard_for_array_tile(array_name: &str, tile_id: u64) -> u32 { let mut buf = Vec::with_capacity(array_name.len() + 8); buf.extend_from_slice(array_name.as_bytes()); diff --git a/nodedb-cluster/src/distributed_array/coordinator/read.rs b/nodedb-cluster/src/distributed_array/coordinator/read.rs index 760c0b184..09ea665fe 100644 --- a/nodedb-cluster/src/distributed_array/coordinator/read.rs +++ b/nodedb-cluster/src/distributed_array/coordinator/read.rs @@ -3,8 +3,8 @@ //! Read-path coordinator for distributed array operations. //! //! [`ArrayCoordinator`] drives fan-out reads (`coord_slice`, `coord_agg`, -//! `coord_surrogate_bitmap_scan`) to the set of vShards whose Hilbert range -//! overlaps the slice predicate. +//! `coord_surrogate_bitmap_scan`) to the vShards that own a tile the slice +//! predicate covers. use std::sync::Arc; @@ -15,8 +15,11 @@ use super::super::merge::{ ArrayAggPartial, any_truncated_before_horizon_agg, merge_read_versions, merge_slice_rows, merge_slice_rows_sorted, reduce_agg_partials, }; +use super::super::routing::shard_bucket_for_vshard; use super::super::rpc::ShardRpcDispatch; -use super::super::scatter::{FanOutParams, FanOutPartitionedParams, fan_out, fan_out_partitioned}; +use super::super::scatter::{ + ArrayCoordError, FanOutParams, FanOutPartitionedParams, fan_out, fan_out_partitioned, +}; use super::super::wire::{ ArrayShardAggReq, ArrayShardAggResp, ArrayShardDeleteReq, ArrayShardDeleteResp, ArrayShardSliceReq, ArrayShardSliceResp, ArrayShardSurrogateBitmapReq, @@ -30,14 +33,13 @@ pub struct ArrayCoordParams { pub shard_ids: Vec, /// Per-shard RPC timeout in milliseconds. pub timeout_ms: u64, - /// Hilbert routing granularity (1–16). 0 means no shard-side routing - /// validation (e.g. when the coordinator was constructed without slice - /// range information, as in tests or unbounded scans). + /// Routing granularity (1–16). 0 skips shard-side routing validation + /// and the per-shard bucket filter. pub prefix_bits: u8, - /// Inclusive Hilbert-prefix ranges `(lo, hi)` that this read covers. - /// Forwarded to each shard so it can verify it still owns the range. - /// Empty means unbounded — the shard skips routing validation. - pub slice_hilbert_ranges: Vec<(u64, u64)>, + /// Tile keys this read covers, at least one per bucket. Forwarded to + /// each shard so it checks that it still owns one. Empty means + /// unbounded, and the shard skips routing validation. + pub slice_tile_keys: Vec, } /// Result of a coordinated slice fan-out. @@ -69,24 +71,13 @@ pub struct CoordAggResult { pub read_versions: Vec, } -/// Compute the inclusive Hilbert-prefix range `[lo, hi]` that vShard `shard_id` -/// owns given the array's routing granularity `prefix_bits`. -/// -/// Each bucket `b = shard_id / stride` covers the Hilbert range -/// `[b << (64 - prefix_bits), ((b + 1) << (64 - prefix_bits)) - 1]`. -/// The stride is `VSHARD_COUNT >> prefix_bits` (floored at 1). -pub(super) fn shard_hilbert_range_for_vshard(shard_id: u32, prefix_bits: u8) -> (u64, u64) { - use crate::routing::VSHARD_COUNT; - let stride = (VSHARD_COUNT >> (prefix_bits as u32)).max(1); - let bucket = shard_id / stride; - let shift = 64u8.saturating_sub(prefix_bits); - let lo = (bucket as u64) << shift; - let hi = if shift == 0 { - u64::MAX - } else { - lo.saturating_add((1u64 << shift).saturating_sub(1)) - }; - (lo, hi) +/// The bucket filter for a read sent to vShard `shard_id`. `None` when +/// `prefix_bits` is 0, which reads every tile. +fn shard_bucket(shard_id: u32, prefix_bits: u8) -> Result> { + if prefix_bits == 0 { + return Ok(None); + } + shard_bucket_for_vshard(shard_id, prefix_bits).map(Some) } /// Coordinator for distributed array read operations. @@ -109,24 +100,24 @@ impl ArrayCoordinator { } } - /// Construct an `ArrayCoordinator` whose target shards are computed from - /// the Hilbert-prefix ranges that overlap a slice predicate. + /// Construct an `ArrayCoordinator` whose target shards own the tiles a + /// slice predicate covers. /// - /// `slice_hilbert_ranges` — `(lo, hi)` pairs computed by the planner from - /// the `Slice` predicate. Pass an empty slice for an unbounded scan. + /// `slice_tile_keys` — tile keys computed by the planner from the + /// `Slice` predicate. Pass an empty slice for an unbounded scan. /// `prefix_bits` — the array's routing granularity from the catalog entry. /// `total_shards` — the number of active vShards in the cluster. pub fn for_slice( source_node: u64, timeout_ms: u64, - slice_hilbert_ranges: &[(u64, u64)], + slice_tile_keys: &[u64], prefix_bits: u8, total_shards: u32, dispatch: Arc, circuit_breaker: Arc, ) -> crate::error::Result { let shard_ids = super::super::routing::array_vshards_for_slice( - slice_hilbert_ranges, + slice_tile_keys, prefix_bits, total_shards, )?; @@ -136,7 +127,7 @@ impl ArrayCoordinator { shard_ids, timeout_ms, prefix_bits, - slice_hilbert_ranges: slice_hilbert_ranges.to_vec(), + slice_tile_keys: slice_tile_keys.to_vec(), }, dispatch, circuit_breaker, @@ -147,10 +138,9 @@ impl ArrayCoordinator { /// /// Each shard receives the full slice request with the caller-supplied /// `limit` pushed down so shards can stop scanning early. The coordinator - /// stamps a per-shard `shard_hilbert_range` so each shard only returns - /// cells whose Hilbert prefix falls within its owned range, preventing - /// duplicate rows in single-node harnesses where all vShards share one - /// Data Plane. The coordinator applies the same `limit` as a final + /// stamps a per-shard `shard_bucket`, so each shard returns only cells + /// whose tile key routes to it. vShards that share one Data Plane then + /// return no duplicate rows. The coordinator applies the same `limit` as a final /// cut-off on the merged result. /// /// When `system_time` is `AllVersions`, the coordinator merge-sorts rows @@ -167,22 +157,17 @@ impl ArrayCoordinator { req: ArrayShardSliceReq, coordinator_limit: u32, system_time: nodedb_types::SystemTimeScope, - ) -> Result { + ) -> std::result::Result { let prefix_bits = self.params.prefix_bits; let per_shard: Vec<(u32, Vec)> = self .params .shard_ids .iter() .map(|&shard_id| { - let shard_hilbert_range = if prefix_bits > 0 { - Some(shard_hilbert_range_for_vshard(shard_id, prefix_bits)) - } else { - None - }; let per_shard_req = ArrayShardSliceReq { prefix_bits, - slice_hilbert_ranges: self.params.slice_hilbert_ranges.clone(), - shard_hilbert_range, + slice_tile_keys: self.params.slice_tile_keys.clone(), + shard_bucket: shard_bucket(shard_id, prefix_bits)?, ..req.clone() }; let bytes = @@ -224,24 +209,21 @@ impl ArrayCoordinator { /// Fan out an aggregate request and reduce partial aggregates from all shards. /// - /// Each shard receives its own `shard_hilbert_range` so it can apply a - /// Hilbert-prefix pre-filter and only count cells in its partition. This - /// prevents double-counting in configurations where multiple vShards share - /// a single Data Plane executor (e.g. single-node harnesses). - pub async fn coord_agg(&self, req: ArrayShardAggReq) -> Result { + /// Each shard receives its own `shard_bucket` and counts only cells whose + /// tile key routes to it. vShards that share one Data Plane executor then + /// count no cell twice. + pub async fn coord_agg( + &self, + req: ArrayShardAggReq, + ) -> std::result::Result { let prefix_bits = self.params.prefix_bits; let per_shard: Vec<(u32, Vec)> = self .params .shard_ids .iter() .map(|&shard_id| { - let hilbert_range = if prefix_bits > 0 { - Some(shard_hilbert_range_for_vshard(shard_id, prefix_bits)) - } else { - None - }; let per_shard_req = ArrayShardAggReq { - shard_hilbert_range: hilbert_range, + shard_bucket: shard_bucket(shard_id, prefix_bits)?, ..req.clone() }; let bytes = @@ -276,7 +258,7 @@ impl ArrayCoordinator { pub async fn coord_delete( &self, req: ArrayShardDeleteReq, - ) -> Result> { + ) -> std::result::Result, ArrayCoordError> { let req_bytes = zerompk::to_msgpack_vec(&req).map_err(|e| ClusterError::Codec { detail: format!("ArrayShardDeleteReq serialise: {e}"), })?; @@ -288,7 +270,7 @@ impl ArrayCoordinator { &self.circuit_breaker, ) .await?; - decode_resps::(&raw) + Ok(decode_resps::(&raw)?) } /// Fan out a surrogate bitmap scan, collect per-shard bitmap bytes, and @@ -298,7 +280,7 @@ impl ArrayCoordinator { pub async fn coord_surrogate_bitmap_scan( &self, req: ArrayShardSurrogateBitmapReq, - ) -> Result> { + ) -> std::result::Result, ArrayCoordError> { let req_bytes = zerompk::to_msgpack_vec(&req).map_err(|e| ClusterError::Codec { detail: format!("ArrayShardSurrogateBitmapReq serialise: {e}"), })?; @@ -310,7 +292,7 @@ impl ArrayCoordinator { &self.circuit_breaker, ) .await?; - decode_resps::(&raw) + Ok(decode_resps::(&raw)?) } pub(super) fn fan_out_params(&self) -> FanOutParams { @@ -414,11 +396,10 @@ mod tests { source_node: 1, shard_ids, timeout_ms: 1000, - // Tests use prefix_bits=0 so shard-side routing validation - // is skipped — mock executors don't need to match Hilbert - // ownership. + // prefix_bits=0 skips shard-side routing validation and the + // bucket filter, so mock executors ignore ownership. prefix_bits: 0, - slice_hilbert_ranges: vec![], + slice_tile_keys: vec![], }, dispatch, Arc::new(CircuitBreaker::new(CircuitBreakerConfig::default())), @@ -440,8 +421,8 @@ mod tests { limit: 100, cell_filter_msgpack: vec![], prefix_bits: 0, - slice_hilbert_ranges: vec![], - shard_hilbert_range: None, + slice_tile_keys: vec![], + shard_bucket: None, system_time: nodedb_types::SystemTimeScope::Current, valid_at_ms: None, txn_id: None, @@ -481,8 +462,8 @@ mod tests { limit: 3, cell_filter_msgpack: vec![], prefix_bits: 0, - slice_hilbert_ranges: vec![], - shard_hilbert_range: None, + slice_tile_keys: vec![], + shard_bucket: None, system_time: nodedb_types::SystemTimeScope::Current, valid_at_ms: None, txn_id: None, @@ -503,7 +484,7 @@ mod tests { reducer_msgpack: vec![0x00], group_by_dim: -1, cell_filter_msgpack: vec![], - shard_hilbert_range: None, + shard_bucket: None, system_as_of: None, valid_at_ms: None, txn_id: None, @@ -662,8 +643,8 @@ mod tests { limit: 0, cell_filter_msgpack: vec![], prefix_bits: 0, - slice_hilbert_ranges: vec![], - shard_hilbert_range: None, + slice_tile_keys: vec![], + shard_bucket: None, system_time: nodedb_types::SystemTimeScope::Current, valid_at_ms: None, txn_id: None, @@ -676,4 +657,85 @@ mod tests { .expect("coord_slice unlimited should succeed"); assert_eq!(result.rows.len(), 20); } + + /// Serves a fixed set of tile keys, each as one row, keeping only the + /// keys the request's `shard_bucket` admits. + struct BucketFilterDispatch { + tile_keys: Vec, + } + + #[async_trait] + impl ShardRpcDispatch for BucketFilterDispatch { + async fn call(&self, req: VShardEnvelope, _timeout_ms: u64) -> Result { + let slice: ArrayShardSliceReq = zerompk::from_msgpack(&req.payload).unwrap(); + let rows = self + .tile_keys + .iter() + .filter(|&&k| nodedb_types::array_route::shard_bucket_admits(slice.shard_bucket, k)) + .map(|k| zerompk::to_msgpack_vec(k).unwrap()) + .collect(); + let resp = ArrayShardSliceResp { + shard_id: req.vshard_id, + rows_msgpack: rows, + truncated: false, + truncated_before_horizon: false, + read_versions: Vec::new(), + }; + let payload = zerompk::to_msgpack_vec(&resp).unwrap(); + Ok(VShardEnvelope::new( + VShardMessageType::ArrayShardSliceResp, + req.target_node, + req.source_node, + req.vshard_id, + payload, + )) + } + } + + /// Over every primary vShard, the bucket filters return each tile once. + #[tokio::test] + async fn bucket_filters_partition_every_tile_across_shards() { + let tile_keys: Vec = (0u64..4096).chain([1 << 40, u64::MAX]).collect(); + for prefix_bits in [1u8, 8, 10, 16] { + let dispatch: Arc = Arc::new(BucketFilterDispatch { + tile_keys: tile_keys.clone(), + }); + let coord = ArrayCoordinator::for_slice( + 1, + 1000, + &[], + prefix_bits, + crate::routing::VSHARD_COUNT, + dispatch, + Arc::new(CircuitBreaker::new(CircuitBreakerConfig::default())), + ) + .expect("for_slice"); + let req = ArrayShardSliceReq { + array_id_msgpack: vec![], + slice_msgpack: vec![], + attr_projection: vec![], + limit: 0, + cell_filter_msgpack: vec![], + prefix_bits: 0, + slice_tile_keys: vec![], + shard_bucket: None, + system_time: nodedb_types::SystemTimeScope::Current, + valid_at_ms: None, + txn_id: None, + }; + let result = coord + .coord_slice(req, 0, nodedb_types::SystemTimeScope::Current) + .await + .expect("coord_slice"); + let mut got: Vec = result + .rows + .iter() + .map(|r| zerompk::from_msgpack::(r).unwrap()) + .collect(); + got.sort_unstable(); + let mut want = tile_keys.clone(); + want.sort_unstable(); + assert_eq!(got, want, "prefix_bits={prefix_bits}"); + } + } } diff --git a/nodedb-cluster/src/distributed_array/coordinator/write.rs b/nodedb-cluster/src/distributed_array/coordinator/write.rs index 80c61d54f..17c284ff3 100644 --- a/nodedb-cluster/src/distributed_array/coordinator/write.rs +++ b/nodedb-cluster/src/distributed_array/coordinator/write.rs @@ -3,7 +3,7 @@ //! Write-path coordinator for distributed array operations. //! //! Provides [`coord_put`], [`coord_put_partitioned`], and [`coord_delete`] -//! — functions that partition a flat cell/coord list by Hilbert tile and +//! — functions that partition a flat cell/coord list by routing bucket and //! fan writes to the owning vShards. use std::sync::Arc; @@ -13,7 +13,7 @@ use crate::error::{ClusterError, Result}; use super::super::partition::{partition_delete_coords, partition_put_cells}; use super::super::rpc::ShardRpcDispatch; -use super::super::scatter::{FanOutPartitionedParams, fan_out_partitioned}; +use super::super::scatter::{ArrayCoordError, FanOutPartitionedParams, fan_out_partitioned}; use super::super::wire::{ ArrayShardDeleteReq, ArrayShardDeleteResp, ArrayShardPutReq, ArrayShardPutResp, }; @@ -27,7 +27,7 @@ pub struct ArrayWriteCoordParams { /// Forward pre-partitioned cell writes to the owning shards. /// -/// The caller groups cells by Hilbert prefix bucket using +/// The caller groups cells by routing bucket using /// `array_vshard_for_tile` and produces one `ArrayShardPutReq` per /// target shard. This function dispatches each batch to its shard via /// `fan_out_partitioned` and collects acknowledgements. @@ -39,7 +39,7 @@ pub async fn coord_put_partitioned( per_shard: Vec<(u32, ArrayShardPutReq)>, dispatch: &Arc, circuit_breaker: &Arc, -) -> Result> { +) -> std::result::Result, ArrayCoordError> { if per_shard.is_empty() { return Ok(Vec::new()); } @@ -69,15 +69,15 @@ pub async fn coord_put_partitioned( ) .await?; - decode_resps::(&raw) + Ok(decode_resps::(&raw)?) } -/// Partition a flat cell list by Hilbert tile and fan out to owning shards. +/// Partition a flat cell list by routing bucket and fan out to owning shards. /// -/// `cells` — each element is `(hilbert_prefix, zerompk-encoded single-cell bytes)`. -/// The Hilbert prefix is computed by the caller (the Control Plane planner) from -/// the cell's coord tuple and the array schema; this function does not decode -/// cell bytes. +/// `cells` — each element is `(tile_key, zerompk-encoded single-cell bytes)`. +/// The caller (the Control Plane planner) computes the tile key from the +/// cell's coord tuple and the array schema. This function decodes no cell +/// bytes. /// /// `prefix_bits` — routing granularity (1–16) from the array catalog entry. /// `wal_lsn` — WAL sequence number allocated by the Control Plane for this batch. @@ -93,7 +93,7 @@ pub async fn coord_put( cells: &[(u64, Vec)], dispatch: &Arc, circuit_breaker: &Arc, -) -> Result> { +) -> std::result::Result, ArrayCoordError> { if cells.is_empty() { return Ok(Vec::new()); } @@ -107,7 +107,7 @@ pub async fn coord_put( array_id_msgpack: array_id_msgpack.clone(), cells_msgpack: b.cells_msgpack, wal_lsn, - representative_hilbert_prefix: b.representative_hilbert_prefix, + representative_tile_key: b.representative_tile_key, prefix_bits, }; (b.vshard_id, req) @@ -117,9 +117,9 @@ pub async fn coord_put( coord_put_partitioned(params, per_shard, dispatch, circuit_breaker).await } -/// Partition a flat coord list by Hilbert tile and fan delete requests to owning shards. +/// Partition a flat coord list by routing bucket and fan delete requests to owning shards. /// -/// `coords` — each element is `(hilbert_prefix, zerompk-encoded single-coord bytes)`. +/// `coords` — each element is `(tile_key, zerompk-encoded single-coord bytes)`. /// `prefix_bits` — routing granularity (1–16). /// `wal_lsn` — WAL sequence number allocated by the Control Plane. /// @@ -132,7 +132,7 @@ pub async fn coord_delete( coords: &[(u64, Vec)], dispatch: &Arc, circuit_breaker: &Arc, -) -> Result> { +) -> std::result::Result, ArrayCoordError> { if coords.is_empty() { return Ok(Vec::new()); } @@ -151,7 +151,7 @@ pub async fn coord_delete( array_id_msgpack: array_id_msgpack.clone(), coords_msgpack: b.coords_msgpack, wal_lsn, - representative_hilbert_prefix: b.representative_hilbert_prefix, + representative_tile_key: b.representative_tile_key, prefix_bits, }; zerompk::to_msgpack_vec(&req) @@ -171,7 +171,7 @@ pub async fn coord_delete( ) .await?; - decode_resps::(&raw) + Ok(decode_resps::(&raw)?) } #[cfg(test)] @@ -179,6 +179,7 @@ mod tests { use async_trait::async_trait; use crate::circuit_breaker::{CircuitBreaker, CircuitBreakerConfig}; + use crate::distributed_array::scatter::ShardFailure; use crate::wire::{VShardEnvelope, VShardMessageType}; use super::*; @@ -263,10 +264,8 @@ mod tests { #[tokio::test] async fn coord_put_partitions_cells_by_tile() { - // prefix_bits=10, stride=1 → vshard == top-10-bit bucket. - // p0 → bucket 0 → vshard 0 - // p1 → bucket 1 → vshard 1 - // p2 → bucket 2 → vshard 2 + // prefix_bits=10, stride=1 → vShard == mixed bucket. + // The three keys mix to three distinct buckets. let p0 = 0x0000_0000_0000_0000u64; let p1 = 0x0040_0000_0000_0000u64; let p2 = 0x0080_0000_0000_0000u64; @@ -285,10 +284,16 @@ mod tests { .expect("coord_put should succeed"); resps.sort_by_key(|r| r.shard_id); + let mut expected: Vec = [p0, p1, p2] + .iter() + .map(|&k| crate::distributed_array::array_vshard_for_tile(k, 10).unwrap()) + .collect(); + expected.sort_unstable(); assert_eq!(resps.len(), 3, "should fan-out to 3 shards"); - assert_eq!(resps[0].shard_id, 0); - assert_eq!(resps[1].shard_id, 1); - assert_eq!(resps[2].shard_id, 2); + assert_eq!( + resps.iter().map(|r| r.shard_id).collect::>(), + expected + ); // Each shard echoes back wal_lsn=42. for r in &resps { assert_eq!(r.applied_lsn, 42); @@ -325,8 +330,14 @@ mod tests { .await .expect_err("coord_put with failing shard should return error"); assert!( - matches!(err, ClusterError::Codec { .. }), - "expected Codec error, got {err:?}" + matches!( + err, + ArrayCoordError::Shard(ShardFailure { + error: ClusterError::Codec { .. }, + .. + }) + ), + "expected a shard's Codec error, got {err:?}" ); } @@ -343,9 +354,16 @@ mod tests { .expect("coord_delete should succeed"); resps.sort_by_key(|r| r.shard_id); + let mut expected: Vec = [p0, p1] + .iter() + .map(|&k| crate::distributed_array::array_vshard_for_tile(k, 10).unwrap()) + .collect(); + expected.sort_unstable(); assert_eq!(resps.len(), 2, "should fan-out to 2 shards"); - assert_eq!(resps[0].shard_id, 0); - assert_eq!(resps[1].shard_id, 1); + assert_eq!( + resps.iter().map(|r| r.shard_id).collect::>(), + expected + ); for r in &resps { assert_eq!(r.applied_lsn, 55); } diff --git a/nodedb-cluster/src/distributed_array/handler.rs b/nodedb-cluster/src/distributed_array/handler.rs index 01cafb3e5..9ac775379 100644 --- a/nodedb-cluster/src/distributed_array/handler.rs +++ b/nodedb-cluster/src/distributed_array/handler.rs @@ -174,15 +174,15 @@ async fn handle_surrogate_bitmap( /// Validate that a PUT request is routed to the correct shard. /// -/// Returns `Ok(())` when routing is correct or when `prefix_bits == 0` -/// (which means the request predates Hilbert routing and skips validation). -/// Returns `Err(ClusterError::WrongOwner)` when the coordinator used a stale -/// routing table; the coordinator should refresh and retry. +/// Returns `Ok(())` when routing is correct or when `prefix_bits == 0`, +/// which skips validation. Returns `Err(ClusterError::WrongOwner)` when the +/// coordinator used a stale routing table. The coordinator then refreshes +/// and retries. fn validate_put_routing(req: &ArrayShardPutReq, local_vshard_id: u32) -> Result<()> { if req.prefix_bits == 0 { return Ok(()); } - let expected = array_vshard_for_tile(req.representative_hilbert_prefix, req.prefix_bits)?; + let expected = array_vshard_for_tile(req.representative_tile_key, req.prefix_bits)?; if expected != local_vshard_id { return Err(ClusterError::WrongOwner { vshard_id: local_vshard_id, @@ -194,18 +194,16 @@ fn validate_put_routing(req: &ArrayShardPutReq, local_vshard_id: u32) -> Result< /// Validate that a SLICE request is routed to the correct shard. /// -/// Returns `Ok(())` when `prefix_bits == 0` (no validation) or when -/// `local_vshard_id` falls within the Hilbert ranges carried by the request. -/// Returns `Err(ClusterError::WrongOwner)` when this shard no longer covers -/// any of the requested ranges, indicating a stale routing table. +/// Returns `Ok(())` when `prefix_bits == 0` or `slice_tile_keys` is empty, +/// which skips validation, or when some tile key routes to +/// `local_vshard_id`. Returns `Err(ClusterError::WrongOwner)` when no key +/// routes here, which means the coordinator used a stale routing table. fn validate_slice_routing(req: &ArrayShardSliceReq, local_vshard_id: u32) -> Result<()> { - if req.prefix_bits == 0 || req.slice_hilbert_ranges.is_empty() { + if req.prefix_bits == 0 || req.slice_tile_keys.is_empty() { return Ok(()); } - // Compute which vShards overlap the slice ranges. If local_vshard_id is not - // among them, this node no longer owns the required Hilbert range. let covered = array_vshards_for_slice( - &req.slice_hilbert_ranges, + &req.slice_tile_keys, req.prefix_bits, crate::routing::VSHARD_COUNT, )?; @@ -226,7 +224,7 @@ fn validate_delete_routing(req: &ArrayShardDeleteReq, local_vshard_id: u32) -> R if req.prefix_bits == 0 { return Ok(()); } - let expected = array_vshard_for_tile(req.representative_hilbert_prefix, req.prefix_bits)?; + let expected = array_vshard_for_tile(req.representative_tile_key, req.prefix_bits)?; if expected != local_vshard_id { return Err(ClusterError::WrongOwner { vshard_id: local_vshard_id, @@ -351,8 +349,8 @@ mod tests { cell_filter_msgpack: vec![], // prefix_bits=0 skips routing validation in the handler. prefix_bits: 0, - slice_hilbert_ranges: vec![], - shard_hilbert_range: None, + slice_tile_keys: vec![], + shard_bucket: None, system_time: nodedb_types::SystemTimeScope::Current, valid_at_ms: None, txn_id: None, @@ -372,7 +370,7 @@ mod tests { reducer_msgpack: reducer_bytes, group_by_dim: -1, cell_filter_msgpack: vec![], - shard_hilbert_range: None, + shard_bucket: None, system_as_of: None, valid_at_ms: None, txn_id: None, @@ -528,7 +526,7 @@ mod tests { assert!((resp.partials[0].sum - 42.0).abs() < f64::EPSILON); } - fn make_put_req_bytes(vshard_id: u32, prefix_bits: u8) -> Vec { + fn make_put_req_bytes(prefix_bits: u8) -> Vec { // Build a minimal ArrayShardPutReq with routing metadata set so // validate_put_routing can confirm the request is destined for // the right shard. Use prefix_bits=0 to skip routing validation @@ -537,17 +535,10 @@ mod tests { array_id_msgpack: vec![], cells_msgpack: vec![], wal_lsn: 77, - representative_hilbert_prefix: if prefix_bits == 0 { - 0 - } else { - // Route to bucket 0 (top bits all zero) → vshard 0 with any prefix_bits. - 0u64 - }, + // Key 0 mixes to 0, so it routes to vShard 0 at any prefix_bits. + representative_tile_key: 0, prefix_bits, }; - // Adjust routing so the request would target vshard_id: set bits so - // bucket = vshard_id when stride = 1 (prefix_bits=10, VSHARD_COUNT=1024). - let _ = vshard_id; // routing byte chosen above; test uses prefix_bits=0. zerompk::to_msgpack_vec(&req).unwrap() } @@ -557,7 +548,7 @@ mod tests { coords_msgpack: vec![], wal_lsn: 88, // prefix_bits=0 skips routing validation in the handler. - representative_hilbert_prefix: 0, + representative_tile_key: 0, prefix_bits: 0, }; zerompk::to_msgpack_vec(&req).unwrap() @@ -573,7 +564,7 @@ mod tests { affected: 3, }); // prefix_bits=0 disables routing validation. - let payload = make_put_req_bytes(0, 0); + let payload = make_put_req_bytes(0); let resp_bytes = handle_array_shard_rpc(ARRAY_SHARD_PUT_REQ, 0, &payload, &executor) .await .expect("put handler should succeed"); @@ -623,14 +614,13 @@ mod tests { truncated_before_horizon: false, affected: 0, }); - // prefix_bits=10, stride=1 → bucket = top 10 bits of hilbert_prefix. - // hilbert_prefix = 0 → bucket 0 → expected vshard 0. - // We tell the handler we're vshard 7 → mismatch → error. + // Key 0 mixes to 0 → bucket 0 → expected vShard 0. + // The handler runs as vShard 7 → mismatch → error. let req = super::super::wire::ArrayShardPutReq { array_id_msgpack: vec![], cells_msgpack: vec![], wal_lsn: 1, - representative_hilbert_prefix: 0, + representative_tile_key: 0, prefix_bits: 10, }; let payload = zerompk::to_msgpack_vec(&req).unwrap(); @@ -646,53 +636,44 @@ mod tests { ); } - #[test] - fn validate_slice_routing_rejects_disjoint_range() { - // prefix_bits=10, stride=1 → vshard == bucket == top 10 bits. - // Hilbert range [0x0040_0000_0000_0000, 0x0040_0000_0000_0000] - // covers bucket 1 → vshard 1. - // Local shard is vshard 5 → disjoint → WrongOwner. - let req = super::super::wire::ArrayShardSliceReq { + fn slice_req_with_keys(slice_tile_keys: Vec) -> super::super::wire::ArrayShardSliceReq { + super::super::wire::ArrayShardSliceReq { array_id_msgpack: vec![], slice_msgpack: vec![], attr_projection: vec![], limit: 10, cell_filter_msgpack: vec![], prefix_bits: 10, - slice_hilbert_ranges: vec![(0x0040_0000_0000_0000, 0x0040_0000_0000_0000)], - shard_hilbert_range: None, + slice_tile_keys, + shard_bucket: None, system_time: nodedb_types::SystemTimeScope::Current, valid_at_ms: None, txn_id: None, - }; - let err = super::validate_slice_routing(&req, 5) - .expect_err("disjoint Hilbert range should reject"); + } + } + + #[test] + fn validate_slice_routing_rejects_a_shard_no_key_routes_to() { + let key = 0x0040_0000_0000_0000u64; + let owner = crate::distributed_array::array_vshard_for_tile(key, 10).unwrap(); + let other = (owner + 1) % crate::routing::VSHARD_COUNT; + let err = super::validate_slice_routing(&slice_req_with_keys(vec![key]), other) + .expect_err("a shard no key routes to should reject"); assert!( matches!( err, - crate::error::ClusterError::WrongOwner { vshard_id: 5, .. } + crate::error::ClusterError::WrongOwner { vshard_id, .. } if vshard_id == other ), - "expected WrongOwner for vshard 5, got {err:?}" + "expected WrongOwner for vshard {other}, got {err:?}" ); } #[test] - fn validate_slice_routing_accepts_overlapping_range() { - // Same range as above → vshard 1. Local shard IS vshard 1 → accept. - let req = super::super::wire::ArrayShardSliceReq { - array_id_msgpack: vec![], - slice_msgpack: vec![], - attr_projection: vec![], - limit: 10, - cell_filter_msgpack: vec![], - prefix_bits: 10, - slice_hilbert_ranges: vec![(0x0040_0000_0000_0000, 0x0040_0000_0000_0000)], - shard_hilbert_range: None, - system_time: nodedb_types::SystemTimeScope::Current, - valid_at_ms: None, - txn_id: None, - }; - super::validate_slice_routing(&req, 1).expect("overlapping range should accept"); + fn validate_slice_routing_accepts_the_owner_of_a_key() { + let key = 0x0040_0000_0000_0000u64; + let owner = crate::distributed_array::array_vshard_for_tile(key, 10).unwrap(); + super::validate_slice_routing(&slice_req_with_keys(vec![key]), owner) + .expect("the owning shard should accept"); } #[tokio::test] diff --git a/nodedb-cluster/src/distributed_array/local_executor.rs b/nodedb-cluster/src/distributed_array/local_executor.rs index 68fcf31ff..971737412 100644 --- a/nodedb-cluster/src/distributed_array/local_executor.rs +++ b/nodedb-cluster/src/distributed_array/local_executor.rs @@ -73,10 +73,9 @@ pub trait ArrayLocalExecutor: Send + Sync + 'static { /// `limit` — maximum rows to return per shard (0 = unlimited). /// `cell_filter_msgpack` — zerompk encoding of `SurrogateBitmap`; empty /// means no filter. - /// `shard_hilbert_range` — optional `[lo, hi]` Hilbert-prefix range; when - /// set only tiles whose prefix falls in this range are returned, preventing - /// duplicate rows in single-node harnesses where all vShards share one - /// Data Plane. `None` = no Hilbert filter. + /// `shard_bucket` — optional bucket filter. When set, only tiles whose + /// key routes to this bucket are returned, so vShards that share one + /// Data Plane return no duplicate rows. `None` = no filter. /// `txn_id` — the reading transaction's id; the executor stamps it on the /// Data Plane request so the shard folds that transaction's staged /// cells into the result. `None` = autocommit read. @@ -116,8 +115,8 @@ pub trait ArrayLocalExecutor: Send + Sync + 'static { /// /// `req.cells_msgpack` is a zerompk encoding of /// `Vec`. All cells belong to - /// the same Hilbert-prefix tile. The shard handler has already validated - /// that this shard owns the tile; the executor dispatches directly to the + /// the same routing bucket. The shard handler has already validated + /// that this shard owns the bucket; the executor dispatches directly to the /// Data Plane without further routing checks. /// /// Returns the applied LSN plus the real cell count the Data Plane diff --git a/nodedb-cluster/src/distributed_array/partition.rs b/nodedb-cluster/src/distributed_array/partition.rs index f6aea11ab..89ce0b32f 100644 --- a/nodedb-cluster/src/distributed_array/partition.rs +++ b/nodedb-cluster/src/distributed_array/partition.rs @@ -3,12 +3,11 @@ //! Cell-to-tile partitioning for distributed array writes. //! //! Both `coord_put` and `coord_delete` receive a flat list of cells (or -//! coord tuples) paired with their pre-computed Hilbert prefixes. This -//! module groups them by tile bucket (the top `prefix_bits` of the Hilbert -//! prefix) and maps each bucket to its owning vShard via -//! `array_vshard_for_tile`. +//! coord tuples) paired with their pre-computed tile keys. This module +//! groups them by owning vShard, the vShard of the key's routing bucket +//! from `array_vshard_for_tile`. //! -//! Callers pre-attach Hilbert prefixes because the query planner already +//! Callers pre-attach tile keys because the query planner already //! holds the schema and can compute them at planning time; doing the same //! computation again inside the coordinator would require an additional //! schema lookup that `nodedb-cluster` intentionally avoids (it carries no @@ -16,13 +15,14 @@ //! //! ## Output shape //! -//! `partition_put_cells` returns a `Vec<(vshard_id, representative_hilbert_prefix, -//! cells_msgpack)>` where `cells_msgpack` is a zerompk-encoded +//! `partition_put_cells` returns one `PutBucket` per vShard: its +//! `representative_tile_key` and `cells_msgpack`, a zerompk-encoded //! `Vec` (the same bytes the shard handler passes to //! `exec_put`). //! -//! `partition_delete_coords` returns a `Vec<(vshard_id, coords_msgpack)>` where -//! `coords_msgpack` is a zerompk-encoded `Vec`. +//! `partition_delete_coords` returns one `DeleteBucket` per vShard: its +//! `representative_tile_key` and `coords_msgpack`, a zerompk-encoded +//! `Vec`. use std::collections::HashMap; @@ -33,9 +33,9 @@ use super::routing::array_vshard_for_tile; /// One bucket produced by `partition_put_cells`. pub struct PutBucket { pub vshard_id: u32, - /// Hilbert prefix of the first cell added to this bucket. Used by the - /// shard handler for routing validation (`validate_put_routing`). - pub representative_hilbert_prefix: u64, + /// Tile key of the first cell added to this bucket. The shard handler + /// routes it to check ownership (`validate_put_routing`). + pub representative_tile_key: u64, /// zerompk-encoded `Vec` for all cells in this bucket. pub cells_msgpack: Vec, } @@ -43,16 +43,16 @@ pub struct PutBucket { /// One bucket produced by `partition_delete_coords`. pub struct DeleteBucket { pub vshard_id: u32, - /// Hilbert prefix of the first coord added to this bucket. Used by the - /// shard handler for routing validation (`validate_delete_routing`). - pub representative_hilbert_prefix: u64, + /// Tile key of the first coord added to this bucket. The shard handler + /// routes it to check ownership (`validate_delete_routing`). + pub representative_tile_key: u64, /// zerompk-encoded `Vec` for all coords in this bucket. pub coords_msgpack: Vec, } -/// Group `(hilbert_prefix, cell_msgpack)` pairs by tile bucket. +/// Group `(tile_key, cell_msgpack)` pairs by owning vShard. /// -/// `cells` — each element is `(hilbert_prefix, zerompk-encoded single cell)`. +/// `cells` — each element is `(tile_key, zerompk-encoded single cell)`. /// `prefix_bits` — routing granularity (1–16). /// /// Returns one `PutBucket` per unique owning vShard. The order of buckets @@ -62,14 +62,12 @@ pub fn partition_put_cells(cells: &[(u64, Vec)], prefix_bits: u8) -> Result< return Ok(Vec::new()); } - // vshard_id → (representative_hilbert_prefix, per-cell byte blobs) + // vshard_id → (representative tile key, per-cell byte blobs) let mut buckets: HashMap>)> = HashMap::new(); - for (hilbert_prefix, cell_bytes) in cells { - let vshard_id = array_vshard_for_tile(*hilbert_prefix, prefix_bits)?; - let entry = buckets - .entry(vshard_id) - .or_insert((*hilbert_prefix, Vec::new())); + for (tile_key, cell_bytes) in cells { + let vshard_id = array_vshard_for_tile(*tile_key, prefix_bits)?; + let entry = buckets.entry(vshard_id).or_insert((*tile_key, Vec::new())); entry.1.push(cell_bytes.clone()); } @@ -79,16 +77,16 @@ pub fn partition_put_cells(cells: &[(u64, Vec)], prefix_bits: u8) -> Result< let cells_msgpack = encode_blob_vec(&cell_blobs)?; Ok(PutBucket { vshard_id, - representative_hilbert_prefix: representative, + representative_tile_key: representative, cells_msgpack, }) }) .collect() } -/// Group `(hilbert_prefix, coord_msgpack)` pairs by tile bucket. +/// Group `(tile_key, coord_msgpack)` pairs by owning vShard. /// -/// `coords` — each element is `(hilbert_prefix, zerompk-encoded single coord)`. +/// `coords` — each element is `(tile_key, zerompk-encoded single coord)`. /// `prefix_bits` — routing granularity (1–16). /// /// Returns one `DeleteBucket` per unique owning vShard. @@ -100,14 +98,12 @@ pub fn partition_delete_coords( return Ok(Vec::new()); } - // vshard_id → (representative_hilbert_prefix, coord byte blobs) + // vshard_id → (representative tile key, coord byte blobs) let mut buckets: HashMap>)> = HashMap::new(); - for (hilbert_prefix, coord_bytes) in coords { - let vshard_id = array_vshard_for_tile(*hilbert_prefix, prefix_bits)?; - let entry = buckets - .entry(vshard_id) - .or_insert((*hilbert_prefix, Vec::new())); + for (tile_key, coord_bytes) in coords { + let vshard_id = array_vshard_for_tile(*tile_key, prefix_bits)?; + let entry = buckets.entry(vshard_id).or_insert((*tile_key, Vec::new())); entry.1.push(coord_bytes.clone()); } @@ -117,7 +113,7 @@ pub fn partition_delete_coords( let coords_msgpack = encode_blob_vec(&coord_blobs)?; Ok(DeleteBucket { vshard_id, - representative_hilbert_prefix: representative, + representative_tile_key: representative, coords_msgpack, }) }) @@ -142,7 +138,7 @@ fn encode_blob_vec(blobs: &[Vec]) -> Result> { mod tests { use super::*; - // Build a fake (hilbert_prefix, cell_bytes) pair. The cell_bytes are + // Build a fake (tile_key, cell_bytes) pair. The cell_bytes are // opaque for the partitioner — any non-empty Vec works. fn cell(prefix: u64, tag: u8) -> (u64, Vec) { (prefix, vec![tag]) @@ -153,7 +149,7 @@ mod tests { (prefix, vec![tag]) } - // Helper: compute expected vshard for a given hilbert prefix and prefix_bits. + // Helper: compute expected vshard for a given tile key and prefix_bits. fn expected_vshard(prefix: u64, prefix_bits: u8) -> u32 { array_vshard_for_tile(prefix, prefix_bits).unwrap() } @@ -172,23 +168,18 @@ mod tests { #[test] fn partition_put_single_cell_one_bucket() { - // prefix_bits=10, stride=1 → vshard == bucket == top 10 bits. - // All-zero hilbert prefix → bucket 0 → vshard 0. + // Key 0 mixes to 0 → bucket 0 → vShard 0. let cells = vec![cell(0, 0xAA)]; let buckets = partition_put_cells(&cells, 10).unwrap(); assert_eq!(buckets.len(), 1); assert_eq!(buckets[0].vshard_id, 0); - assert_eq!(buckets[0].representative_hilbert_prefix, 0); + assert_eq!(buckets[0].representative_tile_key, 0); } #[test] fn partition_put_cells_by_tile_three_buckets() { - // With prefix_bits=10 (stride=1), every distinct top-10-bit value - // is its own vshard. Use three well-separated prefixes. - // - // prefix 0x0000_0000_0000_0000 → bucket 0 → vshard 0 - // prefix 0x0040_0000_0000_0000 → top 10 bits = 1 → vshard 1 - // prefix 0x0080_0000_0000_0000 → top 10 bits = 2 → vshard 2 + // With prefix_bits=10 (stride=1), each mixed bucket is its own + // vShard. The three keys mix to three distinct buckets. let p0 = 0x0000_0000_0000_0000u64; let p1 = 0x0040_0000_0000_0000u64; let p2 = 0x0080_0000_0000_0000u64; @@ -201,22 +192,19 @@ mod tests { cell(p1, 0x05), // same bucket as second ]; - let mut buckets = partition_put_cells(&cells, 10).unwrap(); - // Sort by vshard_id for deterministic assertion. - buckets.sort_by_key(|b| b.vshard_id); + let buckets = partition_put_cells(&cells, 10).unwrap(); assert_eq!(buckets.len(), 3); - assert_eq!(buckets[0].vshard_id, expected_vshard(p0, 10)); - assert_eq!(buckets[1].vshard_id, expected_vshard(p1, 10)); - assert_eq!(buckets[2].vshard_id, expected_vshard(p2, 10)); - - // Bucket 0 and 1 each have 2 cells, bucket 2 has 1. - let blobs0: Vec> = zerompk::from_msgpack(&buckets[0].cells_msgpack).unwrap(); - let blobs1: Vec> = zerompk::from_msgpack(&buckets[1].cells_msgpack).unwrap(); - let blobs2: Vec> = zerompk::from_msgpack(&buckets[2].cells_msgpack).unwrap(); - assert_eq!(blobs0.len(), 2); - assert_eq!(blobs1.len(), 2); - assert_eq!(blobs2.len(), 1); + // p0 and p1 each own 2 cells, p2 owns 1. + for (key, want) in [(p0, 2), (p1, 2), (p2, 1)] { + let vshard = expected_vshard(key, 10); + let bucket = buckets + .iter() + .find(|b| b.vshard_id == vshard) + .expect("bucket for key"); + let blobs: Vec> = zerompk::from_msgpack(&bucket.cells_msgpack).unwrap(); + assert_eq!(blobs.len(), want, "key={key:#x}"); + } } #[test] @@ -226,16 +214,17 @@ mod tests { let coords = vec![coord(p0, 0xAA), coord(p1, 0xBB), coord(p0, 0xCC)]; - let mut buckets = partition_delete_coords(&coords, 10).unwrap(); - buckets.sort_by_key(|b| b.vshard_id); + let buckets = partition_delete_coords(&coords, 10).unwrap(); assert_eq!(buckets.len(), 2); - assert_eq!(buckets[0].vshard_id, expected_vshard(p0, 10)); - assert_eq!(buckets[1].vshard_id, expected_vshard(p1, 10)); - - let blobs0: Vec> = zerompk::from_msgpack(&buckets[0].coords_msgpack).unwrap(); - let blobs1: Vec> = zerompk::from_msgpack(&buckets[1].coords_msgpack).unwrap(); - assert_eq!(blobs0.len(), 2); - assert_eq!(blobs1.len(), 1); + for (key, want) in [(p0, 2), (p1, 1)] { + let vshard = expected_vshard(key, 10); + let bucket = buckets + .iter() + .find(|b| b.vshard_id == vshard) + .expect("bucket for key"); + let blobs: Vec> = zerompk::from_msgpack(&bucket.coords_msgpack).unwrap(); + assert_eq!(blobs.len(), want, "key={key:#x}"); + } } #[test] diff --git a/nodedb-cluster/src/distributed_array/routing.rs b/nodedb-cluster/src/distributed_array/routing.rs index d8a5791f6..29125d5aa 100644 --- a/nodedb-cluster/src/distributed_array/routing.rs +++ b/nodedb-cluster/src/distributed_array/routing.rs @@ -2,264 +2,129 @@ //! vShard routing for array tiles. //! -//! Arrays are sharded by Hilbert-curve prefix: the top `prefix_bits` -//! of a tile's `hilbert_prefix` determine which vShard bucket owns it. +//! A tile routes by the top bits of `fmix64(tile_key)`. The tile-to-vShard +//! functions live in `nodedb_types::array_route`, shared with the Data +//! Plane. This module adds the read-side bucket filter and slice fan-out. //! -//! With `prefix_bits = P` there are `2^P` distinct buckets numbered -//! `0 .. 2^P - 1`. Each bucket maps to a contiguous range of vShards -//! within the cluster's `VSHARD_COUNT`-wide space. The stride is -//! `VSHARD_COUNT / 2^P` (integer division). Bucket `b` owns vShards -//! `[b * stride .. (b + 1) * stride)`. The primary vShard for a bucket -//! is the first one in that range (`b * stride`). - -use crate::error::{ClusterError, Result}; -use crate::routing::VSHARD_COUNT; - -// Array Hilbert-range ownership is derived entirely from vShard ownership: -// bucket→vshard is a deterministic function of `prefix_bits`, and -// vshard→node is maintained by `RoutingTable` (updated by committed -// `RoutingChange::ReassignVShard` entries). No separate array-specific -// routing variant is needed in the metadata log — array migration is "free" -// because a vShard rebalance already atomically re-points all array tile -// ownership through the same `RoutingTable` cut-over path. - -/// Determine the primary owning vShard for a single tile identified by -/// its raw Hilbert-prefix value. -/// -/// `hilbert_prefix` — the tile's u64 Hilbert key. -/// `prefix_bits` — number of high-order bits used for routing (1–16). -/// -/// Returns the vShard ID in `[0, VSHARD_COUNT)`. -pub fn array_vshard_for_tile(hilbert_prefix: u64, prefix_bits: u8) -> Result { - validate_prefix_bits(prefix_bits)?; - let bucket = prefix_bucket(hilbert_prefix, prefix_bits); - Ok(bucket_to_vshard(bucket, prefix_bits)) +//! Bucket-to-vShard is a fixed function of `prefix_bits`, and vShard-to-node +//! lives in `RoutingTable`. A vShard rebalance therefore moves array tiles +//! with no array-specific routing entry in the metadata log. + +pub use nodedb_types::array_route::array_vshard_for_tile; +use nodedb_types::array_route::{ShardBucket, bucket_to_vshard, prefix_bucket, routing_bits}; +use nodedb_types::id::VShardId; + +use crate::error::Result; + +/// The bucket filter a read sent to primary vShard `vshard_id` applies, so +/// the shard returns only the tiles it owns. +pub fn shard_bucket_for_vshard(vshard_id: u32, prefix_bits: u8) -> Result { + let bits = routing_bits(prefix_bits)?; + Ok(ShardBucket { + bucket: vshard_id >> u32::from(VShardId::BITS - bits), + prefix_bits: bits, + }) } -/// Determine all vShards that intersect a set of Hilbert-prefix ranges. +/// The primary vShards that own the tiles in `slice_tile_keys`. /// -/// `slice_hilbert_ranges` — list of `(lo, hi)` inclusive Hilbert-prefix -/// ranges covering the slice's MBR. An empty slice means "unbounded" -/// (returns all vShards up to `total_shards`). +/// `slice_tile_keys` — tile keys covering the slice, at least one per +/// bucket the slice touches. An empty list means an unbounded slice, which +/// returns the primary vShard of every bucket. /// /// `prefix_bits` — routing granularity (1–16). -/// `total_shards` — number of active vShards; returned IDs are -/// clamped to `[0, total_shards)`. +/// `total_shards` — number of active vShards. IDs at or above it are dropped. /// /// Returns a sorted, deduplicated list of vShard IDs. pub fn array_vshards_for_slice( - slice_hilbert_ranges: &[(u64, u64)], + slice_tile_keys: &[u64], prefix_bits: u8, total_shards: u32, ) -> Result> { if total_shards == 0 { return Ok(Vec::new()); } - validate_prefix_bits(prefix_bits)?; - - let stride = vshard_stride(prefix_bits); - let mut out: Vec = Vec::new(); - - // Unbounded slice: return one canonical primary vShard per bucket. - // Within a bucket all vShards serve the same Hilbert range, so fanning - // out to every member would multiply the result by `stride`. - if slice_hilbert_ranges.is_empty() { - let mut vshard_id: u32 = 0; - while vshard_id < total_shards && vshard_id < VSHARD_COUNT { - out.push(vshard_id); - vshard_id = vshard_id.saturating_add(stride); - if stride == 0 { - break; - } - } - return Ok(out); - } - - for &(lo, hi) in slice_hilbert_ranges { - let lo_bucket = prefix_bucket(lo, prefix_bits); - // `hi` is inclusive, so we take the bucket that contains `hi`. - let hi_bucket = prefix_bucket(hi, prefix_bits); - - for bucket in lo_bucket..=hi_bucket { - // One canonical primary vShard per bucket. All other vShards in - // `[vshard_start, vshard_start + stride)` serve identical Hilbert - // ranges; fanning out to all of them double-counts on aggregate. - let vshard_start = bucket_to_vshard(bucket, prefix_bits); - if vshard_start < total_shards { - out.push(vshard_start); - } - } - } - + let bits = routing_bits(prefix_bits)?; + + // One primary vShard per bucket. The other vShards of a bucket serve the + // same tiles, so a fan-out to them double-counts aggregates. + let mut out: Vec = if slice_tile_keys.is_empty() { + (0..(1u32 << bits)) + .map(|bucket| bucket_to_vshard(bucket, bits)) + .collect() + } else { + slice_tile_keys + .iter() + .map(|&key| bucket_to_vshard(prefix_bucket(key, bits), bits)) + .collect() + }; + out.retain(|&vshard| vshard < total_shards); out.sort_unstable(); out.dedup(); Ok(out) } -// ── Internal helpers ────────────────────────────────────────────────────── - -fn validate_prefix_bits(prefix_bits: u8) -> Result<()> { - if prefix_bits == 0 || prefix_bits > 16 { - return Err(ClusterError::Codec { - detail: format!("array routing: prefix_bits must be 1-16, got {prefix_bits}"), - }); - } - Ok(()) -} - -/// Extract the top `prefix_bits` bits of a Hilbert prefix as a bucket index. -fn prefix_bucket(hilbert_prefix: u64, prefix_bits: u8) -> u32 { - // Shift right so the top `prefix_bits` bits are in the low-order position. - let shift = 64u8.saturating_sub(prefix_bits); - (hilbert_prefix >> shift) as u32 -} - -/// Map a bucket index to the primary vShard in that bucket's range. -fn bucket_to_vshard(bucket: u32, prefix_bits: u8) -> u32 { - let stride = vshard_stride(prefix_bits); - bucket.saturating_mul(stride) -} - -/// Number of vShards per Hilbert bucket. -/// -/// With `P` prefix bits there are `2^P` buckets and `VSHARD_COUNT` -/// total vShards. Stride = `VSHARD_COUNT >> P`. When `P >= log2(VSHARD_COUNT)` -/// the stride is 1 (one vShard per bucket or less). -fn vshard_stride(prefix_bits: u8) -> u32 { - // VSHARD_COUNT is a power of two (1024 = 2^10). - // Right-shifting by prefix_bits gives the stride, floored at 1. - let shifted = VSHARD_COUNT >> (prefix_bits as u32); - shifted.max(1) -} - #[cfg(test)] mod tests { use super::*; use crate::routing::VSHARD_COUNT; #[test] - fn tile_routing_invalid_prefix_bits() { - assert!(array_vshard_for_tile(0, 0).is_err()); - assert!(array_vshard_for_tile(0, 17).is_err()); - } - - #[test] - fn tile_routing_top_bit_prefix1() { - // With prefix_bits=1: stride=512. Bucket 0 → vshard 0, bucket 1 → vshard 512. - let shard_low = array_vshard_for_tile(0x0000_0000_0000_0000, 1).unwrap(); - let shard_high = array_vshard_for_tile(0x8000_0000_0000_0000, 1).unwrap(); - assert_eq!(shard_low, 0); - assert_eq!(shard_high, 512); - } - - #[test] - fn tile_routing_prefix10_is_direct() { - // With prefix_bits=10: stride=1, bucket == vshard_id. - // Top 10 bits of 0x0100_0000_0000_0000 = 0b0000_0000_01 = 1. - let vshard = array_vshard_for_tile(0x0040_0000_0000_0000, 10).unwrap(); - // 0x0040... >> (64-10) = 0x0040... >> 54 = 1 - assert_eq!(vshard, 1); - } - - #[test] - fn tile_routing_prefix8_stride4() { - // With prefix_bits=8: stride=4. Bucket 0 → vshard 0, bucket 1 → vshard 4. - let shard0 = array_vshard_for_tile(0x0000_0000_0000_0000, 8).unwrap(); - // Top 8 bits of 0x0100... = 1. - let shard1 = array_vshard_for_tile(0x0100_0000_0000_0000, 8).unwrap(); - assert_eq!(shard0, 0); - assert_eq!(shard1, 4); + fn invalid_prefix_bits_are_refused() { + assert!(shard_bucket_for_vshard(0, 0).is_err()); + assert!(shard_bucket_for_vshard(0, 17).is_err()); + assert!(array_vshards_for_slice(&[], 0, VSHARD_COUNT).is_err()); } #[test] - fn tile_routing_output_in_vshard_range() { - // Every vshard returned must be < VSHARD_COUNT. - for bits in 1u8..=10 { - for &prefix in &[0u64, u64::MAX / 3, u64::MAX / 2, u64::MAX] { - let vshard = array_vshard_for_tile(prefix, bits).unwrap(); - assert!( - vshard < VSHARD_COUNT, - "bits={bits} prefix={prefix:#x} → vshard={vshard} >= {VSHARD_COUNT}" - ); + fn shard_bucket_admits_exactly_the_tiles_routed_to_its_vshard() { + for bits in [1u8, 4, 8, 10, 12, 16] { + let primaries = array_vshards_for_slice(&[], bits, VSHARD_COUNT).unwrap(); + for key in 0u64..512 { + let owner = array_vshard_for_tile(key, bits).unwrap(); + let admitting: Vec = primaries + .iter() + .copied() + .filter(|&v| shard_bucket_for_vshard(v, bits).unwrap().contains(key)) + .collect(); + assert_eq!(admitting, vec![owner], "bits={bits} key={key}"); } } } #[test] fn slice_routing_zero_shards() { - let shards = array_vshards_for_slice(&[(0, u64::MAX)], 8, 0).unwrap(); + let shards = array_vshards_for_slice(&[0, u64::MAX], 8, 0).unwrap(); assert!(shards.is_empty()); } #[test] - fn slice_routing_empty_ranges_returns_all() { - // Empty range list = unbounded slice → one canonical primary per bucket. - // With prefix_bits=8 (stride=4) and total_shards=4, only primary 0 fits. - let shards = array_vshards_for_slice(&[], 8, 4).unwrap(); - assert_eq!(shards, vec![0]); - } - - #[test] - fn slice_routing_full_range_returns_all_active() { - // Full range covers all 256 buckets → one primary per bucket. - // total_shards=4 only fits primary 0. - let shards = array_vshards_for_slice(&[(0, u64::MAX)], 8, 4).unwrap(); - assert_eq!(shards, vec![0]); - } - - #[test] - fn slice_routing_single_bucket_prefix8() { - // Hilbert range [0, 0] covers only bucket 0 → primary vShard 0. - let shards = array_vshards_for_slice(&[(0, 0)], 8, 16).unwrap(); - assert_eq!(shards, vec![0]); - } - - #[test] - fn slice_routing_two_adjacent_buckets() { - // prefix_bits=8, stride=4. One canonical primary per bucket. - // Range 1 → bucket 0 → primary vShard 0. - // Range 2 → bucket 1 → primary vShard 4. - let shards = array_vshards_for_slice( - &[ - (0x0000_0000_0000_0000, 0x00FF_FFFF_FFFF_FFFF), - (0x0100_0000_0000_0000, 0x01FF_FFFF_FFFF_FFFF), - ], - 8, - 1024, - ) - .unwrap(); - assert_eq!(shards, vec![0, 4]); - } - - #[test] - fn slice_routing_clamped_to_total_shards() { - // With total_shards=2, only the primary for bucket 0 (= vShard 0) fits. - let shards = array_vshards_for_slice(&[(0, u64::MAX)], 8, 2).unwrap(); - assert_eq!(shards, vec![0]); + fn slice_routing_empty_keys_returns_every_primary() { + // prefix_bits=8 (stride=4): primaries 0, 4, …, 1020. + let shards = array_vshards_for_slice(&[], 8, VSHARD_COUNT).unwrap(); + assert_eq!(shards.len(), 256); + assert!(shards.iter().all(|v| v % 4 == 0)); + // total_shards=4 keeps only primary 0. + assert_eq!(array_vshards_for_slice(&[], 8, 4).unwrap(), vec![0]); } #[test] - fn slice_routing_dedup_overlapping_ranges() { - // Two ranges that map to the same bucket → one canonical primary vShard. - let shards = array_vshards_for_slice( - &[ - (0x0000_0000_0000_0000, 0x0000_0000_0000_0001), - (0x0000_0000_0000_0002, 0x0000_0000_0000_0003), - ], - 8, - 16, - ) - .unwrap(); - // Both ranges are in bucket 0 → primary vShard 0. - assert_eq!(shards, vec![0]); + fn slice_routing_maps_each_key_to_its_owner() { + let keys = [5u64, 6, 7, 1 << 40]; + let shards = array_vshards_for_slice(&keys, 8, VSHARD_COUNT).unwrap(); + let mut expected: Vec = keys + .iter() + .map(|&k| array_vshard_for_tile(k, 8).unwrap()) + .collect(); + expected.sort_unstable(); + expected.dedup(); + assert_eq!(shards, expected); } #[test] - fn vshard_stride_values() { - // VSHARD_COUNT=1024. - assert_eq!(vshard_stride(1), 512); - assert_eq!(vshard_stride(8), 4); - assert_eq!(vshard_stride(10), 1); - assert_eq!(vshard_stride(16), 1); // floored at 1 + fn slice_routing_dedups_keys_of_one_bucket() { + let key = 42u64; + let shards = array_vshards_for_slice(&[key, key], 8, VSHARD_COUNT).unwrap(); + assert_eq!(shards, vec![array_vshard_for_tile(key, 8).unwrap()]); } } diff --git a/nodedb-cluster/src/distributed_array/scatter.rs b/nodedb-cluster/src/distributed_array/scatter.rs index 4c58f172d..9dd0fc845 100644 --- a/nodedb-cluster/src/distributed_array/scatter.rs +++ b/nodedb-cluster/src/distributed_array/scatter.rs @@ -6,12 +6,12 @@ //! - `fan_out` — broadcast one request to all listed shards (slice, agg, //! delete, surrogate scan). //! - `fan_out_partitioned` — send a different payload to each shard -//! (used by `coord_put` where cells are routed by Hilbert prefix). +//! (used by `coord_put` where cells are routed by tile key). //! //! Both functions dispatch concurrently via `FuturesUnordered`, apply a //! per-shard `tokio::time::timeout`, and wrap each call in the cluster //! `CircuitBreaker`. Any shard failure is propagated as `Err` — partial -//! results are not silently dropped. +//! results are not silently dropped. The error names the failing shard. use std::sync::Arc; @@ -74,6 +74,28 @@ async fn call_with_wrong_owner_retry( use super::rpc::ShardRpcDispatch; +/// A fan-out failure and the shard it names. +#[derive(Debug, thiserror::Error)] +#[error("array shard {vshard_id}: {error}")] +pub struct ShardFailure { + /// The shard whose call failed. + pub vshard_id: u32, + /// Why it failed. + #[source] + pub error: ClusterError, +} + +/// The error of an array coordinator call. +#[derive(Debug, thiserror::Error)] +pub enum ArrayCoordError { + /// A shard call failed. The failure names the shard. + #[error(transparent)] + Shard(#[from] ShardFailure), + /// The coordinator failed outside any shard call, such as an encode. + #[error(transparent)] + Coordinator(#[from] ClusterError), +} + /// Classify whether a shard error should count against the peer circuit breaker. /// /// `WrongOwner` is a transient routing/ownership-churn signal, not a peer-health @@ -92,8 +114,9 @@ fn counts_against_breaker(err: &ClusterError) -> bool { ClusterError::DataPlane { .. } | ClusterError::ShardExecution { .. } | ClusterError::StreamTerminal { .. } => false, - // An unresponsive peer is exactly what the breaker exists to shed - // load from, so a shard timeout counts like any other liveness failure. + // The breaker counts peer liveness, not the send fate of + // `is_lost_answer`. An unresponsive peer is what the breaker sheds + // load from, so a lost answer counts like any other liveness failure. ClusterError::ShardTimeout { .. } | ClusterError::Unanswered { .. } => true, ClusterError::Raft(_) | ClusterError::VShardNotMapped { .. } @@ -166,7 +189,7 @@ pub async fn fan_out( req_bytes: &[u8], dispatch: &Arc, circuit_breaker: &CircuitBreaker, -) -> Result)>> { +) -> std::result::Result)>, ShardFailure> { if params.shard_ids.is_empty() { return Ok(Vec::new()); } @@ -176,11 +199,15 @@ pub async fn fan_out( let mut futs = futures::stream::FuturesUnordered::new(); for &shard_id in ¶ms.shard_ids { + let failed = |error| ShardFailure { + vshard_id: shard_id, + error, + }; // Circuit-breaker gate: treat shard_id as the peer identifier. - let admission = circuit_breaker.check(shard_id as u64)?; + let admission = circuit_breaker.check(shard_id as u64).map_err(failed)?; let env = VShardEnvelope::new( - msg_type_from_opcode(opcode)?, + msg_type_from_opcode(opcode).map_err(failed)?, params.source_node, 0, // target_node resolved by the dispatch impl shard_id, @@ -209,7 +236,10 @@ pub async fn fan_out( if counts_against_breaker(&e) { circuit_breaker.record_failure(shard_id as u64, admission); } - return Err(e); + return Err(ShardFailure { + vshard_id: shard_id, + error: e, + }); } } } @@ -227,7 +257,7 @@ pub async fn fan_out_partitioned( per_shard: &[(u32, Vec)], dispatch: &Arc, circuit_breaker: &CircuitBreaker, -) -> Result)>> { +) -> std::result::Result)>, ShardFailure> { if per_shard.is_empty() { return Ok(Vec::new()); } @@ -235,10 +265,14 @@ pub async fn fan_out_partitioned( let mut futs = futures::stream::FuturesUnordered::new(); for (shard_id, payload) in per_shard { - let admission = circuit_breaker.check(*shard_id as u64)?; + let failed = |error| ShardFailure { + vshard_id: *shard_id, + error, + }; + let admission = circuit_breaker.check(*shard_id as u64).map_err(failed)?; let env = VShardEnvelope::new( - msg_type_from_opcode(opcode)?, + msg_type_from_opcode(opcode).map_err(failed)?, params.source_node, 0, *shard_id, @@ -267,7 +301,10 @@ pub async fn fan_out_partitioned( if counts_against_breaker(&e) { circuit_breaker.record_failure(shard_id as u64, admission); } - return Err(e); + return Err(ShardFailure { + vshard_id: shard_id, + error: e, + }); } } } @@ -425,11 +462,49 @@ mod tests { .await .expect_err("fan_out should propagate shard failure"); assert!( - matches!(err, ClusterError::Transport { .. }), + matches!(err.error, ClusterError::Transport { .. }), "expected Transport error, got {err:?}" ); } + /// A failing shard's id rides with its error, so the caller can name + /// the vShard that refused. + #[tokio::test] + async fn a_failure_names_the_failing_shard() { + let dispatch: Arc = Arc::new(FailDispatch); + let params = FanOutParams { + shard_ids: vec![17], + timeout_ms: 1000, + source_node: 1, + }; + let err = fan_out( + ¶ms, + super::super::opcodes::ARRAY_SHARD_SLICE_REQ, + b"", + &dispatch, + &cb(), + ) + .await + .expect_err("a failing shard fails the fan-out"); + assert_eq!(err.vshard_id, 17); + assert!(err.error.is_unsent(), "got {err:?}"); + + let per_shard = vec![(23u32, b"cells".to_vec())]; + let err = fan_out_partitioned( + &FanOutPartitionedParams { + timeout_ms: 1000, + source_node: 1, + }, + super::super::opcodes::ARRAY_SHARD_PUT_REQ, + &per_shard, + &dispatch, + &cb(), + ) + .await + .expect_err("a failing shard fails the partitioned fan-out"); + assert_eq!(err.vshard_id, 23); + } + #[tokio::test] async fn fan_out_partitioned_dispatches_distinct_payloads() { let dispatch: Arc = Arc::new(EchoDispatch); @@ -486,7 +561,7 @@ mod tests { .await .expect_err("open circuit should block fan_out"); assert!( - matches!(err, ClusterError::CircuitOpen { .. }), + matches!(err.error, ClusterError::CircuitOpen { .. }), "expected CircuitOpen, got {err:?}" ); } @@ -579,7 +654,7 @@ mod tests { .expect_err("fan_out should propagate WrongOwner when both attempts fail"); assert!( - matches!(err, ClusterError::WrongOwner { .. }), + matches!(err.error, ClusterError::WrongOwner { .. }), "expected WrongOwner, got {err:?}" ); assert_eq!( @@ -617,7 +692,7 @@ mod tests { .expect_err("fan_out should propagate WrongOwner"); assert!( - matches!(err, ClusterError::WrongOwner { .. }), + matches!(err.error, ClusterError::WrongOwner { .. }), "expected WrongOwner, got {err:?}" ); // WrongOwner is a routing signal, not a health failure: the breaker @@ -653,7 +728,7 @@ mod tests { .await .expect_err("fan_out should surface a timeout once the shard call never resolves"); - match err { + match err.error { ClusterError::ShardTimeout { vshard_id, elapsed_ms, diff --git a/nodedb-cluster/src/distributed_array/wire.rs b/nodedb-cluster/src/distributed_array/wire.rs index 2183236bb..683f796d5 100644 --- a/nodedb-cluster/src/distributed_array/wire.rs +++ b/nodedb-cluster/src/distributed_array/wire.rs @@ -22,25 +22,20 @@ pub struct ArrayShardSliceReq { /// Optional surrogate pre-filter bitmap bytes (zerompk encoding of `SurrogateBitmap`). /// Empty slice means no filter. pub cell_filter_msgpack: Vec, - /// Prefix bits used for Hilbert routing (1–16). 0 means no routing validation. + /// The array's routing granularity (1–16). 0 skips routing validation. /// - /// When non-zero the shard verifies that its `local_vshard_id` is covered - /// by the Hilbert ranges in `slice_hilbert_ranges` before executing the - /// scan. A mismatch returns `ClusterError::WrongOwner` so the coordinator - /// can retry against a refreshed routing table. + /// When non-zero the shard checks that some key in `slice_tile_keys` + /// routes to its `local_vshard_id`. A mismatch returns + /// `ClusterError::WrongOwner`, so the coordinator retries against a + /// refreshed routing table. pub prefix_bits: u8, - /// Inclusive Hilbert-prefix ranges `(lo, hi)` that this slice covers, - /// pre-computed by the coordinator from the slice predicate. Each entry - /// is encoded as two consecutive `u64` values (little-endian). Empty - /// means unbounded (no routing validation is performed). - pub slice_hilbert_ranges: Vec<(u64, u64)>, - /// Hilbert-prefix range `[lo, hi]` that this shard owns. The shard - /// applies this as a pre-filter so it only returns cells whose Hilbert - /// prefix falls within the range. `None` means unbounded (return all - /// matching cells). Used by the distributed shard handler to prevent - /// duplicate rows in single-node harnesses where all vShards share one - /// Data Plane. - pub shard_hilbert_range: Option<(u64, u64)>, + /// Tile keys covering the slice, at least one per bucket it touches, + /// computed by the planner. Empty means unbounded and skips validation. + pub slice_tile_keys: Vec, + /// The bucket this shard owns. The shard returns only cells whose tile + /// key falls in it, so vShards that share one Data Plane return no + /// duplicate rows. `None` returns every matching cell. + pub shard_bucket: Option, /// Bitemporal system-time scope forwarded from `ArrayOp::Slice::system_time`. /// /// - `Current` = live read. @@ -91,10 +86,9 @@ pub struct ArrayShardAggReq { pub group_by_dim: i32, /// Optional surrogate pre-filter; empty = all cells. pub cell_filter_msgpack: Vec, - /// Hilbert-prefix range `[lo, hi]` that this shard owns. The shard - /// applies this as a pre-filter so it only counts cells whose Hilbert - /// prefix falls within the range. `None` means unbounded (scan all). - pub shard_hilbert_range: Option<(u64, u64)>, + /// The bucket this shard owns. The shard counts only cells whose tile + /// key falls in it. `None` counts every cell. + pub shard_bucket: Option, /// Bitemporal system-time cutoff forwarded from `ArrayOp::Aggregate::system_as_of`. /// `None` = live read. pub system_as_of: Option, @@ -131,16 +125,15 @@ pub struct ArrayShardAggResp { /// /// `cells_msgpack` is a zerompk encoding of /// `Vec`. -/// All cells in this batch belong to the same Hilbert-prefix bucket. +/// All cells in this batch route to the same bucket (`prefix_bucket` of the tile key). #[derive(Debug, Clone, zerompk::ToMessagePack, zerompk::FromMessagePack)] pub struct ArrayShardPutReq { pub array_id_msgpack: Vec, pub cells_msgpack: Vec, pub wal_lsn: u64, - /// Hilbert prefix of any representative cell in this batch (used by - /// the shard handler to validate routing). Set to 0 when routing - /// metadata is unavailable (pre-routing clients). - pub representative_hilbert_prefix: u64, + /// Tile key of any cell in this batch. The shard handler routes it to + /// check that it owns the batch. + pub representative_tile_key: u64, /// Prefix bits used for routing (1–16). 0 means no validation. pub prefix_bits: u8, } @@ -165,11 +158,10 @@ pub struct ArrayShardDeleteReq { pub array_id_msgpack: Vec, pub coords_msgpack: Vec, pub wal_lsn: u64, - /// Hilbert prefix of any representative coord in this batch. Used by the - /// shard handler to validate that it still owns the target Hilbert bucket. - /// Set to 0 when routing metadata is unavailable. - pub representative_hilbert_prefix: u64, - /// Prefix bits used for Hilbert routing (1–16). 0 means no validation. + /// Tile key of any coord in this batch. The shard handler routes it to + /// check that it still owns the batch's bucket. + pub representative_tile_key: u64, + /// Prefix bits used for routing (1–16). 0 means no validation. pub prefix_bits: u8, } diff --git a/nodedb-physical/src/physical_plan/array.rs b/nodedb-physical/src/physical_plan/array.rs index b01e0b039..b0e3606dd 100644 --- a/nodedb-physical/src/physical_plan/array.rs +++ b/nodedb-physical/src/physical_plan/array.rs @@ -13,7 +13,7 @@ use nodedb_array::types::ArrayId; use nodedb_types::sync::wire::SyncProvenance; -use nodedb_types::{SurrogateBitmap, SystemTimeScope}; +use nodedb_types::{ShardBucket, SurrogateBitmap, SystemTimeScope}; /// Reducer for [`ArrayOp::Aggregate`]. Numeric `c_enum` keeps the /// wire encoding to a single byte. @@ -81,7 +81,7 @@ pub enum ArrayOp { array_id: ArrayId, schema_msgpack: Vec, schema_hash: u64, - /// Hilbert-prefix bits for vShard routing. Immutable post-create. + /// Routing bucket bits for vShard routing. Immutable post-create. prefix_bits: u8, /// Array audit retention selected by CREATE ARRAY. This stays on the /// authorized task so planning remains read-only. @@ -150,12 +150,10 @@ pub enum ArrayOp { /// Only cells whose surrogate is present in this bitmap are included. /// `None` = no restriction. cell_filter: Option, - /// Optional Hilbert-prefix range `[lo, hi]`. When set, only cells - /// whose Hilbert prefix falls within this range are included. - /// Used by distributed shard slice to prevent duplicate rows in - /// single-node harnesses where all vShards share one Data Plane. - /// `None` = no Hilbert filter (all cells included). - hilbert_range: Option<(u64, u64)>, + /// Optional shard bucket. When set, only cells whose tile key routes + /// to this bucket are included, so vShards that share one Data + /// Plane return no duplicate rows. `None` includes every cell. + shard_bucket: Option, /// Bitemporal system-time scope. /// /// - `Current`: live read (returns the latest committed state). @@ -193,12 +191,10 @@ pub enum ArrayOp { /// merge partials across shards before finalizing. /// Single-node SQL callers always pass `false`. return_partial: bool, - /// Optional Hilbert-prefix range `[lo, hi]`. When set, only cells - /// whose Hilbert prefix falls within this range are included. - /// Used by distributed shard agg to prevent double-counting in - /// single-node harnesses where all vShards share one Data Plane. - /// `None` = no Hilbert filter (all cells included). - hilbert_range: Option<(u64, u64)>, + /// Optional shard bucket. When set, only cells whose tile key routes + /// to this bucket contribute, so vShards that share one Data Plane + /// count no cell twice. `None` includes every cell. + shard_bucket: Option, /// Bitemporal system-time cutoff. When `Some(t)`, only tile versions /// with `system_from_ms <= t` contribute to the aggregate. `None` = /// live read. @@ -309,7 +305,7 @@ mod tests { group_by_dim: -1, cell_filter: None, return_partial: false, - hilbert_range: None, + shard_bucket: None, system_as_of: None, valid_at_ms: None, }; @@ -327,7 +323,7 @@ mod tests { attr_projection: vec![], limit: 0, cell_filter: None, - hilbert_range: None, + shard_bucket: None, system_time: SystemTimeScope::AllVersions, valid_at_ms: None, }; @@ -355,7 +351,7 @@ mod tests { attr_projection: vec![], limit: 10, cell_filter: None, - hilbert_range: None, + shard_bucket: None, system_time: SystemTimeScope::AsOf(1_700_000_000_000), valid_at_ms: Some(1_600_000_000_000), }; diff --git a/nodedb-physical/src/physical_plan/cluster_array.rs b/nodedb-physical/src/physical_plan/cluster_array.rs index add4e26d0..acf6487c2 100644 --- a/nodedb-physical/src/physical_plan/cluster_array.rs +++ b/nodedb-physical/src/physical_plan/cluster_array.rs @@ -30,9 +30,9 @@ use nodedb_types::SystemTimeScope; pub enum ClusterArrayOp { /// Fan-out a slice query across all target vShards and merge rows. /// - /// `slice_hilbert_ranges` — pre-computed `(lo, hi)` Hilbert-prefix - /// ranges that determine which shards to contact. An empty vec means - /// "scan all shards". + /// `slice_tile_keys` — pre-computed tile keys, at least one per routing + /// bucket the slice touches. They pick the shards to contact. An empty + /// vec scans every shard. /// /// `prefix_bits` — routing granularity from the array catalog entry. Slice { @@ -43,8 +43,8 @@ pub enum ClusterArrayOp { attr_projection: Vec, /// Maximum rows returned. 0 = unlimited. limit: u32, - /// Pre-computed Hilbert-prefix ranges for shard selection. - slice_hilbert_ranges: Vec<(u64, u64)>, + /// Pre-computed tile keys for shard selection. + slice_tile_keys: Vec, /// Routing granularity from the array catalog entry. prefix_bits: u8, /// Bitemporal system-time scope for this slice fan-out. @@ -68,8 +68,8 @@ pub enum ClusterArrayOp { reducer_msgpack: Vec, /// Dimension index to group by (-1 = no grouping). group_by_dim: i32, - /// Pre-computed Hilbert ranges for shard selection (empty = all). - slice_hilbert_ranges: Vec<(u64, u64)>, + /// Pre-computed tile keys for shard selection (empty = all). + slice_tile_keys: Vec, /// Routing granularity. prefix_bits: u8, /// Bitemporal system-time cutoff. `None` = live read. @@ -78,16 +78,16 @@ pub enum ClusterArrayOp { valid_at_ms: Option, }, - /// Fan-out a cell write batch, partitioned by Hilbert tile. + /// Fan-out a cell write batch, partitioned by routing bucket. /// - /// `cells` — list of `(hilbert_prefix, zerompk-encoded single-cell - /// bytes)`. The coordinator partitions these by prefix and sends each - /// shard's subset via `coord_put`. + /// `cells` — list of `(tile_key, zerompk-encoded single-cell bytes)`. + /// The coordinator partitions these by the tile key's vShard and sends + /// each shard's subset via `coord_put`. Put { array_id: ArrayId, /// zerompk-encoded `ArrayId` bytes for the wire request. array_id_msgpack: Vec, - /// Cells as `(hilbert_prefix, cell_msgpack)` pairs. + /// Cells as `(tile_key, cell_msgpack)` pairs. cells: Vec<(u64, Vec)>, /// WAL LSN allocated by the Control Plane for this batch. wal_lsn: u64, @@ -95,14 +95,14 @@ pub enum ClusterArrayOp { prefix_bits: u8, }, - /// Fan-out a coord delete batch, partitioned by Hilbert tile. + /// Fan-out a coord delete batch, partitioned by routing bucket. /// - /// `coords` — `(hilbert_prefix, zerompk-encoded single-coord bytes)`. + /// `coords` — `(tile_key, zerompk-encoded single-coord bytes)`. Delete { array_id: ArrayId, /// zerompk-encoded `ArrayId` bytes for the wire request. array_id_msgpack: Vec, - /// Coords as `(hilbert_prefix, coord_msgpack)` pairs. + /// Coords as `(tile_key, coord_msgpack)` pairs. coords: Vec<(u64, Vec)>, /// WAL LSN. wal_lsn: u64, diff --git a/nodedb-types/src/array_route.rs b/nodedb-types/src/array_route.rs new file mode 100644 index 000000000..506b939eb --- /dev/null +++ b/nodedb-types/src/array_route.rs @@ -0,0 +1,241 @@ +// SPDX-License-Identifier: Apache-2.0 + +//! Routing bucket and vShard of an array tile. +//! +//! A tile key orders tiles on a space-filling curve, so its top bits carry +//! little entropy when tile indices are small. Routing therefore takes the +//! top bits of a bijective mix of the key. Stored keys stay unchanged. +//! The cluster router, the Data Plane shard filter and the snapshot paths +//! use these functions, so a tile has one bucket and one vShard everywhere. +//! +//! The router uses at most [`VShardId::BITS`] bucket bits. Bucket `b` of `R` +//! routing bits owns the vShards `[b * stride, (b + 1) * stride)`, with +//! `stride = VShardId::COUNT >> R`. Its primary vShard is `b * stride`. + +use serde::{Deserialize, Serialize}; + +use crate::id::VShardId; + +/// The largest routing granularity an array declares. +pub const MAX_PREFIX_BITS: u8 = 16; + +/// An array routing granularity outside `1..=MAX_PREFIX_BITS`. +#[derive(Debug, Clone, Copy, PartialEq, Eq, thiserror::Error)] +#[error("array routing: prefix_bits must be 1-{}, got {}", MAX_PREFIX_BITS, .0)] +pub struct PrefixBitsOutOfRange(pub u8); + +/// MurmurHash3 64-bit finalizer. A bijection on `u64`: distinct keys keep +/// distinct values, and every input bit affects every output bit. +pub const fn fmix64(key: u64) -> u64 { + let mut k = key; + k ^= k >> 33; + k = k.wrapping_mul(0xff51_afd7_ed55_8ccd); + k ^= k >> 33; + k = k.wrapping_mul(0xc4ce_b9fe_1a85_ec53); + k ^= k >> 33; + k +} + +/// The routing bucket of tile key `tile_key`: the top `prefix_bits` bits of +/// `fmix64(tile_key)`. Returns a value in `[0, 2^prefix_bits)`. With +/// `prefix_bits == 0` every key maps to bucket 0. +pub const fn prefix_bucket(tile_key: u64, prefix_bits: u8) -> u32 { + let bits = if prefix_bits > 32 { 32 } else { prefix_bits }; + if bits == 0 { + return 0; + } + (fmix64(tile_key) >> (64 - bits as u32)) as u32 +} + +/// The bucket one shard serves. A shard read keeps only the tiles whose +/// key falls in this bucket, so shards that share one Data Plane never +/// return the same tile twice. +#[derive( + Debug, + Clone, + Copy, + PartialEq, + Eq, + Hash, + Serialize, + Deserialize, + zerompk::ToMessagePack, + zerompk::FromMessagePack, +)] +pub struct ShardBucket { + /// Bucket index in `[0, 2^prefix_bits)`. + pub bucket: u32, + /// Number of mixed key bits the bucket is taken from. + pub prefix_bits: u8, +} + +impl ShardBucket { + /// Whether tile key `tile_key` routes to this bucket. + pub const fn contains(&self, tile_key: u64) -> bool { + prefix_bucket(tile_key, self.prefix_bits) == self.bucket + } +} + +/// Whether `tile_key` passes the optional shard filter. `None` keeps every +/// tile. +pub fn shard_bucket_admits(filter: Option, tile_key: u64) -> bool { + filter.is_none_or(|b| b.contains(tile_key)) +} + +/// The bucket bits the router uses for `prefix_bits`: at most +/// [`VShardId::BITS`], since a finer bucket has no vShard of its own. +pub fn routing_bits(prefix_bits: u8) -> Result { + if prefix_bits == 0 || prefix_bits > MAX_PREFIX_BITS { + return Err(PrefixBitsOutOfRange(prefix_bits)); + } + Ok(prefix_bits.min(VShardId::BITS)) +} + +/// The primary vShard of `bucket` at `bits` routing bits. `bits` comes from +/// [`routing_bits`], so it is at most [`VShardId::BITS`]. +pub const fn bucket_to_vshard(bucket: u32, bits: u8) -> u32 { + bucket << (VShardId::BITS - bits) as u32 +} + +/// The primary vShard that owns the tile with key `tile_key`. +/// +/// `prefix_bits` is the array's routing granularity (1–16). Returns a vShard +/// ID in `[0, VShardId::COUNT)`. +pub fn array_vshard_for_tile(tile_key: u64, prefix_bits: u8) -> Result { + let bits = routing_bits(prefix_bits)?; + Ok(bucket_to_vshard(prefix_bucket(tile_key, bits), bits)) +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn fmix64_is_injective_on_a_sample() { + let mut seen = std::collections::HashSet::new(); + for k in 0u64..100_000 { + assert!(seen.insert(fmix64(k)), "collision at {k}"); + } + assert_eq!(fmix64(0), 0); + } + + #[test] + fn prefix_bucket_stays_in_range() { + for bits in 1u8..=16 { + for k in [0u64, 1, 2, 63, 1 << 40, u64::MAX / 3, u64::MAX] { + assert!(prefix_bucket(k, bits) < (1u32 << bits), "bits={bits} k={k}"); + } + } + assert_eq!(prefix_bucket(u64::MAX, 0), 0); + } + + #[test] + fn small_keys_spread_over_buckets() { + // Keys whose top bits are all zero land in most of the 256 buckets. + let buckets: std::collections::HashSet = + (0u64..4096).map(|k| prefix_bucket(k, 8)).collect(); + assert!( + buckets.len() >= 240, + "only {} of 256 buckets used", + buckets.len() + ); + } + + #[test] + fn shard_bucket_partitions_keys() { + let bits = 4u8; + for k in 0u64..2048 { + let owners = (0u32..(1 << bits)) + .filter(|&b| { + ShardBucket { + bucket: b, + prefix_bits: bits, + } + .contains(k) + }) + .count(); + assert_eq!(owners, 1, "key {k} has {owners} owners"); + } + assert!(shard_bucket_admits(None, 7)); + } + + #[test] + fn tile_routing_invalid_prefix_bits() { + assert_eq!(array_vshard_for_tile(0, 0), Err(PrefixBitsOutOfRange(0))); + assert_eq!(array_vshard_for_tile(0, 17), Err(PrefixBitsOutOfRange(17))); + } + + #[test] + fn tile_routing_uses_the_mixed_bucket() { + // prefix_bits=8: stride=4, so the vShard is the mixed bucket times 4. + for key in [0u64, 1, 2, 0x0100_0000_0000_0000, u64::MAX] { + let vshard = array_vshard_for_tile(key, 8).unwrap(); + assert_eq!(vshard, prefix_bucket(key, 8) * 4, "key={key:#x}"); + } + // prefix_bits=10: stride=1, so the vShard is the mixed bucket. + let key = 0x0040_0000_0000_0000; + assert_eq!( + array_vshard_for_tile(key, 10).unwrap(), + prefix_bucket(key, 10) + ); + } + + #[test] + fn tile_routing_output_in_vshard_range() { + // Every vShard returned is below VShardId::COUNT, including + // prefix_bits above VShardId::BITS. + for bits in 1u8..=MAX_PREFIX_BITS { + for &key in &[0u64, 1, u64::MAX / 3, u64::MAX / 2, u64::MAX] { + let vshard = array_vshard_for_tile(key, bits).unwrap(); + assert!( + vshard < VShardId::COUNT, + "bits={bits} key={key:#x} → vshard={vshard}" + ); + } + } + } + + #[test] + fn prefix_bits_above_vshard_bits_route_like_vshard_bits() { + for key in [3u64, 99, 1 << 50, u64::MAX - 7] { + let at_max = array_vshard_for_tile(key, VShardId::BITS).unwrap(); + for bits in VShardId::BITS..=MAX_PREFIX_BITS { + assert_eq!(array_vshard_for_tile(key, bits).unwrap(), at_max); + } + } + } + + #[test] + fn small_tile_keys_spread_over_many_vshards() { + // Keys whose top bits are all zero reach most of the 256 buckets. + let vshards: std::collections::BTreeSet = (0u64..4096) + .map(|key| array_vshard_for_tile(key, 8).unwrap()) + .collect(); + assert!( + vshards.len() >= 240, + "only {} of 256 buckets used", + vshards.len() + ); + } + + #[test] + fn bucket_to_vshard_strides() { + assert_eq!(VShardId::COUNT, 1 << VShardId::BITS); + assert_eq!(VShardId::BITS, 10); + assert_eq!(bucket_to_vshard(1, 1), 512); + assert_eq!(bucket_to_vshard(1, 8), 4); + assert_eq!(bucket_to_vshard(1, 10), 1); + assert_eq!(routing_bits(16), Ok(10)); + } + + #[test] + fn shard_bucket_roundtrips_through_msgpack() { + let b = ShardBucket { + bucket: 41, + prefix_bits: 8, + }; + let bytes = zerompk::to_msgpack_vec(&b).expect("encode"); + let back: ShardBucket = zerompk::from_msgpack(&bytes).expect("decode"); + assert_eq!(b, back); + } +} diff --git a/nodedb-types/src/id/vshard.rs b/nodedb-types/src/id/vshard.rs index 0230f2a07..7732b78c6 100644 --- a/nodedb-types/src/id/vshard.rs +++ b/nodedb-types/src/id/vshard.rs @@ -26,10 +26,15 @@ use serde::{Deserialize, Serialize}; )] pub struct VShardId(pub(super) u32); +const _: () = assert!(VShardId::COUNT.is_power_of_two()); + impl VShardId { /// Total number of virtual shards in the system. pub const COUNT: u32 = 1024; + /// Bits that address every vShard once: `COUNT == 1 << BITS`. + pub const BITS: u8 = Self::COUNT.trailing_zeros() as u8; + pub const fn new(id: u32) -> Self { assert!(id < Self::COUNT, "vShard ID must be < 1024"); Self(id) diff --git a/nodedb-types/src/lib.rs b/nodedb-types/src/lib.rs index 13b965607..f8a7e9372 100644 --- a/nodedb-types/src/lib.rs +++ b/nodedb-types/src/lib.rs @@ -15,6 +15,7 @@ pub mod quota; pub mod approx; pub mod array_cell; +pub mod array_route; pub mod ascii; pub mod backup_envelope; pub mod bbox; @@ -78,6 +79,7 @@ pub mod write_version; pub use approx::{CountMinSketch, HyperLogLog, SpaceSaving, TDigest}; pub use array_cell::ArrayCell; +pub use array_route::ShardBucket; pub use ascii::{ find_ascii_case_insensitive, find_ascii_case_insensitive_from, rfind_ascii_case_insensitive, starts_with_ascii_case_insensitive, strip_prefix_ascii_case_insensitive, diff --git a/nodedb/src/control/backup/restore/array_reissue.rs b/nodedb/src/control/backup/restore/array_reissue.rs index 87b176994..586722c50 100644 --- a/nodedb/src/control/backup/restore/array_reissue.rs +++ b/nodedb/src/control/backup/restore/array_reissue.rs @@ -326,7 +326,8 @@ impl CellWrite<'_> { state, proposer, entry, - crate::control::wal_replication::statement_propose_deadline(state), + crate::control::server::shared::session::StatementDeadline::of_statement(state) + .tokio_instant(), ) .await?; Ok(()) diff --git a/nodedb/src/control/catalog_entry/post_apply/async_dispatch/array.rs b/nodedb/src/control/catalog_entry/post_apply/async_dispatch/array.rs index d71b8f6bf..cc5c9bcd5 100644 --- a/nodedb/src/control/catalog_entry/post_apply/async_dispatch/array.rs +++ b/nodedb/src/control/catalog_entry/post_apply/async_dispatch/array.rs @@ -11,7 +11,7 @@ //! done: the tombstone fences a same-name open until the next `PutArray` //! purges it. //! - A `DeleteArray` with `moved_to` rekeys the store on every core instead. -//! Cells route by Hilbert prefix alone, so the vShard of every cell stays +//! Cells route by tile key alone, so the vShard of every cell stays //! the same, and only the store directory carries the database. //! - Every `DeleteArray` changes the mirror last, under the array //! incarnation's exclusive gate (see `control::write_gate`). diff --git a/nodedb/src/control/cluster/array_cluster_exec/dispatch.rs b/nodedb/src/control/cluster/array_cluster_exec/dispatch.rs index b594003c9..0ff2e34e1 100644 --- a/nodedb/src/control/cluster/array_cluster_exec/dispatch.rs +++ b/nodedb/src/control/cluster/array_cluster_exec/dispatch.rs @@ -15,9 +15,9 @@ use async_trait::async_trait; use nodedb_cluster::distributed_array::ArrayLocalExecutor; use nodedb_cluster::distributed_array::handler::handle_array_shard_rpc; use nodedb_cluster::distributed_array::rpc::ShardRpcDispatch; -use nodedb_cluster::error::Result as ClusterResult; -use nodedb_cluster::rpc_codec::RaftRpc; -use nodedb_cluster::wire::VShardEnvelope; +use nodedb_cluster::error::{ClusterError, Result as ClusterResult}; +use nodedb_cluster::rpc_codec::{RaftRpc, attempt_budget, remaining_budget_ms}; +use nodedb_cluster::wire::{VShardEnvelope, VShardMessageType}; use nodedb_cluster::{NexarTransport, RoutingTable}; use crate::control::cluster::array_cluster_helpers::array_resp_msg_type; @@ -59,25 +59,64 @@ impl NexarArrayDispatch { } } +/// The sends one shard call makes at most: the first, and one resend. +const CALL_ATTEMPTS: usize = 2; + +/// The pause before a resend, so the metadata applier can heal a routing +/// entry that named a stale or fake leader. +const RESEND_PAUSE: std::time::Duration = std::time::Duration::from_millis(50); + #[async_trait] impl ShardRpcDispatch for NexarArrayDispatch { + /// Every send shares the `timeout_ms` budget. A read leaves an RPC + /// timeout for its resend, so a silent shard cannot spend it all. async fn call(&self, req: VShardEnvelope, timeout_ms: u64) -> ClusterResult { - match self.call_once(&req, timeout_ms).await { - Ok(resp) => Ok(resp), - Err(first_err) if is_transport_err(&first_err) => { - // Routing table may have pointed at a stale or fake leader. - // The metadata applier heals it asynchronously; wait briefly, - // re-read routing, and retry once. - tokio::time::sleep(std::time::Duration::from_millis(50)).await; - self.call_once(&req, timeout_ms).await + let deadline = tokio::time::Instant::now() + std::time::Duration::from_millis(timeout_ms); + let resend_safe = is_resend_safe(req.msg_type); + let mut attempts_left = CALL_ATTEMPTS; + loop { + let attempt_ms = attempt_budget( + remaining_budget_ms(deadline), + resend_safe, + attempts_left, + self.transport.rpc_timeout(), + ); + match self.call_once(&req, attempt_ms).await { + Ok(resp) => return Ok(resp), + // A resend starts only while the budget has time left. + Err(e) + if attempts_left > 1 + && remaining_budget_ms(deadline) > 0 + && may_resend(&e, resend_safe) => + { + // `call_once` re-reads routing, so the resend reaches the + // shard's current leader. + attempts_left -= 1; + tokio::time::sleep(RESEND_PAUSE).await; + } + Err(e) => return Err(e), } - Err(e) => Err(e), } } } -fn is_transport_err(e: &nodedb_cluster::error::ClusterError) -> bool { - matches!(e, nodedb_cluster::error::ClusterError::Transport { .. }) +/// Whether a second run of the request leaves the same state and answer. A +/// slice, an aggregate and a surrogate bitmap only read the shard. A put or +/// a delete can have applied when its answer was lost. +fn is_resend_safe(msg_type: VShardMessageType) -> bool { + matches!( + msg_type, + VShardMessageType::ArrayShardSliceReq + | VShardMessageType::ArrayShardAggReq + | VShardMessageType::ArrayShardSurrogateBitmapReq + ) +} + +/// Whether a failed send can go again. A request that never reached the +/// shard always can. A lost answer can only for a resend-safe request: +/// any other one reports its unknown outcome. +fn may_resend(e: &ClusterError, resend_safe: bool) -> bool { + e.is_unsent() || (resend_safe && e.is_lost_answer()) } impl NexarArrayDispatch { @@ -130,8 +169,8 @@ impl NexarArrayDispatch { // Wrap in RaftRpc and send. The remote node's handler decodes the // envelope bytes and dispatches to the appropriate shard handler. let rpc = RaftRpc::VShardEnvelope(envelope_bytes); - // The shard's budget bounds the reply wait, never the transport's fixed - // RPC timeout. + // This attempt's budget bounds the reply wait, never the transport's + // fixed RPC timeout. let reply_wait = std::time::Duration::from_millis(timeout_ms); let resp_rpc = tokio::time::timeout( reply_wait, @@ -139,16 +178,18 @@ impl NexarArrayDispatch { .send_rpc_with_read_timeout(node_id, rpc, reply_wait), ) .await - // The request can have reached the shard, so a timeout is never - // resent: `call` retries only a `Transport` error. + // The request can have reached the shard: `call` resends a timeout + // only for a resend-safe request. .map_err(|_| nodedb_cluster::error::ClusterError::ShardTimeout { vshard_id: req.vshard_id, elapsed_ms: timeout_ms, })??; + // The shard answered, so an answer this node cannot read is a codec + // error: a resend can run a put or a delete twice. match resp_rpc { RaftRpc::VShardEnvelope(bytes) => VShardEnvelope::from_bytes(&bytes).ok_or_else(|| { - nodedb_cluster::error::ClusterError::Transport { + nodedb_cluster::error::ClusterError::Codec { detail: "array shard response: failed to decode VShardEnvelope".into(), } }), @@ -158,7 +199,7 @@ impl NexarArrayDispatch { RaftRpc::VShardRefusal(refusal) => { Err(nodedb_cluster::error::ClusterError::from(refusal.error)) } - other => Err(nodedb_cluster::error::ClusterError::Transport { + other => Err(nodedb_cluster::error::ClusterError::Codec { detail: format!( "array shard RPC: unexpected response type {:?}", std::mem::discriminant(&other) @@ -167,3 +208,63 @@ impl NexarArrayDispatch { } } } + +#[cfg(test)] +mod tests { + use super::*; + + fn timed_out() -> ClusterError { + ClusterError::ShardTimeout { + vshard_id: 3, + elapsed_ms: 1_000, + } + } + + /// A slice, an aggregate and a surrogate bitmap only read the shard. + #[test] + fn only_array_reads_are_resend_safe() { + assert!(is_resend_safe(VShardMessageType::ArrayShardSliceReq)); + assert!(is_resend_safe(VShardMessageType::ArrayShardAggReq)); + assert!(is_resend_safe( + VShardMessageType::ArrayShardSurrogateBitmapReq + )); + assert!(!is_resend_safe(VShardMessageType::ArrayShardPutReq)); + assert!(!is_resend_safe(VShardMessageType::ArrayShardDeleteReq)); + } + + /// A read whose answer was lost goes again. + #[test] + fn a_timed_out_read_resends() { + assert!(may_resend(&timed_out(), true)); + let unanswered = ClusterError::Unanswered { + node_id: 2, + detail: "stream reset".into(), + }; + assert!(may_resend(&unanswered, true)); + } + + /// A put or a delete whose answer was lost reports its unknown outcome. + #[test] + fn a_timed_out_write_is_never_resent() { + assert!(!may_resend(&timed_out(), false)); + } + + /// A request that never reached the shard goes again, read or write. + #[test] + fn an_unsent_request_resends() { + let unsent = ClusterError::Transport { + detail: "connect refused".into(), + }; + assert!(may_resend(&unsent, false)); + assert!(may_resend(&unsent, true)); + } + + /// An answer the shard gave never goes again. + #[test] + fn an_answer_is_never_resent() { + let unreadable = ClusterError::Codec { + detail: "array shard response: failed to decode VShardEnvelope".into(), + }; + assert!(!may_resend(&unreadable, true)); + } +} diff --git a/nodedb/src/control/cluster/array_cluster_exec/executor.rs b/nodedb/src/control/cluster/array_cluster_exec/executor.rs index c3bc8ddbc..ea3680a59 100644 --- a/nodedb/src/control/cluster/array_cluster_exec/executor.rs +++ b/nodedb/src/control/cluster/array_cluster_exec/executor.rs @@ -17,7 +17,7 @@ use nodedb_cluster::routing::VSHARD_COUNT; use nodedb_cluster::{NexarTransport, RoutingTable}; use crate::control::cluster::array_cluster_helpers::{ - cluster_err, encode_err, finalize_agg_partials, + cluster_err, coord_err, encode_err, finalize_agg_partials, }; use crate::control::server::shared::session::pending_shard_reads::{ShardObservation, ShardReads}; use crate::control::server::shared::session::read_set::EngineTag; @@ -48,7 +48,7 @@ struct SliceArgs<'a> { slice_msgpack: &'a [u8], attr_projection: &'a [u32], limit: u32, - slice_hilbert_ranges: &'a [(u64, u64)], + slice_tile_keys: &'a [u64], prefix_bits: u8, system_time: nodedb_types::SystemTimeScope, valid_at_ms: Option, @@ -60,7 +60,7 @@ struct AggArgs<'a> { attr_idx: u32, reducer_msgpack: &'a [u8], group_by_dim: i32, - slice_hilbert_ranges: &'a [(u64, u64)], + slice_tile_keys: &'a [u64], prefix_bits: u8, system_as_of: Option, valid_at_ms: Option, @@ -114,7 +114,7 @@ impl ClusterArrayExecutor { slice_msgpack, attr_projection, limit, - slice_hilbert_ranges, + slice_tile_keys, prefix_bits, system_time, valid_at_ms, @@ -124,7 +124,7 @@ impl ClusterArrayExecutor { slice_msgpack, attr_projection, limit: *limit, - slice_hilbert_ranges, + slice_tile_keys, prefix_bits: *prefix_bits, system_time: *system_time, valid_at_ms: *valid_at_ms, @@ -138,7 +138,7 @@ impl ClusterArrayExecutor { attr_idx, reducer_msgpack, group_by_dim, - slice_hilbert_ranges, + slice_tile_keys, prefix_bits, system_as_of, valid_at_ms, @@ -148,7 +148,7 @@ impl ClusterArrayExecutor { attr_idx: *attr_idx, reducer_msgpack, group_by_dim: *group_by_dim, - slice_hilbert_ranges, + slice_tile_keys, prefix_bits: *prefix_bits, system_as_of: *system_as_of, valid_at_ms: *valid_at_ms, @@ -187,7 +187,7 @@ impl ClusterArrayExecutor { slice_msgpack, attr_projection, limit, - slice_hilbert_ranges, + slice_tile_keys, prefix_bits, system_time, valid_at_ms, @@ -196,7 +196,7 @@ impl ClusterArrayExecutor { let coordinator = ArrayCoordinator::for_slice( self.source_node, ARRAY_RPC_TIMEOUT_MS, - slice_hilbert_ranges, + slice_tile_keys, prefix_bits, self.total_shards, Arc::clone(&self.dispatch), @@ -211,13 +211,11 @@ impl ClusterArrayExecutor { attr_projection: attr_projection.to_vec(), limit, cell_filter_msgpack: vec![], - // prefix_bits, slice_hilbert_ranges, and shard_hilbert_range are - // stamped per-shard by coord_slice from ArrayCoordParams, which was - // populated by for_slice above. Initialise to defaults here; - // coord_slice overwrites them before serialising the request envelope. + // coord_slice stamps prefix_bits, slice_tile_keys and shard_bucket + // per shard from the ArrayCoordParams that for_slice built. prefix_bits: 0, - slice_hilbert_ranges: vec![], - shard_hilbert_range: None, + slice_tile_keys: vec![], + shard_bucket: None, system_time, valid_at_ms, txn_id: txn_id.map(TxnId::as_u64), @@ -225,7 +223,7 @@ impl ClusterArrayExecutor { let result = coordinator .coord_slice(req, limit, system_time) .await - .map_err(cluster_err)?; + .map_err(coord_err)?; note_array_reads(array_id, &result.read_versions); // Encode rows into the structured `ArraySliceResponse` — identical to @@ -265,7 +263,7 @@ impl ClusterArrayExecutor { attr_idx, reducer_msgpack, group_by_dim, - slice_hilbert_ranges, + slice_tile_keys, prefix_bits, system_as_of, valid_at_ms, @@ -274,7 +272,7 @@ impl ClusterArrayExecutor { let coordinator = ArrayCoordinator::for_slice( self.source_node, ARRAY_RPC_TIMEOUT_MS, - slice_hilbert_ranges, + slice_tile_keys, prefix_bits, self.total_shards, Arc::clone(&self.dispatch), @@ -289,12 +287,12 @@ impl ClusterArrayExecutor { reducer_msgpack: reducer_msgpack.to_vec(), group_by_dim, cell_filter_msgpack: vec![], - shard_hilbert_range: None, + shard_bucket: None, system_as_of, valid_at_ms, txn_id: txn_id.map(TxnId::as_u64), }; - let agg = coordinator.coord_agg(req).await.map_err(cluster_err)?; + let agg = coordinator.coord_agg(req).await.map_err(coord_err)?; note_array_reads(array_id, &agg.read_versions); // Decode the reducer so we know which field to finalize. @@ -342,7 +340,7 @@ impl ClusterArrayExecutor { &self.circuit_breaker, ) .await - .map_err(cluster_err)?; + .map_err(coord_err)?; // Sum each shard's real `affected` count — never `cells.len()`, which // counts cells named in the request, not cells the Data Plane @@ -373,7 +371,7 @@ impl ClusterArrayExecutor { &self.circuit_breaker, ) .await - .map_err(cluster_err)?; + .map_err(coord_err)?; // Sum each shard's real `affected` count — never `coords.len()`, // which counts coordinates named, not coordinates that existed and diff --git a/nodedb/src/control/cluster/array_cluster_helpers.rs b/nodedb/src/control/cluster/array_cluster_helpers.rs index 9300a5694..c8a4ddee8 100644 --- a/nodedb/src/control/cluster/array_cluster_helpers.rs +++ b/nodedb/src/control/cluster/array_cluster_helpers.rs @@ -5,6 +5,7 @@ //! fast path. use nodedb_cluster::distributed_array::merge::ArrayAggPartial; +use nodedb_cluster::distributed_array::scatter::{ArrayCoordError, ShardFailure}; use nodedb_cluster::error::ClusterError; use nodedb_cluster::wire::VShardMessageType; @@ -95,17 +96,33 @@ pub(super) fn finalize_agg_partials( rows } +/// Map an array coordinator error to an [`Error`]. +/// +/// A request that never reached its shard ran nothing. It answers the +/// definite, retryable `NoLeader` for that vShard. `NoLeader` never moves +/// the gateway's routing hint. Every other error maps as [`cluster_err`]. +pub(super) fn coord_err(e: ArrayCoordError) -> Error { + match e { + ArrayCoordError::Shard(ShardFailure { vshard_id, error }) if error.is_unsent() => { + tracing::debug!(vshard_id, %error, "array shard request never reached its shard"); + Error::NoLeader { + vshard_id: crate::types::VShardId::new(vshard_id), + } + } + ArrayCoordError::Shard(ShardFailure { error, .. }) + | ArrayCoordError::Coordinator(error) => cluster_err(error), + } +} + pub(super) fn cluster_err(e: ClusterError) -> Error { match e { // A shard did not answer within its timeout: surface as a deterministic // deadline rather than an opaque internal error, matching the // `TypedClusterError::DeadlineExceeded` mapping used elsewhere. A // sent request with no answer can have run, so it takes the same class. - ClusterError::ShardTimeout { .. } | ClusterError::Unanswered { .. } => { - Error::DeadlineExceeded { - request_id: crate::types::RequestId::new(0), - } - } + lost if lost.is_lost_answer() => Error::DeadlineExceeded { + request_id: crate::types::RequestId::new(0), + }, // A shard's Data-Plane verdict keeps its code, so the statement // renders the SQLSTATE a single-node execution renders. ClusterError::DataPlane { code } => Error::DataPlane(code.into()), @@ -168,6 +185,9 @@ pub(super) fn cluster_err(e: ClusterError) -> Error { | ClusterError::SpatialGather(_) | ClusterError::Bm25Gather(_) | ClusterError::TsGather(_) + // The guards above take these. They stay named for exhaustiveness. + | ClusterError::ShardTimeout { .. } + | ClusterError::Unanswered { .. } | ClusterError::ShufflePush(_) | ClusterError::RemoteUntyped { .. }) => Error::Internal { detail: format!("array cluster: {other}"), @@ -202,6 +222,45 @@ mod tests { use super::*; use crate::bridge::envelope::ErrorCode; + /// A shard request that never went out answers `NoLeader` for that + /// shard: nothing ran, so the refusal is definite. + #[test] + fn an_unsent_shard_request_is_no_leader_for_its_vshard() { + let unsent = ArrayCoordError::Shard(ShardFailure { + vshard_id: 41, + error: ClusterError::Transport { + detail: "connect refused".into(), + }, + }); + match coord_err(unsent) { + Error::NoLeader { vshard_id } => assert_eq!(vshard_id.as_u32(), 41), + other => panic!("expected NoLeader for vshard 41, got {other:?}"), + } + } + + /// Every other shard failure maps as `cluster_err` does. + #[test] + fn a_lost_shard_answer_stays_an_unknown_outcome() { + let lost = ArrayCoordError::Shard(ShardFailure { + vshard_id: 41, + error: ClusterError::Unanswered { + node_id: 2, + detail: "stream reset".into(), + }, + }); + assert!(matches!(coord_err(lost), Error::DeadlineExceeded { .. })); + } + + /// A coordinator-side transport error names no shard, so it keeps its + /// `cluster_err` mapping. + #[test] + fn a_coordinator_error_maps_as_cluster_err() { + let local = ArrayCoordError::Coordinator(ClusterError::Codec { + detail: "encode".into(), + }); + assert!(matches!(coord_err(local), Error::Internal { .. })); + } + /// A shard verdict that crossed the cluster keeps its code at the /// coordinator, never `Internal`. #[test] diff --git a/nodedb/src/control/cluster/array_executor/executor.rs b/nodedb/src/control/cluster/array_executor/executor.rs index 2383e4f39..20d4f8b48 100644 --- a/nodedb/src/control/cluster/array_executor/executor.rs +++ b/nodedb/src/control/cluster/array_executor/executor.rs @@ -12,7 +12,7 @@ use nodedb_cluster::rpc_codec::DataPlaneErrorCode; use super::refusal::execution_error; use crate::bridge::envelope::{Priority, Request, Response}; use crate::control::cluster::linearizable_read::{ - confirm_linearizable_read, groups_of_vshards, statement_read_deadline, + StatementDeadline, confirm_linearizable_read, groups_of_vshards, }; use crate::control::server::shared::write_admission::plan_is_write; use crate::control::state::SharedState; @@ -59,9 +59,13 @@ impl DataPlaneArrayExecutor { if !plan_is_write(&plan) { let groups = groups_of_vshards(&self.state, [local_vshard_id.as_u32()]) .map_err(|e| execution_error("array read confirmation", e))?; - confirm_linearizable_read(&self.state, &groups, statement_read_deadline(&self.state)) - .await - .map_err(|e| execution_error("array read confirmation", e))?; + confirm_linearizable_read( + &self.state, + &groups, + StatementDeadline::of_statement(&self.state), + ) + .await + .map_err(|e| execution_error("array read confirmation", e))?; } let request_id = self.state.next_request_id(); let request = local_request(request_id, array_id, local_vshard_id, plan, txn_id); diff --git a/nodedb/src/control/cluster/array_executor/read.rs b/nodedb/src/control/cluster/array_executor/read.rs index 6b6334177..596cd6224 100644 --- a/nodedb/src/control/cluster/array_executor/read.rs +++ b/nodedb/src/control/cluster/array_executor/read.rs @@ -48,7 +48,7 @@ impl DataPlaneArrayExecutor { attr_projection: req.attr_projection.clone(), limit: req.limit, cell_filter, - hilbert_range: req.shard_hilbert_range, + shard_bucket: req.shard_bucket, system_time: req.system_time, valid_at_ms: req.valid_at_ms, }); @@ -119,7 +119,7 @@ impl DataPlaneArrayExecutor { group_by_dim: req.group_by_dim, cell_filter, return_partial: true, - hilbert_range: req.shard_hilbert_range, + shard_bucket: req.shard_bucket, system_as_of: req.system_as_of, valid_at_ms: req.valid_at_ms, }); diff --git a/nodedb/src/control/cluster/array_executor/refusal.rs b/nodedb/src/control/cluster/array_executor/refusal.rs index 79e025fef..cd6805f66 100644 --- a/nodedb/src/control/cluster/array_executor/refusal.rs +++ b/nodedb/src/control/cluster/array_executor/refusal.rs @@ -123,6 +123,8 @@ pub(super) fn execution_error(context: &str, error: crate::Error) -> ClusterErro | crate::Error::GroupMarksUnavailable { .. } | crate::Error::BackupCaptureMoved { .. } | crate::Error::MetadataLeaderUnavailable + | crate::Error::MetadataGroupGone { .. } + | crate::Error::MetadataDdlSuperseded { .. } | crate::Error::AuthorizationStateBehind { .. } | crate::Error::LinearizableReadRefused { .. } | crate::Error::ExecutionLimitExceeded { .. } diff --git a/nodedb/src/control/cluster/array_executor/write.rs b/nodedb/src/control/cluster/array_executor/write.rs index 9be65ad3b..cc1812817 100644 --- a/nodedb/src/control/cluster/array_executor/write.rs +++ b/nodedb/src/control/cluster/array_executor/write.rs @@ -129,7 +129,10 @@ impl DataPlaneArrayExecutor { &self.state, proposer, entry, - crate::control::wal_replication::statement_propose_deadline(&self.state), + crate::control::server::shared::session::StatementDeadline::of_statement( + &self.state, + ) + .tokio_instant(), ) .await .map_err(|e| execution_error(&format!("{op_label} raft propose"), e))?; diff --git a/nodedb/src/control/planner/sql_plan_convert/array_convert/dml.rs b/nodedb/src/control/planner/sql_plan_convert/array_convert/dml.rs index 6e1538742..557824317 100644 --- a/nodedb/src/control/planner/sql_plan_convert/array_convert/dml.rs +++ b/nodedb/src/control/planner/sql_plan_convert/array_convert/dml.rs @@ -86,9 +86,9 @@ pub(in super::super) fn convert_insert_array( let surrogate = ctx.surrogate_for_pk(ctx.collection_key(name), &pk_bytes)?; // A cell routes by its tile's key, the key a slice's shard // fan-out and each shard's tile filter use. - let hilbert = + let tile_key = cell_tile_prefix(&schema, &coord).map_err(|e| crate::Error::PlanError { - detail: format!("INSERT INTO ARRAY {name}: Hilbert prefix: {e}"), + detail: format!("INSERT INTO ARRAY {name}: tile key: {e}"), })?; let cell = ArrayPutCell { coord, @@ -103,7 +103,7 @@ pub(in super::super) fn convert_insert_array( format: "msgpack".into(), detail: format!("array put cell encode: {e}"), })?; - partitioned.push((hilbert, cell_bytes)); + partitioned.push((tile_key, cell_bytes)); } let array_id_msgpack = zerompk::to_msgpack_vec(&aid).map_err(|e| crate::Error::Serialization { @@ -212,9 +212,9 @@ pub(in super::super) fn convert_delete_array( let mut partitioned: Vec<(u64, Vec)> = Vec::with_capacity(coords.len()); for row in coords { let typed = coerce_coords(row, &schema)?; - let hilbert = + let tile_key = cell_tile_prefix(&schema, &typed).map_err(|e| crate::Error::PlanError { - detail: format!("DELETE FROM ARRAY {name}: Hilbert prefix: {e}"), + detail: format!("DELETE FROM ARRAY {name}: tile key: {e}"), })?; let cell = ArrayDeleteCell { coord: typed, @@ -226,7 +226,7 @@ pub(in super::super) fn convert_delete_array( format: "msgpack".into(), detail: format!("array delete cell encode: {e}"), })?; - partitioned.push((hilbert, cell_bytes)); + partitioned.push((tile_key, cell_bytes)); } let array_id_msgpack = zerompk::to_msgpack_vec(&aid).map_err(|e| crate::Error::Serialization { diff --git a/nodedb/src/control/planner/sql_plan_convert/array_fn_convert/aggregate.rs b/nodedb/src/control/planner/sql_plan_convert/array_fn_convert/aggregate.rs index fea8d2ab4..6f51fcda3 100644 --- a/nodedb/src/control/planner/sql_plan_convert/array_fn_convert/aggregate.rs +++ b/nodedb/src/control/planner/sql_plan_convert/array_fn_convert/aggregate.rs @@ -69,7 +69,7 @@ pub(crate) fn convert_agg( attr_idx, reducer_msgpack, group_by_dim: group_by_dim_idx, - slice_hilbert_ranges: vec![], + slice_tile_keys: vec![], prefix_bits: entry.prefix_bits, system_as_of, valid_at_ms, @@ -82,7 +82,7 @@ pub(crate) fn convert_agg( group_by_dim: group_by_dim_idx, cell_filter: None, return_partial: false, - hilbert_range: None, + shard_bucket: None, system_as_of, valid_at_ms, }) diff --git a/nodedb/src/control/planner/sql_plan_convert/array_fn_convert/slice.rs b/nodedb/src/control/planner/sql_plan_convert/array_fn_convert/slice.rs index 453669ffc..5ae48d1d2 100644 --- a/nodedb/src/control/planner/sql_plan_convert/array_fn_convert/slice.rs +++ b/nodedb/src/control/planner/sql_plan_convert/array_fn_convert/slice.rs @@ -10,6 +10,7 @@ use nodedb_array::types::ArrayId; use nodedb_array::types::domain::DomainBound; use nodedb_sql::temporal::TemporalScope; use nodedb_sql::types_array::ArraySliceAst; +use nodedb_types::array_route::prefix_bucket; use crate::bridge::envelope::PhysicalPlan; use crate::types::TenantId; @@ -69,14 +70,14 @@ pub(crate) fn convert_slice( // // The coordinator fans out only to the shards that own a tile the // slice covers. An empty list fans out to every shard. - let slice_hilbert_ranges = - compute_slice_hilbert_ranges(&schema, &slice.dim_ranges, entry.prefix_bits); + let slice_tile_keys = + compute_slice_tile_keys(&schema, &slice.dim_ranges, entry.prefix_bits); PhysicalPlan::ClusterArray(ClusterArrayOp::Slice { array_id: aid, slice_msgpack, attr_projection: attr_indices, limit, - slice_hilbert_ranges, + slice_tile_keys, prefix_bits: entry.prefix_bits, system_time, valid_at_ms, @@ -88,7 +89,7 @@ pub(crate) fn convert_slice( attr_projection: attr_indices, limit, cell_filter: None, - hilbert_range: None, + shard_bucket: None, system_time, valid_at_ms, }) @@ -108,8 +109,8 @@ pub(crate) fn convert_slice( /// tiles fans out to every shard. const MAX_FAN_OUT_TILES: u128 = 4096; -/// The Hilbert ranges a slice's shard fan-out covers: one `(key, key)` pair -/// per routing bucket, each holding the key of one tile in that bucket. +/// The tile keys a slice's shard fan-out covers: one key per routing bucket +/// (`prefix_bucket`), each the key of one tile in that bucket. /// /// Every tile whose cells can match the slice is enumerated, so every shard /// that holds such a cell, or will hold one a later write lands there, is @@ -120,11 +121,11 @@ const MAX_FAN_OUT_TILES: u128 = 4096; /// Returns an empty vec, which fans out to every shard, when the slice /// covers more than [`MAX_FAN_OUT_TILES`] tiles, when the schema has no /// dimension, or when a tile key fails to encode. -fn compute_slice_hilbert_ranges( +fn compute_slice_tile_keys( schema: &ArraySchema, dim_ranges: &[Option], prefix_bits: u8, -) -> Vec<(u64, u64)> { +) -> Vec { if schema.dims.is_empty() || prefix_bits == 0 || prefix_bits > 16 { return Vec::new(); } @@ -145,19 +146,20 @@ fn compute_slice_hilbert_ranges( return Vec::new(); } - let shift = 64 - u32::from(prefix_bits); - let mut buckets: std::collections::BTreeMap = std::collections::BTreeMap::new(); + let mut buckets: std::collections::BTreeMap = std::collections::BTreeMap::new(); let mut indices: Vec = spans.iter().map(|(lo, _)| *lo).collect(); loop { let Ok(key) = tile_prefix(schema, &indices) else { return Vec::new(); }; - buckets.entry(key >> shift).or_insert(key); + buckets + .entry(prefix_bucket(key, prefix_bits)) + .or_insert(key); // Advance the odometer over the per-dimension tile spans. let mut dim = 0; loop { if dim == indices.len() { - return buckets.into_values().map(|key| (key, key)).collect(); + return buckets.into_values().collect(); } if indices[dim] < spans[dim].1 { indices[dim] += 1; @@ -290,9 +292,9 @@ mod tests { Some(DimRange::new(DomainBound::Int64(0), DomainBound::Int64(2))), Some(DimRange::new(DomainBound::Int64(0), DomainBound::Int64(99))), ]; - let ranges = compute_slice_hilbert_ranges(&schema, &box_ranges, 8); - assert!(!ranges.is_empty(), "a three-tile slice fans out by tile"); - let fanned = nodedb_cluster::distributed_array::array_vshards_for_slice(&ranges, 8, 1024) + let keys = compute_slice_tile_keys(&schema, &box_ranges, 8); + assert!(!keys.is_empty(), "a three-tile slice fans out by tile"); + let fanned = nodedb_cluster::distributed_array::array_vshards_for_slice(&keys, 8, 1024) .expect("fan-out"); let mut owners = std::collections::BTreeSet::new(); for chr in 0..=2 { @@ -324,8 +326,8 @@ mod tests { .tile_extents(vec![1]) .build() .expect("schema build"); - assert!(compute_slice_hilbert_ranges(&wide, &[None], 8).is_empty()); - assert!(!compute_slice_hilbert_ranges(&schema, &[None], 8).is_empty()); + assert!(compute_slice_tile_keys(&wide, &[None], 8).is_empty()); + assert!(!compute_slice_tile_keys(&schema, &[None], 8).is_empty()); } #[test] diff --git a/nodedb/src/control/server/shared/ddl/neutral/tenant/move_tenant/arrays.rs b/nodedb/src/control/server/shared/ddl/neutral/tenant/move_tenant/arrays.rs index b377ae33a..01c3f64b2 100644 --- a/nodedb/src/control/server/shared/ddl/neutral/tenant/move_tenant/arrays.rs +++ b/nodedb/src/control/server/shared/ddl/neutral/tenant/move_tenant/arrays.rs @@ -2,7 +2,7 @@ //! The arrays of a `MOVE TENANT`. //! -//! A cell routes to its vShard by Hilbert prefix alone, so moving an array to +//! A cell routes to its vShard by tile key alone, so moving an array to //! another database moves no cell to another vShard. Only the catalog key, //! the surrogate bindings, and each core's store directory carry the //! database. The cutover rekeys all three with a `DeleteArray` at the source diff --git a/nodedb/src/control/server/shared/session/array_fanout_stage.rs b/nodedb/src/control/server/shared/session/array_fanout_stage.rs index 874d0e713..5e5d9bab6 100644 --- a/nodedb/src/control/server/shared/session/array_fanout_stage.rs +++ b/nodedb/src/control/server/shared/session/array_fanout_stage.rs @@ -23,9 +23,11 @@ use futures::future::join_all; use crate::bridge::envelope::{PhysicalPlan, Response, Status}; use crate::control::gateway::RouteDecision; +use crate::control::gateway::retry::Attempt; use crate::control::security::request_scope::RequestAuthScope; use crate::control::server::shared::metering::{PlanMeteringInfo, meter_staged_write}; use crate::control::server::shared::quota_admission::admit_quota_for_dispatch; +use crate::control::server::shared::session::StatementDeadline; use crate::control::server::shared::sql::staging_predicates::require_affected_count; use crate::control::state::SharedState; use crate::types::VShardId; @@ -122,7 +124,9 @@ where // the INNER write, so the un-wrapped plan rides alongside. remote => { let version_plan = inner.clone(); - forward_to_leader(state, remote, wrap(inner), &version_plan).await + let attempt = + Attempt::sole(StatementDeadline::of_statement(state).tokio_instant()); + forward_to_leader(state, remote, wrap(inner), &version_plan, attempt).await } } } diff --git a/nodedb/src/data/executor/array_checkpoint.rs b/nodedb/src/data/executor/array_checkpoint.rs index ff234e7bf..80bd20115 100644 --- a/nodedb/src/data/executor/array_checkpoint.rs +++ b/nodedb/src/data/executor/array_checkpoint.rs @@ -337,7 +337,7 @@ mod tests { attr_projection: vec![], limit: 0, cell_filter: None, - hilbert_range: None, + shard_bucket: None, system_time: nodedb_types::SystemTimeScope::Current, valid_at_ms: None, }); diff --git a/nodedb/src/data/executor/dispatch/array/aggregate.rs b/nodedb/src/data/executor/dispatch/array/aggregate.rs index bd0a03648..3e1582e5a 100644 --- a/nodedb/src/data/executor/dispatch/array/aggregate.rs +++ b/nodedb/src/data/executor/dispatch/array/aggregate.rs @@ -15,7 +15,8 @@ use nodedb_array::segment::TilePayload; use nodedb_array::types::ArrayId; use nodedb_array::types::coord::value::CoordValue; use nodedb_cluster::distributed_array::merge::ArrayAggPartial; -use nodedb_types::SurrogateBitmap; +use nodedb_types::array_route::shard_bucket_admits; +use nodedb_types::{ShardBucket, SurrogateBitmap}; use crate::bridge::envelope::{ErrorCode, Response}; use crate::data::executor::core_loop::CoreLoop; @@ -37,8 +38,8 @@ pub(in crate::data::executor) struct AggParams<'a> { pub group_by_dim_idx: i32, pub cell_filter: Option<&'a SurrogateBitmap>, pub return_partial: bool, - /// Optional Hilbert-prefix range `[lo, hi]` for shard-level partitioning. - pub hilbert_range: Option<(u64, u64)>, + /// Optional shard bucket. Only tiles whose key routes to it contribute. + pub shard_bucket: Option, /// Bitemporal system-time cutoff. `None` = live read. pub system_as_of: Option, /// Bitemporal valid-time point. `None` = no valid-time filter. @@ -49,7 +50,7 @@ pub(in crate::data::executor) struct AggParams<'a> { struct AggTileScan<'a> { array_id: &'a ArrayId, schema: &'a ArraySchema, - hilbert_range: Option<(u64, u64)>, + shard_bucket: Option, system_as_of: Option, valid_at_ms: Option, /// The transaction whose staged cells merge into the scan, or `None` @@ -78,13 +79,6 @@ struct AggEmit<'a> { emit_horizon: bool, } -fn hilbert_prefix_in_range(hp: u64, range: Option<(u64, u64)>) -> bool { - match range { - Some((lo, hi)) => hp >= lo && hp <= hi, - None => true, - } -} - impl CoreLoop { pub(in crate::data::executor) fn dispatch_array_aggregate( &mut self, @@ -98,7 +92,7 @@ impl CoreLoop { group_by_dim_idx, cell_filter, return_partial, - hilbert_range, + shard_bucket, system_as_of, valid_at_ms, } = p; @@ -126,7 +120,7 @@ impl CoreLoop { // shard's result, for the finalizing single-node read and for the // distributed partial alike. Each shard's overlay holds only the // cells partitioned to that shard, and `collect_agg_tiles` applies - // the shard's `hilbert_range` to overlay tiles as to base tiles, so + // the shard's `shard_bucket` to overlay tiles as to base tiles, so // folding them into the shard's partial counts every staged cell // exactly once at the coordinator's merge. `AS OF SYSTEM TIME` is a // snapshot of committed history and skips the overlay. @@ -138,7 +132,7 @@ impl CoreLoop { let (all_tiles, truncated_before_horizon) = match self.collect_agg_tiles(AggTileScan { array_id, schema: &schema, - hilbert_range, + shard_bucket, system_as_of, valid_at_ms, overlay_txn, @@ -172,7 +166,7 @@ impl CoreLoop { /// double-count any cell overwritten across segments in a bitemporal array /// (e.g. v1 in one sealed segment, v2 in another). `system_as_of`/ /// `valid_at_ms` narrow the cutoff for point-in-time queries; the same - /// `hilbert_range` shard filter applies in all cases. + /// `shard_bucket` filter applies in all cases. /// /// Returns an error *detail* string (wrapped into `ErrorCode::Internal` by /// the caller) rather than a `Response`, so it can borrow `&self` cleanly. @@ -180,7 +174,7 @@ impl CoreLoop { let AggTileScan { array_id, schema, - hilbert_range, + shard_bucket, system_as_of, valid_at_ms, overlay_txn, @@ -194,7 +188,7 @@ impl CoreLoop { .scan_tiles_at(cutoff, valid_at_ms) .map_err(|e| format!("array aggregate scan: {e}"))?; // Staged cells join the SAME tile set the base cells reduce through, - // ahead of the shard-range filter, so they fold into the same partial + // ahead of the shard bucket filter, so they fold into the same partial // accumulator under the same surrogate filter. self.merge_array_overlay_tiles( ArrayOverlayMergeParams { @@ -208,7 +202,7 @@ impl CoreLoop { .map_err(|e| format!("array aggregate overlay merge: {e}"))?; let tiles = resolved_tiles .into_iter() - .filter(|(hp, _)| hilbert_prefix_in_range(*hp, hilbert_range)) + .filter(|(hp, _)| shard_bucket_admits(shard_bucket, *hp)) .map(|(_, tile)| TilePayload::Sparse(tile)) .collect(); Ok((tiles, truncated_before_horizon)) @@ -553,7 +547,7 @@ mod tests { group_by_dim: -1, cell_filter: None, return_partial: false, - hilbert_range: None, + shard_bucket: None, system_as_of: None, valid_at_ms: None, }); @@ -587,7 +581,7 @@ mod tests { group_by_dim: -1, cell_filter: None, return_partial: true, - hilbert_range: None, + shard_bucket: None, system_as_of: None, valid_at_ms: None, }); @@ -640,7 +634,7 @@ mod tests { group_by_dim: -1, cell_filter: None, return_partial: false, - hilbert_range: None, + shard_bucket: None, system_as_of: Some(150), valid_at_ms: None, }); @@ -672,7 +666,7 @@ mod tests { group_by_dim: -1, cell_filter: None, return_partial: false, - hilbert_range: None, + shard_bucket: None, system_as_of: Some(50), valid_at_ms: None, }); @@ -707,7 +701,7 @@ mod tests { group_by_dim: 0, cell_filter: None, return_partial: false, - hilbert_range: None, + shard_bucket: None, system_as_of: None, valid_at_ms: None, }); @@ -761,7 +755,7 @@ mod tests { group_by_dim: -1, cell_filter: Some(bm), return_partial: false, - hilbert_range: None, + shard_bucket: None, system_as_of: None, valid_at_ms: None, }); diff --git a/nodedb/src/data/executor/dispatch/array/entry.rs b/nodedb/src/data/executor/dispatch/array/entry.rs index bd2314daa..06bc8e6a1 100644 --- a/nodedb/src/data/executor/dispatch/array/entry.rs +++ b/nodedb/src/data/executor/dispatch/array/entry.rs @@ -88,7 +88,7 @@ impl CoreLoop { attr_projection, limit, cell_filter, - hilbert_range, + shard_bucket, system_time, valid_at_ms, } => self.dispatch_array_slice( @@ -99,7 +99,7 @@ impl CoreLoop { attr_projection, limit: *limit, cell_filter: cell_filter.as_ref(), - hilbert_range: *hilbert_range, + shard_bucket: *shard_bucket, system_time: *system_time, valid_at_ms: *valid_at_ms, }, @@ -119,7 +119,7 @@ impl CoreLoop { group_by_dim, cell_filter, return_partial, - hilbert_range, + shard_bucket, system_as_of, valid_at_ms, } => self.dispatch_array_aggregate( @@ -131,7 +131,7 @@ impl CoreLoop { group_by_dim_idx: *group_by_dim, cell_filter: cell_filter.as_ref(), return_partial: *return_partial, - hilbert_range: *hilbert_range, + shard_bucket: *shard_bucket, system_as_of: *system_as_of, valid_at_ms: *valid_at_ms, }, diff --git a/nodedb/src/data/executor/dispatch/array/read.rs b/nodedb/src/data/executor/dispatch/array/read.rs index 387469017..d622eb0be 100644 --- a/nodedb/src/data/executor/dispatch/array/read.rs +++ b/nodedb/src/data/executor/dispatch/array/read.rs @@ -14,7 +14,8 @@ use nodedb_array::query::slice::{Slice, slice_sparse, tile_overlaps_slice}; use nodedb_array::segment::{MbrQueryPredicate, TilePayload}; use nodedb_array::tile::sparse_tile::SparseTile; use nodedb_array::types::ArrayId; -use nodedb_types::{ArrayCell, SurrogateBitmap, SystemTimeScope, Value}; +use nodedb_types::array_route::shard_bucket_admits; +use nodedb_types::{ArrayCell, ShardBucket, SurrogateBitmap, SystemTimeScope, Value}; /// Slice parameters bundled to avoid exceeding the 7-argument limit. pub(in crate::data::executor) struct SliceParams<'a> { @@ -23,11 +24,10 @@ pub(in crate::data::executor) struct SliceParams<'a> { pub attr_projection: &'a [u32], pub limit: u32, pub cell_filter: Option<&'a SurrogateBitmap>, - /// Optional Hilbert-prefix range `[lo, hi]` for shard-level partitioning. - /// When set, only tiles whose Hilbert prefix falls within this range are - /// included. Used by the distributed shard handler to prevent duplicate - /// rows when all vShards share a single Data Plane. - pub hilbert_range: Option<(u64, u64)>, + /// Optional shard bucket. When set, only tiles whose key routes to this + /// bucket are included, so vShards that share one Data Plane return no + /// duplicate rows. + pub shard_bucket: Option, /// Bitemporal system-time scope. /// /// - `Current`: live read (effective cutoff `i64::MAX`). @@ -59,7 +59,7 @@ impl CoreLoop { attr_projection, limit, cell_filter, - hilbert_range, + shard_bucket, system_time, valid_at_ms, } = p; @@ -126,11 +126,9 @@ impl CoreLoop { } }; - // Apply Hilbert-range and slice coord filters. + // Apply the shard bucket and slice coord filters. versions.retain(|(hp, coord, _sys_ms, _payload)| { - if let Some((lo, hi)) = hilbert_range - && (*hp < lo || *hp > hi) - { + if !shard_bucket_admits(shard_bucket, *hp) { return false; } // Check coord against slice predicate dim-by-dim. @@ -235,9 +233,7 @@ impl CoreLoop { let mut rows: Vec = Vec::new(); 'outer: for (hp, sparse) in resolved_tiles { - if let Some((lo, hi)) = hilbert_range - && (hp < lo || hp > hi) - { + if !shard_bucket_admits(shard_bucket, hp) { continue; } if !tile_overlaps_slice(&sparse.mbr.dim_mins, &sparse.mbr.dim_maxs, &slice) { @@ -742,7 +738,7 @@ mod tests { attr_projection: vec![], limit: 0, cell_filter: None, - hilbert_range: None, + shard_bucket: None, system_time: nodedb_types::SystemTimeScope::Current, valid_at_ms: None, }); @@ -794,7 +790,7 @@ mod tests { attr_projection: vec![], limit: 0, cell_filter: None, - hilbert_range: None, + shard_bucket: None, system_time: nodedb_types::SystemTimeScope::AllVersions, valid_at_ms: None, }); @@ -867,7 +863,7 @@ mod tests { attr_projection: vec![], limit: 0, cell_filter: Some(bm), - hilbert_range: None, + shard_bucket: None, system_time: nodedb_types::SystemTimeScope::Current, valid_at_ms: None, }); @@ -887,4 +883,122 @@ mod tests { // 10.0 + 30.0 = 40.0; 20.0 must have been excluded. assert!((total - 40.0).abs() < 1e-9, "total got {total}"); } + + fn schema_full_range_2d(name: &str) -> ArraySchema { + let full = || Domain::new(DomainBound::Int64(i64::MIN), DomainBound::Int64(i64::MAX)); + ArraySchemaBuilder::new(name) + .dim(DimSpec::new("x", DimType::Int64, full())) + .dim(DimSpec::new("y", DimType::Int64, full())) + .attr(AttrSpec::new("v", AttrType::Float64, true)) + .tile_extents(vec![1, 1]) + .build() + .unwrap() + } + + fn slice_in_bucket(h: &mut Harness, aid: &ArrayId, bucket: Option) -> Vec { + let slice = ArraySlice::new(vec![None, None]); + let r = h.send(ArrayOp::Slice { + array_id: aid.clone(), + slice_msgpack: zerompk::to_msgpack_vec(&slice).unwrap(), + attr_projection: vec![], + limit: 0, + cell_filter: None, + shard_bucket: bucket, + system_time: nodedb_types::SystemTimeScope::Current, + valid_at_ms: None, + }); + assert_eq!(r.status, Status::Ok, "slice failed: {r:?}"); + decode_slice_rows(r.payload.as_ref()) + } + + fn row_coords(row: &Value) -> Vec { + match row { + Value::ArrayCell(ArrayCell { coords, .. }) => coords.clone(), + other => panic!("row not ArrayCell: {other:?}"), + } + } + + /// Over every bucket, the shard filter returns each cell exactly once. + #[test] + fn shard_buckets_partition_the_cells_of_a_full_range_array() { + let mut h = Harness::new(); + let s = schema_full_range_2d("t6_buckets"); + let aid = ArrayId::new(TenantId::new(1), "t6_buckets"); + h.open(&aid, &s, 0xB1); + let cells: Vec = (0..8) + .flat_map(|x| (0..8).map(move |y| cell(x, y, (x * 8 + y) as f64))) + .collect(); + h.put(&aid, cells, 1); + h.flush(&aid); + + let prefix_bits = 8u8; + let mut seen: Vec = Vec::new(); + let mut non_empty = 0usize; + for bucket in 0..(1u32 << prefix_bits) { + let rows = slice_in_bucket( + &mut h, + &aid, + Some(ShardBucket { + bucket, + prefix_bits, + }), + ); + if !rows.is_empty() { + non_empty += 1; + } + seen.extend(rows.iter().map(|r| format!("{:?}", row_coords(r)))); + } + assert_eq!( + seen.len(), + 64, + "every cell is returned by exactly one bucket" + ); + seen.sort(); + seen.dedup(); + assert_eq!(seen.len(), 64, "no cell is returned by two buckets"); + assert!(non_empty >= 32, "64 tiles land in only {non_empty} buckets"); + assert_eq!(slice_in_bucket(&mut h, &aid, None).len(), 64); + } + + /// Two tiles `2^bits` apart share one key, so they share one bucket, and + /// a read of that bucket returns both cells. + #[test] + fn tiles_that_share_a_key_route_together_and_both_cells_read_back() { + let mut h = Harness::new(); + let s = schema_full_range_2d("t6_collide"); + let aid = ArrayId::new(TenantId::new(1), "t6_collide"); + h.open(&aid, &s, 0xB2); + let far = 1i64 << nodedb_array::coord::bits_per_dim(2); + let near_coord = [CoordValue::Int64(3), CoordValue::Int64(5)]; + let far_coord = [CoordValue::Int64(3 + far), CoordValue::Int64(5)]; + let key = nodedb_array::tile::cell_tile_prefix(&s, &near_coord).unwrap(); + assert_eq!( + nodedb_array::tile::cell_tile_prefix(&s, &far_coord).unwrap(), + key, + "the fixture's two tiles share one key" + ); + h.put( + &aid, + vec![cell_sur(3, 5, 1.0, 7), cell_sur(3 + far, 5, 2.0, 8)], + 1, + ); + h.flush(&aid); + + let prefix_bits = 8u8; + let owner = ShardBucket { + bucket: nodedb_types::array_route::prefix_bucket(key, prefix_bits), + prefix_bits, + }; + let rows = slice_in_bucket(&mut h, &aid, Some(owner)); + assert_eq!( + rows.len(), + 2, + "both cells of the shared key read back: {rows:?}" + ); + let other = ShardBucket { + bucket: (owner.bucket + 1) % (1u32 << prefix_bits), + prefix_bits, + }; + assert!(slice_in_bucket(&mut h, &aid, Some(other)).is_empty()); + } } diff --git a/nodedb/src/data/executor/handlers/snapshot/capture_arrays.rs b/nodedb/src/data/executor/handlers/snapshot/capture_arrays.rs index ab14cad02..c216c1d66 100644 --- a/nodedb/src/data/executor/handlers/snapshot/capture_arrays.rs +++ b/nodedb/src/data/executor/handlers/snapshot/capture_arrays.rs @@ -12,7 +12,7 @@ use std::sync::Arc; use nodedb_array::schema::ArraySchema; use nodedb_array::tile::cell_tile_prefix; -use nodedb_cluster::distributed_array::array_vshard_for_tile; +use nodedb_types::array_route::array_vshard_for_tile; use crate::data::executor::core_loop::CoreLoop; use crate::engine::array::export::ArrayCellVersion; @@ -81,18 +81,18 @@ fn by_vshard( ) -> crate::Result>> { let mut grouped: BTreeMap> = BTreeMap::new(); for version in versions { - let prefix = match cell_tile_prefix(schema, &version.coord) { - Ok(prefix) => prefix, + let tile_key = match cell_tile_prefix(schema, &version.coord) { + Ok(tile_key) => tile_key, Err(_) if version.payload.is_none() => continue, Err(e) => { return Err(capture_error( array, - format!("Hilbert prefix of a live cell: {e}"), + format!("tile key of a live cell: {e}"), )); } }; - let vshard = array_vshard_for_tile(prefix, prefix_bits) - .map_err(|e| capture_error(array, format!("vShard of Hilbert prefix {prefix}: {e}")))?; + let vshard = array_vshard_for_tile(tile_key, prefix_bits) + .map_err(|e| capture_error(array, format!("vShard of tile key {tile_key}: {e}")))?; grouped.entry(vshard).or_default().push(version); } Ok(grouped) @@ -152,12 +152,71 @@ mod tests { ); for (vshard, cells) in &grouped { for cell in cells { - let prefix = cell_tile_prefix(&s, &cell.coord).expect("prefix"); - assert_eq!(array_vshard_for_tile(prefix, 8).expect("route"), *vshard); + let tile_key = cell_tile_prefix(&s, &cell.coord).expect("tile key"); + assert_eq!(array_vshard_for_tile(tile_key, 8).expect("route"), *vshard); } } } + fn full_range_2d() -> ArraySchema { + let full = || Domain::new(DomainBound::Int64(i64::MIN), DomainBound::Int64(i64::MAX)); + ArraySchemaBuilder::new("open") + .dim(DimSpec::new("x", DimType::Int64, full())) + .dim(DimSpec::new("y", DimType::Int64, full())) + .attr(AttrSpec::new("v", AttrType::Int64, true)) + .tile_extents(vec![1, 1]) + .build() + .expect("schema") + } + + fn live_2d(x: i64, y: i64) -> ArrayCellVersion { + ArrayCellVersion { + coord: vec![CoordValue::Int64(x), CoordValue::Int64(y)], + ..live(x) + } + } + + /// Cells near the origin of a full-range 2-D array spread over vShards, + /// and a snapshot captured for a group of vShards installs into exactly + /// the cells that group's install replaces. + #[test] + fn a_full_range_array_captures_and_installs_by_the_same_route() { + let s = full_range_2d(); + let versions: Vec = (0..16) + .flat_map(|x| (0..16).map(move |y| live_2d(x, y))) + .collect(); + let grouped = by_vshard("open", &s, 8, versions.clone()).expect("group"); + assert!( + grouped.len() >= 64, + "256 tiles near the origin reach only {} vShards", + grouped.len() + ); + assert_eq!( + grouped.values().map(Vec::len).sum::(), + versions.len(), + "every version is captured once" + ); + + // A Raft group owning every other captured vShard. + let group: std::collections::HashSet = grouped.keys().copied().step_by(2).collect(); + let captured: Vec<&ArrayCellVersion> = grouped + .iter() + .filter(|(vshard, _)| group.contains(vshard)) + .flat_map(|(_, cells)| cells.iter()) + .collect(); + assert!(!captured.is_empty()); + for version in &versions { + let in_snapshot = captured.iter().any(|c| c.coord == version.coord); + let replaced = + super::super::install_arrays::routes_into_group(&s, 8, &group, &version.coord); + assert_eq!( + replaced, in_snapshot, + "coord {:?}: install replaces it iff the snapshot carries it", + version.coord + ); + } + } + #[test] fn an_unroutable_tombstone_is_left_out_and_a_live_cell_fails() { let s = schema(); diff --git a/nodedb/src/data/executor/handlers/snapshot/install_arrays.rs b/nodedb/src/data/executor/handlers/snapshot/install_arrays.rs index 63b16c4e0..4f8db8549 100644 --- a/nodedb/src/data/executor/handlers/snapshot/install_arrays.rs +++ b/nodedb/src/data/executor/handlers/snapshot/install_arrays.rs @@ -13,7 +13,7 @@ use std::sync::Arc; use nodedb_array::schema::ArraySchema; use nodedb_array::tile::cell_tile_prefix; use nodedb_array::types::coord::value::CoordValue; -use nodedb_cluster::distributed_array::array_vshard_for_tile; +use nodedb_types::array_route::array_vshard_for_tile; use crate::data::executor::core_loop::CoreLoop; use crate::engine::array::export::ArrayCellVersion; @@ -28,6 +28,21 @@ fn install_error(detail: String) -> crate::Error { /// `(database, tenant, array)`. type ArrayKey = (u64, u64, String); +/// Whether a cell write at `coord` routes to a vShard in `group`. The install +/// replaces exactly the cells this admits. A coordinate outside the schema's +/// domain has no route and is never replaced. +pub(super) fn routes_into_group( + schema: &ArraySchema, + prefix_bits: u8, + group: &HashSet, + coord: &[CoordValue], +) -> bool { + cell_tile_prefix(schema, coord) + .ok() + .and_then(|tile_key| array_vshard_for_tile(tile_key, prefix_bits).ok()) + .is_some_and(|vshard| group.contains(&vshard)) +} + impl CoreLoop { /// Replace the array cells of `vshards` on this core with `blobs`. /// Returns the versions installed. @@ -92,12 +107,8 @@ impl CoreLoop { let schema = Arc::new(schema); let routes = Arc::clone(&schema); let prefix_bits = entry.prefix_bits; - let replaced = |coord: &[CoordValue]| { - cell_tile_prefix(&routes, coord) - .ok() - .and_then(|prefix| array_vshard_for_tile(prefix, prefix_bits).ok()) - .is_some_and(|vshard| group.contains(&vshard)) - }; + let replaced = + |coord: &[CoordValue]| routes_into_group(&routes, prefix_bits, &group, coord); self.array_engine .replace_cells( &entry.array_id, diff --git a/nodedb/src/data/executor/handlers/transaction/overlay/array_merge.rs b/nodedb/src/data/executor/handlers/transaction/overlay/array_merge.rs index c7bbc056a..3dfb7da53 100644 --- a/nodedb/src/data/executor/handlers/transaction/overlay/array_merge.rs +++ b/nodedb/src/data/executor/handlers/transaction/overlay/array_merge.rs @@ -9,8 +9,8 @@ //! //! - base rows whose coordinate this transaction tombstoned OR re-put are //! dropped (a staged put shadows the base cell, last-writer-wins); -//! - staged puts are appended as synthetic tiles, one per Hilbert prefix, so -//! a shard `hilbert_range` filter, the slice window, the attribute +//! - staged puts are appended as synthetic tiles, one per tile key, so +//! a `shard_bucket` filter, the slice window, the attribute //! projection, the surrogate `cell_filter`, and the row cap all apply to //! staged cells exactly as they apply to base cells. //! @@ -130,8 +130,8 @@ fn merge_overlay( tiles.retain(|(_, tile)| tile.row_count() > 0); } - // Staged puts, grouped by the Hilbert prefix each would land in so a - // shard-range filter treats them like base tiles. + // Staged puts, grouped by the tile key each would land in so a shard + // bucket filter treats them like base tiles. let mut by_prefix: HashMap> = HashMap::new(); for (coord, put) in overlay.staged_cells(array_id) { if let Some(vt) = valid_at_ms From 3d6d6aba1f500c52af00ea1518469d4581685d71 Mon Sep 17 00:00:00 2001 From: Farhan Syah Date: Sun, 11 Oct 2026 13:53:42 +0800 Subject: [PATCH 2/5] feat(wal): log resolved rows for KV and vector predicate writes A KV predicate UPDATE/DELETE and a vector-primary write with a row predicate or patch expression scan state that depends on the clock or on sidecars newer than the checkpoint being replayed. The proposer now resolves them and the log record names the rows and their values, so replay never evaluates a predicate again. --- nodedb/src/control/server/wal_dispatch/mod.rs | 3 + .../server/wal_dispatch/vector/append.rs | 6 +- .../server/wal_dispatch/vector/encode.rs | 41 ++--- .../server/wal_dispatch/vector/logged.rs | 74 ++++++++ .../control/server/wal_dispatch/vector/mod.rs | 4 + .../control/server/wal_dispatch_kv/append.rs | 40 ++--- .../control/server/wal_dispatch_kv/encode.rs | 27 --- .../control/wal_replication/decode/entry.rs | 4 +- .../wal_replication/decode/entry_kv.rs | 28 --- .../src/control/wal_replication/decode/kv.rs | 34 ---- .../control/wal_replication/decode/vector.rs | 8 +- .../wal_replication/decode/vector_direct.rs | 16 +- .../control/wal_replication/encode/entry.rs | 3 +- .../wal_replication/encode/entry_kv.rs | 66 ++----- .../src/control/wal_replication/encode/kv.rs | 31 ---- .../control/wal_replication/encode/vector.rs | 90 +++++++--- .../transaction_redo/chunks/owed.rs | 9 +- .../wal_replication/types/replicated_write.rs | 44 +---- nodedb/src/control/write_resolve/kv.rs | 19 +- nodedb/src/control/write_resolve/vector.rs | 154 ++++++++++++---- .../data/executor/handlers/kv/crud/delete.rs | 2 + .../src/data/executor/handlers/kv/dispatch.rs | 54 ++---- .../executor/handlers/kv/predicate/apply.rs | 166 ------------------ .../executor/handlers/kv/predicate/ctx.rs | 21 +++ .../executor/handlers/kv/predicate/mod.rs | 6 +- .../handlers/kv/resolve/predicate_ops.rs | 4 +- .../stage_write/stage_kv_predicate.rs | 4 +- .../handlers/transaction/write_version_kv.rs | 12 +- .../executor/handlers/vector_direct_delete.rs | 9 +- .../executor/handlers/vector_direct_row.rs | 95 +++++++++- .../handlers/vector_direct_test_rows.rs | 58 ++++++ .../handlers/vector_direct_truncate.rs | 41 +++-- nodedb/src/data/executor/wal_replay/kv.rs | 30 +--- .../data/executor/wal_replay_kv_predicate.rs | 144 --------------- .../data/executor/wal_replay_vector_direct.rs | 44 ++++- 35 files changed, 621 insertions(+), 770 deletions(-) create mode 100644 nodedb/src/control/server/wal_dispatch/vector/logged.rs delete mode 100644 nodedb/src/data/executor/handlers/kv/predicate/apply.rs create mode 100644 nodedb/src/data/executor/handlers/kv/predicate/ctx.rs create mode 100644 nodedb/src/data/executor/handlers/vector_direct_test_rows.rs delete mode 100644 nodedb/src/data/executor/wal_replay_kv_predicate.rs diff --git a/nodedb/src/control/server/wal_dispatch/mod.rs b/nodedb/src/control/server/wal_dispatch/mod.rs index fb8b498e8..e57251fc3 100644 --- a/nodedb/src/control/server/wal_dispatch/mod.rs +++ b/nodedb/src/control/server/wal_dispatch/mod.rs @@ -37,6 +37,8 @@ pub(crate) use text::encode_text_op_record; pub(crate) use timeseries::{ TimeseriesIngestRecord, encode_columnar_truncate_payload, encode_timeseries_ingest_payload, }; +#[cfg(test)] +pub(crate) use vector::encode_vector_direct_delete_payload; pub(crate) use vector::{ VectorDirectDeleteRecord, VectorDirectTruncateRecord, VectorDirectUpdateRecord, VectorDirectUpsertRecord, VectorResolvedDirectWritePayload, VectorResolvedDirectWriteRecord, @@ -46,6 +48,7 @@ pub(crate) use vector::{ encode_vector_delete_payload, encode_vector_direct_truncate_payload, encode_vector_put_payload, encode_vector_resolved_direct_write_payload, }; +pub(crate) use vector::{check_logged_patch, logged_rows}; pub(crate) use super::wal_dispatch_fts_spatial::{ encode_spatial_delete_payload, encode_spatial_put_payload, diff --git a/nodedb/src/control/server/wal_dispatch/vector/append.rs b/nodedb/src/control/server/wal_dispatch/vector/append.rs index 46487dcfd..304d76059 100644 --- a/nodedb/src/control/server/wal_dispatch/vector/append.rs +++ b/nodedb/src/control/server/wal_dispatch/vector/append.rs @@ -231,7 +231,8 @@ pub(crate) fn wal_append_vector_op( rls_filters: _, rls_write_check: _, } => { - let entry = encode_vector_direct_delete_payload(collection.as_str(), field, targets)?; + let rows = super::logged::logged_rows(collection.as_str(), targets)?; + let entry = encode_vector_direct_delete_payload(collection.as_str(), field, rows)?; Some(wal.append_vector_direct_delete(tenant_id, vshard_id, database_id, &entry)?) } // `restart_identity` is applied by the Control Plane after commit, @@ -258,10 +259,11 @@ pub(crate) fn wal_append_vector_op( rls_filters: _, rls_write_check: _, } => { + let rows = super::logged::logged_rows(collection.as_str(), targets)?; let entry = encode_vector_direct_update_payload(VectorDirectUpdatePayload { collection: collection.as_str(), field, - targets, + rows, new_vector: new_vector.as_deref(), payload_patch, quantization: *quantization, diff --git a/nodedb/src/control/server/wal_dispatch/vector/encode.rs b/nodedb/src/control/server/wal_dispatch/vector/encode.rs index 3e413f6d9..586717517 100644 --- a/nodedb/src/control/server/wal_dispatch/vector/encode.rs +++ b/nodedb/src/control/server/wal_dispatch/vector/encode.rs @@ -152,6 +152,7 @@ pub(crate) fn encode_vector_direct_upsert_payload( intent, on_conflict_updates, } = args; + super::logged::check_logged_patch(collection, on_conflict_updates)?; zerompk::to_msgpack_vec(&( collection, field, @@ -173,28 +174,22 @@ pub(crate) fn encode_vector_direct_upsert_payload( /// Encode the payload of a `VectorDirectDelete` WAL record. /// -/// Produces the 3-element shape `(collection, field, targets)`. Replay -/// resolves `targets` against the state it rebuilt; a surrogate with no node -/// is skipped, so re-applying over a restored checkpoint is idempotent. +/// Produces the 3-element shape `(collection, field, rows)`: the rows the +/// delete removed, never a predicate. Replay removes each row whose node it +/// rebuilt, so re-applying over a restored checkpoint is idempotent. pub(crate) fn encode_vector_direct_delete_payload( collection: &str, field: &str, - targets: &nodedb_physical::physical_plan::VectorWriteTargets, + rows: &[nodedb_types::Surrogate], ) -> crate::Result> { - zerompk::to_msgpack_vec(&(collection, field, targets)).map_err(|e| { - crate::Error::Serialization { - format: "msgpack".into(), - detail: format!("wal vector direct delete: {e}"), - } + zerompk::to_msgpack_vec(&(collection, field, rows)).map_err(|e| crate::Error::Serialization { + format: "msgpack".into(), + detail: format!("wal vector direct delete: {e}"), }) } /// The decoded form of a `VectorDirectDelete` record. -pub(crate) type VectorDirectDeleteRecord = ( - String, - String, - nodedb_physical::physical_plan::VectorWriteTargets, -); +pub(crate) type VectorDirectDeleteRecord = (String, String, Vec); /// Encode a `VectorDirectTruncate` record: `(collection, field)`. The /// `restart_identity` flag is a Control-Plane sequence concern and never @@ -216,7 +211,8 @@ pub(crate) type VectorDirectTruncateRecord = (String, String); pub(crate) struct VectorDirectUpdatePayload<'a> { pub collection: &'a str, pub field: &'a str, - pub targets: &'a nodedb_physical::physical_plan::VectorWriteTargets, + /// The rows the update rewrote, never a predicate. + pub rows: &'a [nodedb_types::Surrogate], pub new_vector: Option<&'a [f32]>, pub payload_patch: &'a [(String, nodedb_physical::physical_plan::UpdateValue)], pub quantization: nodedb_types::VectorQuantization, @@ -229,7 +225,7 @@ pub(crate) struct VectorDirectUpdatePayload<'a> { pub(crate) type VectorDirectUpdateRecord = ( String, String, - nodedb_physical::physical_plan::VectorWriteTargets, + Vec, Option>, Vec<(String, nodedb_physical::physical_plan::UpdateValue)>, nodedb_types::VectorQuantization, @@ -239,27 +235,28 @@ pub(crate) type VectorDirectUpdateRecord = ( /// Encode the payload of a `VectorDirectUpdate` WAL record. /// -/// Produces the 8-element shape `(collection, field, targets, new_vector, -/// payload_patch, quantization, storage_dtype, payload_indexes)`. Replay -/// re-runs the update handler, so a row the update already reached before -/// the crash is patched to the same image again. +/// Produces the 8-element shape `(collection, field, rows, new_vector, +/// payload_patch, quantization, storage_dtype, payload_indexes)`. The patch +/// holds literals only, so a row the update already reached before the +/// crash is patched to the same image again. pub(crate) fn encode_vector_direct_update_payload( args: VectorDirectUpdatePayload<'_>, ) -> crate::Result> { let VectorDirectUpdatePayload { collection, field, - targets, + rows, new_vector, payload_patch, quantization, storage_dtype, payload_indexes, } = args; + super::logged::check_logged_patch(collection, payload_patch)?; zerompk::to_msgpack_vec(&( collection, field, - targets, + rows, new_vector, payload_patch, quantization, diff --git a/nodedb/src/control/server/wal_dispatch/vector/logged.rs b/nodedb/src/control/server/wal_dispatch/vector/logged.rs new file mode 100644 index 000000000..2506964b4 --- /dev/null +++ b/nodedb/src/control/server/wal_dispatch/vector/logged.rs @@ -0,0 +1,74 @@ +// SPDX-License-Identifier: BUSL-1.1 + +//! What a vector-primary direct write may put in a log record. +//! +//! The sidecar store is written through, while the vector index restores +//! from its checkpoint. Replay therefore runs against sidecars newer than +//! the index. A record names its rows and their values, never a row +//! predicate or a patch expression replay evaluates again. Such a write +//! resolves to rows first (`control::write_resolve`). + +use nodedb_physical::physical_plan::{UpdateValue, VectorWriteTargets}; +use nodedb_types::Surrogate; + +/// The rows `targets` names. Refuses a row predicate. +pub(crate) fn logged_rows<'a>( + collection: &str, + targets: &'a VectorWriteTargets, +) -> crate::Result<&'a [Surrogate]> { + match targets { + VectorWriteTargets::Surrogates(rows) => Ok(rows), + VectorWriteTargets::Predicate(_) => Err(crate::Error::Internal { + detail: format!( + "internal invariant break: a vector-primary write on '{collection}' reached \ + the log with a row predicate; write_resolve resolves it to rows first" + ), + }), + } +} + +/// Refuse a `patch` that holds an expression. +pub(crate) fn check_logged_patch( + collection: &str, + patch: &[(String, UpdateValue)], +) -> crate::Result<()> { + for (column, value) in patch { + match value { + UpdateValue::Literal(_) => {} + UpdateValue::Expr(_) => { + return Err(crate::Error::Internal { + detail: format!( + "internal invariant break: a vector-primary write on '{collection}' \ + reached the log with an expression for '{column}'; write_resolve \ + resolves it to a value first" + ), + }); + } + } + } + Ok(()) +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn a_row_predicate_is_refused_and_rows_pass() { + let rows = VectorWriteTargets::Surrogates(vec![Surrogate::new(3)]); + assert_eq!(logged_rows("v", &rows).expect("rows"), &[Surrogate::new(3)]); + let predicate = VectorWriteTargets::Predicate(Vec::new()); + assert!(logged_rows("v", &predicate).is_err()); + } + + #[test] + fn an_expression_is_refused_and_a_literal_passes() { + let literal = [("n".to_string(), UpdateValue::Literal(vec![0x01]))]; + assert!(check_logged_patch("v", &literal).is_ok()); + let expression = [( + "n".to_string(), + UpdateValue::Expr(nodedb_query::expr::SqlExpr::Column("n".into())), + )]; + assert!(check_logged_patch("v", &expression).is_err()); + } +} diff --git a/nodedb/src/control/server/wal_dispatch/vector/mod.rs b/nodedb/src/control/server/wal_dispatch/vector/mod.rs index dcd13641b..49572b1be 100644 --- a/nodedb/src/control/server/wal_dispatch/vector/mod.rs +++ b/nodedb/src/control/server/wal_dispatch/vector/mod.rs @@ -5,8 +5,11 @@ pub mod append; pub mod encode; +pub mod logged; pub(crate) use append::wal_append_vector_op; +#[cfg(test)] +pub(crate) use encode::encode_vector_direct_delete_payload; pub(crate) use encode::{ VectorDirectDeleteRecord, VectorDirectTruncateRecord, VectorDirectUpdateRecord, VectorDirectUpsertRecord, encode_multi_vector_delete_payload, encode_multi_vector_put_payload, @@ -18,3 +21,4 @@ pub(crate) use encode::{ VectorResolvedDirectWritePayload, VectorResolvedDirectWriteRecord, encode_vector_resolved_direct_write_payload, }; +pub(crate) use logged::{check_logged_patch, logged_rows}; diff --git a/nodedb/src/control/server/wal_dispatch_kv/append.rs b/nodedb/src/control/server/wal_dispatch_kv/append.rs index 39d89117c..47c289bdf 100644 --- a/nodedb/src/control/server/wal_dispatch_kv/append.rs +++ b/nodedb/src/control/server/wal_dispatch_kv/append.rs @@ -10,10 +10,9 @@ use super::encode::{ KvIncrRecord, KvRegisterSortedIndexFields, KvTransferFields, encode_kv_batch_put, encode_kv_cas, encode_kv_delete, encode_kv_drop_index, encode_kv_drop_sorted_index, encode_kv_expire, encode_kv_field_set, encode_kv_getset, encode_kv_incr, encode_kv_incr_float, - encode_kv_insert_on_conflict_update, encode_kv_persist, encode_kv_predicate_delete, - encode_kv_predicate_update, encode_kv_put, encode_kv_register_index, - encode_kv_register_sorted_index, encode_kv_rewrite, encode_kv_transfer, - encode_kv_transfer_item, encode_kv_truncate, + encode_kv_insert_on_conflict_update, encode_kv_persist, encode_kv_put, + encode_kv_register_index, encode_kv_register_sorted_index, encode_kv_rewrite, + encode_kv_transfer, encode_kv_transfer_item, encode_kv_truncate, }; /// Outcome of [`wal_append_kv_op`]: the allocated WAL LSN and, for a TTL-bearing @@ -314,29 +313,16 @@ pub fn wal_append_kv_op( let entry = encode_kv_truncate(collection.as_str())?; Some(wal.append_delete(tenant_id, vshard_id, database_id, &entry)?) } - // The predicate is the durable record; replay re-executes it via the live handler. - KvOp::PredicateUpdate { - collection, - filters, - updates, - // Per-request authorization input, not part of the durable image. - rls_write_check: _, - // Per-request reply projection, not part of the durable image. - returning: _, - rls_filters: _, - } => { - let entry = encode_kv_predicate_update(collection.as_str(), filters, updates)?; - Some(wal.append_put(tenant_id, vshard_id, database_id, &entry)?) - } - KvOp::PredicateDelete { - collection, - filters, - rls_write_check: _, - returning: _, - rls_filters: _, - } => { - let entry = encode_kv_predicate_delete(collection.as_str(), filters)?; - Some(wal.append_delete(tenant_id, vshard_id, database_id, &entry)?) + // A row set replay scans for itself depends on its clock (TTL expiry), + // so a predicate write resolves to a mutation list first. + KvOp::PredicateUpdate { collection, .. } | KvOp::PredicateDelete { collection, .. } => { + return Err(crate::Error::Internal { + detail: format!( + "internal invariant break: a kv predicate UPDATE/DELETE on '{}' reached \ + the WAL unresolved; write_resolve resolves it to a mutation list first", + collection.as_str() + ), + }); } KvOp::Transfer { collection, diff --git a/nodedb/src/control/server/wal_dispatch_kv/encode.rs b/nodedb/src/control/server/wal_dispatch_kv/encode.rs index be1d77d30..9bf958fcc 100644 --- a/nodedb/src/control/server/wal_dispatch_kv/encode.rs +++ b/nodedb/src/control/server/wal_dispatch_kv/encode.rs @@ -370,33 +370,6 @@ pub(crate) fn encode_kv_drop_sorted_index(index_name: &str) -> crate::Result)], -) -> crate::Result> { - encode( - "predicate update", - &("kv_predicate_update", collection, filters, updates), - ) -} - -/// Encode a `kv_predicate_delete` WAL payload: -/// `("kv_predicate_delete", collection, filters)`. See -/// [`encode_kv_predicate_update`] for why the predicate travels. -pub(crate) fn encode_kv_predicate_delete( - collection: &str, - filters: &[u8], -) -> crate::Result> { - encode( - "predicate delete", - &("kv_predicate_delete", collection, filters), - ) -} - /// Encode a `kv_truncate` WAL payload: `("kv_truncate", collection)`. pub(crate) fn encode_kv_truncate(collection: &str) -> crate::Result> { encode("truncate", &("kv_truncate", collection)) diff --git a/nodedb/src/control/wal_replication/decode/entry.rs b/nodedb/src/control/wal_replication/decode/entry.rs index 7bfdbb723..a107f2239 100644 --- a/nodedb/src/control/wal_replication/decode/entry.rs +++ b/nodedb/src/control/wal_replication/decode/entry.rs @@ -172,9 +172,7 @@ fn to_physical_plan( | ReplicatedWrite::KvFieldSet { .. } | ReplicatedWrite::KvTransfer { .. } | ReplicatedWrite::KvTransferItem { .. } - | ReplicatedWrite::KvResolvedWrite { .. } - | ReplicatedWrite::KvPredicateUpdate { .. } - | ReplicatedWrite::KvPredicateDelete { .. } => entry_kv::decode_arm(write), + | ReplicatedWrite::KvResolvedWrite { .. } => entry_kv::decode_arm(write), // A timeseries ingest carries the proposer's instant for its untimed // rows, installed on every replica as `resolved_now_ms`. ReplicatedWrite::TimeseriesIngest { diff --git a/nodedb/src/control/wal_replication/decode/entry_kv.rs b/nodedb/src/control/wal_replication/decode/entry_kv.rs index f777e5468..f803b07fc 100644 --- a/nodedb/src/control/wal_replication/decode/entry_kv.rs +++ b/nodedb/src/control/wal_replication/decode/entry_kv.rs @@ -264,34 +264,6 @@ pub(super) fn decode_arm(write: &ReplicatedWrite) -> crate::Result<(PhysicalPlan mutations, response_payload, } => kv_resolved::resolved_write(mutations, response_payload)?, - ReplicatedWrite::KvPredicateUpdate { - collection, - filters, - updates, - returning, - rls_filters, - } => kv::predicate_update( - collection, - filters, - updates, - ReturningFields { - returning: decode_returning(returning)?, - rls_filters, - }, - ), - ReplicatedWrite::KvPredicateDelete { - collection, - filters, - returning, - rls_filters, - } => kv::predicate_delete( - collection, - filters, - ReturningFields { - returning: decode_returning(returning)?, - rls_filters, - }, - ), ReplicatedWrite::KvTransferItem { source_collection, dest_collection, diff --git a/nodedb/src/control/wal_replication/decode/kv.rs b/nodedb/src/control/wal_replication/decode/kv.rs index 50ef00a6f..7c26fd7db 100644 --- a/nodedb/src/control/wal_replication/decode/kv.rs +++ b/nodedb/src/control/wal_replication/decode/kv.rs @@ -73,40 +73,6 @@ pub(super) fn delete( }) } -/// Reconstruct a KV predicate `UPDATE` plan. The predicate is re-scanned -/// against this node's own committed state — see `delete` above for why the -/// check slot is stamped rather than re-derived. -pub(super) fn predicate_update( - collection: &str, - filters: &[u8], - updates: &[(String, Vec)], - returning: ReturningFields<'_>, -) -> PhysicalPlan { - PhysicalPlan::Kv(KvOp::PredicateUpdate { - collection: nodedb_types::QualifiedCollection::from_stored(collection.to_owned()), - filters: filters.to_vec(), - updates: updates.to_vec(), - rls_write_check: RlsWriteCheck::already_decided_elsewhere(), - returning: returning.returning, - rls_filters: returning.rls_filters.to_vec(), - }) -} - -/// Reconstruct a KV predicate `DELETE` plan — see [`predicate_update`]. -pub(super) fn predicate_delete( - collection: &str, - filters: &[u8], - returning: ReturningFields<'_>, -) -> PhysicalPlan { - PhysicalPlan::Kv(KvOp::PredicateDelete { - collection: nodedb_types::QualifiedCollection::from_stored(collection.to_owned()), - filters: filters.to_vec(), - rls_write_check: RlsWriteCheck::already_decided_elsewhere(), - returning: returning.returning, - rls_filters: returning.rls_filters.to_vec(), - }) -} - pub(super) fn insert( collection: &str, key: &[u8], diff --git a/nodedb/src/control/wal_replication/decode/vector.rs b/nodedb/src/control/wal_replication/decode/vector.rs index bcc69acb9..170e26c9e 100644 --- a/nodedb/src/control/wal_replication/decode/vector.rs +++ b/nodedb/src/control/wal_replication/decode/vector.rs @@ -157,13 +157,13 @@ pub(super) fn decode_arm(write: &ReplicatedWrite) -> crate::Result ReplicatedWrite::VectorDirectDelete { collection, field, - targets, + rows, returning, rls_filters, } => Ok(super::vector_direct::direct_delete( collection, field, - targets, + rows, decode_sync_engines::decode_returning(returning)?, rls_filters, )), @@ -179,7 +179,7 @@ pub(super) fn decode_arm(write: &ReplicatedWrite) -> crate::Result ReplicatedWrite::VectorDirectUpdate { collection, field, - targets, + rows, new_vector, payload_patch, quantization, @@ -191,7 +191,7 @@ pub(super) fn decode_arm(write: &ReplicatedWrite) -> crate::Result super::vector_direct::DirectUpdateFields { collection, field, - targets, + rows, new_vector, payload_patch, quantization: *quantization, diff --git a/nodedb/src/control/wal_replication/decode/vector_direct.rs b/nodedb/src/control/wal_replication/decode/vector_direct.rs index ac491bacb..5808037d9 100644 --- a/nodedb/src/control/wal_replication/decode/vector_direct.rs +++ b/nodedb/src/control/wal_replication/decode/vector_direct.rs @@ -4,11 +4,9 @@ //! `PhysicalPlan::Vector`. //! //! Every surrogate is rebuilt verbatim from the record; `entry.rs` binds the -//! insert family to its primary key afterwards. `DELETE` / `UPDATE` carry -//! their targets verbatim: a point-targeted write names the leader's -//! surrogates, which every replica bound when the rows were inserted; a -//! predicate-targeted write re-resolves the predicate against the replica's -//! own sidecar rows, exactly as `KvPredicateDelete` does. +//! insert family to its primary key afterwards. `DELETE` / `UPDATE` name the +//! leader's surrogates, which every replica bound when the rows were +//! inserted. A row predicate resolves on the proposer, never here. use nodedb_physical::physical_plan::{ ReturningSpec, UpdateValue, VectorDirectWriteIntent, VectorOp, VectorResolvedMutation, @@ -22,14 +20,14 @@ use crate::bridge::envelope::PhysicalPlan; pub(super) fn direct_delete( collection: &str, field: &str, - targets: &VectorWriteTargets, + rows: &[Surrogate], returning: Option, rls_filters: &[u8], ) -> PhysicalPlan { PhysicalPlan::Vector(VectorOp::DirectDelete { collection: nodedb_types::QualifiedCollection::from_stored(collection.to_owned()), field: field.to_owned(), - targets: targets.clone(), + targets: VectorWriteTargets::Surrogates(rows.to_vec()), returning, rls_filters: rls_filters.to_vec(), // A write-policy collection never reaches this shape: the leader @@ -56,7 +54,7 @@ pub(super) fn direct_truncate( pub(super) struct DirectUpdateFields<'a> { pub(super) collection: &'a str, pub(super) field: &'a str, - pub(super) targets: &'a VectorWriteTargets, + pub(super) rows: &'a [Surrogate], pub(super) new_vector: &'a Option>, pub(super) payload_patch: &'a [(String, UpdateValue)], pub(super) quantization: nodedb_types::VectorQuantization, @@ -71,7 +69,7 @@ pub(super) fn direct_update(f: DirectUpdateFields<'_>) -> PhysicalPlan { PhysicalPlan::Vector(VectorOp::DirectUpdate { collection: nodedb_types::QualifiedCollection::from_stored(f.collection.to_owned()), field: f.field.to_owned(), - targets: f.targets.clone(), + targets: VectorWriteTargets::Surrogates(f.rows.to_vec()), new_vector: f.new_vector.clone(), payload_patch: f.payload_patch.to_vec(), quantization: f.quantization, diff --git a/nodedb/src/control/wal_replication/encode/entry.rs b/nodedb/src/control/wal_replication/encode/entry.rs index a9498560e..69fb9f78c 100644 --- a/nodedb/src/control/wal_replication/encode/entry.rs +++ b/nodedb/src/control/wal_replication/encode/entry.rs @@ -52,8 +52,9 @@ pub fn to_replicated_entry( PhysicalPlan::Document(op) => entry_document::document_write(op), // Fallible: a governed predicate DML refuses rather than encode it bare. PhysicalPlan::Kv(op) => entry_kv::kv_write(op)?, + // Fallible: a row predicate or a patch expression refuses the write. + PhysicalPlan::Vector(op) => vector::encode(op)?, // Exhaustive over their op enums — see their module docs. - PhysicalPlan::Vector(op) => vector::encode(op), PhysicalPlan::Crdt(op) => crdt::encode(op), PhysicalPlan::Graph(op) => entry_graph::graph_write(op), // Fallible for the same reason as the Kv arm above. diff --git a/nodedb/src/control/wal_replication/encode/entry_kv.rs b/nodedb/src/control/wal_replication/encode/entry_kv.rs index 28f42808e..8010a0904 100644 --- a/nodedb/src/control/wal_replication/encode/entry_kv.rs +++ b/nodedb/src/control/wal_replication/encode/entry_kv.rs @@ -13,8 +13,8 @@ use super::kv::{WirePut, WireReturning}; use nodedb_physical::physical_plan::KvOp; /// Encode a `KvOp` write variant, `Ok(None)` when not a single-shard replicated -/// write, or an error when it can't replicate safely. `PredicateUpdate`/ -/// `PredicateDelete` refuse on `rls_write_check.has_predicate()`. +/// write, or an error when it can't replicate safely. `PredicateUpdate` / +/// `PredicateDelete` always refuse: they resolve to a mutation list first. pub(super) fn kv_write(op: &KvOp) -> crate::Result> { Ok(Some(match op { KvOp::Put { @@ -304,43 +304,10 @@ pub(super) fn kv_write(op: &KvOp) -> crate::Result> { rls_write_check: _, } => kv::resolved_write(mutations, response_payload), - // With no write policy, each replica re-scans the predicate — deterministic - // replay. A restricting policy refuses; see `refuse_governed_predicate_dml`. - KvOp::PredicateUpdate { - collection, - filters, - updates, - rls_write_check, - returning, - rls_filters, - } => { - refuse_governed_predicate_dml(collection.as_str(), rls_write_check)?; - kv::predicate_update( - collection.as_str(), - filters, - updates, - WireReturning { - returning, - rls_filters, - }, - ) - } - KvOp::PredicateDelete { - collection, - filters, - rls_write_check, - returning, - rls_filters, - } => { - refuse_governed_predicate_dml(collection.as_str(), rls_write_check)?; - kv::predicate_delete( - collection.as_str(), - filters, - WireReturning { - returning, - rls_filters, - }, - ) + // A row set a replica scans for itself depends on its clock (TTL + // expiry), so a predicate write resolves to rows first. + KvOp::PredicateUpdate { collection, .. } | KvOp::PredicateDelete { collection, .. } => { + return Err(refuse_predicate_dml(collection.as_str())); } // Not a write — reads/scans/sorted-index queries. `ResolveWrite` mutates @@ -364,19 +331,12 @@ pub(super) fn kv_write(op: &KvOp) -> crate::Result> { /// Refuse a KV predicate UPDATE/DELETE on a governed collection: a follower has /// no writing identity to evaluate the predicate. Must resolve to /// `KvOp::ResolvedWrite` before proposing. -fn refuse_governed_predicate_dml( - collection: &str, - rls_write_check: &nodedb_types::RlsWriteCheck, -) -> crate::Result<()> { - if rls_write_check.has_predicate() { - return Err(crate::Error::PlanError { - detail: format!( - "kv predicate UPDATE/DELETE on '{collection}' cannot be replicated as a \ - predicate because it carries an RLS write policy: a follower has no writing \ - identity to evaluate the predicate against. It must be resolved to a concrete \ - mutation list before it is proposed." - ), - }); +fn refuse_predicate_dml(collection: &str) -> crate::Error { + crate::Error::Internal { + detail: format!( + "internal invariant break: a kv predicate UPDATE/DELETE on '{collection}' reached \ + the Raft propose path unresolved; write_resolve resolves it to a mutation list \ + first" + ), } - Ok(()) } diff --git a/nodedb/src/control/wal_replication/encode/kv.rs b/nodedb/src/control/wal_replication/encode/kv.rs index ab5991c96..608281f8a 100644 --- a/nodedb/src/control/wal_replication/encode/kv.rs +++ b/nodedb/src/control/wal_replication/encode/kv.rs @@ -61,37 +61,6 @@ pub(super) fn put( } } -/// Encode a KV predicate `UPDATE`: the predicate itself, never a row set. Only -/// reached for a collection with no write policy. -pub(super) fn predicate_update( - collection: &str, - filters: &[u8], - updates: &[(String, Vec)], - returning: WireReturning<'_>, -) -> ReplicatedWrite { - ReplicatedWrite::KvPredicateUpdate { - collection: collection.to_owned(), - filters: filters.to_vec(), - updates: updates.to_vec(), - returning: encode_returning(returning.returning), - rls_filters: returning.rls_filters.to_vec(), - } -} - -/// Encode a KV predicate `DELETE` — see [`predicate_update`]. -pub(super) fn predicate_delete( - collection: &str, - filters: &[u8], - returning: WireReturning<'_>, -) -> ReplicatedWrite { - ReplicatedWrite::KvPredicateDelete { - collection: collection.to_owned(), - filters: filters.to_vec(), - returning: encode_returning(returning.returning), - rls_filters: returning.rls_filters.to_vec(), - } -} - pub(super) fn delete( collection: &str, keys: &[Vec], diff --git a/nodedb/src/control/wal_replication/encode/vector.rs b/nodedb/src/control/wal_replication/encode/vector.rs index ab9f40c97..cf1958607 100644 --- a/nodedb/src/control/wal_replication/encode/vector.rs +++ b/nodedb/src/control/wal_replication/encode/vector.rs @@ -8,6 +8,8 @@ use nodedb_physical::physical_plan::{ }; use nodedb_types::Surrogate; +use crate::control::server::wal_dispatch::{check_logged_patch, logged_rows}; + /// Encode a `VectorOp` write variant into its `ReplicatedWrite` wire shape. /// /// Returns `None` for the read / DDL-Alter variants (`Search`, @@ -15,8 +17,11 @@ use nodedb_types::Surrogate; /// `SparseSearch`, `MultiVectorScoreSearch`) — none of those are replicated. /// Exhaustive over `VectorOp` (not a catch-all): a new variant forces an /// explicit decision here instead of silently falling through. -pub(super) fn encode(op: &VectorOp) -> Option { - Some(match op { +/// +/// A direct write names its rows and values. A row predicate or a patch +/// expression is refused: `write_resolve` resolves it first. +pub(super) fn encode(op: &VectorOp) -> crate::Result> { + Ok(Some(match op { VectorOp::Insert { collection, vector, @@ -158,7 +163,7 @@ pub(super) fn encode(op: &VectorOp) -> Option { on_conflict_updates, returning, rls_filters, - }), + })?, VectorOp::DirectInsert { collection, field, @@ -185,7 +190,7 @@ pub(super) fn encode(op: &VectorOp) -> Option { on_conflict_updates: &[], returning, rls_filters, - }), + })?, VectorOp::DirectInsertIfAbsent { collection, field, @@ -212,7 +217,7 @@ pub(super) fn encode(op: &VectorOp) -> Option { on_conflict_updates: &[], returning, rls_filters, - }), + })?, VectorOp::DirectDelete { collection, field, @@ -223,7 +228,7 @@ pub(super) fn encode(op: &VectorOp) -> Option { } => ReplicatedWrite::VectorDirectDelete { collection: collection.as_str().to_owned(), field: field.to_owned(), - targets: targets.clone(), + rows: logged_rows(collection.as_str(), targets)?.to_vec(), returning: super::entry::encode_returning(returning), rls_filters: rls_filters.clone(), }, @@ -248,18 +253,21 @@ pub(super) fn encode(op: &VectorOp) -> Option { returning, rls_filters, rls_write_check: _, - } => ReplicatedWrite::VectorDirectUpdate { - collection: collection.as_str().to_owned(), - field: field.to_owned(), - targets: targets.clone(), - new_vector: new_vector.clone(), - payload_patch: payload_patch.clone(), - quantization: *quantization, - storage_dtype: *storage_dtype, - payload_indexes: payload_indexes.clone(), - returning: super::entry::encode_returning(returning), - rls_filters: rls_filters.clone(), - }, + } => { + check_logged_patch(collection.as_str(), payload_patch)?; + ReplicatedWrite::VectorDirectUpdate { + collection: collection.as_str().to_owned(), + field: field.to_owned(), + rows: logged_rows(collection.as_str(), targets)?.to_vec(), + new_vector: new_vector.clone(), + payload_patch: payload_patch.clone(), + quantization: *quantization, + storage_dtype: *storage_dtype, + payload_indexes: payload_indexes.clone(), + returning: super::entry::encode_returning(returning), + rls_filters: rls_filters.clone(), + } + } VectorOp::DropIndex { collection, field_name, @@ -296,8 +304,8 @@ pub(super) fn encode(op: &VectorOp) -> Option { | VectorOp::CompactIndex { .. } | VectorOp::Rebuild { .. } | VectorOp::SparseSearch { .. } - | VectorOp::MultiVectorScoreSearch { .. } => return None, - }) + | VectorOp::MultiVectorScoreSearch { .. } => return Ok(None), + })) } /// Fields shared by the three insert-family ops, bundled so [`direct_write`] @@ -318,8 +326,9 @@ struct DirectWriteFields<'a> { rls_filters: &'a [u8], } -fn direct_write(f: DirectWriteFields<'_>) -> ReplicatedWrite { - ReplicatedWrite::DirectUpsert { +fn direct_write(f: DirectWriteFields<'_>) -> crate::Result { + check_logged_patch(f.collection, f.on_conflict_updates)?; + Ok(ReplicatedWrite::DirectUpsert { collection: f.collection.to_owned(), field: f.field.to_owned(), surrogate: f.surrogate.as_u32(), @@ -333,7 +342,7 @@ fn direct_write(f: DirectWriteFields<'_>) -> ReplicatedWrite { on_conflict_updates: f.on_conflict_updates.to_vec(), returning: super::entry::encode_returning(f.returning), rls_filters: f.rls_filters.to_vec(), - } + }) } pub(super) fn insert( @@ -560,6 +569,41 @@ mod tests { assert!(!rls_write_check.has_predicate()); } + /// A row predicate never reaches the wire, even with no write policy: + /// replay would evaluate it again. A delete naming rows replicates them. + #[test] + fn an_ungoverned_predicate_delete_is_refused_and_named_rows_replicate() { + let tenant = TenantId::new(1); + let vshard = VShardId::new(0); + let by_predicate = predicate_delete(nodedb_types::RlsWriteCheck::NoPolicyApplies); + assert!(to_replicated_entry(tenant, DatabaseId::DEFAULT, vshard, &by_predicate).is_err()); + + let by_rows = PhysicalPlan::Vector(VectorOp::DirectDelete { + collection: QualifiedCollection::new(DatabaseId::DEFAULT, "vecs"), + field: "emb".into(), + targets: nodedb_physical::physical_plan::VectorWriteTargets::Surrogates(vec![ + Surrogate::new(4), + ]), + returning: None, + rls_filters: Vec::new(), + rls_write_check: nodedb_types::RlsWriteCheck::NoPolicyApplies, + }); + let entry = to_replicated_entry(tenant, DatabaseId::DEFAULT, vshard, &by_rows) + .expect("encode must not error") + .expect("a delete naming rows must replicate"); + let (_, _, decoded, _) = + crate::control::wal_replication::decode_replicated_entry(&entry.to_bytes()) + .expect("decode") + .expect("a replicated entry"); + let PhysicalPlan::Vector(VectorOp::DirectDelete { targets, .. }) = decoded else { + panic!("decoded to the wrong shape"); + }; + assert_eq!( + targets, + nodedb_physical::physical_plan::VectorWriteTargets::Surrogates(vec![Surrogate::new(4)]) + ); + } + /// A vector-primary truncate replicates with its `restart_identity` /// flag, so every applying node resets the same sequences. #[test] diff --git a/nodedb/src/control/wal_replication/transaction_redo/chunks/owed.rs b/nodedb/src/control/wal_replication/transaction_redo/chunks/owed.rs index a3ee4317c..15abb36c5 100644 --- a/nodedb/src/control/wal_replication/transaction_redo/chunks/owed.rs +++ b/nodedb/src/control/wal_replication/transaction_redo/chunks/owed.rs @@ -7,13 +7,14 @@ //! hold a later chunk or the final entry of a dropped stream. Here it //! refuses, while every other replica installs the redo. So the drop marks //! the group owed, durably, before the streams release their WAL floor -//! holds: +//! holds. The apply loop also marks a group owed when its apply holds an +//! entry this replica cannot apply as its peers do. //! //! - The group's snapshot requirement holds on this node while it is owed. -//! Its replica refuses log entries, and its leader sends a snapshot. The -//! snapshot carries the group's open streams. +//! Its replica refuses log entries, never leads, and its leader sends a +//! snapshot. The snapshot carries the group's open streams. //! - A refusal of a chunk or final entry of an owed group holds the group's -//! durable floor instead of passing the entry. +//! apply at the entry instead of passing it. //! - A snapshot install of the group settles the debt. //! //! The set is a leaf lock: no other lock is taken while it is held. diff --git a/nodedb/src/control/wal_replication/types/replicated_write.rs b/nodedb/src/control/wal_replication/types/replicated_write.rs index b63fb766b..f0a1bbd55 100644 --- a/nodedb/src/control/wal_replication/types/replicated_write.rs +++ b/nodedb/src/control/wal_replication/types/replicated_write.rs @@ -15,9 +15,9 @@ use crate::wal::RedoStreamId; use nodedb_physical::physical_plan::document::MergeClauseOp; use nodedb_physical::physical_plan::{ ColumnarInsertIntent, CrdtWriteVerb, KvCounterShape, UpdateValue, VectorDirectWriteIntent, - VectorResolvedMutation, VectorWriteTargets, + VectorResolvedMutation, }; -use nodedb_types::{PayloadIndexKind, VectorQuantization, VectorStorageDtype}; +use nodedb_types::{PayloadIndexKind, Surrogate, VectorQuantization, VectorStorageDtype}; #[derive( Debug, @@ -874,36 +874,6 @@ pub enum ReplicatedWrite { response_payload: Vec, }, - /// KV predicate `UPDATE` on a collection with NO write policy — the - /// predicate travels and every replica re-scans it against its own state. - /// A write-policy collection never reaches this shape. - KvPredicateUpdate { - collection: String, - /// Serialized `Vec`. Empty matches every row. - filters: Vec, - updates: Vec<(String, Vec)>, - /// See `ReplicatedWrite::PointPut::returning`. - #[serde(default)] - returning: Option>, - /// See `ReplicatedWrite::PointPut::rls_filters`. - #[serde(default)] - rls_filters: Vec, - }, - - /// KV predicate `DELETE` on a collection with NO write policy — see - /// [`ReplicatedWrite::KvPredicateUpdate`]. - KvPredicateDelete { - collection: String, - /// Serialized `Vec`. Empty matches every row. - filters: Vec, - /// See `ReplicatedWrite::PointPut::returning`. - #[serde(default)] - returning: Option>, - /// See `ReplicatedWrite::PointPut::rls_filters`. - #[serde(default)] - rls_filters: Vec, - }, - /// Resolved form of a deferred document write (`PointUpdate`, /// `PointDelete`, `Upsert`, `BulkUpdate`, `BulkDelete`) on a write-policy /// collection: mutations and reply, already decided against the live @@ -915,12 +885,12 @@ pub enum ReplicatedWrite { response_payload: Vec, }, - /// Vector-primary `DELETE` on a collection with NO write policy — the - /// targets travel and every replica resolves them against its own state. + /// Vector-primary `DELETE` on a collection with NO write policy. It + /// names the rows it removes: a row predicate resolves on the proposer. VectorDirectDelete { collection: String, field: String, - targets: VectorWriteTargets, + rows: Vec, /// See `ReplicatedWrite::PointPut::returning`. #[serde(default)] returning: Option>, @@ -940,11 +910,11 @@ pub enum ReplicatedWrite { }, /// Vector-primary `UPDATE` on a collection with NO write policy — see - /// [`ReplicatedWrite::VectorDirectDelete`]. + /// [`ReplicatedWrite::VectorDirectDelete`]. The patch holds literals only. VectorDirectUpdate { collection: String, field: String, - targets: VectorWriteTargets, + rows: Vec, new_vector: Option>, payload_patch: Vec<(String, UpdateValue)>, quantization: VectorQuantization, diff --git a/nodedb/src/control/write_resolve/kv.rs b/nodedb/src/control/write_resolve/kv.rs index eff276603..2748d90e2 100644 --- a/nodedb/src/control/write_resolve/kv.rs +++ b/nodedb/src/control/write_resolve/kv.rs @@ -3,7 +3,8 @@ //! KV implementation of [`EngineWriteResolver`]. //! //! Resolves a governed state-dependent KV write (increment, CAS, field -//! merge, TTL, transfer, predicate UPDATE/DELETE) into `KvOp::ResolvedWrite` +//! merge, TTL, transfer), and every predicate UPDATE/DELETE, into +//! `KvOp::ResolvedWrite` //! — a mutation list plus response payload, since e.g. a `CAS` miss owes a //! reply while writing nothing. @@ -80,23 +81,17 @@ pub(super) fn resolver_for_kv_op(op: &KvOp) -> Option { if !rls_write_check.has_predicate() { return None; } collection } + // A predicate write always resolves: the row set it scans depends on + // the clock (TTL expiry), so a log record names the rows instead. + KvOp::PredicateUpdate { collection, .. } | KvOp::PredicateDelete { collection, .. } => { + collection + } // Both collections checked: an identity may give a row up but not // receive it — `sole_rls_write_check` would wave the write through. KvOp::TransferItem { diff --git a/nodedb/src/control/write_resolve/vector.rs b/nodedb/src/control/write_resolve/vector.rs index ab1a6735b..9e8c095e6 100644 --- a/nodedb/src/control/write_resolve/vector.rs +++ b/nodedb/src/control/write_resolve/vector.rs @@ -2,11 +2,15 @@ //! Vector-primary implementation of [`EngineWriteResolver`]. //! -//! Resolves a governed vector-primary write — `DirectDelete`, `DirectUpdate`, -//! or a `DirectUpsert` whose conflict patch is decided on the Data Plane — -//! into `VectorOp::ResolvedDirectWrite`: a row-mutation list plus the -//! response payload, each mutation carrying the sidecar it read so the apply -//! refuses a resolution that drifted. +//! Resolves a vector-primary write into `VectorOp::ResolvedDirectWrite`: a +//! row-mutation list plus the response payload, each mutation carrying the +//! sidecar it read so the apply refuses a resolution that drifted. +//! +//! A write resolves when it carries a live write predicate, a row predicate, +//! or a patch expression. The sidecar store is written through while the +//! index restores from its checkpoint, so replay runs against newer sidecars. +//! A log record therefore names rows and their values, never something +//! replay evaluates again. use async_trait::async_trait; use nodedb_types::{QualifiedCollection, RlsWriteCheck}; @@ -14,7 +18,9 @@ use nodedb_types::{QualifiedCollection, RlsWriteCheck}; use crate::bridge::envelope::{PhysicalPlan, Status}; use crate::control::maintenance::clone_materializer::dispatch_local; use crate::control::state::SharedState; -use nodedb_physical::physical_plan::{VectorOp, VectorResolveOutcome}; +use nodedb_physical::physical_plan::{ + UpdateValue, VectorOp, VectorResolveOutcome, VectorWriteTargets, +}; use super::resolved_rows::ResolvedRows; use super::resolver::{EngineWriteResolver, WriteResolveContext}; @@ -41,42 +47,78 @@ pub struct VectorWriteResolver { op: VectorOp, } -/// The resolver for `op`, or `None` when it carries no live write predicate. -/// Exhaustive over `VectorOp` — a new op fails to compile here. +/// Whether `targets` names rows by a predicate replay would evaluate again. +fn by_predicate(targets: &VectorWriteTargets) -> bool { + match targets { + VectorWriteTargets::Predicate(_) => true, + VectorWriteTargets::Surrogates(_) => false, + } +} + +/// Whether `patch` holds an expression replay would evaluate again. +fn has_expression(patch: &[(String, UpdateValue)]) -> bool { + patch.iter().any(|(_, value)| match value { + UpdateValue::Expr(_) => true, + UpdateValue::Literal(_) => false, + }) +} + +/// The resolver for `op`, or `None` when the op logs as it is. Exhaustive +/// over `VectorOp` — a new op fails to compile here. pub(super) fn resolver_for_vector_op(op: &VectorOp) -> Option> { let (collection, field, settings) = match op { VectorOp::DirectDelete { collection, field, + targets, rls_write_check, .. } => { - if !rls_write_check.has_predicate() { + if !rls_write_check.has_predicate() && !by_predicate(targets) { return None; } (collection, field, ResolvedIndexSettings::default()) } - // Only a conflict patch leaves an upsert's predicate live: a - // whole-row upsert is decided Control-Plane-side against its payload. VectorOp::DirectUpdate { collection, field, + targets, + payload_patch, quantization, storage_dtype, payload_indexes, rls_write_check, .. + } => { + if !rls_write_check.has_predicate() + && !by_predicate(targets) + && !has_expression(payload_patch) + { + return None; + } + ( + collection, + field, + ResolvedIndexSettings { + quantization: *quantization, + storage_dtype: *storage_dtype, + payload_indexes: payload_indexes.clone(), + }, + ) } - | VectorOp::DirectUpsert { + // A whole-row upsert is decided Control-Plane-side against its + // payload. A conflict patch is decided against the stored row. + VectorOp::DirectUpsert { collection, field, + on_conflict_updates, quantization, storage_dtype, payload_indexes, rls_write_check, .. } => { - if !rls_write_check.has_predicate() { + if !rls_write_check.has_predicate() && !has_expression(on_conflict_updates) { return None; } ( @@ -204,45 +246,81 @@ impl EngineWriteResolver for VectorWriteResolver { #[cfg(test)] mod tests { - use nodedb_physical::physical_plan::{VectorOp, VectorWriteTargets}; - use nodedb_types::{DatabaseId, QualifiedCollection, RlsWriteCheck}; + use nodedb_physical::physical_plan::{UpdateValue, VectorOp, VectorWriteTargets}; + use nodedb_types::{DatabaseId, QualifiedCollection, RlsWriteCheck, Surrogate}; use super::resolver_for_vector_op; use crate::bridge::envelope::PhysicalPlan; - fn predicate_delete(check: RlsWriteCheck) -> VectorOp { + fn delete(targets: VectorWriteTargets, check: RlsWriteCheck) -> VectorOp { VectorOp::DirectDelete { collection: QualifiedCollection::new(DatabaseId::DEFAULT, "vecs"), field: "emb".into(), - targets: VectorWriteTargets::Predicate(Vec::new()), + targets, returning: None, rls_filters: Vec::new(), rls_write_check: check, } } - /// Only a live predicate needs resolving; a decided or absent policy - /// replicates the write as it is. + fn update(targets: VectorWriteTargets, value: UpdateValue) -> VectorOp { + VectorOp::DirectUpdate { + collection: QualifiedCollection::new(DatabaseId::DEFAULT, "vecs"), + field: "emb".into(), + targets, + new_vector: None, + payload_patch: vec![("n".into(), value)], + quantization: nodedb_types::VectorQuantization::None, + storage_dtype: nodedb_types::VectorStorageDtype::F32, + payload_indexes: Vec::new(), + returning: None, + rls_filters: Vec::new(), + rls_write_check: RlsWriteCheck::NoPolicyApplies, + } + } + + fn rows() -> VectorWriteTargets { + VectorWriteTargets::Surrogates(vec![Surrogate::new(4)]) + } + + /// A write that names its rows and values logs as it is. #[test] - fn only_a_live_predicate_selects_the_resolver() { - assert!( - resolver_for_vector_op(&predicate_delete(RlsWriteCheck::NoPolicyApplies)).is_none() - ); - assert!( - resolver_for_vector_op(&predicate_delete( - RlsWriteCheck::decided_earlier_in_request() - )) - .is_none() + fn a_write_naming_rows_and_values_logs_as_it_is() { + for check in [ + RlsWriteCheck::NoPolicyApplies, + RlsWriteCheck::decided_earlier_in_request(), + ] { + assert!(resolver_for_vector_op(&delete(rows(), check)).is_none()); + } + let literal = UpdateValue::Literal(vec![0x01]); + assert!(resolver_for_vector_op(&update(rows(), literal)).is_none()); + } + + /// A live write predicate, a row predicate, or a patch expression is + /// resolved to rows and values before it is logged. + #[test] + fn anything_replay_evaluates_again_selects_the_resolver() { + let governed = delete(rows(), RlsWriteCheck::from_injected(vec![1, 2, 3])); + let by_predicate = delete( + VectorWriteTargets::Predicate(Vec::new()), + RlsWriteCheck::NoPolicyApplies, ); - let resolver = - resolver_for_vector_op(&predicate_delete(RlsWriteCheck::from_injected(vec![ - 1, 2, 3, - ]))) - .expect("a live predicate selects the resolver"); - assert_eq!(resolver.collection(), "vecs"); - assert!(matches!( - resolver.build_resolve_op(), - PhysicalPlan::Vector(VectorOp::ResolveDirectWrite(_)) - )); + let literal = UpdateValue::Literal(vec![0x01]); + let update_by_predicate = update(VectorWriteTargets::Predicate(Vec::new()), literal); + let expression = UpdateValue::Expr(nodedb_query::expr::SqlExpr::Column("n".into())); + let update_by_expression = update(rows(), expression); + for op in [ + governed, + by_predicate, + update_by_predicate, + update_by_expression, + ] { + let resolver = resolver_for_vector_op(&op).expect("the op resolves before logging"); + assert_eq!(resolver.collection(), "vecs"); + assert!(matches!( + resolver.build_resolve_op(), + PhysicalPlan::Vector(VectorOp::ResolveDirectWrite(_)) + )); + } } } diff --git a/nodedb/src/data/executor/handlers/kv/crud/delete.rs b/nodedb/src/data/executor/handlers/kv/crud/delete.rs index de25fd2e9..a6bd0885d 100644 --- a/nodedb/src/data/executor/handlers/kv/crud/delete.rs +++ b/nodedb/src/data/executor/handlers/kv/crud/delete.rs @@ -107,6 +107,8 @@ impl CoreLoop { ) -> Response { debug!(core = self.core_id, %collection, "kv truncate"); let count = self.kv_engine.truncate(did, tid, collection); + // Every key leaves, and no per-key version names them. + self.note_task_collection_wipe(task, tid, collection); match response_codec::encode_count("deleted", count) { Ok(payload) => self.response_with_payload(task, payload), Err(e) => self.response_error(task, ErrorCode::from(e)), diff --git a/nodedb/src/data/executor/handlers/kv/dispatch.rs b/nodedb/src/data/executor/handlers/kv/dispatch.rs index f4775d5a5..fc917cfc4 100644 --- a/nodedb/src/data/executor/handlers/kv/dispatch.rs +++ b/nodedb/src/data/executor/handlers/kv/dispatch.rs @@ -2,7 +2,7 @@ //! KV operation dispatch: routes `KvOp` variants to their handler methods. -use crate::bridge::envelope::Response; +use crate::bridge::envelope::{ErrorCode, Response}; use crate::data::executor::core_loop::CoreLoop; use crate::data::executor::task::ExecutionTask; use nodedb_physical::physical_plan::KvOp; @@ -441,44 +441,20 @@ impl CoreLoop { cursor, *count, ), - KvOp::PredicateUpdate { - collection, - filters, - updates, - rls_write_check, - returning, - rls_filters, - } => self.execute_kv_predicate_update( - task, - super::predicate::KvPredicateCtx { - did, - tid, - collection: collection.as_str(), - filters, - rls_write_check, - returning: returning.as_ref(), - rls_filters, - }, - updates, - ), - KvOp::PredicateDelete { - collection, - filters, - rls_write_check, - returning, - rls_filters, - } => self.execute_kv_predicate_delete( - task, - super::predicate::KvPredicateCtx { - did, - tid, - collection: collection.as_str(), - filters, - rls_write_check, - returning: returning.as_ref(), - rls_filters, - }, - ), + // A predicate write resolves to a mutation list before it is + // logged, and a transaction stages it. It never applies directly. + KvOp::PredicateUpdate { collection, .. } | KvOp::PredicateDelete { collection, .. } => { + self.response_error( + task, + ErrorCode::Internal { + detail: format!( + "kv predicate UPDATE/DELETE on '{}' reached the apply path \ + unresolved; it resolves to a mutation list first", + collection.as_str() + ), + }, + ) + } KvOp::ResolveWrite(inner) => self.execute_kv_resolve_write(task, did, tid, inner), KvOp::ResolvedWrite { mutations, diff --git a/nodedb/src/data/executor/handlers/kv/predicate/apply.rs b/nodedb/src/data/executor/handlers/kv/predicate/apply.rs deleted file mode 100644 index a623d6718..000000000 --- a/nodedb/src/data/executor/handlers/kv/predicate/apply.rs +++ /dev/null @@ -1,166 +0,0 @@ -// SPDX-License-Identifier: BUSL-1.1 - -//! Live handlers for `KvOp::PredicateUpdate` / `KvOp::PredicateDelete`. Both -//! resolve their row set via [`CoreLoop::kv_predicate_matches`] and reuse the -//! keyed path verbatim — re-deriving either here is how a predicate write -//! would drift from the keyed one. - -use nodedb_physical::physical_plan::ReturningSpec; -use nodedb_types::RlsWriteCheck; -use tracing::debug; - -use crate::bridge::envelope::{ErrorCode, Response}; -use crate::data::executor::core_loop::CoreLoop; -use crate::data::executor::handlers::kv::crud::KvDeleteParams; -use crate::data::executor::handlers::kv::field_compute::merge_field_updates; -use crate::data::executor::handlers::kv::rls::admit_kv_row; -use crate::data::executor::handlers::returning_rows::KvStoredRow; -use crate::data::executor::response_codec; -use crate::data::executor::task::ExecutionTask; -use crate::engine::kv::entry::NO_EXPIRY; -use crate::engine::kv::{KvRewriteParams, current_ms}; - -/// Routing and policy inputs shared by both predicate handlers. -pub(in crate::data::executor) struct KvPredicateCtx<'a> { - pub did: u64, - pub tid: u64, - pub collection: &'a str, - pub filters: &'a [u8], - pub rls_write_check: &'a RlsWriteCheck, - /// When `Some`, project one STORED row per matched key per spec instead - /// of reporting a bare count: the post-image for an update, the - /// pre-image for a delete. - pub returning: Option<&'a ReturningSpec>, - /// Compiled read policy bounding which of those rows may be shown back. - pub rls_filters: &'a [u8], -} - -impl CoreLoop { - /// Merge `updates` into every row the predicate matches. Every post-image - /// is decided against the write policy before the first row is written, - /// so one rejected row leaves the whole statement without effect. - pub(in crate::data::executor) fn execute_kv_predicate_update( - &mut self, - task: &ExecutionTask, - ctx: KvPredicateCtx<'_>, - updates: &[(String, Vec)], - ) -> Response { - let KvPredicateCtx { - did, - tid, - collection, - filters, - rls_write_check, - returning, - rls_filters, - } = ctx; - debug!(core = self.core_id, %collection, "kv predicate update"); - let now_ms = current_ms(); - - let matched = match self.kv_predicate_matches(did, tid, collection, filters, now_ms) { - Ok(rows) => rows, - Err(e) => return self.response_error(task, e), - }; - - let declared = self.declared_columns_of(did, tid, collection); - let mut writes: Vec<(Vec, Vec, Vec)> = Vec::with_capacity(matched.len()); - for (key, body) in matched { - let computed = - match merge_field_updates(collection, Some(body.as_slice()), updates, declared) { - Ok(c) => c, - Err(e) => return self.response_error(task, e), - }; - if let Err(e) = - admit_kv_row(rls_write_check, &computed.new_value, &key, tid, collection) - { - return self.response_error(task, e); - } - writes.push((key, body, computed.new_value)); - } - - for (key, old_body, new_value) in &writes { - // The matched row exists and keeps its bound identity. - self.kv_engine.rewrite_with_absolute_expiry( - KvRewriteParams { - database_id: did, - tenant_id: tid, - collection, - key, - value: new_value, - // Mirrors `execute_kv_field_set`, whose keyed merge this is - // the predicate form of: the merge writes no TTL of its own. - ttl_ms: 0, - now_ms, - }, - NO_EXPIRY, - ); - if let Some(ref m) = self.metrics { - m.record_kv_put(); - } - self.emit_kv_write_event( - task, - collection, - crate::event::WriteOp::Update, - key, - Some(new_value), - Some(old_body), - ); - self.note_kv_write(task, did, tid, collection, key); - } - - if let Some(spec) = returning { - // Each `new_value` IS the stored body: the merge is persisted - // verbatim, so projecting it is projecting the post-image. Zero - // matches ships an EMPTY row set, never a count. - let rows: Vec> = writes - .iter() - .map(|(key, _old_body, new_value)| (key.as_slice(), new_value.as_slice())) - .collect(); - return self.kv_stored_returning_response(task, spec, rls_filters, &rows); - } - - match response_codec::encode_count("affected", writes.len()) { - Ok(payload) => self.response_with_payload(task, payload), - Err(e) => self.response_error(task, ErrorCode::from(e)), - } - } - - /// Delete every row the predicate matches. Resolved keys go through - /// `execute_kv_delete`, so the gate, metrics, and reply are the keyed - /// delete's, not a second copy. - pub(in crate::data::executor) fn execute_kv_predicate_delete( - &mut self, - task: &ExecutionTask, - ctx: KvPredicateCtx<'_>, - ) -> Response { - let KvPredicateCtx { - did, - tid, - collection, - filters, - rls_write_check, - returning, - rls_filters, - } = ctx; - debug!(core = self.core_id, %collection, "kv predicate delete"); - let now_ms = current_ms(); - - let matched = match self.kv_predicate_matches(did, tid, collection, filters, now_ms) { - Ok(rows) => rows, - Err(e) => return self.response_error(task, e), - }; - let keys: Vec> = matched.into_iter().map(|(key, _body)| key).collect(); - self.execute_kv_delete( - task, - KvDeleteParams { - did, - tid, - collection, - keys: &keys, - rls_write_check, - returning, - rls_filters, - }, - ) - } -} diff --git a/nodedb/src/data/executor/handlers/kv/predicate/ctx.rs b/nodedb/src/data/executor/handlers/kv/predicate/ctx.rs new file mode 100644 index 000000000..db4345d58 --- /dev/null +++ b/nodedb/src/data/executor/handlers/kv/predicate/ctx.rs @@ -0,0 +1,21 @@ +// SPDX-License-Identifier: BUSL-1.1 + +//! Inputs of a KV predicate `UPDATE` / `DELETE` resolve. + +use nodedb_physical::physical_plan::ReturningSpec; +use nodedb_types::RlsWriteCheck; + +/// Routing and policy inputs shared by both predicate resolves. +pub(in crate::data::executor) struct KvPredicateCtx<'a> { + pub did: u64, + pub tid: u64, + pub collection: &'a str, + pub filters: &'a [u8], + pub rls_write_check: &'a RlsWriteCheck, + /// When `Some`, project one STORED row per matched key per spec instead + /// of reporting a bare count: the post-image for an update, the + /// pre-image for a delete. + pub returning: Option<&'a ReturningSpec>, + /// Compiled read policy bounding which of those rows may be shown back. + pub rls_filters: &'a [u8], +} diff --git a/nodedb/src/data/executor/handlers/kv/predicate/mod.rs b/nodedb/src/data/executor/handlers/kv/predicate/mod.rs index 84d9919fb..f654b1d8a 100644 --- a/nodedb/src/data/executor/handlers/kv/predicate/mod.rs +++ b/nodedb/src/data/executor/handlers/kv/predicate/mod.rs @@ -1,8 +1,10 @@ // SPDX-License-Identifier: BUSL-1.1 //! KV predicate DML: `UPDATE` / `DELETE` whose `WHERE` names no primary key. +//! The Control Plane resolves both to a mutation list before they are +//! logged, so the Data Plane reads the predicate only to resolve or stage it. -pub(in crate::data::executor) mod apply; +mod ctx; mod matches; -pub(in crate::data::executor) use apply::KvPredicateCtx; +pub(in crate::data::executor) use ctx::KvPredicateCtx; diff --git a/nodedb/src/data/executor/handlers/kv/resolve/predicate_ops.rs b/nodedb/src/data/executor/handlers/kv/resolve/predicate_ops.rs index a86e3445f..d847fc33f 100644 --- a/nodedb/src/data/executor/handlers/kv/resolve/predicate_ops.rs +++ b/nodedb/src/data/executor/handlers/kv/resolve/predicate_ops.rs @@ -63,8 +63,8 @@ impl CoreLoop { collection, key: &key, value: new_value, - // `execute_kv_predicate_update` writes with `ttl_ms: 0`, the - // keyed field merge's behaviour. Preserved verbatim. + // The keyed field merge writes with `ttl_ms: 0`, so the + // resolved rewrite does too. ttl_ms: 0, expire_at_ms: 0, precondition: body, diff --git a/nodedb/src/data/executor/handlers/transaction/stage_write/stage_kv_predicate.rs b/nodedb/src/data/executor/handlers/transaction/stage_write/stage_kv_predicate.rs index c8c0ffa58..7c5251563 100644 --- a/nodedb/src/data/executor/handlers/transaction/stage_write/stage_kv_predicate.rs +++ b/nodedb/src/data/executor/handlers/transaction/stage_write/stage_kv_predicate.rs @@ -66,7 +66,7 @@ impl CoreLoop { /// Stage a KV predicate `UPDATE` at statement time: resolve the current /// BASE ∪ OVERLAY matching set, merge the SET-list into each match, and /// record the post-image as a staged `Put`. Replies `{"affected": N}`, - /// the same shape `execute_kv_predicate_update` reports in autocommit. + /// the same shape the autocommit resolve reports. pub(in crate::data::executor) fn stage_kv_predicate_update( &mut self, params: StageKvPredicateUpdateParams<'_>, @@ -204,6 +204,8 @@ impl CoreLoop { let filters: Vec = decode_scan_filters(filter_bytes, "kv predicate dml filters") .map_err(|e| self.response_error(task, e))?; + // Deterministic: Calvin staging pins the epoch anchor every replica + // shares, and session staging runs on the leader alone. let mut rows = self .kv_predicate_matches(did, tid, collection, filter_bytes, self.kv_read_now_ms()) .map_err(|e| self.response_error(task, e))?; diff --git a/nodedb/src/data/executor/handlers/transaction/write_version_kv.rs b/nodedb/src/data/executor/handlers/transaction/write_version_kv.rs index d2d9ef4b8..add93bf97 100644 --- a/nodedb/src/data/executor/handlers/transaction/write_version_kv.rs +++ b/nodedb/src/data/executor/handlers/transaction/write_version_kv.rs @@ -88,11 +88,13 @@ impl CoreLoop { ); } } - // Whole-collection mutations: key set is every row or predicate- - // resolved at apply time, so only the collection floor applies. - KvOp::Truncate { collection, .. } - | KvOp::PredicateUpdate { collection, .. } - | KvOp::PredicateDelete { collection, .. } => { + // Every row leaves: a wipe bounds every row read of the collection. + KvOp::Truncate { collection, .. } => { + self.note_collection_wipe(db, tenant, collection.as_str(), stamp); + } + // Predicate-resolved at apply time, so only the collection floor + // applies. + KvOp::PredicateUpdate { collection, .. } | KvOp::PredicateDelete { collection, .. } => { self.note_write(db, tenant, collection.as_str(), None, stamp); } // Read-only: nothing written, no version to record. diff --git a/nodedb/src/data/executor/handlers/vector_direct_delete.rs b/nodedb/src/data/executor/handlers/vector_direct_delete.rs index 9aaaa1183..2209612cd 100644 --- a/nodedb/src/data/executor/handlers/vector_direct_delete.rs +++ b/nodedb/src/data/executor/handlers/vector_direct_delete.rs @@ -98,11 +98,16 @@ impl CoreLoop { } let mut removed: Vec<(StorageKey, Vec)> = Vec::with_capacity(surrogates.len()); - for surrogate in &surrogates { + for (at, surrogate) in surrogates.iter().enumerate() { match self.remove_vector_direct_row(&index_key, tid, collection, *surrogate) { Ok(Some(row)) => removed.push((StorageKey::for_surrogate(*surrogate), row.bytes)), Ok(None) => {} - Err(e) => return self.response_error(task, e), + Err(e) => { + // The failed row stays whole. The rows before it are removed. + let before = &surrogates[..at]; + self.finish_vector_direct_write(task, &index_key, tid, collection, before); + return self.response_error(task, e); + } } } if !removed.is_empty() { diff --git a/nodedb/src/data/executor/handlers/vector_direct_row.rs b/nodedb/src/data/executor/handlers/vector_direct_row.rs index 6e79cdd26..acc8a4496 100644 --- a/nodedb/src/data/executor/handlers/vector_direct_row.rs +++ b/nodedb/src/data/executor/handlers/vector_direct_row.rs @@ -190,6 +190,10 @@ impl CoreLoop { /// Remove every trace of `surrogate`'s row: bitmap entries, HNSW node, /// sidecar row, and the cached sidecar. Returns the sidecar that was /// stored, or `None` when no node was bound to the surrogate. + /// + /// The sidecar delete is the only fallible step, so it runs first. A + /// failure leaves the row fully present. A crash after it leaves a node + /// with no sidecar, which replay of the delete record removes. pub(in crate::data::executor) fn remove_vector_direct_row( &mut self, index_key: &VectorIndexKey, @@ -202,19 +206,23 @@ impl CoreLoop { return Ok(None); }; let sidecar = self.vector_sidecar_row(database_id, tid, collection, surrogate)?; - let Some(coll) = self.vector_collections.get_mut(index_key) else { - return Ok(None); - }; - if let Some(row) = &sidecar { - coll.payload.delete_row(node_id, &row.fields); - } - coll.delete(node_id); let key = StorageKey::for_surrogate(surrogate); + crate::fail_point_err!( + self.fail_scope, + "vector_direct::sidecar_delete", + |detail: String| ErrorCode::Internal { detail } + ); self.sparse .delete(database_id, tid, collection, &key) .map_err(ErrorCode::from)?; self.doc_cache .invalidate(database_id, tid, collection, &key); + if let Some(coll) = self.vector_collections.get_mut(index_key) { + if let Some(row) = &sidecar { + coll.payload.delete_row(node_id, &row.fields); + } + coll.delete(node_id); + } Ok(Some(sidecar.unwrap_or_else(|| VectorSidecarRow { bytes: Vec::new(), fields: HashMap::new(), @@ -286,3 +294,76 @@ impl CoreLoop { } } } + +#[cfg(all(test, feature = "failpoints"))] +mod tests { + use nodedb_physical::physical_plan::VectorWriteTargets; + use nodedb_types::fail_point::FailGuard; + use nodedb_types::{RlsWriteCheck, Surrogate}; + + use crate::bridge::envelope::Status; + use crate::data::executor::core_loop::CoreLoop; + use crate::data::executor::core_loop::tests::{make_core_with_dir, make_default_task}; + use crate::data::executor::handlers::vector_direct_delete::VectorDirectDeleteParams; + use crate::data::executor::handlers::vector_direct_test_rows::{ + COLLECTION, FIELD, TID, has_node, has_sidecar, seed, + }; + + const SIDECAR_DELETE: &str = "vector_direct::sidecar_delete"; + + fn delete(core: &mut CoreLoop, surrogate: Surrogate) -> Status { + let task = make_default_task(); + let targets = VectorWriteTargets::Surrogates(vec![surrogate]); + core.execute_vector_direct_delete(VectorDirectDeleteParams { + task: &task, + tid: TID, + collection: COLLECTION, + field: FIELD, + targets: &targets, + returning: None, + rls_filters: &[], + rls_write_check: &RlsWriteCheck::decided_earlier_in_request(), + }) + .status + } + + #[test] + fn a_failed_sidecar_delete_leaves_the_row_whole() { + let dir = tempfile::tempdir().expect("tempdir"); + let (mut core, _req, _resp) = make_core_with_dir(dir.path()); + let row = Surrogate::new(5); + seed(&mut core, row); + + { + let _fail = FailGuard::fail(SIDECAR_DELETE, "disk full"); + assert_ne!(delete(&mut core, row), Status::Ok); + assert!(has_node(&core, row), "a refused delete keeps the node"); + assert!( + has_sidecar(&core, row), + "a refused delete keeps the sidecar" + ); + } + + assert_eq!(delete(&mut core, row), Status::Ok); + assert!(!has_node(&core, row)); + assert!(!has_sidecar(&core, row)); + } + + #[test] + fn a_failed_sidecar_delete_leaves_a_truncated_row_whole() { + let dir = tempfile::tempdir().expect("tempdir"); + let (mut core, _req, _resp) = make_core_with_dir(dir.path()); + let row = Surrogate::new(9); + seed(&mut core, row); + + let _fail = FailGuard::fail(SIDECAR_DELETE, "disk full"); + let task = make_default_task(); + let resp = core.execute_vector_direct_truncate(&task, TID, COLLECTION, FIELD); + assert_ne!(resp.status, Status::Ok); + assert!(has_node(&core, row), "a refused truncate keeps the node"); + assert!( + has_sidecar(&core, row), + "a refused truncate keeps the sidecar" + ); + } +} diff --git a/nodedb/src/data/executor/handlers/vector_direct_test_rows.rs b/nodedb/src/data/executor/handlers/vector_direct_test_rows.rs new file mode 100644 index 000000000..b8e285de5 --- /dev/null +++ b/nodedb/src/data/executor/handlers/vector_direct_test_rows.rs @@ -0,0 +1,58 @@ +// SPDX-License-Identifier: BUSL-1.1 + +//! Vector-primary row fixtures shared by the direct-write and replay tests. + +use std::collections::HashMap; + +use nodedb_physical::physical_plan::VectorDirectWriteIntent; +use nodedb_types::{RlsWriteCheck, Surrogate, Value}; + +use crate::bridge::envelope::Status; +use crate::data::executor::core_loop::CoreLoop; +use crate::data::executor::core_loop::tests::make_default_task; +use crate::data::executor::handlers::vector_upsert::VectorDirectUpsertParams; +use crate::types::DatabaseId; + +pub const TID: u64 = 1; +pub const COLLECTION: &str = "vp_rows"; +pub const FIELD: &str = "emb"; + +/// Store one row under `surrogate`. +pub fn seed(core: &mut CoreLoop, surrogate: Surrogate) { + let mut fields: HashMap = HashMap::new(); + fields.insert("owner".into(), Value::String("a".into())); + let payload = zerompk::to_msgpack_vec(&fields).expect("encode payload"); + let task = make_default_task(); + let resp = core.execute_vector_direct_upsert(VectorDirectUpsertParams { + task: &task, + tid: TID, + collection: COLLECTION, + field: FIELD, + surrogate, + vector: &[1.0, 0.0, 0.0], + payload: &payload, + quantization: nodedb_types::VectorQuantization::None, + storage_dtype: nodedb_types::VectorStorageDtype::F32, + payload_indexes: &[], + intent: VectorDirectWriteIntent::Insert, + on_conflict_updates: &[], + rls_write_check: &RlsWriteCheck::decided_earlier_in_request(), + returning: None, + rls_filters: &[], + }); + assert_eq!(resp.status, Status::Ok, "seed: {:?}", resp.error_code); +} + +/// Whether the index holds a node for `surrogate`. +pub fn has_node(core: &CoreLoop, surrogate: Surrogate) -> bool { + let index_key = + CoreLoop::vector_index_key(DatabaseId::DEFAULT.as_u64(), TID, COLLECTION, FIELD); + core.vector_direct_node(&index_key, surrogate).is_some() +} + +/// Whether the sidecar store holds `surrogate`'s row. +pub fn has_sidecar(core: &CoreLoop, surrogate: Surrogate) -> bool { + core.vector_sidecar_row(DatabaseId::DEFAULT.as_u64(), TID, COLLECTION, surrogate) + .expect("read sidecar") + .is_some() +} diff --git a/nodedb/src/data/executor/handlers/vector_direct_truncate.rs b/nodedb/src/data/executor/handlers/vector_direct_truncate.rs index 20df17085..3e4ff09b4 100644 --- a/nodedb/src/data/executor/handlers/vector_direct_truncate.rs +++ b/nodedb/src/data/executor/handlers/vector_direct_truncate.rs @@ -41,22 +41,32 @@ impl CoreLoop { Err(e) => return self.response_error(task, e), }; - let mut truncated = 0usize; - for surrogate in &surrogates { - match self.remove_vector_direct_row(&index_key, tid, collection, *surrogate) { - Ok(Some(_)) => truncated += 1, - // A sidecar row with no node bound to it: drop the row so the - // collection reads back empty. - Ok(None) => { - let key = StorageKey::for_surrogate(*surrogate); - if let Err(e) = self.sparse.delete(database_id, tid, collection, &key) { - return self.response_error(task, ErrorCode::from(e)); + for (at, surrogate) in surrogates.iter().enumerate() { + let outcome = + match self.remove_vector_direct_row(&index_key, tid, collection, *surrogate) { + Ok(Some(_)) => Ok(()), + // A sidecar row with no node bound to it: drop the row so the + // collection reads back empty. + Ok(None) => { + let key = StorageKey::for_surrogate(*surrogate); + let deleted = self + .sparse + .delete(database_id, tid, collection, &key) + .map(|_| ()) + .map_err(ErrorCode::from); + if deleted.is_ok() { + self.doc_cache + .invalidate(database_id, tid, collection, &key); + } + deleted } - self.doc_cache - .invalidate(database_id, tid, collection, &key); - truncated += 1; - } - Err(e) => return self.response_error(task, e), + Err(e) => Err(e), + }; + if let Err(e) = outcome { + // The failed row stays whole. The rows before it are removed. + let removed = &surrogates[..at]; + self.finish_vector_direct_write(task, &index_key, tid, collection, removed); + return self.response_error(task, e); } } @@ -68,6 +78,7 @@ impl CoreLoop { self.finish_vector_direct_write(task, &index_key, tid, collection, &surrogates); self.invalidate_aggregate_cache_for_collection(database_id, tid, collection); + let truncated = surrogates.len(); debug!(core = self.core_id, %collection, truncated, "vector direct truncate complete"); self.truncate_response(task, truncated) } diff --git a/nodedb/src/data/executor/wal_replay/kv.rs b/nodedb/src/data/executor/wal_replay/kv.rs index d6bf3ab68..702532d78 100644 --- a/nodedb/src/data/executor/wal_replay/kv.rs +++ b/nodedb/src/data/executor/wal_replay/kv.rs @@ -238,20 +238,6 @@ impl CoreLoop { continue; } - // kv_predicate_update (delta): re-resolves the row set against - // current state and re-runs the same merge. - if let Some(applied) = self.try_replay_kv_predicate_update( - &record.payload, - tenant_id, - database_id, - now_ms, - record_lsn, - tombstones, - ) { - puts += applied; - continue; - } - // A KV put-family record of no current shape: refused, never // replayed without its surrogate. Any other `Put` payload // belongs to another engine's replay pass. @@ -336,7 +322,7 @@ impl CoreLoop { }]); } self.kv_engine.truncate(database_id, tenant_id, &collection); - self.note_replay_write(database_id, tenant_id, &collection, None, record_lsn); + self.note_replay_wipe(database_id, tenant_id, &collection, record_lsn); deletes += 1; continue; } @@ -344,20 +330,6 @@ impl CoreLoop { continue; } - // kv_predicate_delete (delta): re-resolves the row set - // against current state. - if let Some(removed) = self.try_replay_kv_predicate_delete( - &record.payload, - tenant_id, - database_id, - now_ms, - record_lsn, - tombstones, - ) { - deletes += removed; - continue; - } - // No tombstone gate: the record carries only `index_name`, // no collection to gate on. if let Some(applied) = diff --git a/nodedb/src/data/executor/wal_replay_kv_predicate.rs b/nodedb/src/data/executor/wal_replay_kv_predicate.rs deleted file mode 100644 index b8a672a26..000000000 --- a/nodedb/src/data/executor/wal_replay_kv_predicate.rs +++ /dev/null @@ -1,144 +0,0 @@ -// SPDX-License-Identifier: BUSL-1.1 - -//! WAL replay for the KV predicate DML records (`kv_predicate_update`, -//! `kv_predicate_delete`). The log carries only the predicate and -//! assignments, never a row set, so replay re-resolves the row set with the -//! same scan and merge the live handlers use. - -use tracing::warn; - -use super::core_loop::CoreLoop; -use super::handlers::kv::field_compute::merge_field_updates; - -impl CoreLoop { - /// Decode + tombstone-gate + replay one `kv_predicate_update` record. - /// - /// Returns `None` when `payload` is not that record shape (the caller - /// tries the next candidate arm), otherwise the number of rows written. - pub(super) fn try_replay_kv_predicate_update( - &mut self, - payload: &[u8], - tenant_id: u64, - database_id: u64, - now_ms: u64, - record_lsn: u64, - tombstones: &nodedb_wal::TombstoneSet, - ) -> Option { - let (disc, collection, filters, updates) = - zerompk::from_msgpack::<(&str, String, Vec, Vec<(String, Vec)>)>(payload) - .ok()?; - if disc != "kv_predicate_update" { - return None; - } - let tombstones = &tombstones.for_database(database_id); - if self.skip_kv_replay_record(tombstones, tenant_id, &collection, record_lsn) { - return Some(0); - } - - let matched = match self.kv_predicate_matches( - database_id, - tenant_id, - &collection, - &filters, - now_ms, - ) { - Ok(rows) => rows, - Err(e) => { - warn!( - core = self.core_id, - %collection, - ?e, - "WAL kv_predicate_update replay: predicate scan failed, skipping record" - ); - return Some(0); - } - }; - - let mut written = 0usize; - for (key, body) in matched { - let computed = match merge_field_updates( - &collection, - Some(body.as_slice()), - &updates, - self.declared_columns_of(database_id, tenant_id, &collection), - ) { - Ok(c) => c, - Err(e) => { - warn!( - core = self.core_id, - %collection, - key = %String::from_utf8_lossy(&key), - ?e, - "WAL kv_predicate_update replay: field merge failed, skipping row" - ); - continue; - } - }; - // The row exists and keeps its bound identity, the same surrogate - // the live merge preserved. - self.kv_engine.rewrite_with_absolute_expiry( - crate::engine::kv::KvRewriteParams { - database_id, - tenant_id, - collection: &collection, - key: &key, - value: &computed.new_value, - ttl_ms: 0, - now_ms, - }, - crate::engine::kv::entry::NO_EXPIRY, - ); - written += 1; - } - // The key set is resolved at replay time rather than carried, so the - // collection floor is the version this can record. - self.note_replay_write(database_id, tenant_id, &collection, None, record_lsn); - Some(written) - } - - /// Decode + tombstone-gate + replay one `kv_predicate_delete` record. - pub(super) fn try_replay_kv_predicate_delete( - &mut self, - payload: &[u8], - tenant_id: u64, - database_id: u64, - now_ms: u64, - record_lsn: u64, - tombstones: &nodedb_wal::TombstoneSet, - ) -> Option { - let (disc, collection, filters) = - zerompk::from_msgpack::<(&str, String, Vec)>(payload).ok()?; - if disc != "kv_predicate_delete" { - return None; - } - let tombstones = &tombstones.for_database(database_id); - if self.skip_kv_replay_record(tombstones, tenant_id, &collection, record_lsn) { - return Some(0); - } - - let matched = match self.kv_predicate_matches( - database_id, - tenant_id, - &collection, - &filters, - now_ms, - ) { - Ok(rows) => rows, - Err(e) => { - warn!( - core = self.core_id, - %collection, - ?e, - "WAL kv_predicate_delete replay: predicate scan failed, skipping record" - ); - return Some(0); - } - }; - let keys: Vec> = matched.into_iter().map(|(key, _body)| key).collect(); - let removed = self - .kv_engine - .delete(database_id, tenant_id, &collection, &keys, now_ms); - self.note_replay_write(database_id, tenant_id, &collection, None, record_lsn); - Some(removed) - } -} diff --git a/nodedb/src/data/executor/wal_replay_vector_direct.rs b/nodedb/src/data/executor/wal_replay_vector_direct.rs index a89d79d8d..a4bf46f77 100644 --- a/nodedb/src/data/executor/wal_replay_vector_direct.rs +++ b/nodedb/src/data/executor/wal_replay_vector_direct.rs @@ -8,7 +8,7 @@ //! gated by the per-collection checkpoint watermark like the direct upsert: //! a restored checkpoint already holds every write at or below it. -use nodedb_physical::physical_plan::VectorOp; +use nodedb_physical::physical_plan::{VectorOp, VectorWriteTargets}; use crate::bridge::envelope::{PhysicalPlan, Status}; use crate::control::server::wal_dispatch::{ @@ -31,7 +31,7 @@ impl CoreLoop { tombstones: &nodedb_wal::TombstoneSet, ) -> bool { let tombstones = tombstones.for_database(database_id); - let Ok((collection, field, targets)) = + let Ok((collection, field, rows)) = zerompk::from_msgpack::(payload) else { self.replay_record_unapplied( @@ -45,6 +45,9 @@ impl CoreLoop { if tombstones.is_tombstoned(tenant_id, &collection, record_lsn) { return false; } + // The record names the rows the delete removed. A row whose sidecar + // delete was durable before a crash still has its restored node. + let targets = VectorWriteTargets::Surrogates(rows); let index_key = CoreLoop::vector_index_key(database_id, tenant_id, &collection, &field); if self.vector_replay_skips(record_lsn) { return false; @@ -192,7 +195,7 @@ impl CoreLoop { let Ok(( collection, field, - targets, + rows, new_vector, payload_patch, quantization, @@ -211,6 +214,7 @@ impl CoreLoop { if tombstones.is_tombstoned(tenant_id, &collection, record_lsn) { return false; } + let targets = VectorWriteTargets::Surrogates(rows); let index_key = CoreLoop::vector_index_key(database_id, tenant_id, &collection, &field); if self.vector_replay_skips(record_lsn) { return false; @@ -283,3 +287,37 @@ impl CoreLoop { true } } + +#[cfg(test)] +mod tests { + use nodedb_types::{StorageKey, Surrogate}; + + use crate::control::server::wal_dispatch::encode_vector_direct_delete_payload; + use crate::data::executor::core_loop::tests::make_core_with_dir; + use crate::data::executor::handlers::vector_direct_test_rows::{ + COLLECTION, FIELD, TID, has_node, has_sidecar, seed, + }; + use crate::types::DatabaseId; + + /// A crash after the sidecar delete was durable leaves the restored node + /// with no sidecar. Replay of the delete record still removes the node. + #[test] + fn replay_removes_a_node_whose_sidecar_delete_was_durable() { + let dir = tempfile::tempdir().expect("tempdir"); + let (mut core, _req, _resp) = make_core_with_dir(dir.path()); + let row = Surrogate::new(7); + seed(&mut core, row); + let db = DatabaseId::DEFAULT.as_u64(); + core.sparse + .delete(db, TID, COLLECTION, &StorageKey::for_surrogate(row)) + .expect("durable sidecar delete"); + assert!(has_node(&core, row) && !has_sidecar(&core, row)); + + let payload = + encode_vector_direct_delete_payload(COLLECTION, FIELD, &[row]).expect("encode"); + let applied = + core.replay_direct_delete(&payload, TID, db, 100, &nodedb_wal::TombstoneSet::new()); + assert!(applied, "the replayed delete applies"); + assert!(!has_node(&core, row), "replay removes the node"); + } +} From 133c805c595154c4c5e072d71cb10f4ff1a08feb Mon Sep 17 00:00:00 2001 From: Farhan Syah Date: Sun, 11 Oct 2026 13:54:14 +0800 Subject: [PATCH 3/5] fix(cluster): bound statements, forwarded reads and apply stalls by deadline Raft, the multi-raft layer, the applier, the gateway and the Calvin submit path now carry one statement deadline end to end. Election and snapshot-debt handling move into their own Raft modules, configuration changes pass a gate, and the apply loop holds, settles and reports an entry it cannot apply instead of stalling the group. The cluster test harness and cases cover a hung node, a held replica and slow catalog DDL. --- .config/nextest.toml | 2 +- Cargo.lock | 1 + .../cases/gateway_hung_node_budget.rs | 217 +++++++++ .../cases/listeners_typed_not_leader.rs | 5 +- .../tests/common_suite/cases/mod.rs | 2 + .../cases/redo_stream_held_replica.rs | 347 +++++++++++++++ .../routing_redirect_hint_unhosted_group.rs | 5 +- .../misc_suite/cases/transport_security.rs | 2 +- .../cases/wire_version_handshake.rs | 2 +- nodedb-cluster/src/applied_watcher/mod.rs | 19 +- .../src/applied_watcher/registry.rs | 64 ++- nodedb-cluster/src/applied_watcher/watcher.rs | 200 ++++++--- .../{validator.rs => validator/batch.rs} | 420 +++++++----------- .../src/calvin/sequencer/validator/graph.rs | 139 ++++++ .../src/calvin/sequencer/validator/keys.rs | 246 ++++++++++ .../src/calvin/sequencer/validator/mod.rs | 8 + .../src/calvin/sequencer/validator/reject.rs | 92 ++++ nodedb-cluster/src/cluster_info.rs | 1 + nodedb-cluster/src/error.rs | 131 ++++++ nodedb-cluster/src/multi_raft/conf_change.rs | 28 +- .../src/multi_raft/conf_change_gate.rs | 265 +++++++++++ nodedb-cluster/src/multi_raft/core.rs | 9 +- nodedb-cluster/src/multi_raft/membership.rs | 41 +- nodedb-cluster/src/multi_raft/mod.rs | 3 + nodedb-cluster/src/multi_raft/mount.rs | 2 +- nodedb-cluster/src/multi_raft/peer_contact.rs | 4 +- nodedb-cluster/src/multi_raft/proposals.rs | 31 +- nodedb-cluster/src/multi_raft/read_index.rs | 19 +- nodedb-cluster/src/multi_raft/status.rs | 4 + .../src/raft_loop/tick/snapshot_dispatch.rs | 2 +- nodedb-cluster/src/rpc_codec/data_propose.rs | 112 +++++ nodedb-cluster/src/rpc_codec/mod.rs | 2 +- nodedb-cluster/src/rpc_codec/raft_msgs.rs | 34 +- nodedb-cluster/src/rpc_codec/raft_rpc.rs | 2 +- .../src/transport/client/transport.rs | 8 +- nodedb-raft/src/lib.rs | 2 +- nodedb-raft/src/message.rs | 7 +- nodedb-raft/src/node/core.rs | 25 +- nodedb-raft/src/node/election.rs | 121 +++++ nodedb-raft/src/node/internal.rs | 101 +---- nodedb-raft/src/node/leader_lease.rs | 4 +- nodedb-raft/src/node/mod.rs | 8 +- nodedb-raft/src/node/peer_contact.rs | 21 +- nodedb-raft/src/node/quorum_contact.rs | 14 +- nodedb-raft/src/node/read_index.rs | 4 +- nodedb-raft/src/node/rpc/append_entries.rs | 85 +++- nodedb-raft/src/node/rpc/install_snapshot.rs | 2 +- nodedb-raft/src/node/rpc/timeout_now.rs | 4 +- nodedb-raft/src/node/snapshot_debt.rs | 341 ++++++++++++++ nodedb-raft/src/node/staleness.rs | 72 ++- nodedb-raft/src/state.rs | 67 ++- nodedb-raft/tests/membership.rs | 4 +- nodedb-test-support/Cargo.toml | 1 + .../src/cluster_harness/cluster/bringup.rs | 36 ++ .../cluster_harness/cluster/spawn_variants.rs | 13 + .../src/cluster_harness/cluster/types.rs | 4 + .../node/lifecycle/spawn_full.rs | 10 + .../node/lifecycle/spawn_variants.rs | 2 + .../src/cluster_harness/node/mod.rs | 1 + .../src/cluster_harness/node/remount.rs | 110 +++++ nodedb/src/bootstrap/cluster_ready.rs | 4 +- nodedb/src/bootstrap/constraint_reconcile.rs | 4 +- nodedb/src/bootstrap/data_group_recovery.rs | 315 +++++++++---- nodedb/src/bridge/envelope/error_code_from.rs | 2 + .../control/backup/restore/crdt_reissue.rs | 3 +- nodedb/src/control/backup/restore/durable.rs | 3 +- nodedb/src/control/backup/schedule/marks.rs | 6 +- .../post_apply/async_dispatch/collection.rs | 40 +- .../src/control/cluster/bootstrap_listener.rs | 16 +- .../scheduler/driver/core/install_gate.rs | 12 +- .../calvin/scheduler/driver/helpers.rs | 51 ++- .../calvin/scheduler/lock/lock_entry.rs | 167 +++++-- .../calvin/scheduler/lock/manager/acquire.rs | 85 +++- .../calvin/scheduler/lock/manager/classify.rs | 35 +- .../scheduler/lock/manager/introspection.rs | 8 +- .../calvin/scheduler/lock/manager/mod.rs | 7 +- .../calvin/scheduler/lock/manager/release.rs | 29 +- .../scheduler/lock/manager/try_acquire.rs | 2 +- .../calvin/scheduler/lock/manager/types.rs | 5 - .../scheduler/lock/manager/wound_wait.rs | 42 +- .../cluster/calvin/scheduler/lock/reap.rs | 11 +- .../control/cluster/calvin_snapshot/mod.rs | 3 +- .../cluster/calvin_snapshot/requirement.rs | 34 +- .../control/cluster/execution_error_wire.rs | 2 + .../src/control/cluster/forwarded_budget.rs | 53 +++ nodedb/src/control/cluster/leased_read.rs | 23 +- .../src/control/cluster/linearizable_read.rs | 223 +++++++--- .../control/cluster/metadata_applier/audit.rs | 7 +- .../cluster/metadata_applier/catalog_ddl.rs | 8 + .../cluster/metadata_applier/dispatch.rs | 6 - .../control/cluster/metadata_applier/wedge.rs | 2 + nodedb/src/control/cluster/mod.rs | 2 + nodedb/src/control/cluster/read_deadline.rs | 67 +++ nodedb/src/control/cluster/read_index.rs | 10 +- .../control/cluster/redo_stream_membership.rs | 26 +- .../src/control/cluster/snapshot_applier.rs | 20 +- .../src/control/cluster/snapshot_builder.rs | 2 + nodedb/src/control/cluster/snapshot_cut.rs | 13 +- .../cluster/start_raft/propose_error.rs | 14 +- .../distributed_applier/apply_loop/context.rs | 6 + .../distributed_applier/apply_loop/driver.rs | 22 +- .../apply_loop/held_entry.rs | 292 ++++++++++++ .../distributed_applier/apply_loop/lane.rs | 168 ++++++- .../apply_loop/metadata_floor.rs | 14 +- .../distributed_applier/apply_loop/mod.rs | 8 + .../apply_loop/panicked.rs | 96 ++++ .../apply_loop/pipeline.rs | 410 ++++++++++++++--- .../apply_loop/proposal_gate.rs | 100 ++++- .../apply_loop/redo_chunk.rs | 46 +- .../distributed_applier/apply_loop/settle.rs | 56 +++ .../distributed_applier/apply_loop/start.rs | 71 ++- .../apply_loop/transaction_redo.rs | 46 +- nodedb/src/control/exec_receiver/executor.rs | 2 + .../control/exec_receiver/metadata_applied.rs | 15 +- nodedb/src/control/fail_gate.rs | 43 ++ nodedb/src/control/gateway/cluster_error.rs | 91 +++- nodedb/src/control/gateway/core.rs | 13 +- nodedb/src/control/gateway/dispatch_local.rs | 3 +- nodedb/src/control/gateway/dispatch_remote.rs | 88 +--- .../control/gateway/dispatch_remote_stream.rs | 148 ++++-- nodedb/src/control/gateway/dispatcher.rs | 21 +- .../error_map/class_parity/error_index.rs | 4 +- .../error_map/class_parity/error_samples.rs | 2 + nodedb/src/control/gateway/error_map/resp.rs | 2 + nodedb/src/control/gateway/live_leaders.rs | 1 + nodedb/src/control/gateway/read_leg.rs | 4 +- nodedb/src/control/gateway/retry.rs | 231 ++++++++-- nodedb/src/control/gateway/stream.rs | 13 +- nodedb/src/control/lease/drain_propose.rs | 21 - .../src/control/lease/drain_propose_async.rs | 19 +- nodedb/src/control/lease/leader_wait.rs | 28 +- nodedb/src/control/lease/mod.rs | 2 +- nodedb/src/control/lease/release.rs | 24 +- nodedb/src/control/lease/self_fence.rs | 13 +- .../maintenance/clone_materializer/chained.rs | 3 +- .../src/control/metadata_proposer/catalog.rs | 157 ++++--- .../metadata_proposer/catalog_batch.rs | 8 +- .../control/metadata_proposer/ddl_prepare.rs | 59 +-- .../control/metadata_proposer/ddl_reclaim.rs | 21 +- .../src/control/metadata_proposer/handle.rs | 90 +++- nodedb/src/control/metadata_proposer/mod.rs | 16 +- .../metadata_proposer/replicated_entries.rs | 33 +- .../src/control/metadata_proposer/timeouts.rs | 6 +- nodedb/src/control/metadata_proposer/wait.rs | 102 +++-- nodedb/src/control/orchestrated_write.rs | 3 +- .../control/planner/calvin/dependent_recon.rs | 6 +- .../planner/calvin/dependent_recon_crdt.rs | 3 +- nodedb/src/control/planner/calvin/mod.rs | 4 +- .../calvin/read_dependent/item_move.rs | 3 +- .../src/control/planner/calvin/reservation.rs | 53 ++- .../src/control/planner/calvin/retry_loop.rs | 155 ++++++- .../control/planner/calvin/submit/assign.rs | 53 ++- .../control/planner/calvin/submit/budget.rs | 17 +- .../control/planner/calvin/submit/local.rs | 137 ++---- .../src/control/planner/calvin/submit/mod.rs | 5 +- .../control/planner/calvin/submit/routed.rs | 15 +- .../control/planner/calvin/submit/stream.rs | 100 ++++- .../src/control/planner/calvin/submit/wait.rs | 138 ++++++ .../planner/calvin/tx_class/static_builder.rs | 10 +- .../control/security/auth_fence/cluster.rs | 10 +- .../security/catalog/redo_snapshot_owed.rs | 10 + .../security/catalog/system_catalog.rs | 14 + .../src/control/server/calvin_submit/hook.rs | 64 ++- .../server/calvin_submit/inbox_hook.rs | 75 +++- .../src/control/server/calvin_submit/mod.rs | 2 +- .../dispatch_utils/change_events/extract.rs | 4 +- .../dispatch_utils/durability_barrier.rs | 20 +- .../server/dispatch_utils/durable_write.rs | 3 +- .../server/dispatch_utils/owner_read.rs | 7 +- .../exchange/all_cores/home_versions.rs | 4 +- .../control/server/exchange/cluster_leaf.rs | 4 +- .../server/graph_dispatch/cluster_resolve.rs | 1 + .../src/control/server/graph_dispatch/hop.rs | 8 +- .../server/graph_dispatch/keyed_read.rs | 5 +- .../match_scatter/round_loop.rs | 1 + .../match_scatter/round_zero.rs | 1 + .../graph_dispatch/rag_fusion/stages.rs | 8 +- .../server/graph_dispatch/read_groups.rs | 5 +- .../server/graph_dispatch/whole_graph.rs | 6 +- .../src/control/server/http/routes/health.rs | 13 +- .../server/native/dispatch/graph_match.rs | 3 +- .../pgwire/handler/dispatch/replicated.rs | 3 +- .../server/pgwire/handler/dispatch/routing.rs | 6 +- .../pgwire/handler/routing/placement.rs | 13 +- .../control/server/pgwire/types/error_map.rs | 4 +- .../src/control/server/reservation/hooks.rs | 62 ++- .../ddl/neutral/collection/purge/dispatch.rs | 15 +- .../server/shared/ddl/neutral/match_ops.rs | 4 +- nodedb/src/control/server/shared/retry.rs | 2 + .../shared/session/commit/read_validation.rs | 8 +- .../shared/session/commit/single_shard.rs | 3 +- .../server/shared/session/ddl_compensate.rs | 5 +- .../server/shared/session/ddl_flush.rs | 36 +- .../control/server/shared/session/deadline.rs | 107 ++++- .../server/shared/session/leader_forward.rs | 13 +- .../src/control/server/shared/session/mod.rs | 4 +- .../server/shared/session/overlay_drop.rs | 99 ++++- .../server/shared/session/staging_gate.rs | 14 +- .../shared/write_admission/admission_keys.rs | 26 +- .../shared/write_admission/leader_gate.rs | 2 - .../predicate/txn_buffering/classify.rs | 43 +- .../server/surrogate_exchange/authority.rs | 4 +- .../sync/async_dispatch/delta/compensation.rs | 4 +- nodedb/src/control/server/sync/refusal.rs | 6 +- nodedb/src/control/state/fields.rs | 4 - nodedb/src/control/state/init.rs | 1 - nodedb/src/control/state/init_prod/open.rs | 1 - .../control/surrogate/assign/core/flush.rs | 8 +- nodedb/src/control/vshard_admission.rs | 2 +- nodedb/src/control/wal_replication/mod.rs | 1 - nodedb/src/control/wal_replication/propose.rs | 8 - nodedb/src/control/write_resolve/propose.rs | 3 +- .../executor/core_loop/write_index/index.rs | 107 ++++- .../executor/core_loop/write_index/record.rs | 189 +++++++- .../core_loop/write_index/validate.rs | 2 +- .../executor/handlers/columnar_truncate.rs | 2 + .../handlers/control/calvin_txn_id.rs | 9 +- nodedb/src/data/executor/handlers/mod.rs | 2 + .../executor/handlers/timeseries/truncate.rs | 2 + .../handlers/transaction/redo_apply/entry.rs | 24 +- .../handlers/transaction/redo_apply/state.rs | 32 +- nodedb/src/data/executor/handlers/truncate.rs | 5 + nodedb/src/data/executor/mod.rs | 1 - nodedb/src/diag/context/data_plane.rs | 7 +- nodedb/src/diag/context/held_entry.rs | 113 +++++ nodedb/src/diag/context/mod.rs | 2 + nodedb/src/diag/context/redo_stream.rs | 14 +- nodedb/src/diag/mod.rs | 17 +- nodedb/src/diag/recording/data_plane.rs | 12 +- nodedb/src/diag/recording/held_entry.rs | 62 +++ nodedb/src/diag/recording/mod.rs | 2 + nodedb/src/error/types.rs | 25 +- nodedb/src/error_classify/public.rs | 9 +- nodedb/src/error_classify/unclassified.rs | 2 + nodedb/src/error_from.rs | 2 + nodedb/src/event/topic/publish.rs | 3 +- nodedb/src/types/record_home.rs | 4 +- nodedb/src/types/snapshot.rs | 2 +- nodedb/tests/crash_harness/vshards.rs | 4 +- nodedb/tests/crash_redo_chunk_stream.rs | 33 +- nodedb/tests/crash_replay_stamp_calvin.rs | 7 +- .../inproc/cases/surrogate_round_trip.rs | 4 +- nodedb/tests/wire/cases/mod.rs | 1 + .../wire/cases/slow_catalog_ddl_apply.rs | 47 ++ 244 files changed, 7612 insertions(+), 2124 deletions(-) create mode 100644 nodedb-cluster-tests/tests/common_suite/cases/gateway_hung_node_budget.rs create mode 100644 nodedb-cluster-tests/tests/common_suite/cases/redo_stream_held_replica.rs rename nodedb-cluster/src/calvin/sequencer/{validator.rs => validator/batch.rs} (66%) create mode 100644 nodedb-cluster/src/calvin/sequencer/validator/graph.rs create mode 100644 nodedb-cluster/src/calvin/sequencer/validator/keys.rs create mode 100644 nodedb-cluster/src/calvin/sequencer/validator/mod.rs create mode 100644 nodedb-cluster/src/calvin/sequencer/validator/reject.rs create mode 100644 nodedb-cluster/src/multi_raft/conf_change_gate.rs create mode 100644 nodedb-raft/src/node/election.rs create mode 100644 nodedb-raft/src/node/snapshot_debt.rs create mode 100644 nodedb-test-support/src/cluster_harness/node/remount.rs create mode 100644 nodedb/src/control/cluster/forwarded_budget.rs create mode 100644 nodedb/src/control/cluster/read_deadline.rs create mode 100644 nodedb/src/control/distributed_applier/apply_loop/held_entry.rs create mode 100644 nodedb/src/control/distributed_applier/apply_loop/panicked.rs create mode 100644 nodedb/src/control/distributed_applier/apply_loop/settle.rs create mode 100644 nodedb/src/control/planner/calvin/submit/wait.rs create mode 100644 nodedb/src/diag/context/held_entry.rs create mode 100644 nodedb/src/diag/recording/held_entry.rs create mode 100644 nodedb/tests/wire/cases/slow_catalog_ddl_apply.rs diff --git a/.config/nextest.toml b/.config/nextest.toml index a24eff7de..9ff3f9904 100644 --- a/.config/nextest.toml +++ b/.config/nextest.toml @@ -157,7 +157,7 @@ threads-required = 'num-test-threads' # Every test in the `wire` target spawns its own real `nodedb` server process. # At full fan-out the box runs one server per core, and each server carries a # Data Plane plus Raft groups — enough CPU starvation that a metadata propose -# misses its 5s deadline and the statement fails. Capping concurrent servers +# misses its statement deadline and the statement fails. Capping concurrent servers # fixes the starvation; widening that deadline would only hide it, and the # deadline is real. # diff --git a/Cargo.lock b/Cargo.lock index 1fa939bb9..53ab4b8cd 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -4679,6 +4679,7 @@ dependencies = [ "smallvec", "sonic-rs", "tempfile", + "thiserror 2.0.20", "tokio", "tokio-postgres", "tokio-tungstenite 0.30.0", diff --git a/nodedb-cluster-tests/tests/common_suite/cases/gateway_hung_node_budget.rs b/nodedb-cluster-tests/tests/common_suite/cases/gateway_hung_node_budget.rs new file mode 100644 index 000000000..38e87b38b --- /dev/null +++ b/nodedb-cluster-tests/tests/common_suite/cases/gateway_hung_node_budget.rs @@ -0,0 +1,217 @@ +// SPDX-License-Identifier: BUSL-1.1 + +//! A node that takes a forwarded request and never answers costs a statement +//! at most its one budget. +//! +//! The execute gate `exec::before_execute` parks every request that the +//! leader of the collection's group receives. A follower forwards its +//! statements on the collection to that leader. +//! +//! - A read waits only its first attempt's share of the budget, then +//! retries. The test releases the gate once the retry starts, and the +//! read returns its row inside the statement timeout. +//! - A write waits its whole budget, then reports an unknown outcome +//! (`57014`). It is never sent again. +//! +//! Requires `--features failpoints`. + +#![cfg(feature = "failpoints")] + +use std::path::Path; +use std::time::{Duration, Instant}; + +use nodedb_cluster::rpc_codec::REPLY_MARGIN; +use nodedb_test_support::fail_point::{FailAction, FailGuard}; +use nodedb_types::id::DatabaseId; + +use crate::common::cluster_harness::shared_steps::db_detail; +use crate::common::cluster_harness::{GroupLeader, TestCluster, TestClusterNode, wait_for}; + +const COLLECTION: &str = "hung_exec"; +const GATE: &str = "exec::before_execute"; +/// Every node's fixed RPC timeout. A read reserves this much plus the reply +/// margin for each attempt after its first. +const RPC_TIMEOUT_SECS: u64 = 3; +/// The read's statement timeout. Its first attempt gets +/// max(12 - 2 × (3 + 2), 3) = 3 s and waits 5 s with the reply margin, +/// which leaves the retry time to answer. +const READ_TIMEOUT: Duration = Duration::from_secs(12); +/// The write's statement timeout. A write waits all of it. +const WRITE_TIMEOUT: Duration = Duration::from_secs(4); +/// How often the test checks the follower's retry count. +const RETRY_POLL: Duration = Duration::from_millis(10); + +/// Release the gate at `release` once `node`'s gateway retries past +/// `retries_before`. Gives up after `give_up`, so a read that never retries +/// fails its own assertions instead of hanging the test. +async fn release_on_retry( + node: &TestClusterNode, + retries_before: u64, + release: &Path, + give_up: Duration, +) { + let started = Instant::now(); + while node.not_leader_retry_count() <= retries_before && started.elapsed() < give_up { + tokio::time::sleep(RETRY_POLL).await; + } + std::fs::write(release, b"release").expect("release the execute gate"); +} + +#[tokio::test(flavor = "multi_thread", worker_threads = 6)] +async fn a_silent_leader_costs_a_read_one_retry_and_a_write_its_budget() { + let cluster = TestCluster::spawn_three_with_rpc_timeout(RPC_TIMEOUT_SECS) + .await + .expect("3-node cluster"); + cluster + .exec_ddl_on_any_leader(&format!( + "CREATE COLLECTION {COLLECTION} (id TEXT PRIMARY KEY, val INT) \ + WITH (engine='document_strict')" + )) + .await + .unwrap_or_else(|e| panic!("CREATE COLLECTION {COLLECTION}: {e}")); + wait_for( + "all 3 nodes see the collection", + Duration::from_secs(10), + Duration::from_millis(50), + || { + cluster + .nodes + .iter() + .all(|n| n.cached_collection_count() >= 1) + }, + ) + .await; + + let vshard = nodedb_cluster::routing::vshard_for_collection( + nodedb_types::CollectionKey::from_bare(DatabaseId::DEFAULT, COLLECTION), + ); + let group = cluster.nodes[0] + .shared + .cluster_routing + .as_ref() + .expect("cluster_routing") + .read() + .unwrap_or_else(|p| p.into_inner()) + .group_for_vshard(vshard) + .expect("the collection's group"); + let leader_id = cluster + .wait_for_group_leader(group, GroupLeader::Any, Duration::from_secs(20)) + .await; + let leader = cluster + .nodes + .iter() + .position(|n| n.node_id == leader_id) + .expect("the group's leader is a cluster node"); + let follower = &cluster.nodes[(leader + 1) % 3]; + + cluster.nodes[leader] + .client + .simple_query(&format!( + "INSERT INTO {COLLECTION} (id, val) VALUES ('k1', 1)" + )) + .await + .unwrap_or_else(|e| panic!("seed row: {}", db_detail(&e))); + cluster + .wait_for_full_apply_convergence(Duration::from_secs(20)) + .await; + + // A read: its first attempt times out, its retry answers. + follower + .client + .simple_query(&format!( + "SET statement_timeout = '{}s'", + READ_TIMEOUT.as_secs() + )) + .await + .expect("SET statement_timeout for the read"); + let read_dir = tempfile::tempdir().expect("read gate directory"); + let read_release = read_dir.path().join("release"); + let read_gate = FailGuard::for_node( + leader_id, + GATE, + FailAction::WaitForFile(read_release.clone()), + ); + let retries_before = follower.not_leader_retry_count(); + let read_sql = format!("SELECT id, val FROM {COLLECTION} WHERE id = 'k1'"); + let started = Instant::now(); + let (read, ()) = tokio::join!( + follower.client.simple_query(&read_sql), + release_on_retry(follower, retries_before, &read_release, READ_TIMEOUT * 2), + ); + let read_elapsed = started.elapsed(); + drop(read_gate); + + let rows = read + .unwrap_or_else(|e| panic!("the read retries past the silent leader: {}", db_detail(&e))) + .into_iter() + .filter(|m| matches!(m, tokio_postgres::SimpleQueryMessage::Row(_))) + .count(); + assert_eq!(rows, 1, "the retried read returns the seeded row"); + assert!( + read_dir.path().join("release.parked").exists(), + "the leader took the read's first attempt and held it" + ); + assert!( + follower.not_leader_retry_count() > retries_before, + "the read retried after its first attempt went unanswered" + ); + assert!( + read_elapsed < READ_TIMEOUT, + "the read finished in {read_elapsed:?}, past its {READ_TIMEOUT:?} statement budget" + ); + + // A write: it waits its whole budget and is never sent again. + follower + .client + .simple_query(&format!( + "SET statement_timeout = '{}s'", + WRITE_TIMEOUT.as_secs() + )) + .await + .expect("SET statement_timeout for the write"); + let write_dir = tempfile::tempdir().expect("write gate directory"); + let write_release = write_dir.path().join("release"); + let write_gate = FailGuard::for_node( + leader_id, + GATE, + FailAction::WaitForFile(write_release.clone()), + ); + let retries_before = follower.not_leader_retry_count(); + let started = Instant::now(); + let write = follower + .client + .simple_query(&format!( + "INSERT INTO {COLLECTION} (id, val) VALUES ('k2', 2)" + )) + .await; + let write_elapsed = started.elapsed(); + std::fs::write(&write_release, b"release").expect("release the execute gate"); + drop(write_gate); + + let err = write.expect_err("a write whose answer never arrives is an unknown outcome"); + assert_eq!( + err.as_db_error().map(|db| db.code().code()), + Some("57014"), + "the write reports its deadline: {}", + db_detail(&err) + ); + assert!( + write_dir.path().join("release.parked").exists(), + "the leader took the write and held it" + ); + assert_eq!( + follower.not_leader_retry_count(), + retries_before, + "a write with a lost answer is never sent again" + ); + assert!( + write_elapsed >= WRITE_TIMEOUT, + "the write waited {write_elapsed:?}, less than its {WRITE_TIMEOUT:?} budget" + ); + assert!( + write_elapsed < (WRITE_TIMEOUT + REPLY_MARGIN) * 2, + "the write waited {write_elapsed:?}, more than one attempt's wait" + ); + + cluster.shutdown().await; +} diff --git a/nodedb-cluster-tests/tests/common_suite/cases/listeners_typed_not_leader.rs b/nodedb-cluster-tests/tests/common_suite/cases/listeners_typed_not_leader.rs index 21819ef65..545704b6f 100644 --- a/nodedb-cluster-tests/tests/common_suite/cases/listeners_typed_not_leader.rs +++ b/nodedb-cluster-tests/tests/common_suite/cases/listeners_typed_not_leader.rs @@ -501,12 +501,13 @@ async fn not_leader_counter_increments_per_retry_attempt() { let counter_inner = Arc::clone(&counter); let call_count_inner = Arc::clone(&call_count); - let result = retry_not_leader(None, move |attempt| { + let deadline = tokio::time::Instant::now() + std::time::Duration::from_secs(30); + let result = retry_not_leader(None, deadline, move |attempt| { let c = Arc::clone(&call_count_inner); let rc = Arc::clone(&counter_inner); async move { let n = c.fetch_add(1, AtomicOrdering::SeqCst); - if attempt > 0 { + if attempt.index > 0 { rc.fetch_add(1, AtomicOrdering::Relaxed); } if n < 2 { diff --git a/nodedb-cluster-tests/tests/common_suite/cases/mod.rs b/nodedb-cluster-tests/tests/common_suite/cases/mod.rs index 1e7a13aa4..3f2f80a5b 100644 --- a/nodedb-cluster-tests/tests/common_suite/cases/mod.rs +++ b/nodedb-cluster-tests/tests/common_suite/cases/mod.rs @@ -89,6 +89,7 @@ mod descriptor_lease_planner_integration; mod descriptor_replay_incarnation; mod descriptor_versioning_cross_node; mod gateway_execute; +mod gateway_hung_node_budget; mod gather_join_in_txn_occ; mod graph_index_edges_replicate_to_followers; mod graph_match_ryow_cross_node; @@ -127,6 +128,7 @@ mod pgwire_gateway_migration; mod planner_local_only; mod prepared_cache_invalidation; mod proposal_committed_twice_applies_once; +mod redo_stream_held_replica; mod resp_gateway_migration; mod retention_policy_cross_node; mod routing_leader_hint_follows_election; diff --git a/nodedb-cluster-tests/tests/common_suite/cases/redo_stream_held_replica.rs b/nodedb-cluster-tests/tests/common_suite/cases/redo_stream_held_replica.rs new file mode 100644 index 000000000..91747b792 --- /dev/null +++ b/nodedb-cluster-tests/tests/common_suite/cases/redo_stream_held_replica.rs @@ -0,0 +1,347 @@ +// SPDX-License-Identifier: BUSL-1.1 + +//! A replica that dropped a chunked redo stream holds its group at the +//! entry it cannot apply, never leads the group, and converges by snapshot. +//! +//! A 3-node cluster caps a redo entry at 64 KiB, so one large commit +//! travels as chunk entries and a final entry. Follower `C` of the +//! collection's group: +//! - applies the stream's first chunk and parks the second chunk on its +//! gate `redo_chunk::before_apply::1`; +//! - receives the commit and one later write in its log while the other +//! replicas apply them; +//! - unmounts the group, which drops the group's streams on `C`, and mounts +//! it again from its log. The group then owes a snapshot on `C`; +//! - releases the chunk. Its stream is gone, so the entry holds the group's +//! apply on `C`. Every snapshot build of the group parks on +//! `snapshot::before_build::`, so the hold lasts. +//! +//! While it holds, `C` stores no row of the collection, its applied index +//! stays below the held entry, a leadership transfer to it never makes it +//! lead, and a read at `C` returns the full count or waits. Once the build +//! gate opens, a snapshot installs and `C`'s content equals its peers'. +//! +//! Requires `--features failpoints`. + +#![cfg(feature = "failpoints")] + +use std::time::{Duration, Instant}; + +use nodedb_raft::ReplicaState; +use nodedb_test_support::fail_point::{FailAction, FailGuard}; +use tokio_postgres::SimpleQueryMessage; + +use super::calvin_replica_content::{local_content, vshard_of, wait_replicas_agree}; +use crate::common::cluster_harness::shared_steps::group_members; +use crate::common::cluster_harness::{ + GroupLeader, TestCluster, TestClusterNode, is_no_serving_leader, wait_for, +}; + +/// The redo entry limit every node runs: the smallest the config takes. +const ENTRY_LIMIT: usize = 64 * 1024; + +/// Rows one statement of the large transaction inserts. +const ROWS_PER_STATEMENT: usize = 100; + +/// Statements the large transaction runs. With 400-byte values its redo +/// holds several times `ENTRY_LIMIT`. +const STATEMENTS: usize = 6; + +/// Rows the collection holds once every write applied: the large +/// transaction's rows and one later write. +const EXPECTED_ROWS: usize = STATEMENTS * ROWS_PER_STATEMENT + 1; + +/// How long each step gets. +const STEP_DEADLINE: Duration = Duration::from_secs(30); + +/// Pause between two polls of a step. +const STEP_POLL: Duration = Duration::from_millis(50); + +/// How long the hold is watched before a snapshot may build. +const HOLD_WINDOW: Duration = Duration::from_secs(4); + +/// How long a read at the held replica gets. +const READ_DEADLINE: Duration = Duration::from_secs(5); + +const COLL: &str = "held_redo_stream"; + +/// The index in `cluster.nodes` of node `node_id`. +fn index_of(cluster: &TestCluster, node_id: u64) -> usize { + cluster + .nodes + .iter() + .position(|node| node.node_id == node_id) + .unwrap_or_else(|| panic!("node {node_id} is live")) +} + +/// A fresh session on `node`. +async fn session(node: &TestClusterNode) -> tokio_postgres::Client { + let (client, connection) = tokio_postgres::connect( + &format!( + "host=127.0.0.1 port={} user=nodedb dbname=default", + node.pg_addr.port() + ), + tokio_postgres::NoTls, + ) + .await + .expect("connect a session"); + tokio::spawn(async move { + let _ = connection.await; + }); + client +} + +/// Commit one transaction whose redo passes `ENTRY_LIMIT` several times. +async fn commit_large_transaction(client: &tokio_postgres::Client) { + let filler = "x".repeat(400); + client.simple_query("BEGIN").await.expect("BEGIN"); + for statement in 0..STATEMENTS { + let values: Vec = (0..ROWS_PER_STATEMENT) + .map(|row| { + let id = statement * ROWS_PER_STATEMENT + row; + format!("('r{id}', '{filler}')") + }) + .collect(); + client + .simple_query(&format!( + "INSERT INTO {COLL} (id, value) VALUES {}", + values.join(", ") + )) + .await + .expect("INSERT inside the transaction"); + } + client.simple_query("COMMIT").await.expect("COMMIT"); +} + +/// The commit index of `group_id` in node `node`'s Raft. +fn raft_committed(node: &TestClusterNode, group_id: u64) -> u64 { + node.shared.multi_raft.get().map_or(0, |multi_raft| { + multi_raft + .lock() + .unwrap_or_else(|p| p.into_inner()) + .commit_index_for(group_id) + .unwrap_or(0) + }) +} + +/// Entries of `group_id` node `node`'s Raft handed to its apply loop. +fn raft_delivered(node: &TestClusterNode, group_id: u64) -> u64 { + node.shared.multi_raft.get().map_or(0, |multi_raft| { + multi_raft + .lock() + .unwrap_or_else(|p| p.into_inner()) + .last_applied(group_id) + .unwrap_or(0) + }) +} + +/// What `node`'s replica of `group_id` needs before it takes log entries. +fn replica_state(node: &TestClusterNode, group_id: u64) -> Option { + node.shared.multi_raft.get().and_then(|multi_raft| { + multi_raft + .lock() + .unwrap_or_else(|p| p.into_inner()) + .replica_state(group_id) + }) +} + +/// Documents of `COLL` `node` stores, read from its own cores. +async fn stored_rows(node: &TestClusterNode) -> usize { + local_content(node, &[COLL]) + .await + .get(&vshard_of(COLL)) + .map_or(0, |content| { + content.keys().filter(|key| key.starts_with("doc:")).count() + }) +} + +/// Read `COUNT(*)` at `node`. A count must be the full count. A read that +/// waits past `READ_DEADLINE`, or finds no serving leader, returns no count. +async fn assert_read_is_whole_or_waits(node: &TestClusterNode) { + let client = session(node).await; + let count_sql = format!("SELECT COUNT(*) FROM {COLL}"); + let read = client.simple_query(&count_sql); + match tokio::time::timeout(READ_DEADLINE, read).await { + Err(_) => {} + Ok(Err(error)) if is_no_serving_leader(&error) => {} + Ok(Err(error)) => panic!("node {}: COUNT(*) failed: {error}", node.node_id), + Ok(Ok(messages)) => { + let count: Vec = messages + .iter() + .filter_map(|message| match message { + SimpleQueryMessage::Row(row) => row.get(0).map(str::to_owned), + _ => None, + }) + .collect(); + assert_eq!( + count, + vec![EXPECTED_ROWS.to_string()], + "node {}: a read served while the node's apply holds must count every \ + committed row", + node.node_id + ); + } + } +} + +#[tokio::test(flavor = "multi_thread", worker_threads = 6)] +async fn a_replica_that_lost_a_redo_stream_holds_its_group_until_a_snapshot() { + let cluster = TestCluster::spawn_three_with_redo_entry_limit(ENTRY_LIMIT) + .await + .expect("3-node cluster"); + cluster + .exec_ddl_on_any_leader(&format!( + "CREATE COLLECTION {COLL} (id TEXT PRIMARY KEY, value TEXT) \ + WITH (engine='document_strict')" + )) + .await + .unwrap_or_else(|e| panic!("create {COLL}: {e}")); + wait_for( + "every node replicates the collection's group", + STEP_DEADLINE, + STEP_POLL, + || { + cluster.nodes.iter().all(|node| { + node.group_id_for_collection(COLL) + .is_some_and(|group| node.replicates_data_group(group)) + }) + }, + ) + .await; + cluster.wait_for_preferred_leaders(STEP_DEADLINE).await; + + let group = cluster.nodes[0] + .group_id_for_collection(COLL) + .expect("the collection's data group"); + let leader = cluster + .wait_for_group_leader(group, GroupLeader::Any, STEP_DEADLINE) + .await; + let held_id = group_members(&cluster.nodes[0], group) + .into_iter() + .find(|member| *member != leader) + .expect("the group has a follower"); + let held = &cluster.nodes[index_of(&cluster, held_id)]; + let writer = &cluster.nodes[index_of(&cluster, leader)]; + + let gate_dir = tempfile::tempdir().expect("gate directory"); + let chunk_release = gate_dir.path().join("chunk"); + let chunk_parked = gate_dir.path().join("chunk.parked"); + let build_release = gate_dir.path().join("build"); + let chunk_gate = FailGuard::for_node( + held_id, + "redo_chunk::before_apply::1", + FailAction::WaitForFile(chunk_release.clone()), + ); + let build_gate = FailGuard::install( + &format!("snapshot::before_build::{group}"), + FailAction::WaitForFile(build_release.clone()), + ); + + let client = session(writer).await; + commit_large_transaction(&client).await; + client + .simple_query(&format!( + "INSERT INTO {COLL} (id, value) VALUES ('after', 'later write')" + )) + .await + .expect("a write after the large transaction"); + let leader_applied = raft_committed(writer, group); + wait_for( + "the leader applies every committed entry", + STEP_DEADLINE, + STEP_POLL, + || writer.shared.applied_index_watcher(group).current() >= leader_applied, + ) + .await; + + wait_for( + "the held node parks the stream's second chunk and its log holds every write", + STEP_DEADLINE, + STEP_POLL, + || chunk_parked.exists() && raft_delivered(held, group) >= leader_applied, + ) + .await; + let tracker = held.shared.propose_tracker.get().expect("propose tracker"); + let held_index = tracker + .applying(group) + .expect("the held node's group waits behind the parked chunk") + .log_index; + assert!( + held_index < leader_applied, + "the parked chunk at {held_index} comes before the later write at {leader_applied}" + ); + + held.remount_data_group(group) + .await + .unwrap_or_else(|e| panic!("{e}")); + assert!( + held.shared.redo_chunks.owes_snapshot(group), + "node {held_id} dropped the group's stream: it owes a snapshot" + ); + assert_eq!( + replica_state(held, group), + Some(ReplicaState::AwaitingSnapshot), + "node {held_id}'s data diverged: it must not lead" + ); + + std::fs::write(&chunk_release, b"release").expect("release the parked chunk"); + drop(chunk_gate); + + // A transfer to the held node starts its campaign. A refused transfer + // starts nothing. Either way it must never lead. + let _ = writer.transfer_group_leadership(group, held_id); + let watch_until = Instant::now() + HOLD_WINDOW; + while Instant::now() < watch_until { + let applied = held.shared.applied_index_watcher(group).current(); + assert!( + applied < held_index, + "node {held_id} applied through {applied}, past the held entry at {held_index}" + ); + assert_eq!( + tracker.applying(group).map(|entry| entry.log_index), + Some(held_index), + "node {held_id}'s group waits behind the held entry" + ); + assert!( + !held.leads_group(group), + "node {held_id} leads group {group} while it awaits a snapshot" + ); + assert_eq!( + replica_state(held, group), + Some(ReplicaState::AwaitingSnapshot) + ); + tokio::time::sleep(STEP_POLL).await; + } + assert_eq!( + stored_rows(held).await, + 0, + "node {held_id} stores rows the held stream's final entry or the later write carry" + ); + assert_read_is_whole_or_waits(held).await; + + std::fs::write(&build_release, b"release").expect("release the snapshot build"); + drop(build_gate); + + wait_for( + "the held node installs a snapshot and settles its debt", + STEP_DEADLINE, + STEP_POLL, + || { + !held.shared.redo_chunks.owes_snapshot(group) + && replica_state(held, group) == Some(ReplicaState::Whole) + && held.shared.applied_index_watcher(group).current() >= leader_applied + }, + ) + .await; + let content = wait_replicas_agree(&cluster, &[COLL]).await; + let rows = content.get(&vshard_of(COLL)).map_or(0, |stored| { + stored.keys().filter(|key| key.starts_with("doc:")).count() + }); + assert_eq!( + rows, EXPECTED_ROWS, + "every replica stores every committed row" + ); + assert_eq!(stored_rows(held).await, EXPECTED_ROWS); + + cluster.shutdown().await; +} diff --git a/nodedb-cluster-tests/tests/common_suite/cases/routing_redirect_hint_unhosted_group.rs b/nodedb-cluster-tests/tests/common_suite/cases/routing_redirect_hint_unhosted_group.rs index 6fe3b9af1..33919eb3f 100644 --- a/nodedb-cluster-tests/tests/common_suite/cases/routing_redirect_hint_unhosted_group.rs +++ b/nodedb-cluster-tests/tests/common_suite/cases/routing_redirect_hint_unhosted_group.rs @@ -197,8 +197,9 @@ async fn an_outsider_follows_a_redirect_to_the_new_leader_and_ignores_a_stale_on // A stale redirect naming the old leader at the old term, through the // gateway's retry path, leaves the hint on the new leader. - let _ = retry_not_leader(Some(&**routing), |attempt| async move { - if attempt == 0 { + let deadline = tokio::time::Instant::now() + std::time::Duration::from_secs(30); + let _ = retry_not_leader(Some(&**routing), deadline, |attempt| async move { + if attempt.index == 0 { Err(nodedb::Error::NotLeader { vshard_id: VShardId::new(vshard_id), leader_node: old_leader, diff --git a/nodedb-cluster-tests/tests/misc_suite/cases/transport_security.rs b/nodedb-cluster-tests/tests/misc_suite/cases/transport_security.rs index 8ae10a2e1..4b4edac70 100644 --- a/nodedb-cluster-tests/tests/misc_suite/cases/transport_security.rs +++ b/nodedb-cluster-tests/tests/misc_suite/cases/transport_security.rs @@ -65,7 +65,7 @@ impl RaftRpcHandler for EchoHandler { success: true, last_log_index: req.prev_log_index + req.entries.len() as u64, round: req.round, - needs_snapshot: false, + replica: nodedb_raft::ReplicaState::Whole, })) } RaftRpc::RequestVoteRequest(req) => { diff --git a/nodedb-cluster-tests/tests/misc_suite/cases/wire_version_handshake.rs b/nodedb-cluster-tests/tests/misc_suite/cases/wire_version_handshake.rs index ed7e01bf5..b1ef14b75 100644 --- a/nodedb-cluster-tests/tests/misc_suite/cases/wire_version_handshake.rs +++ b/nodedb-cluster-tests/tests/misc_suite/cases/wire_version_handshake.rs @@ -37,7 +37,7 @@ impl RaftRpcHandler for EchoHandler { success: true, last_log_index: req.prev_log_index, round: req.round, - needs_snapshot: false, + replica: nodedb_raft::ReplicaState::Whole, })) } other => Err(ClusterError::Transport { diff --git a/nodedb-cluster/src/applied_watcher/mod.rs b/nodedb-cluster/src/applied_watcher/mod.rs index 681ff9289..64232ec58 100644 --- a/nodedb-cluster/src/applied_watcher/mod.rs +++ b/nodedb-cluster/src/applied_watcher/mod.rs @@ -9,18 +9,15 @@ //! checks) can wait on the apply watermark of *their* group rather //! than only the metadata group. //! -//! Bump points (single source of truth for "applied on this node"): +//! Bump points: //! -//! 1. [`crate::raft_loop::tick`] after the per-group apply phase -//! — covers regular committed-entry application for every locally -//! mounted group, including the metadata group. -//! 2. [`crate::raft_loop::handle_rpc`] after `handle_install_snapshot` -//! — covers the snapshot-install path where `last_applied` jumps -//! to `last_included_index` without going through the apply loop. -//! -//! Both bump points read the canonical applied index from the Raft -//! node itself rather than mirroring it from a side channel, so the -//! watcher cannot drift from Raft's own state. +//! 1. The metadata lane, once a metadata entry applied. +//! 2. The tick loop, for the Calvin sequencer group, which the host applies +//! inline. +//! 3. The host apply loop, for data groups, once an entry's apply concluded +//! and every earlier entry of the group settled. +//! 4. A snapshot install, for every group whose state it restored. +//! 5. A group mount, seeded with the durable applied index. pub mod registry; pub mod watcher; diff --git a/nodedb-cluster/src/applied_watcher/registry.rs b/nodedb-cluster/src/applied_watcher/registry.rs index 666c875fd..d571458ad 100644 --- a/nodedb-cluster/src/applied_watcher/registry.rs +++ b/nodedb-cluster/src/applied_watcher/registry.rs @@ -20,7 +20,7 @@ use std::collections::HashMap; use std::sync::{Arc, RwLock}; -use std::time::Duration; +use std::time::Instant; use super::watcher::{AppliedIndexWatcher, WaitOutcome}; @@ -77,12 +77,18 @@ impl GroupAppliedWatchers { } } - /// Block until `group_id`'s applied watermark reaches `target`, - /// the timeout elapses, or the group is removed. Lazily creates - /// the watcher if it does not yet exist (waiters proposed before - /// the first apply on this node would otherwise miss the wake-up). - pub fn wait_for(&self, group_id: u64, target: u64, timeout: Duration) -> WaitOutcome { - self.get_or_create(group_id).wait_for(target, timeout) + /// Wait until `group_id`'s applied watermark reaches `target`, + /// `deadline` passes, or the group is removed. Creates the watcher + /// when absent, so a wait that starts before the first apply here + /// still wakes. + pub async fn wait_for_async( + &self, + group_id: u64, + target: u64, + deadline: Instant, + ) -> WaitOutcome { + let watcher = self.get_or_create(group_id); + watcher.wait_for_async(target, deadline).await } /// Number of groups currently registered. Useful for metrics. @@ -117,7 +123,11 @@ impl GroupAppliedWatchers { #[cfg(test)] mod tests { use super::*; - use std::thread; + use std::time::Duration; + + fn after(duration: Duration) -> Instant { + Instant::now() + duration + } #[test] fn separate_groups_have_independent_watermarks() { @@ -128,32 +138,40 @@ mod tests { assert_eq!(r.get(2).unwrap().current(), 10); } - #[test] - fn wait_returns_reached_after_bump() { + #[tokio::test] + async fn wait_returns_reached_after_bump() { let r = Arc::new(GroupAppliedWatchers::new()); - let r2 = r.clone(); - let h = thread::spawn(move || r2.wait_for(7, 5, Duration::from_secs(2))); - thread::sleep(Duration::from_millis(20)); + let waiter = { + let r = Arc::clone(&r); + tokio::spawn(async move { r.wait_for_async(7, 5, after(Duration::from_secs(2))).await }) + }; + tokio::time::sleep(Duration::from_millis(20)).await; r.bump(7, 5); - assert_eq!(h.join().unwrap(), WaitOutcome::Reached); + assert_eq!(waiter.await.unwrap(), WaitOutcome::Reached); } - #[test] - fn remove_wakes_waiter_with_group_gone() { + #[tokio::test] + async fn remove_wakes_waiter_with_group_gone() { let r = Arc::new(GroupAppliedWatchers::new()); - let r2 = r.clone(); // Pre-create so the waiter watches the same Arc that remove() closes. r.get_or_create(3); - let h = thread::spawn(move || r2.wait_for(3, 99, Duration::from_secs(2))); - thread::sleep(Duration::from_millis(20)); + let waiter = { + let r = Arc::clone(&r); + tokio::spawn( + async move { r.wait_for_async(3, 99, after(Duration::from_secs(2))).await }, + ) + }; + tokio::time::sleep(Duration::from_millis(20)).await; r.remove(3); - assert_eq!(h.join().unwrap(), WaitOutcome::GroupGone); + assert_eq!(waiter.await.unwrap(), WaitOutcome::GroupGone); } - #[test] - fn missing_group_creates_watcher_lazily() { + #[tokio::test] + async fn missing_group_creates_watcher_lazily() { let r = GroupAppliedWatchers::new(); - let outcome = r.wait_for(42, 1, Duration::from_millis(20)); + let outcome = r + .wait_for_async(42, 1, after(Duration::from_millis(20))) + .await; assert_eq!(outcome, WaitOutcome::TimedOut); assert!(r.get(42).is_some()); } diff --git a/nodedb-cluster/src/applied_watcher/watcher.rs b/nodedb-cluster/src/applied_watcher/watcher.rs index 080c769fa..62845a0d0 100644 --- a/nodedb-cluster/src/applied_watcher/watcher.rs +++ b/nodedb-cluster/src/applied_watcher/watcher.rs @@ -1,19 +1,18 @@ // SPDX-License-Identifier: BUSL-1.1 -//! Sync wait primitive used to block on a Raft group's local apply -//! watermark. +//! Async wait on a Raft group's local apply watermark. //! -//! Uses `std::sync::Condvar` so it is safe to call from synchronous -//! pgwire handler code without entering a tokio reactor. A `closed` -//! flag distinguishes "still waiting for apply" from "the group is -//! no longer hosted on this node" — a follower that gets removed via -//! conf-change must wake outstanding waiters with a terminal status -//! rather than time them out. +//! A waiter parks as a task on a [`Notify`], so it holds no thread while it +//! waits. A `closed` flag tells "still waiting for apply" apart from "the +//! group is no longer hosted on this node": a removed group wakes its +//! waiters with a terminal outcome instead of timing them out. -use std::sync::{Condvar, Mutex}; -use std::time::{Duration, Instant}; +use std::sync::Mutex; +use std::time::Instant; -/// Outcome of [`AppliedIndexWatcher::wait_for`]. +use tokio::sync::Notify; + +/// Outcome of [`AppliedIndexWatcher::wait_for_async`]. #[derive(Debug, Clone, Copy, PartialEq, Eq)] pub enum WaitOutcome { /// The applied watermark reached the requested target. @@ -35,7 +34,8 @@ impl WaitOutcome { #[derive(Debug, Default)] pub struct AppliedIndexWatcher { state: Mutex, - cv: Condvar, + /// Woken on every watermark advance and on close. + changed: Notify, } #[derive(Debug, Default)] @@ -53,29 +53,36 @@ impl AppliedIndexWatcher { Self::default() } + fn state(&self) -> std::sync::MutexGuard<'_, State> { + self.state.lock().unwrap_or_else(|p| p.into_inner()) + } + /// Advance the watermark. Idempotent: smaller indices are ignored. /// Bumps on a closed watcher are also ignored (the group is gone). pub fn bump(&self, applied_index: u64) { - let mut guard = self.state.lock().unwrap_or_else(|p| p.into_inner()); - if guard.closed { - return; - } - if applied_index > guard.applied { - guard.applied = applied_index; - self.cv.notify_all(); + let advanced = { + let mut guard = self.state(); + let advanced = !guard.closed && applied_index > guard.applied; + if advanced { + guard.applied = applied_index; + } + advanced + }; + if advanced { + self.changed.notify_waiters(); } } /// Read the current watermark without blocking. pub fn current(&self) -> u64 { - self.state.lock().unwrap_or_else(|p| p.into_inner()).applied + self.state().applied } /// Declare that the applier starts a batch ending at `last_index`. The /// applier calls it before the batch mutates any state a reader can see. /// Idempotent: smaller indices are ignored. pub fn begin_batch(&self, last_index: u64) { - let mut guard = self.state.lock().unwrap_or_else(|p| p.into_inner()); + let mut guard = self.state(); guard.applying_through = guard.applying_through.max(last_index); } @@ -84,51 +91,60 @@ impl AppliedIndexWatcher { /// /// A reader that sees an effect of the batch in progress gets a floor at /// or above the index of the entry that made it visible. A waiter for - /// that floor waits on [`Self::wait_for`], which the finished batch + /// that floor waits on [`Self::wait_for_async`], which the finished batch /// satisfies. pub fn floor(&self) -> u64 { - let guard = self.state.lock().unwrap_or_else(|p| p.into_inner()); + let guard = self.state(); guard.applied.max(guard.applying_through) } /// True once [`Self::close`] has been called. pub fn is_closed(&self) -> bool { - self.state.lock().unwrap_or_else(|p| p.into_inner()).closed + self.state().closed } /// Mark this watcher closed and wake every waiter with /// [`WaitOutcome::GroupGone`]. Idempotent. pub fn close(&self) { - let mut guard = self.state.lock().unwrap_or_else(|p| p.into_inner()); - if !guard.closed { + let closed_now = { + let mut guard = self.state(); + let closed_now = !guard.closed; guard.closed = true; - self.cv.notify_all(); + closed_now + }; + if closed_now { + self.changed.notify_waiters(); } } - /// Block until the watermark reaches `target`, the timeout elapses, - /// or the watcher is closed. - pub fn wait_for(&self, target: u64, timeout: Duration) -> WaitOutcome { - let deadline = Instant::now() + timeout; - let mut guard = self.state.lock().unwrap_or_else(|p| p.into_inner()); + /// The outcome of a wait for `target` as the state stands, `None` while + /// it is still pending. + fn settled(&self, target: u64) -> Option { + let guard = self.state(); + if guard.applied >= target { + Some(WaitOutcome::Reached) + } else if guard.closed { + Some(WaitOutcome::GroupGone) + } else { + None + } + } + + /// Wait until the watermark reaches `target`, `deadline` passes, or the + /// watcher is closed. The waiting task holds no thread. + pub async fn wait_for_async(&self, target: u64, deadline: Instant) -> WaitOutcome { + let deadline = tokio::time::Instant::from_std(deadline); loop { - if guard.applied >= target { - return WaitOutcome::Reached; + let notified = self.changed.notified(); + tokio::pin!(notified); + // Registered before the state is read, so a bump between the + // read and the await still wakes this waiter. + notified.as_mut().enable(); + if let Some(outcome) = self.settled(target) { + return outcome; } - if guard.closed { - return WaitOutcome::GroupGone; - } - let remaining = match deadline.checked_duration_since(Instant::now()) { - Some(r) if !r.is_zero() => r, - _ => return WaitOutcome::TimedOut, - }; - let wait = self - .cv - .wait_timeout(guard, remaining) - .unwrap_or_else(|p| p.into_inner()); - guard = wait.0; - if wait.1.timed_out() && guard.applied < target && !guard.closed { - return WaitOutcome::TimedOut; + if tokio::time::timeout_at(deadline, notified).await.is_err() { + return self.settled(target).unwrap_or(WaitOutcome::TimedOut); } } } @@ -138,47 +154,83 @@ impl AppliedIndexWatcher { mod tests { use super::*; use std::sync::Arc; - use std::thread; + use std::time::Duration; - #[test] - fn bump_notifies_waiter() { + fn after(duration: Duration) -> Instant { + Instant::now() + duration + } + + #[tokio::test] + async fn a_bump_wakes_the_waiter() { let w = Arc::new(AppliedIndexWatcher::new()); - let w2 = w.clone(); - let h = thread::spawn(move || w2.wait_for(5, Duration::from_secs(2))); - thread::sleep(Duration::from_millis(20)); + let waiter = { + let w = Arc::clone(&w); + tokio::spawn(async move { w.wait_for_async(5, after(Duration::from_secs(5))).await }) + }; + tokio::time::sleep(Duration::from_millis(20)).await; w.bump(3); - thread::sleep(Duration::from_millis(20)); + tokio::time::sleep(Duration::from_millis(20)).await; w.bump(5); - assert_eq!(h.join().unwrap(), WaitOutcome::Reached); + let started = Instant::now(); + assert_eq!(waiter.await.expect("join"), WaitOutcome::Reached); + assert!(started.elapsed() < Duration::from_secs(2)); } - #[test] - fn times_out_if_never_bumped() { + #[tokio::test] + async fn the_wait_times_out_at_its_deadline() { let w = AppliedIndexWatcher::new(); - assert_eq!( - w.wait_for(1, Duration::from_millis(30)), - WaitOutcome::TimedOut - ); + let started = Instant::now(); + let outcome = w.wait_for_async(1, after(Duration::from_millis(200))).await; + assert_eq!(outcome, WaitOutcome::TimedOut); + let waited = started.elapsed(); + assert!(waited >= Duration::from_millis(200), "{waited:?}"); + assert!(waited < Duration::from_secs(2), "{waited:?}"); } - #[test] - fn already_past_target_returns_immediately() { + #[tokio::test] + async fn a_passed_deadline_still_reports_a_reached_target() { let w = AppliedIndexWatcher::new(); w.bump(10); - assert_eq!( - w.wait_for(5, Duration::from_millis(10)), - WaitOutcome::Reached - ); + let outcome = w.wait_for_async(5, Instant::now()).await; + assert_eq!(outcome, WaitOutcome::Reached); } - #[test] - fn close_wakes_waiter_with_group_gone() { + #[tokio::test] + async fn close_wakes_the_waiter_with_group_gone() { let w = Arc::new(AppliedIndexWatcher::new()); - let w2 = w.clone(); - let h = thread::spawn(move || w2.wait_for(99, Duration::from_secs(2))); - thread::sleep(Duration::from_millis(20)); + let waiter = { + let w = Arc::clone(&w); + tokio::spawn(async move { w.wait_for_async(99, after(Duration::from_secs(5))).await }) + }; + tokio::time::sleep(Duration::from_millis(20)).await; w.close(); - assert_eq!(h.join().unwrap(), WaitOutcome::GroupGone); + assert_eq!(waiter.await.expect("join"), WaitOutcome::GroupGone); + } + + /// A bump that races the start of a wait is never lost: every waiter + /// ends `Reached` well before its deadline. + #[tokio::test(flavor = "multi_thread", worker_threads = 4)] + async fn a_bump_racing_the_wait_start_is_not_lost() { + for round in 1..=200u64 { + let w = Arc::new(AppliedIndexWatcher::new()); + let waiter = { + let w = Arc::clone(&w); + tokio::spawn(async move { + w.wait_for_async(round, after(Duration::from_secs(10))) + .await + }) + }; + let bumper = { + let w = Arc::clone(&w); + tokio::spawn(async move { w.bump(round) }) + }; + bumper.await.expect("join bumper"); + let outcome = tokio::time::timeout(Duration::from_secs(5), waiter) + .await + .expect("the waiter woke on the bump") + .expect("join waiter"); + assert_eq!(outcome, WaitOutcome::Reached, "round {round}"); + } } #[test] diff --git a/nodedb-cluster/src/calvin/sequencer/validator.rs b/nodedb-cluster/src/calvin/sequencer/validator/batch.rs similarity index 66% rename from nodedb-cluster/src/calvin/sequencer/validator.rs rename to nodedb-cluster/src/calvin/sequencer/validator/batch.rs index d2e45f60f..b273323db 100644 --- a/nodedb-cluster/src/calvin/sequencer/validator.rs +++ b/nodedb-cluster/src/calvin/sequencer/validator/batch.rs @@ -20,7 +20,7 @@ //! # Conflict handling (read-aware) //! //! The scheduler locks a txn's write keys exclusive and its read keys shared, -//! so a reader and a writer of one key serialize by `position`. Per-key last-write-LSN +//! so a reader and a writer of conflicting keys serialize by `position`. Per-key last-write-LSN //! validation means a reader `R` of key `K` aborts at commit only if a //! *lower-position* writer `W` of `K` committed first (its write LSN exceeds //! `R`'s read version). Giving `R` a lower position than `W` converts that @@ -29,9 +29,14 @@ //! So the validator: //! //! - Builds, per key `K`, the set of transactions that read `K` and the set -//! that write `K`. -//! - For every `(reader r, writer w)` pair on the same key with `r != w`, adds -//! a directed edge `r -> w` ("`r` must get a lower position than `w`"). +//! that write `K`, and per collection its collection-scope and row keys. +//! - For every `(reader r, writer w)` pair with `r != w` whose keys conflict, +//! adds a directed edge `r -> w` ("`r` must get a lower position than +//! `w`"). Two keys conflict when they are the same key, or when they share +//! a collection and either covers the whole collection (see +//! [`super::keys::conflicts`]). The Data Plane validates a collection read +//! against every row write and a row read against every whole-collection +//! write, so these pairs order like same-key pairs. //! - Emits positions in a deterministic topological order of those edges, //! which places every reader ahead of the conflicting writers it can — the //! read-after-write → write-after-read reorder that avoids the abort. @@ -46,146 +51,17 @@ //! mirroring "the later transaction loses". Positions on the admitted set are //! dense `0..(n - num_rejected)` in emission order. -use std::collections::{BTreeMap, BTreeSet}; +use std::collections::BTreeSet; use xxhash_rust::xxh3::xxh3_64; use crate::calvin::sequencer::error::SequencerError; use crate::calvin::sequencer::inbox::{AdmittedTx, RejectedTx}; -use crate::calvin::sequencer::metrics::ConflictKey; -use crate::calvin::types::{EngineKeySet, ReadWriteSet, SequencedTxn}; - -/// Canonical, order-comparable identity of a single key across engines: -/// `(engine_discriminant, collection, key_bytes)`. Used as the `BTreeMap` key -/// for the reader/writer maps so all iteration feeding output is deterministic. -type KeyId = (u8, String, Vec); - -/// A single flattened key drawn from one transaction's read or write set. -#[derive(Debug)] -struct FlatKey { - /// Discriminant tag for the key-set variant (Document=0, Vector=1, Kv=2, - /// Edge=3, Array=4, Collection=5, Unique=6). - discriminant: u8, - /// Static engine name used when building conflict-context keys. - engine_name: &'static str, - /// Collection name. - collection: String, - /// Serialized key bytes. - key_bytes: Vec, -} - -impl FlatKey { - /// The order-comparable identity of this key. - fn key_id(&self) -> KeyId { - ( - self.discriminant, - self.collection.clone(), - self.key_bytes.clone(), - ) - } -} +use crate::calvin::types::SequencedTxn; -/// Flatten one read-or-write set into its constituent per-key entries. -fn flatten_set(set: &ReadWriteSet) -> Vec { - let mut out = Vec::new(); - for key_set in &set.0 { - match key_set { - EngineKeySet::Document { - collection, - surrogates, - } => { - for &s in surrogates.as_slice() { - out.push(FlatKey { - discriminant: 0, - engine_name: "document", - collection: collection.clone(), - key_bytes: s.to_le_bytes().to_vec(), - }); - } - } - EngineKeySet::Vector { - collection, - surrogates, - } => { - for &s in surrogates.as_slice() { - out.push(FlatKey { - discriminant: 1, - engine_name: "vector", - collection: collection.clone(), - key_bytes: s.to_le_bytes().to_vec(), - }); - } - } - EngineKeySet::Kv { collection, keys } => { - for k in keys.as_slice() { - out.push(FlatKey { - discriminant: 2, - engine_name: "kv", - collection: collection.clone(), - key_bytes: k.clone(), - }); - } - } - EngineKeySet::Edge { - collection, edges, .. - } => { - for &(src, dst) in edges.as_slice() { - let mut key_bytes = src.to_le_bytes().to_vec(); - key_bytes.extend_from_slice(&dst.to_le_bytes()); - out.push(FlatKey { - discriminant: 3, - engine_name: "edge", - collection: collection.clone(), - key_bytes, - }); - } - } - // The whole array on each vShard it names. - EngineKeySet::Array { - collection, - vshards, - } => { - for &vshard in vshards.as_slice() { - out.push(FlatKey { - discriminant: 4, - engine_name: "array", - collection: collection.clone(), - key_bytes: vshard.to_le_bytes().to_vec(), - }); - } - } - // The whole collection: one key with no row bytes. - EngineKeySet::Collection { collection, .. } => { - out.push(FlatKey { - discriminant: 5, - engine_name: "collection", - collection: collection.clone(), - key_bytes: Vec::new(), - }); - } - // One key per claimed value: the index name, a zero byte, the value. - EngineKeySet::Unique { - collection, - index, - values, - } => { - for value in values.as_slice() { - let mut key_bytes = Vec::with_capacity(index.len() + 1 + value.len()); - key_bytes.extend_from_slice(index.as_bytes()); - key_bytes.push(0); - key_bytes.extend_from_slice(value); - out.push(FlatKey { - discriminant: 6, - engine_name: "unique", - collection: collection.clone(), - key_bytes, - }); - } - } - } - } - out -} +use super::graph::ConflictIndex; +use super::keys::flatten_set; +use super::reject::reject_context; /// Sort key for admitted transactions: `(inbox_seq, tenant_id, hash(plans))`. fn admitted_sort_key(tx: &AdmittedTx) -> (u64, u64, u64) { @@ -218,49 +94,19 @@ pub fn validate_batch_with_assignments( candidates.sort_by_key(admitted_sort_key); let n = candidates.len(); - // Flatten both sets of every transaction and build the reader/writer maps - // keyed by canonical key identity. BTreeMap + ascending index vectors keep - // every iteration that feeds output deterministic. - let mut readers_of: BTreeMap> = BTreeMap::new(); - let mut writers_of: BTreeMap> = BTreeMap::new(); - let mut tx_reads: Vec> = Vec::with_capacity(n); - let mut tx_writes: Vec> = Vec::with_capacity(n); - + // Flatten both sets of every transaction into the conflict index, in + // sorted-index order. + let mut index = ConflictIndex::default(); for (i, tx) in candidates.iter().enumerate() { - let reads = flatten_set(&tx.tx_class.read_set); - let writes = flatten_set(&tx.tx_class.write_set); - for fk in &reads { - let entry = readers_of.entry(fk.key_id()).or_default(); - if entry.last() != Some(&i) { - entry.push(i); - } - } - for fk in &writes { - let entry = writers_of.entry(fk.key_id()).or_default(); - if entry.last() != Some(&i) { - entry.push(i); - } - } - tx_reads.push(reads); - tx_writes.push(writes); + index.add( + i, + flatten_set(&tx.tx_class.read_set), + flatten_set(&tx.tx_class.write_set), + ); } - // Build RAW edges `reader -> writer` (reader gets the lower position). A - // BTreeSet of successors deduplicates edges arising from multiple shared - // keys and keeps successor iteration deterministic. - let mut out: Vec> = vec![BTreeSet::new(); n]; - let mut in_degree: Vec = vec![0; n]; - for (key, readers) in &readers_of { - if let Some(writers) = writers_of.get(key) { - for &r in readers { - for &w in writers { - if r != w && out[r].insert(w) { - in_degree[w] += 1; - } - } - } - } - } + // RAW edges `reader -> writer`: the reader gets the lower position. + let (out, mut in_degree) = index.edges(n); // Deterministic topological emission with cycle-breaking. `ready` is a // BTreeSet so `pop_first` always yields the smallest index — i.e. the @@ -334,15 +180,8 @@ pub fn validate_batch_with_assignments( } None => { let tenant = tx.tx_class.tenant_id.as_u64(); - let (position_admitted, conflict_context) = reject_context( - idx, - tenant, - &tx_reads, - &tx_writes, - &readers_of, - &writers_of, - &emitted_position, - ); + let (position_admitted, conflict_context) = + reject_context(idx, tenant, &index, &emitted_position); rejected_out.push(RejectedTx { admitted: tx, reason: SequencerError::Conflict { position_admitted }, @@ -363,67 +202,6 @@ pub fn validate_batch( (admitted.into_iter().map(|(_, txn)| txn).collect(), rejected) } -/// Build the `(position_admitted, conflict_context)` for a cycle-rejected txn. -/// -/// A rejected txn conflicts with any admitted txn that writes a key it reads or -/// reads a key it writes. We select the conflict with the smallest admitted -/// position (ties broken by key identity) so both the reported position and the -/// [`ConflictKey`] are deterministic. If no conflicting txn was admitted, the -/// position defaults to `0` and the context to `None`. -fn reject_context( - idx: usize, - tenant: u64, - tx_reads: &[Vec], - tx_writes: &[Vec], - readers_of: &BTreeMap>, - writers_of: &BTreeMap>, - emitted_position: &[Option], -) -> (u32, Option) { - // Best conflict so far, ordered by (admitted position, key identity). - let mut best: Option<(u32, KeyId, &'static str)> = None; - let mut consider = |fk: &FlatKey, others: Option<&Vec>| { - let Some(others) = others else { return }; - for &other in others { - if other == idx { - continue; - } - let Some(pos) = emitted_position[other] else { - continue; - }; - let key = fk.key_id(); - let candidate = (pos, key, fk.engine_name); - let better = match &best { - None => true, - Some((bpos, bkey, _)) => (pos, &candidate.1) < (*bpos, bkey), - }; - if better { - best = Some(candidate); - } - } - }; - - // A writer of a key this txn reads. - for fk in &tx_reads[idx] { - consider(fk, writers_of.get(&fk.key_id())); - } - // A reader of a key this txn writes. - for fk in &tx_writes[idx] { - consider(fk, readers_of.get(&fk.key_id())); - } - - match best { - Some((pos, key, engine)) => ( - pos, - Some(ConflictKey { - tenant, - engine, - collection: key.1, - }), - ), - None => (0, None), - } -} - #[cfg(test)] mod tests { use super::*; @@ -694,4 +472,150 @@ mod tests { assert_eq!(a.position, b.position); } } + + /// A transaction that reads `read` and writes `write`, both whole key + /// sets. Single-vShard: the validator works on set contents only. + fn make_set_tx( + inbox_seq: u64, + read: Vec, + write: Vec, + ) -> AdmittedTx { + let tx_class = TxClass::new_single_vshard( + ReadWriteSet::new(read), + ReadWriteSet::new(write), + vec![inbox_seq as u8], + TenantId::new(1), + None, + crate::calvin::types::VersionedReadSet::default(), + ) + .expect("valid TxClass"); + AdmittedTx { + inbox_seq, + tx_class, + } + } + + fn rows(collection: &str, surrogates: Vec) -> EngineKeySet { + EngineKeySet::Document { + collection: collection.to_owned(), + surrogates: SortedVec::new(surrogates), + } + } + + fn whole(collection: &str) -> EngineKeySet { + EngineKeySet::Collection { + collection: collection.to_owned(), + vshards: SortedVec::new(vec![]), + } + } + + fn position_of(admitted: &[(u64, SequencedTxn)], inbox_seq: u64) -> u32 { + admitted + .iter() + .find(|(seq, _)| *seq == inbox_seq) + .map(|(_, t)| t.position) + .expect("txn admitted") + } + + #[test] + fn a_collection_reader_precedes_a_row_writer_of_its_collection() { + let (col_k, col_other) = find_two_distinct_collections(); + // tx0 writes one row of K. tx1 scans K and writes elsewhere. + let writer = make_set_tx(0, vec![], vec![rows(&col_k, vec![7])]); + let reader = make_set_tx(1, vec![whole(&col_k)], vec![rows(&col_other, vec![9])]); + let (admitted, rejected) = validate_batch_with_assignments(11, vec![writer, reader]); + assert!(rejected.is_empty()); + assert!( + position_of(&admitted, 1) < position_of(&admitted, 0), + "the collection reader must precede the row writer" + ); + } + + #[test] + fn a_row_reader_precedes_a_collection_writer() { + let (col_k, col_other) = find_two_distinct_collections(); + // tx0 truncates K. tx1 reads one row of K and writes elsewhere. + let truncate = make_set_tx(0, vec![], vec![whole(&col_k)]); + let reader = make_set_tx( + 1, + vec![rows(&col_k, vec![7])], + vec![rows(&col_other, vec![9])], + ); + let (admitted, rejected) = validate_batch_with_assignments(12, vec![truncate, reader]); + assert!(rejected.is_empty()); + assert!( + position_of(&admitted, 1) < position_of(&admitted, 0), + "the row reader must precede the truncate" + ); + + // An array write covers its collection on its vShard as well. + let array_write = make_set_tx( + 0, + vec![], + vec![EngineKeySet::Array { + collection: col_k.clone(), + vshards: SortedVec::new(vec![3]), + }], + ); + let reader = make_set_tx( + 1, + vec![rows(&col_k, vec![7])], + vec![rows(&col_other, vec![9])], + ); + let (admitted, rejected) = validate_batch_with_assignments(13, vec![array_write, reader]); + assert!(rejected.is_empty()); + assert!(position_of(&admitted, 1) < position_of(&admitted, 0)); + } + + #[test] + fn a_collection_scope_cycle_rejects_the_later_txn() { + let (col_a, col_b) = find_two_distinct_collections(); + // tx0 scans A and writes a row of B. tx1 reads a row of B and + // truncates A. Each must precede the other: no serial order exists. + let tx0 = make_set_tx(0, vec![whole(&col_a)], vec![rows(&col_b, vec![1])]); + let tx1 = make_set_tx(1, vec![rows(&col_b, vec![1])], vec![whole(&col_a)]); + let (admitted, rejected) = validate_batch_with_assignments(14, vec![tx0, tx1]); + assert_eq!(admitted.len(), 1); + assert_eq!(rejected.len(), 1); + assert_eq!(rejected[0].admitted.inbox_seq, 1, "the later txn loses"); + assert_eq!( + rejected[0].reason, + SequencerError::Conflict { + position_admitted: 0 + } + ); + let ctx = rejected[0] + .conflict_context + .as_ref() + .expect("a cycle rejection names its conflict"); + assert!( + ctx.collection == col_a || ctx.collection == col_b, + "collection {} is one of the cycle's", + ctx.collection + ); + + // A row-only variant of the same shapes is no cycle: tx1 reads a row + // of B that tx0 does not write. + let tx0 = make_set_tx(0, vec![rows(&col_a, vec![5])], vec![rows(&col_b, vec![1])]); + let tx1 = make_set_tx(1, vec![rows(&col_b, vec![2])], vec![rows(&col_a, vec![6])]); + let (admitted, rejected) = validate_batch_with_assignments(15, vec![tx0, tx1]); + assert_eq!(admitted.len(), 2); + assert!(rejected.is_empty()); + } + + #[test] + fn array_writes_on_distinct_vshards_add_no_edge() { + let (col_k, col_other) = find_two_distinct_collections(); + let array = |vshard: u32| EngineKeySet::Array { + collection: col_k.clone(), + vshards: SortedVec::new(vec![vshard]), + }; + // tx0 reads K on vShard 1 and writes a row of O. tx1 reads that row + // and writes K on vShard 2. Only tx1 -> tx0 holds: no cycle. + let tx0 = make_set_tx(0, vec![array(1)], vec![rows(&col_other, vec![1])]); + let tx1 = make_set_tx(1, vec![rows(&col_other, vec![1])], vec![array(2)]); + let (admitted, rejected) = validate_batch_with_assignments(16, vec![tx0, tx1]); + assert!(rejected.is_empty()); + assert!(position_of(&admitted, 1) < position_of(&admitted, 0)); + } } diff --git a/nodedb-cluster/src/calvin/sequencer/validator/graph.rs b/nodedb-cluster/src/calvin/sequencer/validator/graph.rs new file mode 100644 index 000000000..b458a2da1 --- /dev/null +++ b/nodedb-cluster/src/calvin/sequencer/validator/graph.rs @@ -0,0 +1,139 @@ +// SPDX-License-Identifier: BUSL-1.1 + +//! The reader/writer index of a candidate batch and the reorder edges it +//! implies. +//! +//! Every map is a `BTreeMap` and every list holds ascending transaction +//! indices, so every iteration that feeds output is deterministic. + +use std::collections::{BTreeMap, BTreeSet}; + +use super::keys::{FlatKey, KeyId, KeyScope, conflicts}; + +/// One key of one transaction: `(transaction index, key index)`. The key +/// index points into that transaction's flattened read or write keys. +pub(super) type KeyRef = (usize, usize); + +/// One collection's keys, split by scope. +#[derive(Debug, Default)] +pub(super) struct CollectionKeys { + /// Every collection-scope read key. + pub(super) scoped_reads: Vec, + /// Every collection-scope write key. + pub(super) scoped_writes: Vec, + /// One row read key per transaction that reads rows of the collection. + /// It stands for all of them: a row key conflicts with a collection-scope + /// key of its collection whatever its bytes. + pub(super) row_reads: Vec, + /// One row write key per transaction that writes rows of the collection. + pub(super) row_writes: Vec, +} + +/// The flattened keys of every candidate and the maps that find conflicting +/// pairs without comparing every key against every other. +#[derive(Debug, Default)] +pub(super) struct ConflictIndex { + /// Flattened read keys, per transaction. + pub(super) reads: Vec>, + /// Flattened write keys, per transaction. + pub(super) writes: Vec>, + /// Transactions that read each exact key. + pub(super) readers_of: BTreeMap>, + /// Transactions that write each exact key. + pub(super) writers_of: BTreeMap>, + /// Keys per collection, for the pairs a collection-scope key makes. + pub(super) by_collection: BTreeMap, +} + +impl ConflictIndex { + /// Add transaction `i`'s keys. Transactions are added in index order. + pub(super) fn add(&mut self, i: usize, reads: Vec, writes: Vec) { + for (k, fk) in reads.iter().enumerate() { + push_txn(self.readers_of.entry(fk.key_id()).or_default(), i); + let coll = self.by_collection.entry(fk.collection.clone()).or_default(); + match fk.scope { + KeyScope::Row => push_key(&mut coll.row_reads, (i, k)), + KeyScope::Collection => coll.scoped_reads.push((i, k)), + } + } + for (k, fk) in writes.iter().enumerate() { + push_txn(self.writers_of.entry(fk.key_id()).or_default(), i); + let coll = self.by_collection.entry(fk.collection.clone()).or_default(); + match fk.scope { + KeyScope::Row => push_key(&mut coll.row_writes, (i, k)), + KeyScope::Collection => coll.scoped_writes.push((i, k)), + } + } + self.reads.push(reads); + self.writes.push(writes); + } + + /// The read key `r` names. + pub(super) fn read(&self, (txn, key): KeyRef) -> &FlatKey { + &self.reads[txn][key] + } + + /// The write key `w` names. + pub(super) fn write(&self, (txn, key): KeyRef) -> &FlatKey { + &self.writes[txn][key] + } + + /// The reorder edges `reader -> writer` of every conflicting pair, as a + /// successor set per transaction, and each transaction's in-degree. + /// + /// A `BTreeSet` of successors deduplicates edges that several keys imply + /// and keeps successor iteration deterministic. + pub(super) fn edges(&self, n: usize) -> (Vec>, Vec) { + let mut out: Vec> = vec![BTreeSet::new(); n]; + let mut in_degree: Vec = vec![0; n]; + let mut add = |r: usize, w: usize| { + if r != w && out[r].insert(w) { + in_degree[w] += 1; + } + }; + + // A reader and a writer of one exact key. + for (key, readers) in &self.readers_of { + if let Some(writers) = self.writers_of.get(key) { + for &r in readers { + for &w in writers { + add(r, w); + } + } + } + } + + // A collection-scope key against the keys of its collection. + for coll in self.by_collection.values() { + for &r in &coll.scoped_reads { + for &w in coll.scoped_writes.iter().chain(&coll.row_writes) { + if conflicts(self.read(r), self.write(w)) { + add(r.0, w.0); + } + } + } + for &r in &coll.row_reads { + for &w in &coll.scoped_writes { + if conflicts(self.read(r), self.write(w)) { + add(r.0, w.0); + } + } + } + } + (out, in_degree) + } +} + +/// Append transaction `i` unless it is already the last entry. +fn push_txn(list: &mut Vec, i: usize) { + if list.last() != Some(&i) { + list.push(i); + } +} + +/// Append key `key` unless its transaction already has an entry. +fn push_key(list: &mut Vec, key: KeyRef) { + if list.last().map(|(txn, _)| *txn) != Some(key.0) { + list.push(key); + } +} diff --git a/nodedb-cluster/src/calvin/sequencer/validator/keys.rs b/nodedb-cluster/src/calvin/sequencer/validator/keys.rs new file mode 100644 index 000000000..cc98e86bd --- /dev/null +++ b/nodedb-cluster/src/calvin/sequencer/validator/keys.rs @@ -0,0 +1,246 @@ +// SPDX-License-Identifier: BUSL-1.1 + +//! Flattened per-key view of a transaction's read and write sets, and the +//! one predicate that decides whether a read key and a write key conflict. + +use crate::calvin::types::{EngineKeySet, ReadWriteSet}; + +/// Canonical, order-comparable identity of a single key across engines: +/// `(engine_discriminant, collection, key_bytes)`. Used as the `BTreeMap` key +/// for the reader/writer maps so all iteration feeding output is deterministic. +pub(super) type KeyId = (u8, String, Vec); + +/// Discriminant of an array key: one array on one vShard. +const ARRAY: u8 = 4; + +/// How much of its collection one key covers. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub(super) enum KeyScope { + /// One row: a surrogate, a KV key, an edge pair or a UNIQUE value. + Row, + /// Every row: a whole-collection key set, or an array on one vShard. + Collection, +} + +/// A single flattened key drawn from one transaction's read or write set. +#[derive(Debug)] +pub(super) struct FlatKey { + /// Discriminant tag for the key-set variant (Document=0, Vector=1, Kv=2, + /// Edge=3, Array=4, Collection=5, Unique=6). + pub(super) discriminant: u8, + /// Static engine name used when building conflict-context keys. + pub(super) engine_name: &'static str, + /// Collection name. + pub(super) collection: String, + /// Serialized key bytes. + pub(super) key_bytes: Vec, + /// How much of the collection the key covers. + pub(super) scope: KeyScope, +} + +impl FlatKey { + /// The order-comparable identity of this key. + pub(super) fn key_id(&self) -> KeyId { + ( + self.discriminant, + self.collection.clone(), + self.key_bytes.clone(), + ) + } +} + +/// Whether a reader of `read` and a writer of `write` conflict, so the +/// reader must take the lower position. +/// +/// Mirrors the scheduler's lock compatibility: +/// - One key conflicts with itself. +/// - A collection-scope key conflicts with every key of its collection. A row +/// key announces itself on its collection, and a whole-collection key +/// excludes every row announcement. +/// - Two array keys of one collection on distinct vShards are disjoint: each +/// vShard's lock table holds its own part of the array. +/// - Two distinct row keys never conflict. +pub(super) fn conflicts(read: &FlatKey, write: &FlatKey) -> bool { + if read.collection != write.collection { + return false; + } + if read.discriminant == write.discriminant && read.key_bytes == write.key_bytes { + return true; + } + match (read.scope, write.scope) { + (KeyScope::Row, KeyScope::Row) => false, + (KeyScope::Collection, KeyScope::Collection) + if read.discriminant == ARRAY && write.discriminant == ARRAY => + { + false + } + (KeyScope::Collection, KeyScope::Row | KeyScope::Collection) + | (KeyScope::Row, KeyScope::Collection) => true, + } +} + +/// Flatten one read-or-write set into its constituent per-key entries. +pub(super) fn flatten_set(set: &ReadWriteSet) -> Vec { + let mut out = Vec::new(); + for key_set in &set.0 { + match key_set { + EngineKeySet::Document { + collection, + surrogates, + } => { + for &s in surrogates.as_slice() { + out.push(FlatKey { + discriminant: 0, + engine_name: "document", + collection: collection.clone(), + key_bytes: s.to_le_bytes().to_vec(), + scope: KeyScope::Row, + }); + } + } + EngineKeySet::Vector { + collection, + surrogates, + } => { + for &s in surrogates.as_slice() { + out.push(FlatKey { + discriminant: 1, + engine_name: "vector", + collection: collection.clone(), + key_bytes: s.to_le_bytes().to_vec(), + scope: KeyScope::Row, + }); + } + } + EngineKeySet::Kv { collection, keys } => { + for k in keys.as_slice() { + out.push(FlatKey { + discriminant: 2, + engine_name: "kv", + collection: collection.clone(), + key_bytes: k.clone(), + scope: KeyScope::Row, + }); + } + } + EngineKeySet::Edge { + collection, edges, .. + } => { + for &(src, dst) in edges.as_slice() { + let mut key_bytes = src.to_le_bytes().to_vec(); + key_bytes.extend_from_slice(&dst.to_le_bytes()); + out.push(FlatKey { + discriminant: 3, + engine_name: "edge", + collection: collection.clone(), + key_bytes, + scope: KeyScope::Row, + }); + } + } + // The whole array on each vShard it names. + EngineKeySet::Array { + collection, + vshards, + } => { + for &vshard in vshards.as_slice() { + out.push(FlatKey { + discriminant: ARRAY, + engine_name: "array", + collection: collection.clone(), + key_bytes: vshard.to_le_bytes().to_vec(), + scope: KeyScope::Collection, + }); + } + } + // The whole collection: one key with no row bytes. + EngineKeySet::Collection { collection, .. } => { + out.push(FlatKey { + discriminant: 5, + engine_name: "collection", + collection: collection.clone(), + key_bytes: Vec::new(), + scope: KeyScope::Collection, + }); + } + // One key per claimed value: the index name, a zero byte, the value. + EngineKeySet::Unique { + collection, + index, + values, + } => { + for value in values.as_slice() { + let mut key_bytes = Vec::with_capacity(index.len() + 1 + value.len()); + key_bytes.extend_from_slice(index.as_bytes()); + key_bytes.push(0); + key_bytes.extend_from_slice(value); + out.push(FlatKey { + discriminant: 6, + engine_name: "unique", + collection: collection.clone(), + key_bytes, + scope: KeyScope::Row, + }); + } + } + } + } + out +} + +#[cfg(test)] +mod tests { + use super::*; + + fn key(discriminant: u8, collection: &str, bytes: &[u8], scope: KeyScope) -> FlatKey { + FlatKey { + discriminant, + engine_name: "test", + collection: collection.to_owned(), + key_bytes: bytes.to_vec(), + scope, + } + } + + fn row(collection: &str, surrogate: u32) -> FlatKey { + key(0, collection, &surrogate.to_le_bytes(), KeyScope::Row) + } + + fn whole(collection: &str) -> FlatKey { + key(5, collection, &[], KeyScope::Collection) + } + + fn array(collection: &str, vshard: u32) -> FlatKey { + key( + ARRAY, + collection, + &vshard.to_le_bytes(), + KeyScope::Collection, + ) + } + + #[test] + fn a_key_conflicts_with_itself_only_among_rows() { + assert!(conflicts(&row("c", 1), &row("c", 1))); + assert!(!conflicts(&row("c", 1), &row("c", 2))); + assert!(!conflicts(&row("c", 1), &row("d", 1))); + } + + #[test] + fn a_collection_scope_key_conflicts_with_every_key_of_its_collection() { + assert!(conflicts(&whole("c"), &row("c", 1))); + assert!(conflicts(&row("c", 1), &whole("c"))); + assert!(conflicts(&whole("c"), &whole("c"))); + assert!(conflicts(&array("c", 3), &row("c", 1))); + assert!(conflicts(&row("c", 1), &array("c", 3))); + assert!(conflicts(&whole("c"), &array("c", 3))); + assert!(!conflicts(&whole("c"), &row("d", 1))); + assert!(!conflicts(&row("d", 1), &whole("c"))); + } + + #[test] + fn array_keys_on_distinct_vshards_are_disjoint() { + assert!(conflicts(&array("c", 3), &array("c", 3))); + assert!(!conflicts(&array("c", 3), &array("c", 4))); + } +} diff --git a/nodedb-cluster/src/calvin/sequencer/validator/mod.rs b/nodedb-cluster/src/calvin/sequencer/validator/mod.rs new file mode 100644 index 000000000..f948cb9f5 --- /dev/null +++ b/nodedb-cluster/src/calvin/sequencer/validator/mod.rs @@ -0,0 +1,8 @@ +// SPDX-License-Identifier: BUSL-1.1 + +pub mod batch; +mod graph; +mod keys; +mod reject; + +pub use batch::{validate_batch, validate_batch_with_assignments}; diff --git a/nodedb-cluster/src/calvin/sequencer/validator/reject.rs b/nodedb-cluster/src/calvin/sequencer/validator/reject.rs new file mode 100644 index 000000000..8a960f89c --- /dev/null +++ b/nodedb-cluster/src/calvin/sequencer/validator/reject.rs @@ -0,0 +1,92 @@ +// SPDX-License-Identifier: BUSL-1.1 + +//! The reported position and conflict context of a cycle-rejected +//! transaction. + +use crate::calvin::sequencer::metrics::ConflictKey; + +use super::graph::{ConflictIndex, KeyRef}; +use super::keys::{FlatKey, KeyId, KeyScope, conflicts}; + +/// Build the `(position_admitted, conflict_context)` for a cycle-rejected txn. +/// +/// A rejected txn conflicts with any admitted txn that writes a key it reads +/// or reads a key it writes, by [`conflicts`]. We select the conflict with +/// the smallest admitted position (ties broken by key identity) so both the +/// reported position and the [`ConflictKey`] are deterministic. If no +/// conflicting txn was admitted, the position defaults to `0` and the context +/// to `None`. +pub(super) fn reject_context( + idx: usize, + tenant: u64, + index: &ConflictIndex, + emitted_position: &[Option], +) -> (u32, Option) { + // Best conflict so far, ordered by (admitted position, key identity). + let mut best: Option<(u32, KeyId, &'static str)> = None; + let mut consider = |fk: &FlatKey, other: usize| { + if other == idx { + return; + } + let Some(pos) = emitted_position[other] else { + return; + }; + let key = fk.key_id(); + let better = match &best { + None => true, + Some((bpos, bkey, _)) => (pos, &key) < (*bpos, bkey), + }; + if better { + best = Some((pos, key, fk.engine_name)); + } + }; + + for fk in &index.reads[idx] { + // A writer of a key this txn reads. + for &other in index.writers_of.get(&fk.key_id()).into_iter().flatten() { + consider(fk, other); + } + // A writer whose key's scope covers this read, or the reverse. + if let Some(coll) = index.by_collection.get(&fk.collection) { + let row_writes: &[KeyRef] = match fk.scope { + KeyScope::Row => &[], + KeyScope::Collection => &coll.row_writes, + }; + for &w in coll.scoped_writes.iter().chain(row_writes) { + if conflicts(fk, index.write(w)) { + consider(fk, w.0); + } + } + } + } + for fk in &index.writes[idx] { + // A reader of a key this txn writes. + for &other in index.readers_of.get(&fk.key_id()).into_iter().flatten() { + consider(fk, other); + } + // A reader whose key's scope covers this write, or the reverse. + if let Some(coll) = index.by_collection.get(&fk.collection) { + let row_reads: &[KeyRef] = match fk.scope { + KeyScope::Row => &[], + KeyScope::Collection => &coll.row_reads, + }; + for &r in coll.scoped_reads.iter().chain(row_reads) { + if conflicts(index.read(r), fk) { + consider(fk, r.0); + } + } + } + } + + match best { + Some((pos, key, engine)) => ( + pos, + Some(ConflictKey { + tenant, + engine, + collection: key.1, + }), + ), + None => (0, None), + } +} diff --git a/nodedb-cluster/src/cluster_info.rs b/nodedb-cluster/src/cluster_info.rs index 9d466d8d8..56fe44334 100644 --- a/nodedb-cluster/src/cluster_info.rs +++ b/nodedb-cluster/src/cluster_info.rs @@ -267,6 +267,7 @@ mod tests { last_applied: 5, last_log_index: 5, snapshot_index: 0, + replica_state: nodedb_raft::ReplicaState::Whole, member_count: 3, learner_count: 0, vshard_count: 512, diff --git a/nodedb-cluster/src/error.rs b/nodedb-cluster/src/error.rs index 26e716a99..ee107278e 100644 --- a/nodedb-cluster/src/error.rs +++ b/nodedb-cluster/src/error.rs @@ -271,7 +271,78 @@ pub enum ClusterError { }, } +/// What a failed send says about the request it carried. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +enum SendFate { + /// The request never reached the peer whole. A resend cannot run it twice. + Unsent, + /// The request reached the peer and no answer came back. The peer can + /// have run it, so its outcome is unknown. + LostAnswer, + /// An answer, or a refusal the caller handles itself. + Answered, +} + impl ClusterError { + /// Whether the request never reached the peer whole. + /// + /// The peer ran nothing, so any request can be sent again. + pub fn is_unsent(&self) -> bool { + self.send_fate() == SendFate::Unsent + } + + /// Whether the request reached the peer and its answer was lost. + /// + /// The peer can have run it. Only a resend-safe request goes again. Any + /// other request reports an unknown outcome. + pub fn is_lost_answer(&self) -> bool { + self.send_fate() == SendFate::LostAnswer + } + + fn send_fate(&self) -> SendFate { + match self { + Self::Transport { .. } => SendFate::Unsent, + Self::ShardTimeout { .. } | Self::Unanswered { .. } => SendFate::LostAnswer, + Self::Raft(_) + | Self::VShardNotMapped { .. } + | Self::GroupNotFound { .. } + | Self::LearnerNotCaughtUp { .. } + | Self::MigrationInProgress { .. } + | Self::MigrationPauseBudgetExceeded { .. } + | Self::NodeUnreachable { .. } + | Self::GhostNotFound { .. } + | Self::StreamTerminal { .. } + | Self::Storage { .. } + | Self::DataPlane { .. } + | Self::Codec { .. } + | Self::UnsupportedWireVersion { .. } + | Self::CircuitOpen { .. } + | Self::JoinGroupDisappeared { .. } + | Self::JoinCommitTimeout { .. } + | Self::ReadIndexNotLeader { .. } + | Self::ReadIndexTimeout { .. } + | Self::Config { .. } + | Self::MigrationCheckpoint(_) + | Self::MigrationRecovery(_) + | Self::WrongOwner { .. } + | Self::Calvin(_) + | Self::SnapshotCrcMismatch { .. } + | Self::SnapshotOffsetRegression { .. } + | Self::PartialSnapshotCorrupt { .. } + | Self::PartialSnapshotCleanupFailed { .. } + | Self::SnapshotApplyFailed { .. } + | Self::Mirror(_) + | Self::BspBarrier(_) + | Self::VectorGather(_) + | Self::SpatialGather(_) + | Self::Bm25Gather(_) + | Self::TsGather(_) + | Self::ShufflePush(_) + | Self::RemoteUntyped { .. } + | Self::ShardExecution { .. } => SendFate::Answered, + } + } + /// Whether the error means the link to the peer failed. /// /// A link failure counts against the peer's circuit breaker and ends a @@ -288,6 +359,16 @@ impl ClusterError { } } +/// An array routing granularity out of range is a codec error: it arrives +/// on a wire request or a catalog entry. +impl From for ClusterError { + fn from(e: nodedb_types::array_route::PrefixBitsOutOfRange) -> Self { + ClusterError::Codec { + detail: e.to_string(), + } + } +} + #[cfg(test)] mod tests { use super::*; @@ -316,4 +397,54 @@ mod tests { .is_link_failure() ); } + + #[test] + fn only_a_transport_error_is_unsent() { + let unsent = ClusterError::Transport { + detail: "connect refused".into(), + }; + assert!(unsent.is_unsent()); + assert!(!unsent.is_lost_answer()); + } + + #[test] + fn a_timeout_or_a_missing_answer_is_a_lost_answer() { + let lost = [ + ClusterError::ShardTimeout { + vshard_id: 3, + elapsed_ms: 50, + }, + ClusterError::Unanswered { + node_id: 2, + detail: "stream reset".into(), + }, + ]; + for error in &lost { + assert!(error.is_lost_answer(), "{error}"); + assert!(!error.is_unsent(), "{error}"); + } + } + + #[test] + fn an_answer_is_neither_unsent_nor_lost() { + let answered = [ + ClusterError::Raft(nodedb_raft::RaftError::NotLeader { + leader_hint: Some(2), + term: 3, + }), + ClusterError::CircuitOpen { + node_id: 1, + failures: 5, + }, + ClusterError::NodeUnreachable { node_id: 1 }, + ClusterError::RemoteUntyped { + detail: "refused".into(), + }, + ClusterError::Calvin(CalvinError::AdmissionTimedOut), + ]; + for error in &answered { + assert!(!error.is_unsent(), "{error}"); + assert!(!error.is_lost_answer(), "{error}"); + } + } } diff --git a/nodedb-cluster/src/multi_raft/conf_change.rs b/nodedb-cluster/src/multi_raft/conf_change.rs index cca910be2..4ac4dd356 100644 --- a/nodedb-cluster/src/multi_raft/conf_change.rs +++ b/nodedb-cluster/src/multi_raft/conf_change.rs @@ -62,34 +62,16 @@ impl MultiRaft { group_id: u64, change: &ConfChange, ) -> Result<(u64, u64)> { + // Promoting a learner that has not caught up grows the voter set + // while it cannot ack, shrinking the effective quorum. Removing the + // last whole voter leaves no replica to lead and send a snapshot. + self.admit_conf_change(group_id, change)?; let (log_index, committed_immediately) = { let node = self .groups .get_mut(&group_id) .ok_or(ClusterError::GroupNotFound { group_id })?; - // PromoteLearner safety gate (propose side): a learner must have - // caught up to the group's commit index BEFORE the promote entry - // enters the log. Promoting an un-caught-up learner grows the - // voter set while it cannot yet ack anything, shrinking the - // effective quorum (e.g. 3 voters + 1 stale learner -> no - // majority) and risking cluster unavailability. The check cannot - // live in `apply_conf_change` — at apply time the commit index - // has already advanced past the promote entry itself, so it - // would spuriously reject promotions that were valid when - // proposed. - if change.change_type == ConfChangeType::PromoteLearner - && node.role() == nodedb_raft::NodeRole::Leader - && node.match_index_for(change.node_id).unwrap_or(0) < node.commit_index() - { - return Err(ClusterError::LearnerNotCaughtUp { - group_id, - node_id: change.node_id, - match_index: node.match_index_for(change.node_id).unwrap_or(0), - commit_index: node.commit_index(), - }); - } - let data = change.to_entry_data()?; let log_index = node.propose(data)?; // A single-voter group commits once its own disk holds the @@ -376,7 +358,7 @@ mod tests { success: true, last_log_index: last, round: nodedb_raft::node::leader_lease::UNTRACKED_ROUND, - needs_snapshot: false, + replica: nodedb_raft::ReplicaState::Whole, }; mr.handle_append_entries_response(0, 2, &resp).unwrap(); assert_eq!(mr.match_index_for(0, 2), Some(last)); diff --git a/nodedb-cluster/src/multi_raft/conf_change_gate.rs b/nodedb-cluster/src/multi_raft/conf_change_gate.rs new file mode 100644 index 000000000..b226a064e --- /dev/null +++ b/nodedb-cluster/src/multi_raft/conf_change_gate.rs @@ -0,0 +1,265 @@ +// SPDX-License-Identifier: BUSL-1.1 + +//! The leader's check on a configuration change before it enters the log. +//! +//! A replica awaiting a snapshot holds no state to lead from, and only a +//! leader sends it one. So a group must keep a whole voter: +//! - A learner is promoted only once it caught up and awaits no snapshot. +//! - A voter is removed only when a voter that may lead stays. The leader +//! may lead: a replica whose data diverged steps down and never +//! campaigns. A peer counts by the state its latest reply this term +//! reported: one that only wants a snapshot may lead, one whose data +//! diverged may not. A peer not heard this term does not count: its +//! state is unknown. +//! +//! Every proposal path runs the check: a change proposed as a conf change, +//! and conf-change bytes proposed as a plain entry. + +use nodedb_raft::{NodeRole, RaftError, ReplicaState}; + +use crate::conf_change::{ConfChange, ConfChangeType}; +use crate::error::{ClusterError, Result}; + +use super::core::MultiRaft; + +impl MultiRaft { + /// Refuse `change` to `group_id` when it would promote a learner that is + /// not caught up, or remove the last whole voter. A node that does not + /// lead the group admits it: its propose refuses with `NotLeader`. + pub(super) fn admit_conf_change(&self, group_id: u64, change: &ConfChange) -> Result<()> { + let node = self + .groups + .get(&group_id) + .ok_or(ClusterError::GroupNotFound { group_id })?; + if node.role() != NodeRole::Leader { + return Ok(()); + } + match change.change_type { + // At apply time the commit index has already passed the promote + // entry, so the check runs here, before the entry enters the log. + ConfChangeType::PromoteLearner => { + if self.peer_caught_up(group_id, change.node_id) { + return Ok(()); + } + Err(ClusterError::LearnerNotCaughtUp { + group_id, + node_id: change.node_id, + match_index: node.match_index_for(change.node_id).unwrap_or(0), + commit_index: node.commit_index(), + }) + } + ConfChangeType::RemoveNode => { + let whole_voter_stays = std::iter::once(self.node_id) + .chain(node.voters().iter().copied()) + .filter(|&voter| voter != change.node_id) + .any(|voter| { + if voter == self.node_id { + node.replica_state().may_lead() + } else { + node.peer_replica_state(voter) + .is_some_and(ReplicaState::may_lead) + } + }); + if whole_voter_stays { + return Ok(()); + } + Err(ClusterError::Raft(RaftError::ProposalRejected { + reason: format!( + "removing voter {} from group {group_id} leaves no voter that may \ + lead; every other voter is unheard this term or its data diverged; \ + retry once another voter answers that it may lead", + change.node_id + ), + })) + } + ConfChangeType::AddNode + | ConfChangeType::AddLearner + | ConfChangeType::RemoveLearner => Ok(()), + } + } +} + +#[cfg(test)] +mod tests { + use std::time::{Duration, Instant}; + + use nodedb_raft::message::AppendEntriesResponse; + use nodedb_raft::node::leader_lease::UNTRACKED_ROUND; + + use super::*; + use crate::routing::RoutingTable; + + /// Node 1 leads group 0 alone, with its election no-op committed. + fn leader_mr() -> (MultiRaft, tempfile::TempDir) { + let dir = tempfile::tempdir().expect("tempdir"); + let rt = RoutingTable::uniform(1, &[1], 1); + let mut mr = MultiRaft::new(1, rt, dir.path().to_path_buf()); + mr.add_group(0, vec![]).expect("add group"); + for node in mr.groups.values_mut() { + node.election_deadline_override(Instant::now() - Duration::from_millis(1)); + } + mr.tick().expect("tick"); + mr.wait_all_durable_blocking(); + mr.tick().expect("tick"); + assert!(mr.group_role_is_leader(0)); + (mr, dir) + } + + fn reply(mr: &MultiRaft, last_log_index: u64, replica: ReplicaState) -> AppendEntriesResponse { + AppendEntriesResponse { + term: mr.groups.get(&0).map_or(0, |node| node.current_term()), + success: !replica.wants_snapshot(), + last_log_index, + round: UNTRACKED_ROUND, + replica, + } + } + + fn change(change_type: ConfChangeType, node_id: u64) -> ConfChange { + ConfChange { + change_type, + node_id, + } + } + + /// The leader cannot remove itself while every other voter awaits a + /// snapshot: no replica would be left to lead and send it one. + #[test] + fn removing_the_last_whole_voter_is_refused() { + let (mut mr, _dir) = leader_mr(); + mr.apply_conf_change(0, &change(ConfChangeType::AddNode, 2)) + .expect("add voter 2"); + let awaiting = reply(&mr, 0, ReplicaState::AwaitingSnapshot); + mr.handle_append_entries_response(0, 2, &awaiting) + .expect("reply"); + + let refused = mr.propose_conf_change(0, &change(ConfChangeType::RemoveNode, 1)); + assert!( + matches!( + refused, + Err(ClusterError::Raft(RaftError::ProposalRejected { .. })) + ), + "got {refused:?}" + ); + let bytes = change(ConfChangeType::RemoveNode, 1) + .to_entry_data() + .expect("encode"); + assert!( + mr.propose_to_group(0, bytes).is_err(), + "the plain proposal path runs the same check" + ); + + assert!( + mr.admit_conf_change(0, &change(ConfChangeType::RemoveNode, 2)) + .is_ok(), + "removing the awaiting voter keeps the whole leader" + ); + } + + /// A fresh leader has heard no other voter this term: their state is + /// unknown, so it cannot remove itself. + #[test] + fn a_leader_that_heard_no_voter_cannot_remove_itself() { + let (mut mr, _dir) = leader_mr(); + for voter in [2, 3] { + mr.apply_conf_change(0, &change(ConfChangeType::AddNode, voter)) + .expect("add voter"); + } + let refused = mr.admit_conf_change(0, &change(ConfChangeType::RemoveNode, 1)); + assert!( + matches!( + refused, + Err(ClusterError::Raft(RaftError::ProposalRejected { .. })) + ), + "got {refused:?}" + ); + + let answer = reply(&mr, 0, ReplicaState::WantsSnapshot); + mr.handle_append_entries_response(0, 3, &answer) + .expect("reply"); + assert!( + mr.admit_conf_change(0, &change(ConfChangeType::RemoveNode, 1)) + .is_ok(), + "voter 3 answered and may lead" + ); + } + + /// Node 1 leads with voters 2 and 3, which both answered with `state`. + fn leader_with_two_voters_in(state: ReplicaState) -> (MultiRaft, tempfile::TempDir) { + let (mut mr, dir) = leader_mr(); + for voter in [2, 3] { + mr.apply_conf_change(0, &change(ConfChangeType::AddNode, voter)) + .expect("add voter"); + } + let answer = reply(&mr, 0, state); + for voter in [2, 3] { + mr.handle_append_entries_response(0, voter, &answer) + .expect("reply"); + } + (mr, dir) + } + + /// Voters that only want a snapshot for state beside their log may + /// lead: the leader can remove itself. Voters whose data diverged may + /// not: the removal is refused. + #[test] + fn a_voter_that_wants_a_snapshot_may_lead_and_a_diverged_one_may_not() { + let (mr, _dir) = leader_with_two_voters_in(ReplicaState::WantsSnapshot); + assert!( + mr.admit_conf_change(0, &change(ConfChangeType::RemoveNode, 1)) + .is_ok() + ); + assert!( + !mr.peer_caught_up(0, 2), + "a peer that wants a snapshot is behind" + ); + + let (mr, _dir) = leader_with_two_voters_in(ReplicaState::AwaitingSnapshot); + assert!(matches!( + mr.admit_conf_change(0, &change(ConfChangeType::RemoveNode, 1)), + Err(ClusterError::Raft(RaftError::ProposalRejected { .. })) + )); + assert!(!mr.peer_caught_up(0, 2)); + } + + #[test] + fn removing_a_voter_while_a_whole_voter_stays_is_admitted() { + let (mut mr, _dir) = leader_mr(); + mr.apply_conf_change(0, &change(ConfChangeType::AddNode, 2)) + .expect("add voter 2"); + let last = mr.groups.get(&0).map_or(0, |node| node.last_log_index()); + let whole = reply(&mr, last, ReplicaState::Whole); + mr.handle_append_entries_response(0, 2, &whole) + .expect("reply"); + assert!( + mr.admit_conf_change(0, &change(ConfChangeType::RemoveNode, 1)) + .is_ok() + ); + } + + /// A remounted learner whose log matches still awaits its snapshot: it + /// is promoted only once the snapshot installs. + #[test] + fn a_learner_awaiting_a_snapshot_is_not_promoted() { + let (mut mr, _dir) = leader_mr(); + mr.apply_conf_change(0, &change(ConfChangeType::AddLearner, 2)) + .expect("add learner 2"); + let last = mr.groups.get(&0).map_or(0, |node| node.last_log_index()); + let matched = reply(&mr, last, ReplicaState::Whole); + mr.handle_append_entries_response(0, 2, &matched) + .expect("reply"); + let awaiting = reply(&mr, last, ReplicaState::AwaitingSnapshot); + mr.handle_append_entries_response(0, 2, &awaiting) + .expect("reply"); + + assert!(mr.ready_learners(0).is_empty()); + assert!(matches!( + mr.admit_conf_change(0, &change(ConfChangeType::PromoteLearner, 2)), + Err(ClusterError::LearnerNotCaughtUp { .. }) + )); + + mr.handle_append_entries_response(0, 2, &matched) + .expect("reply"); + assert_eq!(mr.ready_learners(0), vec![2]); + } +} diff --git a/nodedb-cluster/src/multi_raft/core.rs b/nodedb-cluster/src/multi_raft/core.rs index beaefed7d..9665f9a8e 100644 --- a/nodedb-cluster/src/multi_raft/core.rs +++ b/nodedb-cluster/src/multi_raft/core.rs @@ -74,11 +74,12 @@ pub struct MultiRaft { pub(super) snapshot_requirement: Option, } -/// Whether this node's replica of a data group must take a snapshot before -/// it takes log entries. Called with the group id and the first index the +/// What this node's replica of a data group needs before it takes log +/// entries: nothing, a snapshot it may lead without, or a snapshot it must +/// not lead without. Called with the group id and the first index the /// Calvin sequencer log still holds here, under the `MultiRaft` lock: it /// must not take that lock. -pub type SnapshotRequirement = Arc bool + Send + Sync>; +pub type SnapshotRequirement = Arc nodedb_raft::ReplicaState + Send + Sync>; /// Aggregated output from all Raft groups after a tick. #[derive(Debug, Default)] @@ -148,7 +149,7 @@ impl MultiRaft { let sequencer_first = self.sequencer_first_available(); for (&group_id, node) in &mut self.groups { if is_data_group(group_id) { - node.set_snapshot_required(requirement(group_id, sequencer_first)); + node.set_replica_state(requirement(group_id, sequencer_first)); } } self.snapshot_requirement = Some(requirement); diff --git a/nodedb-cluster/src/multi_raft/membership.rs b/nodedb-cluster/src/multi_raft/membership.rs index 59c83fa5e..118fa7252 100644 --- a/nodedb-cluster/src/multi_raft/membership.rs +++ b/nodedb-cluster/src/multi_raft/membership.rs @@ -9,8 +9,8 @@ //! - `ready_learners(group)`: used by the tick loop's "promote //! caught-up learners" phase — returns every learner in the group //! whose `match_index` on this (leader) node is at least the current -//! `commit_index`, i.e. learners that have replicated enough log to be -//! safely promoted. +//! `commit_index` and that answered whole this term, i.e. learners that hold +//! the group's state and enough log to be safely promoted. //! - `group_leader(group)`: leader id observed by this node's local //! RaftNode state, used by the join flow to decide redirect vs admit. //! - `group_role_is_leader(group)`: cheap leader-check helper. @@ -36,8 +36,10 @@ impl MultiRaft { self.groups.get(&group_id).map(|n| n.commit_index()) } - /// Learners in `group_id` whose `match_index` on this leader has - /// caught up to the current `commit_index` — safe to promote. + /// Learners in `group_id` that caught up to this leader's commit index + /// and answered whole this term — safe to promote. A remounted learner whose + /// log matches still lacks the group's state until its snapshot + /// installs. /// /// Returns an empty vec if this node is not the leader of the group /// or the group is not hosted here. @@ -45,14 +47,10 @@ impl MultiRaft { let Some(node) = self.groups.get(&group_id) else { return Vec::new(); }; - if node.role() != NodeRole::Leader { - return Vec::new(); - } - let commit = node.commit_index(); node.learners() .iter() .copied() - .filter(|&learner| node.match_index_for(learner).unwrap_or(0) >= commit) + .filter(|&learner| self.peer_caught_up(group_id, learner)) .collect() } @@ -89,15 +87,16 @@ impl MultiRaft { } /// Whether `peer` holds every entry this leader has committed in - /// `group_id`, and did not ask for a snapshot last. `false` when this - /// node does not lead the group. + /// `group_id`, and its latest reply this term reported it whole. `false` + /// when this node does not lead the group, or has not heard `peer` this + /// term. pub fn peer_caught_up(&self, group_id: u64, peer: u64) -> bool { let Some(node) = self.groups.get(&group_id) else { return false; }; node.role() == NodeRole::Leader && node.match_index_for(peer).unwrap_or(0) >= node.commit_index() - && !node.peer_awaits_snapshot(peer) + && node.peer_answered_whole(peer) } /// Initiate a leadership transfer for `group_id` to `target`. @@ -114,20 +113,12 @@ impl MultiRaft { node.transfer_leadership(target).map_err(ClusterError::Raft) } - /// Hand this node's leadership of `group_id` to another voter that holds - /// every committed entry and did not ask for a snapshot last. Returns the - /// target. `None` when this node does not lead the group, no voter - /// qualifies, or the transfer did not start. + /// Hand this node's leadership of `group_id` to the voter + /// `RaftNode::hand_off_target` names. Returns the target. `None` when + /// this node does not lead the group, no voter qualifies, or the transfer + /// did not start. pub fn hand_off_leadership(&mut self, group_id: u64) -> Option { - let membership = self.group_membership(group_id)?; - if membership.leader_id != self.node_id { - return None; - } - let target = membership - .voters - .iter() - .copied() - .find(|&voter| voter != self.node_id && self.peer_caught_up(group_id, voter))?; + let target = self.groups.get(&group_id)?.hand_off_target()?; self.transfer_leadership(group_id, target).ok()?; Some(target) } diff --git a/nodedb-cluster/src/multi_raft/mod.rs b/nodedb-cluster/src/multi_raft/mod.rs index e285def53..493d92a59 100644 --- a/nodedb-cluster/src/multi_raft/mod.rs +++ b/nodedb-cluster/src/multi_raft/mod.rs @@ -14,6 +14,8 @@ //! corresponding response handlers. //! - [`conf_change`]: `propose_conf_change` / `apply_conf_change` with //! proper voter / learner / promotion semantics. +//! - [`conf_change_gate`]: the leader's check that a change keeps a voter +//! that holds the group's state. //! - [`membership`]: learner catch-up / promotion helpers //! (`commit_index_for`, `ready_learners`, etc.) driven by the tick loop. //! - [`membership_sync`]: set a group's voters and learners from routing @@ -21,6 +23,7 @@ //! - [`read_index`]: leadership confirmation for linearizable reads. pub mod conf_change; +pub mod conf_change_gate; pub mod core; pub mod membership; pub mod membership_sync; diff --git a/nodedb-cluster/src/multi_raft/mount.rs b/nodedb-cluster/src/multi_raft/mount.rs index 3f277a650..c149c3909 100644 --- a/nodedb-cluster/src/multi_raft/mount.rs +++ b/nodedb-cluster/src/multi_raft/mount.rs @@ -121,7 +121,7 @@ impl MultiRaft { if super::core::is_data_group(group_id) && let Some(requirement) = self.snapshot_requirement.as_ref() { - node.set_snapshot_required(requirement(group_id, self.sequencer_first_available())); + node.set_replica_state(requirement(group_id, self.sequencer_first_available())); } self.groups.insert(group_id, node); self.apply_gates.mount(group_id); diff --git a/nodedb-cluster/src/multi_raft/peer_contact.rs b/nodedb-cluster/src/multi_raft/peer_contact.rs index 6adfbae23..cb10491fd 100644 --- a/nodedb-cluster/src/multi_raft/peer_contact.rs +++ b/nodedb-cluster/src/multi_raft/peer_contact.rs @@ -4,7 +4,7 @@ use std::time::Duration; -use nodedb_raft::{NodeRole, StalenessVerdict}; +use nodedb_raft::NodeRole; use crate::multi_raft::core::MultiRaft; @@ -43,7 +43,7 @@ impl MultiRaft { pub fn leader_contact_within(&self, group_id: u64, window: Duration) -> bool { self.groups .get(&group_id) - .is_some_and(|node| node.staleness_verdict(window) != StalenessVerdict::NoRecentContact) + .is_some_and(|node| node.leader_contact_within(window)) } /// Response counts from every peer of `group_id`. `None` when the group diff --git a/nodedb-cluster/src/multi_raft/proposals.rs b/nodedb-cluster/src/multi_raft/proposals.rs index 5d8033ee7..31f8f0082 100644 --- a/nodedb-cluster/src/multi_raft/proposals.rs +++ b/nodedb-cluster/src/multi_raft/proposals.rs @@ -4,6 +4,7 @@ use nodedb_raft::RaftNode; +use crate::conf_change::ConfChange; use crate::error::{ClusterError, Result}; use crate::group_disk::StagedLogStorage; use crate::multi_raft::core::MultiRaft; @@ -53,6 +54,9 @@ impl MultiRaft { .read() .unwrap_or_else(|p| p.into_inner()) .group_for_vshard(vshard_id)?; + if let Some(change) = ConfChange::from_entry_data(&data) { + self.admit_conf_change(group_id, &change)?; + } let node = self .groups .get_mut(&group_id) @@ -78,6 +82,9 @@ impl MultiRaft { /// /// Returns the committed log index on success. pub fn propose_to_group(&mut self, group_id: u64, data: Vec) -> Result { + if let Some(change) = ConfChange::from_entry_data(&data) { + self.admit_conf_change(group_id, &change)?; + } let node = self .groups .get_mut(&group_id) @@ -123,13 +130,19 @@ impl MultiRaft { .map(|n| n.first_available_index()) } - /// Make this node's replica of `group_id` refuse log entries until a - /// snapshot installs, or lift that. Its leader answers the refusal with - /// a snapshot. Returns whether this node hosts the group. - pub fn set_snapshot_required(&mut self, group_id: u64, required: bool) -> bool { + /// Raise what this node's replica of `group_id` needs to `state`, never + /// lowering it: it refuses log entries until a snapshot installs, and + /// its leader answers the refusal with a snapshot. A replica whose data + /// diverged also steps down from leadership and never campaigns. Only an + /// install lifts it (see [`Self::refresh_snapshot_requirement`]). + /// Returns whether this node hosts the group. + pub fn require_snapshot(&mut self, group_id: u64, state: nodedb_raft::ReplicaState) -> bool { match self.groups.get_mut(&group_id) { Some(node) => { - node.set_snapshot_required(required); + let raised = node.replica_state().raised_to(state); + if raised != node.replica_state() { + node.set_replica_state(raised); + } true } None => false, @@ -159,7 +172,7 @@ impl MultiRaft { let sequencer_first = self.sequencer_first_available(); for group_id in groups { if let Some(node) = self.groups.get_mut(&group_id) { - node.set_snapshot_required(requirement(group_id, sequencer_first)); + node.set_replica_state(requirement(group_id, sequencer_first)); } } } @@ -182,6 +195,12 @@ impl MultiRaft { .is_some_and(|node| node.snapshot_required()) } + /// What this node's replica of `group_id` needs before it takes log + /// entries. `None` when the group is not mounted here. + pub fn replica_state(&self, group_id: u64) -> Option { + self.groups.get(&group_id).map(|node| node.replica_state()) + } + /// Auto-compact a group's log if its configured threshold has been /// reached, given the DATA-PLANE applied watermark `applied_index`. /// diff --git a/nodedb-cluster/src/multi_raft/read_index.rs b/nodedb-cluster/src/multi_raft/read_index.rs index 92edcdc36..92601186d 100644 --- a/nodedb-cluster/src/multi_raft/read_index.rs +++ b/nodedb-cluster/src/multi_raft/read_index.rs @@ -51,11 +51,20 @@ impl MultiRaft { } /// Whether this node's replica of `group_id` is within `max_staleness` - /// of the leader. False for a group not hosted here. - pub fn within_staleness_bound(&self, group_id: u64, max_staleness: Duration) -> bool { - self.groups - .get(&group_id) - .is_some_and(|node| node.within_staleness_bound(max_staleness)) + /// of the leader. `applied` is the highest index whose state reads see + /// here, the host's applied watermark, never Raft's hand-off index. + /// False for a group not hosted here, and for a replica that awaits a + /// snapshot install: its data diverged from the log. + pub fn within_staleness_bound( + &self, + group_id: u64, + max_staleness: Duration, + applied: u64, + ) -> bool { + self.groups.get(&group_id).is_some_and(|node| { + node.replica_state() != nodedb_raft::ReplicaState::AwaitingSnapshot + && node.within_staleness_bound(max_staleness, applied) + }) } } diff --git a/nodedb-cluster/src/multi_raft/status.rs b/nodedb-cluster/src/multi_raft/status.rs index 2a019f7bc..863053cc1 100644 --- a/nodedb-cluster/src/multi_raft/status.rs +++ b/nodedb-cluster/src/multi_raft/status.rs @@ -22,6 +22,9 @@ pub struct GroupStatus { /// lagging peer below this index can only be caught up via /// `InstallSnapshot`, never `AppendEntries`. pub snapshot_index: u64, + /// What this replica needs before it takes log entries. A replica that + /// awaits a snapshot applies nothing until one installs. + pub replica_state: nodedb_raft::ReplicaState, pub member_count: usize, pub learner_count: usize, pub vshard_count: usize, @@ -100,6 +103,7 @@ impl MultiRaft { last_applied: node.last_applied(), last_log_index: node.last_log_index(), snapshot_index: node.log_snapshot_index(), + replica_state: node.replica_state(), member_count: node.voters().len() + usize::from(self_is_voter), learner_count: node.learners().len() + usize::from(node.role() == nodedb_raft::NodeRole::Learner), diff --git a/nodedb-cluster/src/raft_loop/tick/snapshot_dispatch.rs b/nodedb-cluster/src/raft_loop/tick/snapshot_dispatch.rs index dfd6c0be1..ceaae2a9f 100644 --- a/nodedb-cluster/src/raft_loop/tick/snapshot_dispatch.rs +++ b/nodedb-cluster/src/raft_loop/tick/snapshot_dispatch.rs @@ -326,7 +326,7 @@ async fn send_snapshot( success: false, last_log_index: 0, round: nodedb_raft::node::leader_lease::UNTRACKED_ROUND, - needs_snapshot: false, + replica: nodedb_raft::ReplicaState::Whole, }, ) { tracing::error!(group_id, peer, resp_term, error = %e, "apply higher term from install_snapshot response"); diff --git a/nodedb-cluster/src/rpc_codec/data_propose.rs b/nodedb-cluster/src/rpc_codec/data_propose.rs index 6bd70dae7..b650d04ec 100644 --- a/nodedb-cluster/src/rpc_codec/data_propose.rs +++ b/nodedb-cluster/src/rpc_codec/data_propose.rs @@ -53,6 +53,42 @@ pub fn reply_wait(deadline_remaining_ms: u64) -> std::time::Duration { std::time::Duration::from_millis(deadline_remaining_ms).saturating_add(REPLY_MARGIN) } +/// The budget one send attempt gets out of the `remaining_ms` a statement has +/// left. `attempts_left` counts this attempt. +/// +/// A request that is not resend-safe gets all of `remaining_ms`: a lost +/// answer is an unknown outcome, so no later attempt follows. +/// +/// A resend-safe request reserves `rpc_timeout + REPLY_MARGIN` for each later +/// attempt: that attempt's floor plus its reply margin. It still gets at +/// least `rpc_timeout`, capped at `remaining_ms`. The last attempt gets all +/// that is left. +/// +/// Bound: each attempt waits [`reply_wait`] of its budget. A caller that +/// starts no attempt once the deadline passed waits at most the statement +/// budget plus one `REPLY_MARGIN` across every attempt. Each attempt's +/// budget is at most what is left, so only the attempt that reaches the +/// deadline runs past it, by its margin. +pub fn attempt_budget( + remaining_ms: u64, + resend_safe: bool, + attempts_left: usize, + rpc_timeout: std::time::Duration, +) -> u64 { + if !resend_safe || attempts_left <= 1 { + return remaining_ms; + } + let rpc_timeout_ms = u64::try_from(rpc_timeout.as_millis()).unwrap_or(u64::MAX); + let margin_ms = u64::try_from(REPLY_MARGIN.as_millis()).unwrap_or(u64::MAX); + let later_attempts = u64::try_from(attempts_left - 1).unwrap_or(u64::MAX); + let reserve = rpc_timeout_ms + .saturating_add(margin_ms) + .saturating_mul(later_attempts); + remaining_ms + .saturating_sub(reserve) + .max(remaining_ms.min(rpc_timeout_ms)) +} + /// The leader's deadline for a forward that carries `deadline_remaining_ms`. /// `None` when the budget is spent. /// @@ -291,6 +327,82 @@ mod tests { std::time::Duration::from_millis(u64::MAX) + REPLY_MARGIN ); } + + const RPC_TIMEOUT: std::time::Duration = std::time::Duration::from_secs(5); + + /// A write can have run on a node that never answered, so it never + /// retries and keeps the whole remainder on every attempt. + #[test] + fn a_write_attempt_gets_the_full_remainder() { + assert_eq!(attempt_budget(30_000, false, 3, RPC_TIMEOUT), 30_000); + assert_eq!(attempt_budget(30_000, false, 2, RPC_TIMEOUT), 30_000); + assert_eq!(attempt_budget(1_000, false, 3, RPC_TIMEOUT), 1_000); + } + + /// A read leaves one RPC timeout plus one reply margin for each later + /// attempt. + #[test] + fn a_read_attempt_reserves_an_rpc_timeout_and_margin_per_later_attempt() { + // 30 s - 2 × (5 s + 2 s) = 16 s; 30 s - (5 s + 2 s) = 23 s. + assert_eq!(attempt_budget(30_000, true, 3, RPC_TIMEOUT), 16_000); + assert_eq!(attempt_budget(30_000, true, 2, RPC_TIMEOUT), 23_000); + } + + /// A read with little left still gets one RPC timeout, never more than + /// what is left. + #[test] + fn a_read_attempt_keeps_at_least_one_rpc_timeout() { + assert_eq!(attempt_budget(15_000, true, 3, RPC_TIMEOUT), 5_000); + assert_eq!(attempt_budget(3_000, true, 3, RPC_TIMEOUT), 3_000); + assert_eq!(attempt_budget(0, true, 3, RPC_TIMEOUT), 0); + } + + /// Every attempt of a read sent to a node that never answers waits out + /// its reply. The waits together stay within the budget plus one margin, + /// for an RPC timeout above and below the margin. + #[test] + fn hung_read_attempts_fit_the_budget_plus_one_margin() { + let margin_ms = u64::try_from(REPLY_MARGIN.as_millis()).unwrap_or(u64::MAX); + for rpc_timeout_ms in [500, 1_000, 5_000] { + let rpc_timeout = std::time::Duration::from_millis(rpc_timeout_ms); + for budget_ms in [1_000, 4_000, 12_000, 30_000] { + let mut remaining = budget_ms; + let mut waited = 0; + for attempts_left in (1..=3).rev() { + if remaining == 0 { + break; + } + let attempt = attempt_budget(remaining, true, attempts_left, rpc_timeout); + assert!(attempt <= remaining); + let wait = attempt + margin_ms; + waited += wait; + remaining = remaining.saturating_sub(wait); + } + assert!( + waited <= budget_ms + margin_ms, + "rpc timeout {rpc_timeout_ms} ms, budget {budget_ms} ms: waited {waited} ms" + ); + } + } + } + + /// An RPC timeout below the reply margin still leaves each later + /// attempt its whole floor and margin. + #[test] + fn a_short_rpc_timeout_reserves_its_margin_too() { + let rpc_timeout = std::time::Duration::from_millis(500); + // 12 s - 2 × (0.5 s + 2 s) = 7 s. + assert_eq!(attempt_budget(12_000, true, 3, rpc_timeout), 7_000); + // 12 s - (0.5 s + 2 s) = 9.5 s. + assert_eq!(attempt_budget(12_000, true, 2, rpc_timeout), 9_500); + } + + /// The last attempt of a read gets all that is left. + #[test] + fn the_last_read_attempt_gets_all_that_is_left() { + assert_eq!(attempt_budget(30_000, true, 1, RPC_TIMEOUT), 30_000); + assert_eq!(attempt_budget(30_000, true, 0, RPC_TIMEOUT), 30_000); + } use crate::rpc_codec::{decode, encode}; fn roundtrip(target: ProposeTarget) -> DataProposeRequest { diff --git a/nodedb-cluster/src/rpc_codec/mod.rs b/nodedb-cluster/src/rpc_codec/mod.rs index e75b05d7c..79d320230 100644 --- a/nodedb-cluster/src/rpc_codec/mod.rs +++ b/nodedb-cluster/src/rpc_codec/mod.rs @@ -55,7 +55,7 @@ pub use data_plane_error::{ }; pub use data_propose::{ DataProposeRequest, DataProposeResponse, ForwardedProposeRefusal, ProposeTarget, REPLY_MARGIN, - forwarded_deadline, remaining_budget_ms, reply_wait, + attempt_budget, forwarded_deadline, remaining_budget_ms, reply_wait, }; pub use execute::{ DescriptorVersionEntry, ExecuteRequest, ExecuteResponse, ExecuteStreamChunk, ExecuteStreamEnd, diff --git a/nodedb-cluster/src/rpc_codec/raft_msgs.rs b/nodedb-cluster/src/rpc_codec/raft_msgs.rs index 4518f7c98..8d57a2c1f 100644 --- a/nodedb-cluster/src/rpc_codec/raft_msgs.rs +++ b/nodedb-cluster/src/rpc_codec/raft_msgs.rs @@ -214,21 +214,27 @@ mod tests { #[test] fn roundtrip_append_entries_response() { - let resp = AppendEntriesResponse { - term: 5, - success: true, - last_log_index: 100, - round: 11, - needs_snapshot: true, - }; - match roundtrip(RaftRpc::AppendEntriesResponse(resp)) { - RaftRpc::AppendEntriesResponse(d) => { - assert_eq!(d.term, 5); - assert_eq!(d.round, 11); - assert!(d.success); - assert!(d.needs_snapshot); + for replica in [ + nodedb_raft::ReplicaState::Whole, + nodedb_raft::ReplicaState::WantsSnapshot, + nodedb_raft::ReplicaState::AwaitingSnapshot, + ] { + let resp = AppendEntriesResponse { + term: 5, + success: true, + last_log_index: 100, + round: 11, + replica, + }; + match roundtrip(RaftRpc::AppendEntriesResponse(resp)) { + RaftRpc::AppendEntriesResponse(d) => { + assert_eq!(d.term, 5); + assert_eq!(d.round, 11); + assert!(d.success); + assert_eq!(d.replica, replica, "the follower's state crosses the wire"); + } + other => panic!("expected AppendEntriesResponse, got {other:?}"), } - other => panic!("expected AppendEntriesResponse, got {other:?}"), } } diff --git a/nodedb-cluster/src/rpc_codec/raft_rpc.rs b/nodedb-cluster/src/rpc_codec/raft_rpc.rs index 5e6ebb452..414e9c598 100644 --- a/nodedb-cluster/src/rpc_codec/raft_rpc.rs +++ b/nodedb-cluster/src/rpc_codec/raft_rpc.rs @@ -472,7 +472,7 @@ mod tests { success: true, last_log_index: 5, round: 3, - needs_snapshot: false, + replica: nodedb_raft::ReplicaState::Whole, }); let encoded = encode(&rpc, &crate::cluster_epoch::ClusterEpochState::default()).unwrap(); let header: [u8; HEADER_SIZE] = encoded[..HEADER_SIZE].try_into().unwrap(); diff --git a/nodedb-cluster/src/transport/client/transport.rs b/nodedb-cluster/src/transport/client/transport.rs index 74c569bf7..c635f8d89 100644 --- a/nodedb-cluster/src/transport/client/transport.rs +++ b/nodedb-cluster/src/transport/client/transport.rs @@ -248,6 +248,12 @@ impl NexarTransport { &self.circuit_breaker } + /// The fixed timeout of one RPC on this transport, from + /// `ClusterTransportTuning::rpc_timeout_secs`. + pub fn rpc_timeout(&self) -> Duration { + self.rpc_timeout + } + /// The local address this transport is listening on. pub fn local_addr(&self) -> SocketAddr { self.listener.local_addr() @@ -471,7 +477,7 @@ pub(super) mod tests { success: true, last_log_index: req.prev_log_index + req.entries.len() as u64, round: req.round, - needs_snapshot: false, + replica: nodedb_raft::ReplicaState::Whole, })) } RaftRpc::RequestVoteRequest(req) if req.group_id == STALLED_GROUP => { diff --git a/nodedb-raft/src/lib.rs b/nodedb-raft/src/lib.rs index 203c93d80..d54544787 100644 --- a/nodedb-raft/src/lib.rs +++ b/nodedb-raft/src/lib.rs @@ -35,6 +35,6 @@ pub use snapshot_framing::{ SNAPSHOT_FORMAT_VERSION, SNAPSHOT_MAGIC, SnapshotEngineId, SnapshotFramingError, decode_snapshot_chunk, encode_snapshot_chunk, }; -pub use state::{HardState, NodeRole, PeerRole}; +pub use state::{HardState, NodeRole, PeerRole, ReplicaState}; pub use storage::LogStorage; pub use transport::RaftTransport; diff --git a/nodedb-raft/src/message.rs b/nodedb-raft/src/message.rs index 368a054b3..e83281f63 100644 --- a/nodedb-raft/src/message.rs +++ b/nodedb-raft/src/message.rs @@ -89,9 +89,10 @@ pub struct AppendEntriesResponse { pub last_log_index: u64, /// `round` of the request this answers. pub round: u64, - /// Set on a rejection from a follower that holds no state it can resume - /// the log from. The leader sends it a snapshot instead of entries. - pub needs_snapshot: bool, + /// What the follower needs before it takes entries. A rejection from a + /// follower that wants a snapshot gets one instead of entries. The + /// leader records it per peer: a peer whose data diverged cannot lead. + pub replica: crate::state::ReplicaState, } /// RequestVote RPC (Raft paper Figure 2). diff --git a/nodedb-raft/src/node/core.rs b/nodedb-raft/src/node/core.rs index d895ec21a..f2b319664 100644 --- a/nodedb-raft/src/node/core.rs +++ b/nodedb-raft/src/node/core.rs @@ -16,7 +16,7 @@ use crate::error::{RaftError, Result}; use crate::log::RaftLog; use crate::message::{AppendEntriesRequest, LogEntry, PreVoteRequest, TimeoutNowRequest}; use crate::state::{ - HardState, LeaderState, LeadershipTransfer, NodeRole, PreVoteRound, VolatileState, + HardState, LeaderState, LeadershipTransfer, NodeRole, PreVoteRound, ReplicaState, VolatileState, }; use crate::storage::LogStorage; @@ -134,10 +134,12 @@ pub struct RaftNode { /// An outside bound on compaction: the log never discards an entry above /// it. An archiver that must copy entries before they go raises it. pub(super) compaction_ceiling: Option>, - /// Whether this follower holds no state it can resume the log from. It - /// then refuses every `AppendEntries` with `needs_snapshot`, and its - /// leader sends a snapshot instead. The driver sets and clears it. - pub(super) snapshot_required: bool, + /// Whether this replica holds the state it can resume the log from. One + /// that wants a snapshot refuses every `AppendEntries`, reporting this + /// state, and its leader sends a snapshot instead. One whose + /// data diverged also never campaigns. The driver sets and clears it + /// (see [`super::snapshot_debt`]). + pub(super) replica: ReplicaState, /// The highest `replicated_floor` a leader sent this node, or this /// node's own as leader. See [`Self::replicated_floor`]. pub(super) replicated_floor: u64, @@ -189,23 +191,12 @@ impl RaftNode { lease: LeaseState::new(), boot_vote_fence: now + config.election_timeout_max, compaction_ceiling: None, - snapshot_required: false, + replica: ReplicaState::Whole, replicated_floor: 0, config, } } - /// Require a snapshot before this follower takes log entries again, or - /// lift the requirement. See [`Self::snapshot_required`]. - pub fn set_snapshot_required(&mut self, required: bool) { - self.snapshot_required = required; - } - - /// Whether this follower refuses log entries until a snapshot installs. - pub fn snapshot_required(&self) -> bool { - self.snapshot_required - } - /// Restore state from persistent storage. Must be called before ticking. /// /// Seeds `volatile.last_applied` from the durable applied index so diff --git a/nodedb-raft/src/node/election.rs b/nodedb-raft/src/node/election.rs new file mode 100644 index 000000000..2cff12512 --- /dev/null +++ b/nodedb-raft/src/node/election.rs @@ -0,0 +1,121 @@ +// SPDX-License-Identifier: BUSL-1.1 + +//! Election entry points: the pre-vote round and the vote round. A replica +//! whose data diverged starts neither. + +use std::collections::HashSet; + +use tracing::{debug, info}; + +use crate::message::{PreVoteRequest, RequestVoteRequest}; +use crate::state::{NodeRole, PreVoteRound}; +use crate::storage::LogStorage; + +use super::core::RaftNode; + +impl RaftNode { + /// Ask peers whether they WOULD vote for this node at `current_term + 1`. + /// + /// Touches neither `current_term` nor `voted_for` and persists nothing, so + /// a node cut off from the cluster can retry indefinitely without inflating + /// its term and deposing the healthy leader when the partition heals. + pub(super) fn start_pre_election(&mut self) { + match self.role { + NodeRole::Learner | NodeRole::Observer => return, + NodeRole::Follower | NodeRole::Candidate | NodeRole::Leader => {} + } + // A replica whose data diverged cannot lead. + if !self.replica.may_lead() { + return; + } + + // Single voter: there is nobody to probe, and the existing immediate + // win must be preserved. + if self.config.peers.is_empty() { + self.start_election(); + return; + } + + // A round that wins nothing retries on the usual randomized backoff. + self.reset_election_timeout(); + + let term = self.hard_state.current_term + 1; + self.pre_vote = Some(PreVoteRound { + term, + granted: HashSet::new(), + }); + + debug!( + node = self.config.node_id, + group = self.config.group_id, + term, + "starting pre-vote round" + ); + + for &peer in &self.config.peers { + self.ready.pre_vote_requests.push(( + peer, + PreVoteRequest { + term, + candidate_id: self.config.node_id, + last_log_index: self.log.last_index(), + last_log_term: self.log.last_term(), + group_id: self.config.group_id, + }, + )); + } + } + + pub(super) fn start_election(&mut self) { + // Learners and observers never stand for election — defensive check + // in case `tick()` is bypassed (e.g., tests forcing a deadline). + match self.role { + NodeRole::Learner | NodeRole::Observer => return, + NodeRole::Follower | NodeRole::Candidate | NodeRole::Leader => {} + } + // A replica whose data diverged cannot lead, so it never asks for + // votes, and a single voter never elects itself. + if !self.replica.may_lead() { + return; + } + + self.pre_vote = None; + self.hard_state.current_term += 1; + self.role = NodeRole::Candidate; + self.hard_state.voted_for = self.config.node_id; + self.votes_received.clear(); + self.leader_id = 0; + + self.persist_hard_state(); + self.reset_election_timeout(); + + info!( + node = self.config.node_id, + group = self.config.group_id, + term = self.hard_state.current_term, + "starting election" + ); + + // Single-voter cluster: win immediately. (Learners in the group do + // not count — single voter + N learners still elects the single + // voter as leader.) + if self.config.peers.is_empty() { + self.become_leader(); + return; + } + + for &peer in &self.config.peers { + self.ready.vote_requests.push(( + peer, + RequestVoteRequest { + term: self.hard_state.current_term, + candidate_id: self.config.node_id, + last_log_index: self.log.last_index(), + last_log_term: self.log.last_term(), + group_id: self.config.group_id, + transfer: false, + }, + )); + } + } +} diff --git a/nodedb-raft/src/node/internal.rs b/nodedb-raft/src/node/internal.rs index cc9eb796a..006b96104 100644 --- a/nodedb-raft/src/node/internal.rs +++ b/nodedb-raft/src/node/internal.rs @@ -2,116 +2,19 @@ //! Internal state transitions and replication logic. -use std::collections::HashSet; use std::time::{Duration, Instant}; use rand::RngExt; use tracing::{debug, info}; use crate::error::{RaftError, Result}; -use crate::message::{AppendEntriesRequest, LogEntry, PreVoteRequest, RequestVoteRequest}; -use crate::state::{LeaderState, NodeRole, PreVoteRound}; +use crate::message::{AppendEntriesRequest, LogEntry}; +use crate::state::{LeaderState, NodeRole}; use crate::storage::LogStorage; use super::core::RaftNode; impl RaftNode { - /// Ask peers whether they WOULD vote for this node at `current_term + 1`. - /// - /// Touches neither `current_term` nor `voted_for` and persists nothing, so - /// a node cut off from the cluster can retry indefinitely without inflating - /// its term and deposing the healthy leader when the partition heals. - pub(super) fn start_pre_election(&mut self) { - match self.role { - NodeRole::Learner | NodeRole::Observer => return, - NodeRole::Follower | NodeRole::Candidate | NodeRole::Leader => {} - } - - // Single voter: there is nobody to probe, and the existing immediate - // win must be preserved. - if self.config.peers.is_empty() { - self.start_election(); - return; - } - - // A round that wins nothing retries on the usual randomized backoff. - self.reset_election_timeout(); - - let term = self.hard_state.current_term + 1; - self.pre_vote = Some(PreVoteRound { - term, - granted: HashSet::new(), - }); - - debug!( - node = self.config.node_id, - group = self.config.group_id, - term, - "starting pre-vote round" - ); - - for &peer in &self.config.peers { - self.ready.pre_vote_requests.push(( - peer, - PreVoteRequest { - term, - candidate_id: self.config.node_id, - last_log_index: self.log.last_index(), - last_log_term: self.log.last_term(), - group_id: self.config.group_id, - }, - )); - } - } - - pub(super) fn start_election(&mut self) { - // Learners and observers never stand for election — defensive check - // in case `tick()` is bypassed (e.g., tests forcing a deadline). - match self.role { - NodeRole::Learner | NodeRole::Observer => return, - NodeRole::Follower | NodeRole::Candidate | NodeRole::Leader => {} - } - - self.pre_vote = None; - self.hard_state.current_term += 1; - self.role = NodeRole::Candidate; - self.hard_state.voted_for = self.config.node_id; - self.votes_received.clear(); - self.leader_id = 0; - - self.persist_hard_state(); - self.reset_election_timeout(); - - info!( - node = self.config.node_id, - group = self.config.group_id, - term = self.hard_state.current_term, - "starting election" - ); - - // Single-voter cluster: win immediately. (Learners in the group do - // not count — single voter + N learners still elects the single - // voter as leader.) - if self.config.peers.is_empty() { - self.become_leader(); - return; - } - - for &peer in &self.config.peers { - self.ready.vote_requests.push(( - peer, - RequestVoteRequest { - term: self.hard_state.current_term, - candidate_id: self.config.node_id, - last_log_index: self.log.last_index(), - last_log_term: self.log.last_term(), - group_id: self.config.group_id, - transfer: false, - }, - )); - } - } - /// Step down to follower (or keep learner/observer role if the node is one). /// /// Learners and observers that receive an `AppendEntries` with a higher diff --git a/nodedb-raft/src/node/leader_lease.rs b/nodedb-raft/src/node/leader_lease.rs index bfc7ca729..047303958 100644 --- a/nodedb-raft/src/node/leader_lease.rs +++ b/nodedb-raft/src/node/leader_lease.rs @@ -305,7 +305,7 @@ mod tests { success: true, last_log_index, round, - needs_snapshot: false, + replica: crate::state::ReplicaState::Whole, }, ); } @@ -422,7 +422,7 @@ mod tests { success: false, last_log_index: 0, round: stale, - needs_snapshot: false, + replica: crate::state::ReplicaState::Whole, }, ); assert_eq!(node.role(), NodeRole::Follower); diff --git a/nodedb-raft/src/node/mod.rs b/nodedb-raft/src/node/mod.rs index 75c0ac82b..ae6c6a2e4 100644 --- a/nodedb-raft/src/node/mod.rs +++ b/nodedb-raft/src/node/mod.rs @@ -7,7 +7,9 @@ //! - [`core`]: `RaftNode` struct, constructors, simple accessors, `tick`, //! `propose`, and the `Ready` output type. //! - [`durability`]: Applied-index durability floor and log compaction. -//! - [`internal`]: Internal state transitions (elections, replication, +//! - [`election`]: The pre-vote and vote rounds a replica starts, and the +//! guard that keeps a replica whose data diverged out of both. +//! - [`internal`]: Internal state transitions (role changes, replication, //! commit advancement) and timeout math. //! - [`leader_lease`]: Serving a linearizable read from the leader's commit //! index without a quorum round, and the vote refusal that makes it safe. @@ -20,12 +22,15 @@ //! - [`rpc`]: Incoming RPC handlers (`AppendEntries`, `PreVote`, //! `RequestVote`, `InstallSnapshot`, `TimeoutNow`, and their response //! handlers). +//! - [`snapshot_debt`]: A replica awaiting a snapshot refuses entries and +//! never leads. //! - [`staleness`]: How far behind the leader a replica is, for //! bounded-staleness reads. pub mod config; pub mod core; pub mod durability; +mod election; mod internal; pub mod leader_lease; pub mod membership; @@ -33,6 +38,7 @@ pub mod peer_contact; pub mod quorum_contact; pub mod read_index; pub mod rpc; +pub mod snapshot_debt; pub mod staleness; pub use self::config::RaftConfig; diff --git a/nodedb-raft/src/node/peer_contact.rs b/nodedb-raft/src/node/peer_contact.rs index c8dcfe5bf..eb54a6d0d 100644 --- a/nodedb-raft/src/node/peer_contact.rs +++ b/nodedb-raft/src/node/peer_contact.rs @@ -6,7 +6,7 @@ use std::time::Instant; use crate::node::core::RaftNode; -use crate::state::NodeRole; +use crate::state::{NodeRole, ReplicaState}; use crate::storage::LogStorage; impl RaftNode { @@ -57,12 +57,19 @@ impl RaftNode { .max(lowest_live_voter.min(self.volatile.commit_index)) } - /// Whether `peer`'s latest response to this leader asked for a snapshot. - /// Such a peer holds no state to lead from. `false` off the leader. - pub fn peer_awaits_snapshot(&self, peer: u64) -> bool { + /// Whether `peer`'s latest response this term reported it whole. `false` + /// for a peer not heard this term, and off the leader. + pub fn peer_answered_whole(&self, peer: u64) -> bool { + self.peer_replica_state(peer) == Some(ReplicaState::Whole) + } + + /// `peer`'s state as its latest response this term reported it. `None` + /// for a peer not heard this term, and off the leader: its state is + /// unknown. + pub fn peer_replica_state(&self, peer: u64) -> Option { self.leader_state .as_ref() - .is_some_and(|leader| leader.awaiting_snapshot.contains(&peer)) + .and_then(|leader| leader.peer_states.get(&peer).copied()) } /// The leader this node names at `now`, when it has proof the leader is @@ -178,7 +185,7 @@ mod tests { success: true, last_log_index: 1, round: 1, - needs_snapshot: false, + replica: crate::state::ReplicaState::Whole, }; node.handle_append_entries_response(2, &ack); assert_eq!(node.replicated_floor(), 0, "voter 3 holds nothing yet"); @@ -216,7 +223,7 @@ mod tests { success, last_log_index, round: node.lease.next_round - 1, - needs_snapshot: false, + replica: crate::state::ReplicaState::Whole, }; node.handle_append_entries_response(peer, &resp); } diff --git a/nodedb-raft/src/node/quorum_contact.rs b/nodedb-raft/src/node/quorum_contact.rs index 74f32f7a2..f57c43feb 100644 --- a/nodedb-raft/src/node/quorum_contact.rs +++ b/nodedb-raft/src/node/quorum_contact.rs @@ -206,7 +206,7 @@ mod tests { success: true, last_log_index: node.last_log_index(), round, - needs_snapshot: false, + replica: crate::state::ReplicaState::Whole, }, ); } @@ -226,7 +226,7 @@ mod tests { success: false, last_log_index: 0, round: latest_round(&node), - needs_snapshot: false, + replica: crate::state::ReplicaState::Whole, }, ); @@ -257,7 +257,7 @@ mod tests { success: true, last_log_index: 1, round: latest_round(&node), - needs_snapshot: false, + replica: crate::state::ReplicaState::Whole, }, ); @@ -296,7 +296,7 @@ mod tests { success: true, last_log_index: node.last_log_index(), round: latest_round(&node), - needs_snapshot: false, + replica: crate::state::ReplicaState::Whole, }, ); @@ -380,7 +380,7 @@ mod tests { success: true, last_log_index: node.last_log_index(), round: latest_round(&node), - needs_snapshot: false, + replica: crate::state::ReplicaState::Whole, }, ); node.tick(); @@ -468,7 +468,7 @@ mod tests { success: false, last_log_index: 0, round: stale, - needs_snapshot: false, + replica: crate::state::ReplicaState::Whole, }, ); assert_eq!(node.role(), NodeRole::Follower); @@ -520,7 +520,7 @@ mod tests { success: true, last_log_index: node.last_log_index(), round: latest_round(&node), - needs_snapshot: false, + replica: crate::state::ReplicaState::Whole, }, ); diff --git a/nodedb-raft/src/node/read_index.rs b/nodedb-raft/src/node/read_index.rs index ca3111af0..ab1b970e5 100644 --- a/nodedb-raft/src/node/read_index.rs +++ b/nodedb-raft/src/node/read_index.rs @@ -190,7 +190,7 @@ mod tests { success, last_log_index: node.last_log_index(), round: node.lease.next_round - 1, - needs_snapshot: false, + replica: crate::state::ReplicaState::Whole, }, ); } @@ -268,7 +268,7 @@ mod tests { success: false, last_log_index: 0, round: 0, - needs_snapshot: false, + replica: crate::state::ReplicaState::Whole, }, ); diff --git a/nodedb-raft/src/node/rpc/append_entries.rs b/nodedb-raft/src/node/rpc/append_entries.rs index 95637f3fa..1bb0ca819 100644 --- a/nodedb-raft/src/node/rpc/append_entries.rs +++ b/nodedb-raft/src/node/rpc/append_entries.rs @@ -18,7 +18,7 @@ impl RaftNode { success: false, last_log_index: self.log.last_index(), round: req.round, - needs_snapshot: false, + replica: self.replica, }; } @@ -49,13 +49,13 @@ impl RaftNode { // A follower with no state to resume from takes no entries. The // leader answers the refusal with a snapshot. - if self.snapshot_required { + if self.replica.wants_snapshot() { return AppendEntriesResponse { term: self.hard_state.current_term, success: false, last_log_index: self.log.last_index(), round: req.round, - needs_snapshot: true, + replica: self.replica, }; } @@ -68,7 +68,7 @@ impl RaftNode { success: false, last_log_index: self.volatile.last_applied, round: req.round, - needs_snapshot: false, + replica: self.replica, }; } @@ -82,7 +82,7 @@ impl RaftNode { success: false, last_log_index: self.log.last_index(), round: req.round, - needs_snapshot: false, + replica: self.replica, }; } } @@ -97,7 +97,7 @@ impl RaftNode { success: false, last_log_index: self.log.last_index(), round: req.round, - needs_snapshot: false, + replica: self.replica, }; } }; @@ -116,7 +116,7 @@ impl RaftNode { success: true, last_log_index: self.claimable_match(matched, wrote), round: req.round, - needs_snapshot: false, + replica: self.replica, } } @@ -149,6 +149,11 @@ impl RaftNode { self.become_follower(resp.term); return; } + // A reply to a request of an earlier term says nothing about this + // term's log, acks, or peer states. + if resp.term < self.hard_state.current_term { + return; + } if self.role != NodeRole::Leader { return; @@ -205,7 +210,7 @@ impl RaftNode { // the follower recorded this leader's contact before its log check. // The lease settles at the round's send time. Check-quorum dates the // answer at its arrival. - if peer_is_voter && resp.term == self.hard_state.current_term { + if peer_is_voter { let now = std::time::Instant::now(); self.record_lease_ack(peer, resp.round, now); self.settle_lease(); @@ -216,11 +221,7 @@ impl RaftNode { None => return, }; - if resp.needs_snapshot { - leader.awaiting_snapshot.insert(peer); - } else { - leader.awaiting_snapshot.remove(&peer); - } + leader.peer_states.insert(peer, resp.replica); if resp.success { let new_match = resp.last_log_index; if new_match > leader.match_index_for(peer) { @@ -231,7 +232,7 @@ impl RaftNode { self.try_advance_commit_index(); self.replicated_floor = self.replicated_floor(); } - } else if resp.needs_snapshot { + } else if resp.replica.wants_snapshot() { // The peer holds no state to resume from: entries from any index // leave it as it is, so it gets a snapshot. if !self.ready.snapshots_needed.contains(&peer) { @@ -365,12 +366,12 @@ mod tests { } /// A follower that requires a snapshot refuses entries with - /// `needs_snapshot`, and its leader flags it for a snapshot even though + /// the state it wants, and its leader flags it for a snapshot even though /// the leader's log holds every entry. #[test] fn a_follower_that_requires_a_snapshot_gets_one() { let mut follower = RaftNode::new(test_config(2, vec![1]), MemStorage::new()); - follower.set_snapshot_required(true); + follower.set_replica_state(crate::state::ReplicaState::WantsSnapshot); let req = AppendEntriesRequest { term: 1, leader_id: 1, @@ -388,7 +389,7 @@ mod tests { }; let refusal = follower.handle_append_entries(&req); assert!(!refusal.success); - assert!(refusal.needs_snapshot); + assert_eq!(refusal.replica, crate::state::ReplicaState::WantsSnapshot); assert_eq!(follower.last_log_index(), 0, "no entry was appended"); assert_eq!(follower.leader_id(), 1, "the leader's contact still counts"); @@ -406,7 +407,7 @@ mod tests { leader.handle_append_entries_response(2, &refusal); assert!(leader.take_ready().snapshots_needed.contains(&2)); - follower.set_snapshot_required(false); + follower.set_replica_state(crate::state::ReplicaState::Whole); assert!(follower.handle_append_entries(&req).success); } @@ -430,7 +431,7 @@ mod tests { success, last_log_index, round: 1, - needs_snapshot: false, + replica: crate::state::ReplicaState::Whole, }; node.handle_append_entries_response(2, &answer(true, 1)); node.handle_append_entries_response(2, &answer(false, 0)); @@ -463,6 +464,44 @@ mod tests { assert_eq!(node.hard_state.voted_for, 0, "a new term frees the vote"); } + /// A reply carrying an earlier term answers a request of that term. It + /// records no ack, match, or peer state in this term. + #[test] + fn a_reply_from_an_earlier_term_records_nothing() { + let mut node = RaftNode::new(test_config(1, vec![2, 3]), MemStorage::new()); + force_election(&mut node); + let _ = node.take_ready(); + node.handle_request_vote_response( + 2, + &RequestVoteResponse { + term: 1, + vote_granted: true, + }, + ); + assert_eq!(node.role(), NodeRole::Leader); + + let stale = AppendEntriesResponse { + term: 0, + success: true, + last_log_index: node.last_log_index(), + round: UNTRACKED_ROUND, + replica: crate::state::ReplicaState::AwaitingSnapshot, + }; + node.handle_append_entries_response(2, &stale); + assert_eq!(node.peer_replica_state(2), None); + assert_eq!(node.peer_ack_count(2), Some(0)); + assert_eq!(node.match_index_for(2), Some(0)); + assert!(node.take_ready().snapshots_needed.is_empty()); + + let current = AppendEntriesResponse { term: 1, ..stale }; + node.handle_append_entries_response(2, ¤t); + assert_eq!( + node.peer_replica_state(2), + Some(crate::state::ReplicaState::AwaitingSnapshot) + ); + assert_eq!(node.peer_ack_count(2), Some(1)); + } + #[test] fn leader_steps_down_on_higher_term() { let config = test_config(1, vec![2, 3]); @@ -564,7 +603,7 @@ mod tests { success: true, last_log_index: 2, round: UNTRACKED_ROUND, - needs_snapshot: false, + replica: crate::state::ReplicaState::Whole, }; node.handle_append_entries_response(4, &ae_ok); assert_eq!( @@ -669,7 +708,7 @@ mod tests { success: true, last_log_index: idx, round: UNTRACKED_ROUND, - needs_snapshot: false, + replica: crate::state::ReplicaState::Whole, }; leader.handle_append_entries_response(2, &ae_ok); assert_eq!(leader.commit_index(), idx); @@ -723,7 +762,7 @@ mod tests { success: true, last_log_index: idx, round: UNTRACKED_ROUND, - needs_snapshot: false, + replica: crate::state::ReplicaState::Whole, }; node1.handle_append_entries_response(5, &obs_ack); assert_eq!( @@ -749,7 +788,7 @@ mod tests { success: true, last_log_index: idx, round: UNTRACKED_ROUND, - needs_snapshot: false, + replica: crate::state::ReplicaState::Whole, }; leader.handle_append_entries_response(2, &voter_ack); assert_eq!( diff --git a/nodedb-raft/src/node/rpc/install_snapshot.rs b/nodedb-raft/src/node/rpc/install_snapshot.rs index 278c757b6..62d2c2207 100644 --- a/nodedb-raft/src/node/rpc/install_snapshot.rs +++ b/nodedb-raft/src/node/rpc/install_snapshot.rs @@ -261,7 +261,7 @@ mod tests { success: true, last_log_index: tip, round: UNTRACKED_ROUND, - needs_snapshot: false, + replica: crate::state::ReplicaState::Whole, }, ); assert_eq!(leader.commit_index(), tip); diff --git a/nodedb-raft/src/node/rpc/timeout_now.rs b/nodedb-raft/src/node/rpc/timeout_now.rs index 60fcdd492..a59248964 100644 --- a/nodedb-raft/src/node/rpc/timeout_now.rs +++ b/nodedb-raft/src/node/rpc/timeout_now.rs @@ -115,7 +115,7 @@ mod tests { success: true, last_log_index: idx, round: UNTRACKED_ROUND, - needs_snapshot: false, + replica: crate::state::ReplicaState::Whole, } } @@ -261,7 +261,7 @@ mod tests { success: false, last_log_index: 0, round: UNTRACKED_ROUND, - needs_snapshot: false, + replica: crate::state::ReplicaState::Whole, }, ); assert_eq!(leader.role(), NodeRole::Follower); diff --git a/nodedb-raft/src/node/snapshot_debt.rs b/nodedb-raft/src/node/snapshot_debt.rs new file mode 100644 index 000000000..52d786e61 --- /dev/null +++ b/nodedb-raft/src/node/snapshot_debt.rs @@ -0,0 +1,341 @@ +// SPDX-License-Identifier: BUSL-1.1 + +//! A replica that takes a snapshot before it takes log entries again. +//! +//! - [`ReplicaState::WantsSnapshot`]: its data matches its log, and state +//! its host keeps beside the log lags. It refuses entries so its leader +//! sends a snapshot. It still campaigns and leads. +//! - [`ReplicaState::AwaitingSnapshot`]: its data diverged. Only a leader +//! sends a snapshot, so a leader awaiting one would wait forever. Entering +//! this state steps a leader or candidate down, and the election paths +//! refuse to start while it holds. A leader hands its term to the voter +//! [`RaftNode::hand_off_target`] names, when one qualifies. + +use tracing::info; + +use crate::message::TimeoutNowRequest; +use crate::node::core::RaftNode; +use crate::state::{NodeRole, ReplicaState}; +use crate::storage::LogStorage; + +impl RaftNode { + /// Set what this replica needs before it takes log entries again. A + /// leader or candidate whose data diverged steps down. + pub fn set_replica_state(&mut self, state: ReplicaState) { + self.replica = state; + if !state.may_lead() { + self.stand_down_for_snapshot(); + } + } + + /// Whether this replica refuses log entries until a snapshot installs. + pub fn snapshot_required(&self) -> bool { + self.replica.wants_snapshot() + } + + /// Whether this replica holds the state it serves and leads from. + pub fn replica_state(&self) -> ReplicaState { + self.replica + } + + /// Leave a leadership or candidacy this replica can no longer hold. + fn stand_down_for_snapshot(&mut self) { + let term = self.hard_state.current_term; + match self.role { + NodeRole::Leader => { + let target = self.hand_off_target(); + if let Some(target) = target { + // The target campaigns at once. Voters grant it although + // they still hear this leader, so no lease holds past + // here. + self.lease.revoke(); + self.ready.timeout_now.push(( + target, + TimeoutNowRequest { + term, + leader_id: self.config.node_id, + group_id: self.config.group_id, + }, + )); + } + info!( + node = self.config.node_id, + group = self.config.group_id, + term, + ?target, + "replica awaits a snapshot; stepping down from leader" + ); + self.become_follower(term); + } + NodeRole::Candidate => self.become_follower(term), + NodeRole::Follower => self.pre_vote = None, + NodeRole::Learner | NodeRole::Observer => {} + } + } + + /// The voter peer this leader hands its leadership to. `None` off the + /// leader, or when no voter qualifies. + /// + /// A target answered whole this term and holds every committed entry. A + /// voter missing a committed entry cannot win an election. Of the + /// qualified voters, the highest match goes first. One at the log + /// frontier wins its campaign against every voter. + pub fn hand_off_target(&self) -> Option { + let commit = self.volatile.commit_index; + self.config + .peers + .iter() + .copied() + .filter(|&peer| self.peer_answered_whole(peer)) + .filter_map(|peer| Some((peer, self.match_index_for(peer)?))) + .filter(|&(_, matched)| matched >= commit) + .max_by_key(|&(_, matched)| matched) + .map(|(peer, _)| peer) + } +} + +#[cfg(test)] +mod tests { + use std::time::{Duration, Instant}; + + use crate::message::{AppendEntriesRequest, AppendEntriesResponse, RequestVoteResponse}; + use crate::node::core::RaftNode; + use crate::node::leader_lease::UNTRACKED_ROUND; + use crate::state::{NodeRole, ReplicaState}; + use crate::storage::MemStorage; + use crate::test_support::{force_election, test_config}; + + fn leader_3voter() -> RaftNode { + let mut node = RaftNode::new(test_config(1, vec![2, 3]), MemStorage::new()); + force_election(&mut node); + let _ = node.take_ready(); + node.handle_request_vote_response( + 2, + &RequestVoteResponse { + term: 1, + vote_granted: true, + }, + ); + assert_eq!(node.role(), NodeRole::Leader); + let _ = node.take_ready(); + node + } + + fn reply(last_log_index: u64, replica: ReplicaState) -> AppendEntriesResponse { + AppendEntriesResponse { + term: 1, + success: !replica.wants_snapshot(), + last_log_index, + round: UNTRACKED_ROUND, + replica, + } + } + + #[test] + fn a_replica_awaiting_a_snapshot_never_campaigns() { + let mut node = RaftNode::new(test_config(2, vec![1, 3]), MemStorage::new()); + node.set_replica_state(ReplicaState::AwaitingSnapshot); + assert_eq!(node.replica_state(), ReplicaState::AwaitingSnapshot); + + force_election(&mut node); + let ready = node.take_ready(); + assert!(ready.pre_vote_requests.is_empty(), "no pre-vote round"); + assert!(ready.vote_requests.is_empty(), "no vote request"); + assert_eq!(node.role(), NodeRole::Follower); + assert_eq!(node.current_term(), 0); + + node.set_replica_state(ReplicaState::Whole); + force_election(&mut node); + assert_eq!( + node.role(), + NodeRole::Candidate, + "a whole replica campaigns" + ); + } + + #[test] + fn a_sole_voter_awaiting_a_snapshot_never_elects_itself() { + let mut node = RaftNode::new(test_config(1, vec![]), MemStorage::new()); + node.set_replica_state(ReplicaState::AwaitingSnapshot); + node.election_deadline_override(Instant::now() - Duration::from_millis(1)); + node.tick(); + assert_eq!(node.role(), NodeRole::Follower); + assert_eq!(node.leader_id(), 0); + } + + #[test] + fn a_transfer_never_makes_an_awaiting_replica_campaign() { + let mut node = RaftNode::new(test_config(2, vec![1, 3]), MemStorage::new()); + node.handle_append_entries(&AppendEntriesRequest { + term: 1, + leader_id: 1, + prev_log_index: 0, + prev_log_term: 0, + entries: vec![], + leader_commit: 0, + group_id: 1, + round: 1, + replicated_floor: 0, + }); + node.set_replica_state(ReplicaState::AwaitingSnapshot); + node.handle_timeout_now(&crate::message::TimeoutNowRequest { + term: 1, + leader_id: 1, + group_id: 1, + }); + assert_eq!(node.role(), NodeRole::Follower); + assert_eq!(node.current_term(), 1); + assert!(node.take_ready().vote_requests.is_empty()); + } + + #[test] + fn a_leader_given_the_requirement_steps_down_and_hands_off() { + let mut leader = leader_3voter(); + let frontier = leader.last_log_index(); + leader.handle_append_entries_response(2, &reply(frontier, ReplicaState::Whole)); + let _ = leader.take_ready(); + + leader.set_replica_state(ReplicaState::AwaitingSnapshot); + assert_eq!(leader.role(), NodeRole::Follower); + assert_eq!(leader.leader_id(), 0); + let ready = leader.take_ready(); + assert_eq!(ready.timeout_now.len(), 1); + let (target, request) = ready.timeout_now[0].clone(); + assert_eq!(target, 2); + assert_eq!((request.term, request.leader_id), (1, 1)); + + leader.election_deadline_override(Instant::now() - Duration::from_millis(1)); + leader.tick(); + assert_eq!(leader.role(), NodeRole::Follower, "it never campaigns back"); + } + + #[test] + fn a_leader_never_hands_off_to_a_peer_awaiting_a_snapshot() { + let mut leader = leader_3voter(); + let frontier = leader.last_log_index(); + // Peer 2 matched the frontier, then asked for a snapshot. Peer 3 + // never answered. + leader.handle_append_entries_response(2, &reply(frontier, ReplicaState::Whole)); + leader.handle_append_entries_response(2, &reply(frontier, ReplicaState::AwaitingSnapshot)); + let _ = leader.take_ready(); + + leader.set_replica_state(ReplicaState::AwaitingSnapshot); + assert_eq!(leader.role(), NodeRole::Follower); + assert!(leader.take_ready().timeout_now.is_empty()); + } + + /// A replica that wants a snapshot only for state beside its log, such + /// as a lost Calvin base, still campaigns and leads, and a leader that + /// takes that state keeps leading. + #[test] + fn a_replica_that_wants_a_snapshot_still_campaigns_and_leads() { + let mut node = RaftNode::new(test_config(1, vec![2, 3]), MemStorage::new()); + node.set_replica_state(ReplicaState::WantsSnapshot); + assert!(node.snapshot_required()); + force_election(&mut node); + assert_eq!(node.role(), NodeRole::Candidate); + node.handle_request_vote_response( + 2, + &RequestVoteResponse { + term: 1, + vote_granted: true, + }, + ); + assert_eq!(node.role(), NodeRole::Leader); + + let mut leader = leader_3voter(); + leader.set_replica_state(ReplicaState::WantsSnapshot); + assert_eq!(leader.role(), NodeRole::Leader); + assert!(leader.take_ready().timeout_now.is_empty()); + + let mut sole = RaftNode::new(test_config(1, vec![]), MemStorage::new()); + sole.set_replica_state(ReplicaState::WantsSnapshot); + sole.election_deadline_override(Instant::now() - Duration::from_millis(1)); + sole.tick(); + assert_eq!(sole.role(), NodeRole::Leader); + } + + #[test] + fn a_stricter_state_wins() { + assert_eq!( + ReplicaState::AwaitingSnapshot.raised_to(ReplicaState::WantsSnapshot), + ReplicaState::AwaitingSnapshot + ); + assert_eq!( + ReplicaState::Whole.raised_to(ReplicaState::WantsSnapshot), + ReplicaState::WantsSnapshot + ); + } + + #[test] + fn a_candidate_given_the_requirement_stands_down() { + let mut node = RaftNode::new(test_config(1, vec![2, 3]), MemStorage::new()); + force_election(&mut node); + assert_eq!(node.role(), NodeRole::Candidate); + node.set_replica_state(ReplicaState::AwaitingSnapshot); + assert_eq!(node.role(), NodeRole::Follower); + node.handle_request_vote_response( + 2, + &RequestVoteResponse { + term: 1, + vote_granted: true, + }, + ); + assert_eq!( + node.role(), + NodeRole::Follower, + "a late grant elects nothing" + ); + } + + /// A voter not heard this term is never a hand-off target. + #[test] + fn a_voter_not_heard_this_term_is_no_hand_off_target() { + let mut leader = leader_3voter(); + assert_eq!(leader.hand_off_target(), None); + leader.handle_append_entries_response(3, &reply(0, ReplicaState::AwaitingSnapshot)); + assert_eq!(leader.hand_off_target(), None, "voter 2 is still unheard"); + } + + /// The target holds every committed entry, and the highest match goes + /// first. + #[test] + fn the_hand_off_target_holds_every_committed_entry() { + let mut node = RaftNode::new(test_config(1, vec![2, 3, 4, 5]), MemStorage::new()); + force_election(&mut node); + let _ = node.take_ready(); + for peer in [2, 3] { + node.handle_request_vote_response( + peer, + &RequestVoteResponse { + term: 1, + vote_granted: true, + }, + ); + } + assert_eq!(node.role(), NodeRole::Leader); + let frontier = node.last_log_index(); + for peer in [2, 3, 4] { + node.handle_append_entries_response(peer, &reply(frontier, ReplicaState::Whole)); + } + assert_eq!(node.commit_index(), frontier); + node.handle_append_entries_response(5, &reply(0, ReplicaState::Whole)); + for peer in [2, 3] { + node.handle_append_entries_response( + peer, + &reply(frontier, ReplicaState::AwaitingSnapshot), + ); + } + assert_eq!(node.hand_off_target(), Some(4)); + + node.handle_append_entries_response(4, &reply(frontier, ReplicaState::WantsSnapshot)); + assert_eq!( + node.hand_off_target(), + None, + "voter 5 answered whole but misses a committed entry" + ); + + node.handle_append_entries_response(5, &reply(frontier, ReplicaState::Whole)); + assert_eq!(node.hand_off_target(), Some(5)); + } +} diff --git a/nodedb-raft/src/node/staleness.rs b/nodedb-raft/src/node/staleness.rs index 98ba6be76..591093c8f 100644 --- a/nodedb-raft/src/node/staleness.rs +++ b/nodedb-raft/src/node/staleness.rs @@ -31,14 +31,19 @@ pub enum StalenessVerdict { } impl RaftNode { - /// Whether this replica is within `max_staleness` of the leader. + /// Whether this replica is within `max_staleness` of the leader, given + /// `applied`: the highest index whose state is visible to reads here. + /// + /// `applied` is the caller's, not Raft's `last_applied`. Raft advances + /// `last_applied` when it hands an entry to the state machine, before + /// the entry's state is visible. /// /// The leader is always fresh — it is the source of the bound, not a /// consumer of it. A follower is fresh when it has applied everything /// the leader had committed as of a contact no older than the bound: /// anything committed since is at most `max_staleness` old, which is /// precisely what was promised. - pub fn staleness_verdict(&self, max_staleness: Duration) -> StalenessVerdict { + pub fn staleness_verdict(&self, max_staleness: Duration, applied: u64) -> StalenessVerdict { if self.role == NodeRole::Leader { return StalenessVerdict::Fresh; } @@ -48,18 +53,30 @@ impl RaftNode { if contact.at.elapsed() > max_staleness { return StalenessVerdict::NoRecentContact; } - if self.volatile.last_applied < contact.leader_commit { + if applied < contact.leader_commit { return StalenessVerdict::Behind { leader_commit: contact.leader_commit, - applied: self.volatile.last_applied, + applied, }; } StalenessVerdict::Fresh } - /// Whether a read with `max_staleness` can be served from this replica. - pub fn within_staleness_bound(&self, max_staleness: Duration) -> bool { - self.staleness_verdict(max_staleness) == StalenessVerdict::Fresh + /// Whether a read with `max_staleness` can be served from this replica, + /// whose reads see state through `applied`. + pub fn within_staleness_bound(&self, max_staleness: Duration, applied: u64) -> bool { + self.staleness_verdict(max_staleness, applied) == StalenessVerdict::Fresh + } + + /// Whether this replica heard from its leader within `window`. The + /// leader itself always has. How far behind the replica applies does not + /// count. + pub fn leader_contact_within(&self, window: Duration) -> bool { + if self.role == NodeRole::Leader { + return true; + } + self.leader_contact + .is_some_and(|contact| contact.at.elapsed() <= window) } } @@ -115,16 +132,17 @@ mod tests { fn a_replica_that_never_heard_from_a_leader_is_not_fresh() { let node = follower(); assert_eq!( - node.staleness_verdict(BOUND), + node.staleness_verdict(BOUND, 0), StalenessVerdict::NoRecentContact ); + assert!(!node.leader_contact_within(BOUND)); } #[test] fn a_caught_up_replica_in_recent_contact_is_fresh() { let mut node = follower(); heartbeat(&mut node, 0); - assert_eq!(node.staleness_verdict(BOUND), StalenessVerdict::Fresh); + assert_eq!(node.staleness_verdict(BOUND, 0), StalenessVerdict::Fresh); } /// The failure the old measure could not see: applying steadily while @@ -134,9 +152,8 @@ mod tests { fn a_replica_behind_the_leader_is_not_fresh_however_recent_the_contact() { let mut node = follower(); heartbeat(&mut node, 500); - node.advance_applied(10); assert_eq!( - node.staleness_verdict(BOUND), + node.staleness_verdict(BOUND, 10), StalenessVerdict::Behind { leader_commit: 500, applied: 10, @@ -150,8 +167,31 @@ mod tests { fn an_idle_cluster_does_not_make_a_caught_up_replica_look_stale() { let mut node = follower(); heartbeat(&mut node, 7); - node.advance_applied(7); - assert_eq!(node.staleness_verdict(BOUND), StalenessVerdict::Fresh); + assert_eq!(node.staleness_verdict(BOUND, 7), StalenessVerdict::Fresh); + } + + /// Raft's own `last_applied` at the leader's commit proves nothing: the + /// entries can still be applying. The verdict reads the caller's + /// watermark, so a lagging one is stale and one at the commit is fresh. + #[test] + fn the_verdict_reads_the_callers_watermark_not_the_hand_off_index() { + let mut node = follower(); + heartbeat(&mut node, 9); + node.advance_applied(9); + assert_eq!(node.last_applied(), 9); + assert_eq!( + node.staleness_verdict(BOUND, 4), + StalenessVerdict::Behind { + leader_commit: 9, + applied: 4, + } + ); + assert!(!node.within_staleness_bound(BOUND, 4)); + assert!(node.within_staleness_bound(BOUND, 9)); + assert!( + node.leader_contact_within(BOUND), + "a lagging apply keeps contact" + ); } #[test] @@ -160,9 +200,10 @@ mod tests { heartbeat(&mut node, 0); node.leader_contact_at_override(Instant::now() - Duration::from_secs(30)); assert_eq!( - node.staleness_verdict(BOUND), + node.staleness_verdict(BOUND, 0), StalenessVerdict::NoRecentContact ); + assert!(!node.leader_contact_within(BOUND)); } /// The leader is the source of the bound, so it needs no contact of its @@ -173,6 +214,7 @@ mod tests { node.election_deadline_override(Instant::now() - Duration::from_millis(1)); node.tick(); assert_eq!(node.role(), NodeRole::Leader); - assert_eq!(node.staleness_verdict(BOUND), StalenessVerdict::Fresh); + assert_eq!(node.staleness_verdict(BOUND, 0), StalenessVerdict::Fresh); + assert!(node.leader_contact_within(BOUND)); } } diff --git a/nodedb-raft/src/state.rs b/nodedb-raft/src/state.rs index 18faf94b4..f7bd867c2 100644 --- a/nodedb-raft/src/state.rs +++ b/nodedb-raft/src/state.rs @@ -50,6 +50,63 @@ pub enum NodeRole { Observer, } +/// Whether a replica holds the state beside its log that it serves and +/// leads from. A follower reports it on every `AppendEntries` reply. +/// +/// Variants order from least to most strict. +#[repr(u8)] +#[derive( + Debug, + Clone, + Copy, + PartialEq, + Eq, + PartialOrd, + Ord, + serde::Serialize, + serde::Deserialize, + rkyv::Archive, + rkyv::Serialize, + rkyv::Deserialize, + zerompk::ToMessagePack, + zerompk::FromMessagePack, +)] +#[msgpack(c_enum)] +pub enum ReplicaState { + /// Its state matches the log it holds. + Whole = 0, + /// Its data matches its log, but state its host keeps beside the log + /// lags. It refuses log entries so its leader sends a snapshot. It + /// still campaigns and leads. + WantsSnapshot = 1, + /// Its data diverged from the log. It refuses log entries, never stands + /// for election, and leads nothing until a snapshot installs. + AwaitingSnapshot = 2, +} + +impl ReplicaState { + /// The stricter of `self` and `other`. + pub fn raised_to(self, other: Self) -> Self { + self.max(other) + } + + /// Whether the replica refuses log entries until a snapshot installs. + pub fn wants_snapshot(self) -> bool { + match self { + Self::Whole => false, + Self::WantsSnapshot | Self::AwaitingSnapshot => true, + } + } + + /// Whether the replica may campaign and lead. + pub fn may_lead(self) -> bool { + match self { + Self::Whole | Self::WantsSnapshot => true, + Self::AwaitingSnapshot => false, + } + } +} + /// Role of a remote peer as seen by the local leader. /// /// Used to classify tracked peers so the leader can send entries to both @@ -183,9 +240,9 @@ pub struct LeaderState { /// timed: leadership is confirmed by a quorum of these rising after a /// heartbeat round. pub ack_count: Vec<(u64, u64)>, - /// Peers whose latest `AppendEntries` response asked for a snapshot. - /// Such a peer holds no state to lead from. - pub awaiting_snapshot: std::collections::HashSet, + /// Each peer's state as its latest `AppendEntries` response this term + /// reported it. A peer not heard this term has no entry. + pub peer_states: std::collections::HashMap, } impl LeaderState { @@ -195,7 +252,7 @@ impl LeaderState { next_index: peers.iter().map(|&id| (id, last_log_index + 1)).collect(), match_index: peers.iter().map(|&id| (id, 0)).collect(), ack_count: peers.iter().map(|&id| (id, 0)).collect(), - awaiting_snapshot: std::collections::HashSet::new(), + peer_states: std::collections::HashMap::new(), observer_states: observers .iter() .map(|&id| { @@ -254,7 +311,7 @@ impl LeaderState { self.next_index.retain(|&(id, _)| id != peer); self.match_index.retain(|&(id, _)| id != peer); self.ack_count.retain(|&(id, _)| id != peer); - self.awaiting_snapshot.remove(&peer); + self.peer_states.remove(&peer); } /// Responses received from `peer` in this term. diff --git a/nodedb-raft/tests/membership.rs b/nodedb-raft/tests/membership.rs index 57ecd2089..2698ac7ad 100644 --- a/nodedb-raft/tests/membership.rs +++ b/nodedb-raft/tests/membership.rs @@ -192,7 +192,7 @@ fn learner_catchup_then_promotion_lifecycle() { success: true, last_log_index: idx, round: 0, - needs_snapshot: false, + replica: nodedb_raft::ReplicaState::Whole, }; node.handle_append_entries_response(2, &voter_ack); assert_eq!(node.commit_index(), idx); @@ -203,7 +203,7 @@ fn learner_catchup_then_promotion_lifecycle() { success: true, last_log_index: idx, round: 0, - needs_snapshot: false, + replica: nodedb_raft::ReplicaState::Whole, }; node.handle_append_entries_response(3, &learner_ack); assert_eq!(node.match_index_for(3), Some(idx)); diff --git a/nodedb-test-support/Cargo.toml b/nodedb-test-support/Cargo.toml index 385462da9..ba22e37a3 100644 --- a/nodedb-test-support/Cargo.toml +++ b/nodedb-test-support/Cargo.toml @@ -42,6 +42,7 @@ futures = { workspace = true } smallvec = { workspace = true } uuid = { workspace = true } tracing-subscriber = { workspace = true } +thiserror = { workspace = true } # JWT fixture: mint RS256 tokens against an in-process JWKS endpoint. rsa = { workspace = true } diff --git a/nodedb-test-support/src/cluster_harness/cluster/bringup.rs b/nodedb-test-support/src/cluster_harness/cluster/bringup.rs index d1bea0ea7..2d5c94dc5 100644 --- a/nodedb-test-support/src/cluster_harness/cluster/bringup.rs +++ b/nodedb-test-support/src/cluster_harness/cluster/bringup.rs @@ -36,6 +36,7 @@ impl TestCluster { backup_storage: None, pitr: None, node_timeseries_tuning: std::collections::HashMap::new(), + redo_chunk_limits: None, }; Self::spawn_three_with_config(config).await } @@ -60,6 +61,7 @@ impl TestCluster { backup_storage: None, pitr: None, node_timeseries_tuning: std::collections::HashMap::new(), + redo_chunk_limits: None, }) .await } @@ -83,6 +85,7 @@ impl TestCluster { backup_storage: None, pitr: None, node_timeseries_tuning: std::collections::HashMap::new(), + redo_chunk_limits: None, }) .await } @@ -106,6 +109,7 @@ impl TestCluster { backup_storage: None, pitr: None, node_timeseries_tuning: std::collections::HashMap::from([(node_id, timeseries_tuning)]), + redo_chunk_limits: None, }) .await } @@ -131,6 +135,7 @@ impl TestCluster { }), pitr: None, node_timeseries_tuning: std::collections::HashMap::new(), + redo_chunk_limits: None, }) .await } @@ -153,6 +158,37 @@ impl TestCluster { backup_storage: None, pitr: Some(pitr), node_timeseries_tuning: std::collections::HashMap::new(), + redo_chunk_limits: None, + }) + .await + } + + /// Spawn a 3-node cluster whose nodes cap a redo entry at + /// `max_entry_bytes`, so a commit larger than it travels as a chunked + /// redo stream. Uses the standard fast-election tuning and 1 core per + /// node. + pub async fn spawn_three_with_redo_entry_limit( + max_entry_bytes: usize, + ) -> Result> { + let calvin = nodedb_types::config::tuning::CalvinTuning::default(); + Self::spawn_three_with_config(ClusterSpawnConfig { + tuning: super::types::fast_cluster_tuning(), + graph_tuning: nodedb_types::config::tuning::GraphTuning::default(), + query_tuning: nodedb_types::config::tuning::QueryTuning::default(), + num_cores: 1, + log_compaction_threshold: None, + replication_factor: 3, + num_groups: DEFAULT_NUM_GROUPS, + single_node_calvin: false, + backup_storage: None, + pitr: None, + node_timeseries_tuning: std::collections::HashMap::new(), + redo_chunk_limits: Some( + nodedb::control::wal_replication::transaction_redo::chunks::RedoChunkLimits { + max_entry_bytes, + max_open_bytes: calvin.max_open_redo_bytes, + }, + ), }) .await } diff --git a/nodedb-test-support/src/cluster_harness/cluster/spawn_variants.rs b/nodedb-test-support/src/cluster_harness/cluster/spawn_variants.rs index a1c57240d..b9f2e5855 100644 --- a/nodedb-test-support/src/cluster_harness/cluster/spawn_variants.rs +++ b/nodedb-test-support/src/cluster_harness/cluster/spawn_variants.rs @@ -69,6 +69,19 @@ impl TestCluster { .await } + /// [`Self::spawn_three`] with `rpc_timeout_secs` as every node's fixed + /// RPC timeout. A test sets it to size the time a read reserves for its + /// retries. + pub async fn spawn_three_with_rpc_timeout( + rpc_timeout_secs: u64, + ) -> Result> { + Self::spawn_three_with_tuning(ClusterTransportTuning { + rpc_timeout_secs, + ..fast_cluster_tuning() + }) + .await + } + /// Spawn a 3-node cluster with a custom `ClusterTransportTuning`. /// Used by the descriptor-lease renewal tests to drive the /// renewal loop on a much faster cadence than the production diff --git a/nodedb-test-support/src/cluster_harness/cluster/types.rs b/nodedb-test-support/src/cluster_harness/cluster/types.rs index 23983f3cd..e27a2b87b 100644 --- a/nodedb-test-support/src/cluster_harness/cluster/types.rs +++ b/nodedb-test-support/src/cluster_harness/cluster/types.rs @@ -49,6 +49,10 @@ pub(crate) struct ClusterSpawnConfig { /// runs `TimeseriesToning::default()`. pub(crate) node_timeseries_tuning: std::collections::HashMap, + /// Chunked-redo bounds every node runs. `None` keeps the default + /// `[tuning.calvin]` bounds. + pub(crate) redo_chunk_limits: + Option, } impl ClusterSpawnConfig { diff --git a/nodedb-test-support/src/cluster_harness/node/lifecycle/spawn_full.rs b/nodedb-test-support/src/cluster_harness/node/lifecycle/spawn_full.rs index 004496689..36954469c 100644 --- a/nodedb-test-support/src/cluster_harness/node/lifecycle/spawn_full.rs +++ b/nodedb-test-support/src/cluster_harness/node/lifecycle/spawn_full.rs @@ -200,6 +200,16 @@ impl TestClusterNode { // cloned. Before `EventPlane::spawn` below: the Event Plane starts the // cross-shard drain only when its sender is wired. if let Some(state) = Arc::get_mut(&mut shared) { + // No stream is open yet: the store with the test's bounds + // replaces the default one before any entry applies. + if let Some(limits) = config.redo_chunk_limits { + state.redo_chunks = Arc::new( + nodedb::control::wal_replication::transaction_redo::RedoChunkStore::new( + Arc::clone(&wal), + limits, + ), + ); + } // Consumer offsets, job history, MV state and the array-sync // stores live in the data directory in the production layout. A // restart reopens them there, since no metadata entry below the diff --git a/nodedb-test-support/src/cluster_harness/node/lifecycle/spawn_variants.rs b/nodedb-test-support/src/cluster_harness/node/lifecycle/spawn_variants.rs index 1f46d3508..3f816eae0 100644 --- a/nodedb-test-support/src/cluster_harness/node/lifecycle/spawn_variants.rs +++ b/nodedb-test-support/src/cluster_harness/node/lifecycle/spawn_variants.rs @@ -108,6 +108,7 @@ impl TestClusterNode { backup_storage: None, pitr: None, node_timeseries_tuning: std::collections::HashMap::new(), + redo_chunk_limits: None, }; Self::spawn_with_full_config(node_id, seed_nodes, &config).await } @@ -166,6 +167,7 @@ impl TestClusterNode { backup_storage: None, pitr: None, node_timeseries_tuning: std::collections::HashMap::new(), + redo_chunk_limits: None, }; Self::spawn_with_full_config_at(1, vec![], &config, data_dir_path, None).await } diff --git a/nodedb-test-support/src/cluster_harness/node/mod.rs b/nodedb-test-support/src/cluster_harness/node/mod.rs index 19ac65c78..5768ebde8 100644 --- a/nodedb-test-support/src/cluster_harness/node/mod.rs +++ b/nodedb-test-support/src/cluster_harness/node/mod.rs @@ -9,5 +9,6 @@ pub mod leadership; pub mod lifecycle; pub mod native_client; pub mod raw_pgwire; +pub mod remount; pub use lifecycle::TestClusterNode; diff --git a/nodedb-test-support/src/cluster_harness/node/remount.rs b/nodedb-test-support/src/cluster_harness/node/remount.rs new file mode 100644 index 000000000..51bab42e6 --- /dev/null +++ b/nodedb-test-support/src/cluster_harness/node/remount.rs @@ -0,0 +1,110 @@ +// SPDX-License-Identifier: BUSL-1.1 + +//! Unmount one data group on a [`TestClusterNode`] and mount it again from +//! its log on disk, as a node that left a group and rejoined it does. + +use std::sync::Arc; + +use nodedb::control::state::SharedState; +use nodedb_cluster::ClusterError; + +use crate::cluster_harness::node::lifecycle::TestClusterNode; + +/// Why a remount of a data group failed. Every case names the node and the +/// group. +#[derive(Debug, thiserror::Error)] +pub enum RemountError { + #[error("node {node_id}: the remount task of group {group_id} did not finish: {source}")] + Task { + node_id: u64, + group_id: u64, + source: tokio::task::JoinError, + }, + #[error("node {node_id}: its Raft groups are not started, so group {group_id} has none")] + NotStarted { node_id: u64, group_id: u64 }, + #[error("node {node_id}: group {group_id} is not mounted")] + NotMounted { node_id: u64, group_id: u64 }, + #[error("node {node_id}: dropping the redo streams of group {group_id}: {source}")] + DropStreams { + node_id: u64, + group_id: u64, + source: nodedb::Error, + }, + #[error("node {node_id}: reopening group {group_id}: {source}")] + Reopen { + node_id: u64, + group_id: u64, + source: ClusterError, + }, + #[error("node {node_id}: group {group_id} mounted again while its lock was held")] + MountedTwice { node_id: u64, group_id: u64 }, +} + +impl TestClusterNode { + /// Unmount `group_id` here, drop its redo streams as the membership pass + /// drops the streams of a group this node no longer mounts, and mount the + /// group again from its log on disk with the same voters and learners. + /// + /// The node's `MultiRaft` lock is held throughout, so no tick mounts the + /// group in between. + pub async fn remount_data_group(&self, group_id: u64) -> Result<(), RemountError> { + let shared = Arc::clone(&self.shared); + let node_id = self.node_id; + tokio::task::spawn_blocking(move || remount(&shared, node_id, group_id)) + .await + .map_err(|source| RemountError::Task { + node_id, + group_id, + source, + })? + } +} + +/// Run the remount on a blocking thread: closing and opening a group's log +/// blocks on disk. +fn remount(shared: &SharedState, node_id: u64, group_id: u64) -> Result<(), RemountError> { + let multi_raft = shared + .multi_raft + .get() + .ok_or(RemountError::NotStarted { node_id, group_id })?; + let mut mr = multi_raft.lock().unwrap_or_else(|p| p.into_inner()); + let membership = mr + .group_membership(group_id) + .ok_or(RemountError::NotMounted { node_id, group_id })?; + let starts_as_learner = membership.learners.contains(&node_id); + let peers: Vec = membership + .voters + .into_iter() + .filter(|&id| id != node_id) + .collect(); + let learners: Vec = membership + .learners + .into_iter() + .filter(|&id| id != node_id) + .collect(); + // Dropping the replica closes its log file, which the mount reopens. + drop(mr.unmount_group(group_id)); + shared + .redo_chunks + .drop_unhosted_groups( + |group| mr.contains_group(group), + shared.credentials.catalog(), + ) + .map_err(|source| RemountError::DropStreams { + node_id, + group_id, + source, + })?; + let opened = mr + .mount_spec(group_id, peers, learners, starts_as_learner) + .open() + .map_err(|source| RemountError::Reopen { + node_id, + group_id, + source, + })?; + if mr.insert_opened(opened).is_some() { + return Err(RemountError::MountedTwice { node_id, group_id }); + } + Ok(()) +} diff --git a/nodedb/src/bootstrap/cluster_ready.rs b/nodedb/src/bootstrap/cluster_ready.rs index f6b24d6e0..b3627e86b 100644 --- a/nodedb/src/bootstrap/cluster_ready.rs +++ b/nodedb/src/bootstrap/cluster_ready.rs @@ -44,8 +44,8 @@ pub async fn await_cluster_ready( // Boot-time readiness gate: wait until the metadata raft group has // applied its first entry on this node before opening any // client-facing listener. This eliminates the restart-window race - // where the first DDL observes `metadata propose: not leader` - // because the election has not yet completed. + // where the first DDL finds no metadata leader because the election + // has not yet completed. wait_for_raft_ready( shared, &mut raft_ready_rx, diff --git a/nodedb/src/bootstrap/constraint_reconcile.rs b/nodedb/src/bootstrap/constraint_reconcile.rs index 90e2b28b1..b1f3e671e 100644 --- a/nodedb/src/bootstrap/constraint_reconcile.rs +++ b/nodedb/src/bootstrap/constraint_reconcile.rs @@ -169,7 +169,9 @@ pub async fn reconcile_once( }, ); - let deadline = crate::control::wal_replication::statement_propose_deadline(shared); + let deadline = + crate::control::server::shared::session::StatementDeadline::of_statement(shared) + .tokio_instant(); match propose_replicated_entry(shared, &proposer, entry, deadline).await { Ok(_) => { // Record only on commit. A transient / NotLeader error diff --git a/nodedb/src/bootstrap/data_group_recovery.rs b/nodedb/src/bootstrap/data_group_recovery.rs index 950a0e87e..7c86cabae 100644 --- a/nodedb/src/bootstrap/data_group_recovery.rs +++ b/nodedb/src/bootstrap/data_group_recovery.rs @@ -7,23 +7,28 @@ //! (KV hash index, columnar/timeseries/spatial in-memory state, graph node //! labels, …) is rebuilt only by re-applying that log into the Data Plane. //! -//! Raft's `commit_index` / `last_applied` are volatile: after a restart every -//! locally hosted group must win its own randomized election before it commits -//! a no-op and carries its retained log forward to the applier. Those elections -//! run independently of the metadata group's, so waiting only on metadata +//! Raft's `commit_index` is volatile: after a restart every locally hosted +//! group must win its own randomized election before it commits a no-op and +//! carries its retained log forward to the applier. Those elections run +//! independently of the metadata group's, so waiting only on metadata //! readiness lets the gateway open while a data group has replayed nothing — an //! acknowledged write then reads back as if it never happened, with no error to //! explain it. //! //! This gate closes that window: startup is held until every locally hosted //! data group has elected a leader, committed through the end of its retained -//! log, and applied that log. It fails closed — a timeout aborts startup rather -//! than opening a gateway that would serve incomplete state. +//! log, and applied that log on this node. It fails closed — a timeout aborts +//! startup rather than opening a gateway that would serve incomplete state. +//! +//! A group whose replica here awaits a snapshot install does not hold startup. +//! Its data diverged from the log, so it applies nothing until a snapshot +//! installs, and its reads are refused until then. use std::sync::Arc; use std::time::{Duration, Instant}; -use tracing::{debug, info}; +use nodedb_raft::ReplicaState; +use tracing::{debug, info, warn}; use crate::control::state::SharedState; @@ -47,7 +52,7 @@ fn is_data_group(group_id: u64) -> bool { && group_id != nodedb_cluster::calvin::SEQUENCER_GROUP_ID } -/// Why a data group is not yet recovered, or `None` when it is. +/// Why a data group is not yet recovered. #[derive(Debug, Clone, Copy, PartialEq, Eq)] enum GroupWait { /// No leader yet — the post-restart election is still running, so the group @@ -59,21 +64,46 @@ enum GroupWait { commit_index: u64, last_log_index: u64, }, - /// The committed log is known, but has not been fully applied yet. + /// The committed log is known, but this node has not applied all of it. ReplayLagging { commit_index: u64, - last_applied: u64, + host_applied: u64, }, } -/// Recovery predicate for one group, from a single `GroupStatus` snapshot. +/// Where one data group stands against the gate. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +enum GroupRecovery { + /// Applied on this node through its committed log. + Replayed, + /// Its replica here awaits a snapshot install. It applies nothing until + /// one installs and serves no reads meanwhile, so it does not hold + /// startup. + AwaitingSnapshot, + /// Still holding startup. + Pending(GroupWait), +} + +/// One group's progress, from its Raft status and this node's watermark. +#[derive(Debug, Clone, Copy)] +struct GroupProgress { + leader_id: u64, + commit_index: u64, + last_log_index: u64, + /// The highest index whose apply concluded on this node: the watermark + /// reads wait on. + host_applied: u64, + replica_state: ReplicaState, +} + +/// Recovery verdict for one group. /// -/// `last_applied` is Raft's own applied watermark, and is the only usable -/// progress signal here. The Data-Plane-side apply watermark cannot be the -/// target: a leader's election no-op is committed but is never delivered to the -/// distributed applier, so a Data-Plane watermark stops one short of -/// `commit_index` and never converges — and every post-restart log ends in -/// exactly such a no-op. +/// The progress signal is the host watermark, not Raft's `last_applied`. Raft +/// advances `last_applied` when it hands an entry to the apply loop, before +/// the entry's apply concludes. Reads wait on the host watermark, so a gate on +/// `last_applied` opens while replay still runs. The election no-op reaches +/// the apply loop and concludes like any entry, so the host watermark reaches +/// `commit_index`. /// /// `commit_index` alone is not a sufficient target either. It is volatile and /// starts at 0 after a restart, and Raft publishes `leader_id` before the new @@ -87,32 +117,32 @@ enum GroupWait { /// Once `commit_index >= last_log_index` the target is static while the gateway /// is closed — no new entries can be proposed — so this converges rather than /// chasing a moving head. -fn group_wait( - leader_id: u64, - commit_index: u64, - last_log_index: u64, - last_applied: u64, -) -> Option { - if leader_id == 0 { - return Some(GroupWait::NoLeader); +fn group_recovery(progress: GroupProgress) -> GroupRecovery { + // Checked first: such a replica never leads, so a group it holds alone + // never elects. + if progress.replica_state == ReplicaState::AwaitingSnapshot { + return GroupRecovery::AwaitingSnapshot; + } + if progress.leader_id == 0 { + return GroupRecovery::Pending(GroupWait::NoLeader); } // Empty log: nothing was ever written to this group, so there is nothing to // replay once it has a leader. - if last_log_index == 0 { - return None; + if progress.last_log_index == 0 { + return GroupRecovery::Replayed; } - if commit_index < last_log_index { - return Some(GroupWait::LogUncommitted { - commit_index, - last_log_index, + if progress.commit_index < progress.last_log_index { + return GroupRecovery::Pending(GroupWait::LogUncommitted { + commit_index: progress.commit_index, + last_log_index: progress.last_log_index, }); } - if last_applied >= commit_index { - return None; + if progress.host_applied >= progress.commit_index { + return GroupRecovery::Replayed; } - Some(GroupWait::ReplayLagging { - commit_index, - last_applied, + GroupRecovery::Pending(GroupWait::ReplayLagging { + commit_index: progress.commit_index, + host_applied: progress.host_applied, }) } @@ -137,38 +167,52 @@ impl std::fmt::Display for PendingGroup { ), GroupWait::ReplayLagging { commit_index, - last_applied, + host_applied, } => write!( f, - "group {} applied {last_applied} of {commit_index} committed entries", + "group {} applied {host_applied} of {commit_index} committed entries on this node", self.group_id ), } } } -/// Collect the data groups that are not yet recovered. -fn pending_groups(statuses: Vec) -> Vec { - statuses - .into_iter() - .filter(|s| is_data_group(s.group_id)) - .filter_map(|s| { - group_wait( - s.leader_id, - s.commit_index, - s.last_log_index, - s.last_applied, - ) - .map(|wait| PendingGroup { - group_id: s.group_id, +/// The data groups still holding startup, and the groups awaiting a snapshot. +#[derive(Debug, Default)] +struct GateView { + pending: Vec, + awaiting_snapshot: Vec, +} + +/// Sort every data group in `statuses` against the gate. `host_applied` +/// reads this node's watermark of a group. +fn gate_view( + statuses: Vec, + host_applied: impl Fn(u64) -> u64, +) -> GateView { + let mut view = GateView::default(); + for status in statuses.into_iter().filter(|s| is_data_group(s.group_id)) { + let progress = GroupProgress { + leader_id: status.leader_id, + commit_index: status.commit_index, + last_log_index: status.last_log_index, + host_applied: host_applied(status.group_id), + replica_state: status.replica_state, + }; + match group_recovery(progress) { + GroupRecovery::Replayed => {} + GroupRecovery::AwaitingSnapshot => view.awaiting_snapshot.push(status.group_id), + GroupRecovery::Pending(wait) => view.pending.push(PendingGroup { + group_id: status.group_id, wait, - }) - }) - .collect() + }), + } + } + view } /// Hold startup until every locally hosted data group has replayed its retained -/// Raft log. +/// Raft log on this node. A group awaiting a snapshot install does not hold it. /// /// Refuses when `start_raft` installed no Raft status source. pub async fn await_data_group_recovery(shared: &Arc) -> anyhow::Result<()> { @@ -179,14 +223,24 @@ pub async fn await_data_group_recovery(shared: &Arc) -> anyhow::Res let deadline = Instant::now() + DATA_GROUP_RECOVERY_TIMEOUT; loop { - let pending = pending_groups(status_fn()); - if pending.is_empty() { + let view = gate_view(status_fn(), |group_id| { + shared.applied_index_watcher(group_id).current() + }); + if view.pending.is_empty() { + for group_id in &view.awaiting_snapshot { + warn!( + group_id, + "data raft group awaits a snapshot install on this node; it serves no reads \ + until one installs" + ); + } info!("all local data raft groups replayed — opening client gateway"); return Ok(()); } if Instant::now() >= deadline { - let detail = pending + let detail = view + .pending .iter() .map(|p| p.to_string()) .collect::>() @@ -197,7 +251,7 @@ pub async fn await_data_group_recovery(shared: &Arc) -> anyhow::Res } debug!( - pending = pending.len(), + pending = view.pending.len(), "waiting for data raft group replay" ); tokio::time::sleep(POLL_INTERVAL).await; @@ -208,6 +262,42 @@ pub async fn await_data_group_recovery(shared: &Arc) -> anyhow::Res mod tests { use super::*; + fn progress( + leader_id: u64, + commit_index: u64, + last_log_index: u64, + host_applied: u64, + ) -> GroupProgress { + GroupProgress { + leader_id, + commit_index, + last_log_index, + host_applied, + replica_state: ReplicaState::Whole, + } + } + + fn status( + group_id: u64, + raft_applied: u64, + replica_state: ReplicaState, + ) -> nodedb_cluster::GroupStatus { + nodedb_cluster::GroupStatus { + group_id, + role: "Leader".into(), + leader_id: 1, + term: 2, + commit_index: 12, + last_applied: raft_applied, + last_log_index: 12, + snapshot_index: 0, + replica_state, + member_count: 1, + learner_count: 0, + vshard_count: 1, + } + } + #[test] fn metadata_and_sequencer_groups_are_not_data_groups() { assert!(!is_data_group(nodedb_cluster::METADATA_GROUP_ID)); @@ -222,13 +312,22 @@ mod tests { #[test] fn leaderless_group_waits_for_election() { - assert_eq!(group_wait(0, 0, 0, 0), Some(GroupWait::NoLeader)); - assert_eq!(group_wait(0, 9, 9, 9), Some(GroupWait::NoLeader)); + assert_eq!( + group_recovery(progress(0, 0, 0, 0)), + GroupRecovery::Pending(GroupWait::NoLeader) + ); + assert_eq!( + group_recovery(progress(0, 9, 9, 9)), + GroupRecovery::Pending(GroupWait::NoLeader) + ); } #[test] fn empty_log_is_recovered_once_leader_exists() { - assert_eq!(group_wait(1, 0, 0, 0), None); + assert_eq!( + group_recovery(progress(1, 0, 0, 0)), + GroupRecovery::Replayed + ); } /// The window this gate exists to close: a restart resets the volatile @@ -238,8 +337,8 @@ mod tests { #[test] fn fresh_leader_with_uncommitted_retained_log_waits() { assert_eq!( - group_wait(1, 0, 12, 0), - Some(GroupWait::LogUncommitted { + group_recovery(progress(1, 0, 12, 0)), + GroupRecovery::Pending(GroupWait::LogUncommitted { commit_index: 0, last_log_index: 12, }) @@ -249,8 +348,8 @@ mod tests { #[test] fn partially_committed_retained_log_waits() { assert_eq!( - group_wait(1, 5, 12, 5), - Some(GroupWait::LogUncommitted { + group_recovery(progress(1, 5, 12, 5)), + GroupRecovery::Pending(GroupWait::LogUncommitted { commit_index: 5, last_log_index: 12, }) @@ -260,26 +359,84 @@ mod tests { #[test] fn committed_but_unapplied_log_waits() { assert_eq!( - group_wait(1, 12, 12, 4), - Some(GroupWait::ReplayLagging { + group_recovery(progress(1, 12, 12, 4)), + GroupRecovery::Pending(GroupWait::ReplayLagging { commit_index: 12, - last_applied: 4, + host_applied: 4, }) ); } - /// A freshly elected leader whose only committed entry is its own election - /// no-op is recovered once Raft has applied it. The no-op never reaches the - /// Data Plane, which is why Raft's `last_applied` — not a Data-Plane - /// watermark — is the progress signal here. + /// Raft hands every entry to the apply loop, so its `last_applied` reaches + /// `commit_index` while replay still runs. The host watermark below it + /// holds the gate. #[test] - fn election_noop_only_log_converges() { - assert_eq!(group_wait(1, 1, 1, 1), None); + fn a_handed_off_log_waits_for_the_host_watermark() { + let view = gate_view(vec![status(4, 12, ReplicaState::Whole)], |_| 7); + assert!(view.awaiting_snapshot.is_empty()); + assert_eq!(view.pending.len(), 1); + assert_eq!( + view.pending[0].wait, + GroupWait::ReplayLagging { + commit_index: 12, + host_applied: 7, + } + ); } + /// The host watermark at `commit_index` opens the gate, the election no-op + /// included: it concludes on the apply loop like any entry. #[test] - fn caught_up_group_is_recovered() { - assert_eq!(group_wait(1, 12, 12, 12), None); - assert_eq!(group_wait(1, 12, 12, 13), None); + fn a_host_watermark_at_the_commit_index_is_recovered() { + let view = gate_view(vec![status(4, 12, ReplicaState::Whole)], |_| 12); + assert!(view.pending.is_empty()); + assert!(view.awaiting_snapshot.is_empty()); + assert_eq!( + group_recovery(progress(1, 1, 1, 1)), + GroupRecovery::Replayed + ); + assert_eq!( + group_recovery(progress(1, 12, 12, 13)), + GroupRecovery::Replayed + ); + } + + /// A replica that awaits a snapshot install applies nothing until one + /// installs. It does not hold startup, even with no leader, and is + /// reported apart. + #[test] + fn a_group_awaiting_a_snapshot_does_not_hold_startup() { + let awaiting = GroupProgress { + replica_state: ReplicaState::AwaitingSnapshot, + ..progress(0, 0, 12, 3) + }; + assert_eq!(group_recovery(awaiting), GroupRecovery::AwaitingSnapshot); + + let view = gate_view( + vec![ + status(4, 12, ReplicaState::AwaitingSnapshot), + status(5, 12, ReplicaState::Whole), + ], + |group_id| if group_id == 4 { 3 } else { 12 }, + ); + assert!(view.pending.is_empty()); + assert_eq!(view.awaiting_snapshot, vec![4]); + } + + /// A replica that only wants a snapshot keeps its data whole and can + /// lead, so it holds startup like any other group until it replays. + #[test] + fn a_group_wanting_a_snapshot_still_replays() { + let wanting = GroupProgress { + replica_state: ReplicaState::WantsSnapshot, + ..progress(1, 12, 12, 4) + }; + assert_eq!( + group_recovery(wanting), + GroupRecovery::Pending(GroupWait::ReplayLagging { + commit_index: 12, + host_applied: 4, + }) + ); } } diff --git a/nodedb/src/bridge/envelope/error_code_from.rs b/nodedb/src/bridge/envelope/error_code_from.rs index c94d78be3..c1862b0fa 100644 --- a/nodedb/src/bridge/envelope/error_code_from.rs +++ b/nodedb/src/bridge/envelope/error_code_from.rs @@ -298,6 +298,8 @@ impl From for ErrorCode { | crate::Error::CommittedResultUnavailable { .. } | crate::Error::ProposalOutcomeUnknown { .. } | crate::Error::MetadataLeaderUnavailable + | crate::Error::MetadataGroupGone { .. } + | crate::Error::MetadataDdlSuperseded { .. } | crate::Error::Wal(_) | crate::Error::Dispatch { .. } | crate::Error::Storage { .. } diff --git a/nodedb/src/control/backup/restore/crdt_reissue.rs b/nodedb/src/control/backup/restore/crdt_reissue.rs index 9016067ab..9caf721b9 100644 --- a/nodedb/src/control/backup/restore/crdt_reissue.rs +++ b/nodedb/src/control/backup/restore/crdt_reissue.rs @@ -51,7 +51,8 @@ async fn reissue_crdt_collection( })? .with_event_source(EventSource::Restore) .with_restore_id(target.restore_id); - let deadline = crate::control::wal_replication::statement_propose_deadline(state); + let deadline = crate::control::server::shared::session::StatementDeadline::of_statement(state) + .tokio_instant(); crate::control::wal_replication::propose_replicated_entry(state, proposer, entry, deadline) .await?; Ok(()) diff --git a/nodedb/src/control/backup/restore/durable.rs b/nodedb/src/control/backup/restore/durable.rs index 3fc14e7d1..2c7d19715 100644 --- a/nodedb/src/control/backup/restore/durable.rs +++ b/nodedb/src/control/backup/restore/durable.rs @@ -67,7 +67,8 @@ pub(crate) async fn reissue_plan_durably( // fire again for it. .with_event_source(crate::event::EventSource::Restore) .with_restore_id(target.restore_id); - let deadline = crate::control::wal_replication::statement_propose_deadline(state); + let deadline = crate::control::server::shared::session::StatementDeadline::of_statement(state) + .tokio_instant(); let (_, write_versions) = crate::control::wal_replication::propose_replicated_entry(state, proposer, entry, deadline) .await?; diff --git a/nodedb/src/control/backup/schedule/marks.rs b/nodedb/src/control/backup/schedule/marks.rs index bd98f7b9a..5217b88b3 100644 --- a/nodedb/src/control/backup/schedule/marks.rs +++ b/nodedb/src/control/backup/schedule/marks.rs @@ -12,12 +12,12 @@ //! lags therefore never mistakes a finished minute for a due one. use std::sync::Arc; -use std::time::{Duration, Instant}; +use std::time::Duration; use super::blocking::off_runtime; use crate::config::server::BackupScheduleSettings; use crate::control::catalog_entry::CatalogEntry; -use crate::control::cluster::linearizable_read::confirm_linearizable_read; +use crate::control::cluster::linearizable_read::{StatementDeadline, confirm_linearizable_read}; use crate::control::metadata_proposer::propose_catalog_entry_async; use crate::control::security::catalog::backup_schedule_marks::StoredBackupScheduleMark; use crate::control::state::SharedState; @@ -52,7 +52,7 @@ pub async fn settled_through_linearizable( confirm_linearizable_read( state, &[nodedb_cluster::METADATA_GROUP_ID], - Instant::now() + MARK_READ_TIMEOUT, + StatementDeadline::after(MARK_READ_TIMEOUT), ) .await?; let (state, schedule) = (Arc::clone(state), schedule.clone()); diff --git a/nodedb/src/control/catalog_entry/post_apply/async_dispatch/collection.rs b/nodedb/src/control/catalog_entry/post_apply/async_dispatch/collection.rs index e80b09e54..7a8b0fdb1 100644 --- a/nodedb/src/control/catalog_entry/post_apply/async_dispatch/collection.rs +++ b/nodedb/src/control/catalog_entry/post_apply/async_dispatch/collection.rs @@ -186,7 +186,10 @@ pub(crate) async fn reclaim_collection_storage( // performance, requirement. let hold = (!drain_already_held).then(|| shared.quiesce.begin_drain(database_id, tenant_id, name)); - wait_drained_reporting_progress(shared, database_id, tenant_id, name).await; + shared + .quiesce + .wait_until_drained(database_id, tenant_id, name) + .await; // 4. Reclaim on local Data Plane. RESULT-CHECKED: the redb + // versioned engine purge is correctness-critical (the catalog @@ -282,41 +285,6 @@ pub(crate) async fn reclaim_collection_storage( } } -/// How often a quiesce drain checks for closed scans. -const DRAIN_PROGRESS_TICK: std::time::Duration = std::time::Duration::from_millis(100); - -/// Wait until every open scan of the collection closes. Each closed scan -/// counts as apply progress, so a proposer waiting on this purge keeps -/// waiting while scans close and times out only when none does. -/// -/// A draining collection admits no new scan, so the open count only falls. -async fn wait_drained_reporting_progress( - shared: &SharedState, - database_id: u64, - tenant_id: u64, - name: &str, -) { - let drained = shared - .quiesce - .wait_until_drained(database_id, tenant_id, name); - tokio::pin!(drained); - let mut open = shared.quiesce.open_scans(database_id, tenant_id, name); - loop { - tokio::select! { - () = &mut drained => return, - () = tokio::time::sleep(DRAIN_PROGRESS_TICK) => { - let now = shared.quiesce.open_scans(database_id, tenant_id, name); - if now < open { - shared - .metadata_apply_progress - .fetch_add(1, std::sync::atomic::Ordering::Release); - } - open = now; - } - } - } -} - /// Clear this node's storage under `name` before a new incarnation of the /// collection registers there. /// diff --git a/nodedb/src/control/cluster/bootstrap_listener.rs b/nodedb/src/control/cluster/bootstrap_listener.rs index 140b9f82d..0b526962d 100644 --- a/nodedb/src/control/cluster/bootstrap_listener.rs +++ b/nodedb/src/control/cluster/bootstrap_listener.rs @@ -61,14 +61,14 @@ impl nodedb_cluster::decommission::MetadataProposer for BootstrapMetadataPropose let index = raft_loop .propose_stamped_to_metadata_group_via_leader(bytes) .await?; - let watchers = Arc::clone(&self.watchers); - let outcome = tokio::task::spawn_blocking(move || { - watchers.wait_for(METADATA_GROUP_ID, index, Duration::from_secs(10)) - }) - .await - .map_err(|error| nodedb_cluster::ClusterError::Transport { - detail: format!("bootstrap metadata apply wait failed: {error}"), - })?; + let outcome = self + .watchers + .wait_for_async( + METADATA_GROUP_ID, + index, + std::time::Instant::now() + Duration::from_secs(10), + ) + .await; match outcome { WaitOutcome::Reached => Ok(index), WaitOutcome::TimedOut => Err(nodedb_cluster::ClusterError::Transport { diff --git a/nodedb/src/control/cluster/calvin/scheduler/driver/core/install_gate.rs b/nodedb/src/control/cluster/calvin/scheduler/driver/core/install_gate.rs index f195d9abe..053614780 100644 --- a/nodedb/src/control/cluster/calvin/scheduler/driver/core/install_gate.rs +++ b/nodedb/src/control/cluster/calvin/scheduler/driver/core/install_gate.rs @@ -44,7 +44,8 @@ impl InstallGate { impl Scheduler { /// Record that this replica's Calvin state of the vShard has a hole, and /// make its data-group replica refuse log entries until a snapshot - /// brings the state back. + /// brings the state back. The group's data is whole here, so the + /// replica keeps campaigning and leading. pub(in crate::control::cluster::calvin::scheduler::driver::core) fn lose_calvin_base(&self) { if let Err(error) = self .shared @@ -60,10 +61,11 @@ impl Scheduler { } match group_of_vshard(&self.shared, self.vshard_id) { Ok(group_id) => { - self.multi_raft - .lock() - .unwrap_or_else(|p| p.into_inner()) - .set_snapshot_required(group_id, true); + let mut multi_raft = self.multi_raft.lock().unwrap_or_else(|p| p.into_inner()); + crate::control::cluster::calvin_snapshot::require_calvin_snapshot( + &mut multi_raft, + group_id, + ); } Err(error) => tracing::error!( vshard_id = self.vshard_id, diff --git a/nodedb/src/control/cluster/calvin/scheduler/driver/helpers.rs b/nodedb/src/control/cluster/calvin/scheduler/driver/helpers.rs index 7eaaf45e7..d7569e1dd 100644 --- a/nodedb/src/control/cluster/calvin/scheduler/driver/helpers.rs +++ b/nodedb/src/control/cluster/calvin/scheduler/driver/helpers.rs @@ -64,17 +64,18 @@ pub(super) fn decode_lock_key(wire: &LockKeyWire) -> LockKey { /// `Exclusive`, and its collection key is `Intent`. /// - A write [`EngineKeySet::Collection`] and a write array are `Exclusive` /// on the collection key. -/// - A read row key is `Shared`, with no collection key. -/// - A read [`EngineKeySet::Collection`] is `Shared` on the collection key. +/// - A read row key is `Shared`, and its collection key is `IntentShared`. +/// - A read [`EngineKeySet::Collection`] and a read array are `Shared` on the +/// collection key. /// /// A key named twice takes the merged mode (see [`LockMode::merge`]). pub(crate) fn expand_rw_set(txn: &SequencedTxn) -> BTreeMap { let mut keys = BTreeMap::new(); for ks in &txn.tx_class.read_set.0 { - add_key_set(&mut keys, ks, LockMode::Shared); + add_key_set(&mut keys, ks, Access::Read); } for ks in &txn.tx_class.write_set.0 { - add_key_set(&mut keys, ks, LockMode::Exclusive); + add_key_set(&mut keys, ks, Access::Write); } keys } @@ -86,14 +87,39 @@ pub(crate) fn expand_rw_set(txn: &SequencedTxn) -> BTreeMap { pub(crate) fn expand_write_key_sets(sets: &[EngineKeySet]) -> BTreeMap { let mut keys = BTreeMap::new(); for ks in sets { - add_key_set(&mut keys, ks, LockMode::Exclusive); + add_key_set(&mut keys, ks, Access::Write); } keys } -/// Add the keys of `ks` to `keys`. `row_mode` is `Shared` for a read set and -/// `Exclusive` for a write set. -fn add_key_set(keys: &mut BTreeMap, ks: &EngineKeySet, row_mode: LockMode) { +/// Whether a key set comes from a read set or a write set. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +enum Access { + Read, + Write, +} + +impl Access { + /// The mode of a row key, and of a whole-collection key set. + fn row_mode(self) -> LockMode { + match self { + Access::Read => LockMode::Shared, + Access::Write => LockMode::Exclusive, + } + } + + /// The mode a row key announces on its collection key. + fn announce_mode(self) -> LockMode { + match self { + Access::Read => LockMode::IntentShared, + Access::Write => LockMode::Intent, + } + } +} + +/// Add the keys of `ks` to `keys` for `access`. +fn add_key_set(keys: &mut BTreeMap, ks: &EngineKeySet, access: Access) { + let row_mode = access.row_mode(); let coll: Arc = Arc::from(ks.collection()); let whole = LockKey::Collection { collection: Arc::clone(&coll), @@ -145,11 +171,10 @@ fn add_key_set(keys: &mut BTreeMap, ks: &EngineKeySet, row_mo return; } } - // A row writer announces itself on its collection, so a collection-wide - // writer or a predicate reader of that collection orders against it. - if row_mode == LockMode::Exclusive { - request(keys, whole, LockMode::Intent); - } + // A row key announces itself on its collection, so a collection-wide + // writer orders against every row reader and writer of the collection, + // and a predicate reader orders against every row writer. + request(keys, whole, access.announce_mode()); } /// Request `key` in `mode`, merged with any mode already requested for it. diff --git a/nodedb/src/control/cluster/calvin/scheduler/lock/lock_entry.rs b/nodedb/src/control/cluster/calvin/scheduler/lock/lock_entry.rs index 314ae76aa..c25ee65d0 100644 --- a/nodedb/src/control/cluster/calvin/scheduler/lock/lock_entry.rs +++ b/nodedb/src/control/cluster/calvin/scheduler/lock/lock_entry.rs @@ -17,16 +17,22 @@ use super::lock_key::TxnId; /// /// Compatibility: /// -/// | held \ requested | Intent | Shared | Exclusive | -/// |------------------|--------|--------|-----------| -/// | Intent | yes | no | no | -/// | Shared | no | yes | no | -/// | Exclusive | no | no | no | +/// | held \ requested | IntentShared | Intent | Shared | Exclusive | +/// |------------------|--------------|--------|--------|-----------| +/// | IntentShared | yes | yes | yes | no | +/// | Intent | yes | yes | no | no | +/// | Shared | yes | no | yes | no | +/// | Exclusive | no | no | no | no | #[derive(Debug, Clone, Copy, PartialEq, Eq)] pub enum LockMode { - /// A row writer's hold on its collection key. Compatible only with other - /// `Intent` holders, so row writers run together and exclude a predicate - /// reader or a collection-wide writer. + /// A row reader's hold on its collection key. Excludes only a + /// collection-wide writer, so a TRUNCATE orders against the row reads of + /// its collection while row readers, row writers and predicate readers + /// share the key. + IntentShared, + /// A row writer's hold on its collection key. Compatible with `Intent` + /// and `IntentShared` holders, so row writers run together and exclude a + /// predicate reader or a collection-wide writer. Intent, /// Compatible with other shared holders on the same key. Shared, @@ -39,25 +45,39 @@ impl LockMode { /// together. pub fn compatible(self, other: LockMode) -> bool { match (self, other) { - (LockMode::Intent, LockMode::Intent) | (LockMode::Shared, LockMode::Shared) => true, - (LockMode::Intent, LockMode::Shared | LockMode::Exclusive) - | (LockMode::Shared, LockMode::Intent | LockMode::Exclusive) - | (LockMode::Exclusive, LockMode::Intent | LockMode::Shared | LockMode::Exclusive) => { - false - } + ( + LockMode::IntentShared, + LockMode::IntentShared | LockMode::Intent | LockMode::Shared, + ) + | (LockMode::Intent | LockMode::Shared, LockMode::IntentShared) + | (LockMode::Intent, LockMode::Intent) + | (LockMode::Shared, LockMode::Shared) => true, + (LockMode::IntentShared | LockMode::Intent | LockMode::Shared, LockMode::Exclusive) + | ( + LockMode::Exclusive, + LockMode::IntentShared | LockMode::Intent | LockMode::Shared | LockMode::Exclusive, + ) + | (LockMode::Intent, LockMode::Shared) + | (LockMode::Shared, LockMode::Intent) => false, } } /// The one mode that covers both `self` and `other` for one transaction. /// - /// Equal modes merge to themselves. Any other pair merges to `Exclusive`: - /// `Shared` plus `Intent` means the transaction both reads the whole - /// collection and writes rows of it. + /// Equal modes merge to themselves. `IntentShared` merges to the other + /// mode, which covers it. Any other pair merges to `Exclusive`: `Shared` + /// plus `Intent` means the transaction both reads the whole collection + /// and writes rows of it. pub fn merge(self, other: LockMode) -> LockMode { if self == other { - self - } else { - LockMode::Exclusive + return self; + } + match (self, other) { + (LockMode::IntentShared, stronger) | (stronger, LockMode::IntentShared) => stronger, + ( + LockMode::Intent | LockMode::Shared | LockMode::Exclusive, + LockMode::Intent | LockMode::Shared | LockMode::Exclusive, + ) => LockMode::Exclusive, } } } @@ -80,27 +100,87 @@ pub enum AcquireOutcome { /// Per-key lock state. /// /// Invariants maintained by [`LockManager`](super::manager::LockManager): -/// - Every holder holds the key in `mode`. Two modes never mix, because no -/// two distinct modes are compatible. +/// - Every pair of holder modes is compatible. `IntentShared` holders mix +/// with `Intent` or with `Shared` holders, never with both. +/// - `mode` is the merge of every holder's mode: the strongest one held. /// - `Exclusive` entries have exactly one holder. /// - `waiters` is FIFO; each waiter carries the mode it requested so that /// promotion on release is mode-aware. pub(super) struct LockEntry { - /// The mode the current holders hold this lock under. + /// The strongest mode a current holder holds this lock under. A request + /// is compatible with every holder iff it is compatible with this mode. pub(super) mode: LockMode, - /// The transactions currently holding this lock. Exclusive: exactly one; - /// Shared or Intent: one or more. - pub(super) holders: SmallVec<[TxnId; 2]>, + /// The transactions currently holding this lock, each with its own mode. + /// Exclusive: exactly one; otherwise one or more. + pub(super) holders: SmallVec<[(TxnId, LockMode); 2]>, /// Transactions waiting for this lock, in FIFO order, each tagged with the /// mode it requested. pub(super) waiters: VecDeque<(TxnId, LockMode)>, } +impl LockEntry { + /// Whether `txn` holds this lock. + pub(super) fn holds(&self, txn: TxnId) -> bool { + self.holders.iter().any(|(holder, _)| *holder == txn) + } + + /// The mode `txn` holds this lock in, `None` when it holds none. + pub(super) fn mode_of(&self, txn: TxnId) -> Option { + self.holders + .iter() + .find(|(holder, _)| *holder == txn) + .map(|(_, mode)| *mode) + } + + /// The holders' transactions, in grant order. + pub(super) fn holder_ids(&self) -> Vec { + self.holders.iter().map(|(holder, _)| *holder).collect() + } + + /// Whether every holder other than `txn` is compatible with `mode`. + pub(super) fn others_compatible(&self, txn: TxnId, mode: LockMode) -> bool { + self.holders + .iter() + .filter(|(holder, _)| *holder != txn) + .all(|(_, held)| held.compatible(mode)) + } + + /// Add `txn` in `mode`, or raise the mode it already holds to cover + /// `mode`, then recompute the group mode. + pub(super) fn hold(&mut self, txn: TxnId, mode: LockMode) { + match self.holders.iter_mut().find(|(holder, _)| *holder == txn) { + Some((_, held)) => *held = held.merge(mode), + None => self.holders.push((txn, mode)), + } + self.regroup(); + } + + /// Drop `txn` from the holders, then recompute the group mode. + pub(super) fn drop_holder(&mut self, txn: TxnId) { + self.holders.retain(|(holder, _)| *holder != txn); + self.regroup(); + } + + /// Set `mode` to the merge of every holder's mode. An entry with no + /// holder keeps its last mode until a waiter's grant replaces it. + fn regroup(&mut self) { + let mut held = self.holders.iter().map(|(_, mode)| *mode); + if let Some(first) = held.next() { + self.mode = held.fold(first, LockMode::merge); + } + } +} + #[cfg(test)] mod tests { use super::*; - const ALL: [LockMode; 3] = [LockMode::Intent, LockMode::Shared, LockMode::Exclusive]; + const ALL: [LockMode; 4] = [ + LockMode::IntentShared, + LockMode::Intent, + LockMode::Shared, + LockMode::Exclusive, + ]; #[test] fn compatibility_matrix() { @@ -111,9 +191,42 @@ mod tests { for mode in ALL { assert!(!mode.compatible(LockMode::Exclusive)); assert!(!LockMode::Exclusive.compatible(mode)); + assert_eq!( + mode.compatible(LockMode::IntentShared), + mode != LockMode::Exclusive + ); + assert_eq!( + LockMode::IntentShared.compatible(mode), + mode != LockMode::Exclusive + ); + } + } + + #[test] + fn intent_shared_merges_to_the_mode_that_covers_it() { + for mode in ALL { + assert_eq!(LockMode::IntentShared.merge(mode), mode); + assert_eq!(mode.merge(LockMode::IntentShared), mode); } } + #[test] + fn the_group_mode_drops_when_the_strongest_holder_leaves() { + let (reader, writer) = (TxnId::new(1, 0), TxnId::new(1, 1)); + let mut entry = LockEntry { + mode: LockMode::IntentShared, + holders: SmallVec::from_slice(&[(reader, LockMode::IntentShared)]), + waiters: VecDeque::new(), + }; + entry.hold(writer, LockMode::Intent); + assert_eq!(entry.mode, LockMode::Intent); + assert!(!entry.mode.compatible(LockMode::Shared)); + entry.drop_holder(writer); + assert_eq!(entry.mode, LockMode::IntentShared); + assert!(entry.mode.compatible(LockMode::Shared)); + assert_eq!(entry.holder_ids(), vec![reader]); + } + #[test] fn merge_keeps_the_strongest_mode() { for mode in ALL { diff --git a/nodedb/src/control/cluster/calvin/scheduler/lock/manager/acquire.rs b/nodedb/src/control/cluster/calvin/scheduler/lock/manager/acquire.rs index 0d13629bc..10c384087 100644 --- a/nodedb/src/control/cluster/calvin/scheduler/lock/manager/acquire.rs +++ b/nodedb/src/control/cluster/calvin/scheduler/lock/manager/acquire.rs @@ -163,7 +163,7 @@ mod tests { ); let entry = lm.table.get(&coll("c")).unwrap(); assert_eq!(entry.mode, LockMode::Intent); - assert_eq!(entry.holders.as_slice(), &[t1, t2]); + assert_eq!(entry.holder_ids(), vec![t1, t2]); } #[test] @@ -195,6 +195,85 @@ mod tests { } } + /// A row read: its row `Shared`, its collection `IntentShared`. + fn row_read(name: &str) -> BTreeMap { + BTreeMap::from([ + (key(name), LockMode::Shared), + (coll(name), LockMode::IntentShared), + ]) + } + + #[test] + fn a_row_reader_orders_against_a_collection_writer() { + // The reader holds first: the truncate waits for it. + let mut lm = LockManager::new(); + let (reader, truncate) = (txn(1, 0), txn(1, 1)); + assert_eq!(lm.acquire(reader, row_read("c")), AcquireOutcome::Ready); + assert_eq!( + lm.acquire(truncate, one(coll("c"), LockMode::Exclusive)), + AcquireOutcome::Blocked, + "a truncate waits for the row reads of its collection" + ); + assert_eq!(lm.release(reader), vec![truncate]); + + // The truncate holds first: the reader waits for it. + let mut lm = LockManager::new(); + let (truncate, reader) = (txn(1, 0), txn(1, 1)); + assert_eq!( + lm.acquire(truncate, one(coll("c"), LockMode::Exclusive)), + AcquireOutcome::Ready + ); + assert_eq!( + lm.acquire(reader, row_read("c")), + AcquireOutcome::Blocked, + "a row read waits for the truncate sequenced before it" + ); + assert_eq!(lm.release(truncate), vec![reader]); + } + + #[test] + fn a_row_reader_shares_its_collection_with_row_writers_and_predicate_readers() { + let mut lm = LockManager::new(); + let (reader, writer, scan) = (txn(1, 0), txn(1, 1), txn(1, 2)); + assert_eq!(lm.acquire(reader, row_read("c")), AcquireOutcome::Ready); + assert_eq!( + lm.acquire( + writer, + BTreeMap::from([ + (coll("c"), LockMode::Intent), + (key("d"), LockMode::Exclusive) + ]) + ), + AcquireOutcome::Ready, + "a row writer joins a row reader" + ); + assert_eq!( + lm.acquire(scan, one(coll("c"), LockMode::Shared)), + AcquireOutcome::Blocked, + "a predicate reader waits for the row writer" + ); + // The writer leaves: the group drops to the reader's mode, which the + // predicate reader shares. + assert_eq!(lm.release(writer), vec![scan]); + let entry = lm.table.get(&coll("c")).unwrap(); + assert_eq!(entry.mode, LockMode::Shared); + assert_eq!(entry.holder_ids(), vec![reader, scan]); + } + + #[test] + fn a_row_read_and_a_row_write_of_one_collection_merge_to_intent() { + let mut lm = LockManager::new(); + let (t, other) = (txn(1, 0), txn(1, 1)); + let mut keys = row_read("c"); + keys.insert(coll("c"), LockMode::IntentShared.merge(LockMode::Intent)); + assert_eq!(keys.get(&coll("c")), Some(&LockMode::Intent)); + assert_eq!(lm.acquire(t, keys), AcquireOutcome::Ready); + assert_eq!( + lm.acquire(other, one(coll("c"), LockMode::Intent)), + AcquireOutcome::Ready + ); + } + #[test] fn a_compatible_request_queues_behind_an_earlier_waiter() { let mut lm = LockManager::new(); @@ -325,7 +404,7 @@ mod tests { let entry = lm.table.get(&key("k")).unwrap(); assert_eq!(entry.mode, LockMode::Exclusive); - assert_eq!(entry.holders.as_slice(), &[t]); + assert_eq!(entry.holder_ids(), vec![t]); } #[test] @@ -352,7 +431,7 @@ mod tests { for k in ["k1", "k2"] { let entry = lm.table.get(&key(k)).unwrap(); assert_eq!(entry.mode, LockMode::Exclusive); - assert_eq!(entry.holders.as_slice(), &[t]); + assert_eq!(entry.holder_ids(), vec![t]); } } diff --git a/nodedb/src/control/cluster/calvin/scheduler/lock/manager/classify.rs b/nodedb/src/control/cluster/calvin/scheduler/lock/manager/classify.rs index 42492d644..77b406c5d 100644 --- a/nodedb/src/control/cluster/calvin/scheduler/lock/manager/classify.rs +++ b/nodedb/src/control/cluster/calvin/scheduler/lock/manager/classify.rs @@ -18,7 +18,8 @@ pub(super) enum KeyState { /// No entry: the requester can take the key. Free, /// The requester already holds the key in a mode that covers the request, - /// or holds it alone and can raise its mode in place. + /// or can raise its mode in place because every other holder is + /// compatible with the raised mode. Held, /// Other holders hold the key in a compatible mode and nobody waits: the /// requester can join them. @@ -34,8 +35,9 @@ impl LockManager { let Some(entry) = self.table.get(key) else { return KeyState::Free; }; - if entry.holders.contains(&txn) { - if entry.mode.merge(mode) == entry.mode || entry.held_solely_by(txn) { + if let Some(own) = entry.mode_of(txn) { + let raised = own.merge(mode); + if raised == own || entry.others_compatible(txn, raised) { return KeyState::Held; } return KeyState::Conflict; @@ -58,22 +60,14 @@ impl LockManager { key.clone(), LockEntry { mode, - holders: smallvec![txn], + holders: smallvec![(txn, mode)], waiters: VecDeque::new(), }, ); } - Some(entry) => { - if entry.holders.contains(&txn) { - // A sole holder raises its mode in place. A co-holder only - // reaches here when its mode already covers the request. - if entry.held_solely_by(txn) { - entry.mode = entry.mode.merge(mode); - } - } else { - entry.holders.push(txn); - } - } + // A holder raises its own mode in place. Classification admitted + // the raised mode against every other holder. + Some(entry) => entry.hold(txn, mode), } } @@ -89,11 +83,12 @@ impl LockManager { if entry.has_waiter(txn) { return; } - let wanted = if entry.holders.contains(&txn) { - entry.holders.retain(|h| *h != txn); - entry.mode.merge(mode) - } else { - mode + let wanted = match entry.mode_of(txn) { + Some(own) => { + entry.drop_holder(txn); + own.merge(mode) + } + None => mode, }; entry.waiters.push_back((txn, wanted)); } diff --git a/nodedb/src/control/cluster/calvin/scheduler/lock/manager/introspection.rs b/nodedb/src/control/cluster/calvin/scheduler/lock/manager/introspection.rs index d4b0018a5..480d35d24 100644 --- a/nodedb/src/control/cluster/calvin/scheduler/lock/manager/introspection.rs +++ b/nodedb/src/control/cluster/calvin/scheduler/lock/manager/introspection.rs @@ -35,7 +35,7 @@ impl LockManager { key: key.clone(), requested: *mode, held: entry.mode, - holders: entry.holders.to_vec(), + holders: entry.holder_ids(), waiters: entry.waiters.iter().copied().collect(), }) }) @@ -56,8 +56,8 @@ impl LockManager { pub fn is_ready(&self, txn: TxnId, keys: &BTreeMap) -> bool { keys.keys().all(|key| { match self.table.get(key) { - None => true, // key is free - Some(entry) => entry.holders.contains(&txn), // txn is a current holder + None => true, // key is free + Some(entry) => entry.holds(txn), // txn is a current holder } }) } @@ -69,7 +69,7 @@ impl LockManager { pub fn reservation_holder_count(&self, key: &LockKey) -> usize { self.table .get(key) - .map(|e| e.holders.iter().filter(|h| h.is_reservation()).count()) + .map(|e| e.holders.iter().filter(|(h, _)| h.is_reservation()).count()) .unwrap_or(0) } diff --git a/nodedb/src/control/cluster/calvin/scheduler/lock/manager/mod.rs b/nodedb/src/control/cluster/calvin/scheduler/lock/manager/mod.rs index 619b57359..4f850cfe0 100644 --- a/nodedb/src/control/cluster/calvin/scheduler/lock/manager/mod.rs +++ b/nodedb/src/control/cluster/calvin/scheduler/lock/manager/mod.rs @@ -7,10 +7,11 @@ //! The lock manager provides a deterministic, totally-ordered lock table over //! per-key entries keyed by //! [`LockKey`](crate::control::cluster::calvin::scheduler::lock::LockKey). -//! Locks come in three modes (see +//! Locks come in four modes (see //! [`LockMode`](crate::control::cluster::calvin::scheduler::lock::LockMode)): -//! `Exclusive` (one holder), `Shared` (many readers), and `Intent` (many row -//! writers of one collection). A transaction acquires one map of key to mode. +//! `Exclusive` (one holder), `Shared` (many readers), `Intent` (many row +//! writers of one collection), and `IntentShared` (many row readers of one +//! collection). A transaction acquires one map of key to mode. //! Read reservations take one `Shared` key via [`LockManager::acquire_shared`]. //! //! # Determinism diff --git a/nodedb/src/control/cluster/calvin/scheduler/lock/manager/release.rs b/nodedb/src/control/cluster/calvin/scheduler/lock/manager/release.rs index f7f1ca556..c47055bb2 100644 --- a/nodedb/src/control/cluster/calvin/scheduler/lock/manager/release.rs +++ b/nodedb/src/control/cluster/calvin/scheduler/lock/manager/release.rs @@ -29,7 +29,7 @@ impl LockManager { let Some(entry) = self.table.get_mut(key) else { continue; }; - entry.holders.retain(|h| *h != txn); + entry.drop_holder(txn); entry.waiters.retain(|(w, _)| *w != txn); self.promote_waiters(key, &mut newly_promoted); } @@ -56,10 +56,7 @@ impl LockManager { break; } entry.waiters.pop_front(); - if entry.holders.is_empty() { - entry.mode = mode; - } - entry.holders.push(next); + entry.hold(next, mode); granted.push(next); } if entry.holders.is_empty() && entry.waiters.is_empty() { @@ -70,11 +67,9 @@ impl LockManager { let Some(pending) = self.pending_keys.remove(&next) else { continue; }; - let all_held = pending.keys().all(|k| { - self.table - .get(k) - .is_some_and(|entry| entry.holders.contains(&next)) - }); + let all_held = pending + .keys() + .all(|k| self.table.get(k).is_some_and(|entry| entry.holds(next))); if all_held { self.finish_ready(next, pending); newly_promoted.insert(next); @@ -187,12 +182,12 @@ mod tests { // Release t1 — t2 should become holder (FIFO). lm.release(t1); - let holder = lm.table.get(&key("x")).unwrap().holders[0]; + let holder = lm.table.get(&key("x")).unwrap().holder_ids()[0]; assert_eq!(holder, t2); // Release t2 — t3 should become holder. lm.release(t2); - let holder = lm.table.get(&key("x")).unwrap().holders[0]; + let holder = lm.table.get(&key("x")).unwrap().holder_ids()[0]; assert_eq!(holder, t3); } @@ -228,8 +223,8 @@ mod tests { let entry = lm.table.get(&key("k")).unwrap(); assert_eq!(entry.mode, LockMode::Shared); - assert!(entry.holders.contains(&s1)); - assert!(entry.holders.contains(&s2)); + assert!(entry.holds(s1)); + assert!(entry.holds(s2)); } #[test] @@ -250,7 +245,7 @@ mod tests { let entry = lm.table.get(&key("k")).unwrap(); assert_eq!(entry.mode, LockMode::Exclusive); assert_eq!(entry.holders.len(), 1); - assert_eq!(entry.holders[0], x1); + assert_eq!(entry.holder_ids()[0], x1); // x2 is still waiting behind x1. assert!(entry.has_waiter(x2)); } @@ -268,8 +263,8 @@ mod tests { // Releasing one shared holder leaves the other holding the key. lm.release(t1); let entry = lm.table.get(&key("k")).unwrap(); - assert!(!entry.holders.contains(&t1)); - assert!(entry.holders.contains(&t2)); + assert!(!entry.holds(t1)); + assert!(entry.holds(t2)); assert_eq!(lm.lock_count(), 1); // Releasing the last shared holder frees the key. diff --git a/nodedb/src/control/cluster/calvin/scheduler/lock/manager/try_acquire.rs b/nodedb/src/control/cluster/calvin/scheduler/lock/manager/try_acquire.rs index 1455a67a9..838380a6c 100644 --- a/nodedb/src/control/cluster/calvin/scheduler/lock/manager/try_acquire.rs +++ b/nodedb/src/control/cluster/calvin/scheduler/lock/manager/try_acquire.rs @@ -60,7 +60,7 @@ mod tests { assert!(lm.try_acquire(b, BTreeMap::from([(coll(), LockMode::Intent)]))); assert!(!lm.try_acquire(c, BTreeMap::from([(coll(), LockMode::Exclusive)]))); let entry = lm.table.get(&coll()).unwrap(); - assert_eq!(entry.holders.as_slice(), &[a, b]); + assert_eq!(entry.holder_ids(), vec![a, b]); assert!(entry.waiters.is_empty(), "a refused probe leaves no waiter"); } } diff --git a/nodedb/src/control/cluster/calvin/scheduler/lock/manager/types.rs b/nodedb/src/control/cluster/calvin/scheduler/lock/manager/types.rs index 0ec4039d6..a21674224 100644 --- a/nodedb/src/control/cluster/calvin/scheduler/lock/manager/types.rs +++ b/nodedb/src/control/cluster/calvin/scheduler/lock/manager/types.rs @@ -81,11 +81,6 @@ impl Default for LockManager { } impl LockEntry { - /// Whether `txn` is the only holder of this entry. - pub(super) fn held_solely_by(&self, txn: TxnId) -> bool { - self.holders.len() == 1 && self.holders[0] == txn - } - /// Whether `txn` is already enqueued as a waiter on this entry. pub(super) fn has_waiter(&self, txn: TxnId) -> bool { self.waiters.iter().any(|(w, _)| *w == txn) diff --git a/nodedb/src/control/cluster/calvin/scheduler/lock/manager/wound_wait.rs b/nodedb/src/control/cluster/calvin/scheduler/lock/manager/wound_wait.rs index 00d2579c3..cb7da3fa8 100644 --- a/nodedb/src/control/cluster/calvin/scheduler/lock/manager/wound_wait.rs +++ b/nodedb/src/control/cluster/calvin/scheduler/lock/manager/wound_wait.rs @@ -36,14 +36,16 @@ impl LockManager { let Some(entry) = self.table.get(key) else { return ConflictResolution::Block; }; - if !entry.waiters.is_empty() || entry.mode != LockMode::Shared { + if !entry.waiters.is_empty() { return ConflictResolution::Block; } - let mut others = entry.holders.iter().filter(|h| **h != txn).peekable(); + let mut others = entry.holders.iter().filter(|(h, _)| *h != txn).peekable(); if others.peek().is_none() { return ConflictResolution::Block; } - if others.any(|holder| !holder.is_reservation() || txn >= *holder) { + if others.any(|(holder, held)| { + *held != LockMode::Shared || !holder.is_reservation() || txn >= *holder + }) { return ConflictResolution::Block; } } @@ -58,13 +60,9 @@ impl LockManager { let Some(entry) = self.table.get_mut(key) else { return; }; - let wanted = if entry.holders.contains(&txn) { - entry.mode.merge(mode) - } else { - mode - }; + let wanted = entry.mode_of(txn).map_or(mode, |own| own.merge(mode)); entry.holders.clear(); - entry.holders.push(txn); + entry.holders.push((txn, wanted)); entry.mode = wanted; } @@ -117,8 +115,8 @@ mod tests { let entry = lm.table.get(&key("s")).unwrap(); assert_eq!(entry.mode, LockMode::Shared); - assert!(entry.holders.contains(&r1)); - assert!(entry.holders.contains(&r2)); + assert!(entry.holds(r1)); + assert!(entry.holds(r2)); } #[test] @@ -162,7 +160,7 @@ mod tests { let entry = lm.table.get(&key("k")).unwrap(); assert_eq!(entry.mode, LockMode::Exclusive); - assert_eq!(entry.holders.as_slice(), &[writer]); + assert_eq!(entry.holder_ids(), vec![writer]); } #[test] @@ -181,7 +179,7 @@ mod tests { ); let entry = lm.table.get(&coll).unwrap(); assert_eq!(entry.mode, LockMode::Intent); - assert_eq!(entry.holders.as_slice(), &[writer]); + assert_eq!(entry.holder_ids(), vec![writer]); } #[test] @@ -197,8 +195,8 @@ mod tests { ); let entry = lm.table.get(&key("k")).unwrap(); - assert!(entry.holders.contains(&r), "the shared holder still holds"); - assert!(!entry.holders.contains(&writer)); + assert!(entry.holds(r), "the shared holder still holds"); + assert!(!entry.holds(writer)); assert!(entry.has_waiter(writer)); } @@ -212,7 +210,7 @@ mod tests { assert_eq!(lm.acquire(t1, exclusive(&["k"])), AcquireOutcome::Blocked); let entry = lm.table.get(&key("k")).unwrap(); - assert_eq!(entry.holders.as_slice(), &[t2]); + assert_eq!(entry.holder_ids(), vec![t2]); assert!(entry.has_waiter(t1)); } @@ -234,7 +232,7 @@ mod tests { for k in ["k1", "k2"] { let entry = lm.table.get(&key(k)).unwrap(); assert_eq!(entry.mode, LockMode::Exclusive); - assert_eq!(entry.holders.as_slice(), &[r]); + assert_eq!(entry.holder_ids(), vec![r]); } } @@ -256,7 +254,7 @@ mod tests { ); for (k, holder) in [("k1", s1), ("k2", s2)] { let entry = lm.table.get(&key(k)).unwrap(); - assert_eq!(entry.holders.as_slice(), &[holder]); + assert_eq!(entry.holder_ids(), vec![holder]); assert!(entry.has_waiter(r)); } } @@ -272,7 +270,7 @@ mod tests { assert_eq!(lm.acquire(r_old, exclusive(&["k"])), AcquireOutcome::Ready); let entry = lm.table.get(&key("k")).unwrap(); assert_eq!(entry.mode, LockMode::Exclusive); - assert_eq!(entry.holders.as_slice(), &[r_old]); + assert_eq!(entry.holder_ids(), vec![r_old]); // The younger one's self-upgrade drops its own hold and waits. let mut lm = LockManager::new(); @@ -284,13 +282,13 @@ mod tests { ); let entry = lm.table.get(&key("k")).unwrap(); assert_eq!(entry.mode, LockMode::Shared); - assert_eq!(entry.holders.as_slice(), &[r_old]); + assert_eq!(entry.holder_ids(), vec![r_old]); assert!(entry.has_waiter(r_young)); assert_eq!(lm.release(r_old), vec![r_young]); let entry = lm.table.get(&key("k")).unwrap(); assert_eq!(entry.mode, LockMode::Exclusive); - assert_eq!(entry.holders.as_slice(), &[r_young]); + assert_eq!(entry.holder_ids(), vec![r_young]); } #[test] @@ -311,6 +309,6 @@ mod tests { let k2 = lm.table.get(&key("k2")).unwrap(); assert_eq!(k2.mode, LockMode::Exclusive); - assert_eq!(k2.holders.as_slice(), &[writer]); + assert_eq!(k2.holder_ids(), vec![writer]); } } diff --git a/nodedb/src/control/cluster/calvin/scheduler/lock/reap.rs b/nodedb/src/control/cluster/calvin/scheduler/lock/reap.rs index a47ff6470..bc6ce9110 100644 --- a/nodedb/src/control/cluster/calvin/scheduler/lock/reap.rs +++ b/nodedb/src/control/cluster/calvin/scheduler/lock/reap.rs @@ -62,7 +62,8 @@ impl LockManager { keys.iter().all(|key| { self.table .get(key) - .is_none_or(|e| !e.holders.contains(&owner) || e.mode == LockMode::Shared) + .and_then(|e| e.mode_of(owner)) + .is_none_or(|mode| mode == LockMode::Shared) }) }); let waits_shared = self @@ -128,7 +129,7 @@ mod tests { let promoted = lm.reap_expired_shared(50); assert!(promoted.is_empty()); assert_eq!(lm.lock_count(), 1, "still within the lease, not reaped"); - assert!(lm.table.get(&key("k")).unwrap().holders.contains(&r)); + assert!(lm.table.get(&key("k")).unwrap().holds(r)); } #[test] @@ -144,7 +145,7 @@ mod tests { 1, "a real txn's exclusive lock is never reaped" ); - assert!(lm.table.get(&key("k")).unwrap().holders.contains(&t)); + assert!(lm.table.get(&key("k")).unwrap().holds(t)); } #[test] @@ -164,7 +165,7 @@ mod tests { "mid-commit reservation is left alone" ); assert_eq!(entry.holders.len(), 1); - assert_eq!(entry.holders[0], r); + assert_eq!(entry.holder_ids()[0], r); } #[test] @@ -196,7 +197,7 @@ mod tests { let entry = lm.table.get(&key("k")).unwrap(); assert_eq!(entry.mode, LockMode::Exclusive); assert_eq!(entry.holders.len(), 1); - assert_eq!(entry.holders[0], writer); + assert_eq!(entry.holder_ids()[0], writer); } #[test] diff --git a/nodedb/src/control/cluster/calvin_snapshot/mod.rs b/nodedb/src/control/cluster/calvin_snapshot/mod.rs index 1f7b75a92..206f3654f 100644 --- a/nodedb/src/control/cluster/calvin_snapshot/mod.rs +++ b/nodedb/src/control/cluster/calvin_snapshot/mod.rs @@ -7,5 +7,6 @@ pub mod requirement; pub use capture::{await_calvin_cut, capture_calvin_cut}; pub use install::install_calvin_cut; pub use requirement::{ - forget_left, install_snapshot_requirement, may_start, note_kept, rebased, retain_mounted, + forget_left, install_snapshot_requirement, may_start, note_kept, rebased, + require_calvin_snapshot, retain_mounted, }; diff --git a/nodedb/src/control/cluster/calvin_snapshot/requirement.rs b/nodedb/src/control/cluster/calvin_snapshot/requirement.rs index a337335f8..69f4767ca 100644 --- a/nodedb/src/control/cluster/calvin_snapshot/requirement.rs +++ b/nodedb/src/control/cluster/calvin_snapshot/requirement.rs @@ -22,11 +22,20 @@ //! A data group whose chunked redo streams this node dropped when it left //! the group requires a snapshot too, until one installs (see //! `wal_replication::transaction_redo::chunks::owed`). +//! +//! The two needs differ in kind: +//! - A missing Calvin base leaves the group's data whole. The replica wants +//! a snapshot ([`ReplicaState::WantsSnapshot`]) and may still lead, so a +//! group whose every replica lost the base keeps a leader, and Calvin +//! stays stopped there. +//! - A snapshot debt means this replica's data diverged from the log. It +//! awaits a snapshot ([`ReplicaState::AwaitingSnapshot`]) and never leads. use std::collections::BTreeSet; use std::sync::{Arc, Mutex, RwLock}; use nodedb_cluster::multi_raft::MultiRaft; +use nodedb_raft::ReplicaState; use crate::control::security::catalog::calvin_base::CalvinBase; use crate::control::state::SharedState; @@ -50,7 +59,7 @@ pub fn install_snapshot_requirement( let weak = Arc::downgrade(shared); multi_raft.set_snapshot_requirement(Arc::new(move |group_id, sequencer_first| { let Some(shared) = weak.upgrade() else { - return false; + return ReplicaState::Whole; }; let vshards = routing .read() @@ -58,7 +67,13 @@ pub fn install_snapshot_requirement( .vshards_for_group(group_id); // Evaluated first and always: it registers the vShards as waiting. let reaches = shared.calvin.bases.group_reaches(&vshards, sequencer_first); - !reaches || shared.redo_chunks.owes_snapshot(group_id) + if shared.redo_chunks.owes_snapshot(group_id) { + ReplicaState::AwaitingSnapshot + } else if reaches { + ReplicaState::Whole + } else { + ReplicaState::WantsSnapshot + } })); Ok(()) } @@ -95,11 +110,11 @@ pub fn may_start( } if let Some(group_id) = group_id { let mut multi_raft = multi_raft.lock().unwrap_or_else(|p| p.into_inner()); - multi_raft.set_snapshot_required(group_id, true); + require_calvin_snapshot(&mut multi_raft, group_id); // Without a scheduler here the vShard casts no vote, so a leader here - // blocks every Calvin transaction on it. A replica whose state is - // whole takes the leadership. When none qualifies, the transactions - // end at their deadline with an error. + // blocks every Calvin transaction on it. A replica whose Calvin state + // is whole takes the leadership. When none qualifies, the + // transactions end at their deadline with an error. let sole_replica = multi_raft .group_membership(group_id) .is_some_and(|m| m.voters.len() + m.learners.len() <= 1); @@ -134,6 +149,13 @@ pub fn may_start( false } +/// Make this node's replica of `group_id` refuse log entries until a +/// snapshot brings its Calvin state back. Its data is whole, so it keeps +/// campaigning and leading. +pub fn require_calvin_snapshot(multi_raft: &mut MultiRaft, group_id: u64) { + multi_raft.require_snapshot(group_id, ReplicaState::WantsSnapshot); +} + /// Record that the schedulers of `vshards`, each `(vshard_id, from)`, /// started from bases that reach the sequencer log and caught up from /// index `from`. They keep them whole from here. diff --git a/nodedb/src/control/cluster/execution_error_wire.rs b/nodedb/src/control/cluster/execution_error_wire.rs index 89323fa95..a37053b3a 100644 --- a/nodedb/src/control/cluster/execution_error_wire.rs +++ b/nodedb/src/control/cluster/execution_error_wire.rs @@ -133,6 +133,8 @@ pub(crate) fn execution_error_to_typed(err: crate::Error) -> TypedClusterError { | crate::Error::GroupMarksUnavailable { .. } | crate::Error::BackupCaptureMoved { .. } | crate::Error::MetadataLeaderUnavailable + | crate::Error::MetadataGroupGone { .. } + | crate::Error::MetadataDdlSuperseded { .. } | crate::Error::AuthorizationStateBehind { .. } | crate::Error::LinearizableReadRefused { .. } | crate::Error::ExecutionLimitExceeded { .. } diff --git a/nodedb/src/control/cluster/forwarded_budget.rs b/nodedb/src/control/cluster/forwarded_budget.rs new file mode 100644 index 000000000..82b9911a7 --- /dev/null +++ b/nodedb/src/control/cluster/forwarded_budget.rs @@ -0,0 +1,53 @@ +// SPDX-License-Identifier: BUSL-1.1 + +//! The deadline a leader-side hook works to for a forwarded request. + +use nodedb_cluster::TypedClusterError; +use nodedb_cluster::rpc_codec::forwarded_deadline; + +use crate::bridge::envelope::ErrorCode; +use crate::control::cluster::execution_error_wire::execution_error_to_typed; + +/// The deadline for a forwarded request that carries `deadline_remaining_ms`. +/// +/// A spent budget refuses the request before it runs. The refusal is +/// `ExpiredBeforeExecution`, so the coordinator reports a definite refusal, +/// never an unknown outcome. +pub(crate) fn hook_deadline( + deadline_remaining_ms: u64, +) -> Result { + match forwarded_deadline(deadline_remaining_ms) { + Some(deadline) => Ok(deadline), + None => Err(execution_error_to_typed(crate::Error::DataPlane( + ErrorCode::ExpiredBeforeExecution, + ))), + } +} + +#[cfg(test)] +mod tests { + use std::time::Duration; + + use super::*; + use crate::control::server::dispatch_utils::write_definitely_not_applied; + + /// A spent budget comes back to the coordinator as a definite refusal. + #[test] + fn a_spent_budget_is_a_definite_refusal() { + let refusal = hook_deadline(0).expect_err("a spent budget is refused"); + let crate::Error::DataPlane(code) = crate::Error::from(refusal) else { + panic!("expected a Data-Plane verdict"); + }; + assert_eq!(code, ErrorCode::ExpiredBeforeExecution); + assert!(write_definitely_not_applied(&code)); + } + + /// A live budget is used as it arrived, never stretched or floored. + #[tokio::test] + async fn a_live_budget_ends_within_what_arrived() { + let before = tokio::time::Instant::now(); + let deadline = hook_deadline(50).expect("a live budget"); + assert!(deadline >= before + Duration::from_millis(50)); + assert!(deadline <= tokio::time::Instant::now() + Duration::from_millis(50)); + } +} diff --git a/nodedb/src/control/cluster/leased_read.rs b/nodedb/src/control/cluster/leased_read.rs index 0d7150fd3..9e0db6e6c 100644 --- a/nodedb/src/control/cluster/leased_read.rs +++ b/nodedb/src/control/cluster/leased_read.rs @@ -10,11 +10,9 @@ //! misses. A group whose lease this node does not hold is refused, with the //! leader and term its routing table names. -use std::time::Instant; - use crate::control::state::SharedState; -use super::linearizable_read::wait_applied_through; +use super::linearizable_read::{StatementDeadline, wait_applied_through}; /// A group this node does not serve a leased read of. #[derive(Debug, Clone, Copy, PartialEq, Eq)] @@ -31,13 +29,13 @@ pub struct LeaseRefusal { /// leases. /// /// Returns the groups whose lease this node does not hold. Every other group -/// is applied here through its lease read index before this returns. A node -/// with no routing table serves every group: without a cluster there is one -/// copy and nothing to prove. +/// is applied here through its lease read index, within `deadline`, before +/// this returns. A node with no routing table serves every group: without a +/// cluster there is one copy and nothing to prove. pub async fn confirm_leased_read( state: &SharedState, groups: &[u64], - deadline: Instant, + deadline: StatementDeadline, ) -> crate::Result> { let Some(routing) = state.cluster_routing.as_ref() else { return Ok(Vec::new()); @@ -91,7 +89,7 @@ pub async fn confirm_leased_read( mod tests { use std::sync::atomic::{AtomicBool, Ordering}; use std::sync::{Arc, RwLock}; - use std::time::Duration; + use std::time::{Duration, Instant}; use async_trait::async_trait; use nodedb_cluster::RoutingTable; @@ -130,7 +128,12 @@ mod tests { Err(ReadIndexRefusal::NotLeader) } - fn within_staleness_bound(&self, _group_id: u64, _max_staleness: Duration) -> bool { + fn within_staleness_bound( + &self, + _group_id: u64, + _max_staleness: Duration, + _applied: u64, + ) -> bool { self.probe(); false } @@ -178,7 +181,7 @@ mod tests { ); let groups = routing.read().expect("routing lock").group_ids(); - let refusals = confirm_leased_read(&state, &groups, Instant::now()) + let refusals = confirm_leased_read(&state, &groups, StatementDeadline::at(Instant::now())) .await .expect("a refused lease is not an error"); assert!( diff --git a/nodedb/src/control/cluster/linearizable_read.rs b/nodedb/src/control/cluster/linearizable_read.rs index 369026938..d63f7feb1 100644 --- a/nodedb/src/control/cluster/linearizable_read.rs +++ b/nodedb/src/control/cluster/linearizable_read.rs @@ -8,18 +8,19 @@ //! leads (under its lease or a quorum round), the leader over the transport //! otherwise. Every path that serves a strong read calls //! [`confirm_linearizable_read`] on the serving node before it reads. - -use std::time::{Duration, Instant}; +//! +//! The two steps run under different bounds (see [`super::read_deadline`]): +//! - The quorum round for the read index ends at the transport RPC timeout, +//! so a partition refuses the read quickly. +//! - This node's wait to apply through the index follows the statement +//! deadline, so a lagging apply delays the read instead of refusing it. use nodedb_cluster::WaitOutcome; use crate::control::cluster::read_index::ReadIndexRefusal; use crate::control::state::SharedState; -/// Budget for one linearizable read to get each group's read index and apply -/// through it. Several production election timeouts (150-300ms), so an -/// ordinary round trip fits and a partition is refused rather than hung on. -pub const LINEARIZABLE_READ_TIMEOUT: Duration = Duration::from_millis(750); +pub use super::read_deadline::{ReadIndexDeadline, StatementDeadline}; /// The Raft groups that hold `vshard_ids`, deduplicated. Empty on a node with /// no routing table: without a cluster there is one copy and nothing to prove. @@ -87,29 +88,16 @@ fn hosts_group(state: &SharedState, routing: &nodedb_cluster::RoutingTable, grou }) } -/// The deadline a linearizable read in the running statement gets: never past -/// what is left of the statement's budget. -pub fn statement_read_deadline(state: &SharedState) -> Instant { - let remaining_ms = crate::control::server::shared::session::statement_deadline_ms( - state.tuning.network.default_deadline_secs, - ); - linearizable_read_deadline(Duration::from_millis(remaining_ms)) -} - -/// The deadline a linearizable read gets, never past `statement_budget`. -pub fn linearizable_read_deadline(statement_budget: Duration) -> Instant { - Instant::now() + LINEARIZABLE_READ_TIMEOUT.min(statement_budget) -} - /// Make a linearizable read of `groups` safe to serve on this node. /// /// Each group gets a read index taken after this call, then this node waits /// until it has applied the group through that index. The groups run -/// concurrently under one `deadline`. +/// concurrently. Each quorum round ends at the transport RPC timeout, and the +/// apply waits end at `deadline`. pub async fn confirm_linearizable_read( state: &SharedState, groups: &[u64], - deadline: Instant, + deadline: StatementDeadline, ) -> crate::Result<()> { if groups.is_empty() { return Ok(()); @@ -118,44 +106,55 @@ pub async fn confirm_linearizable_read( // A routing table without a gate: `start_raft` has not published it. return Err(refused(groups[0], "raft is not serving on this node yet")); }; + let round = ReadIndexDeadline::of_round(state, deadline); let reads = groups.iter().map(|&group_id| async move { - let remaining = deadline.saturating_duration_since(Instant::now()); - let read_index = - gate.read_index(group_id, remaining) - .await - .map_err(|refusal| match refusal { - ReadIndexRefusal::NotLeader => { - refused(group_id, "no leader confirmed a read index") - } - ReadIndexRefusal::Timeout { waited_ms } => refused( - group_id, - &format!("no quorum confirmed a read index within {waited_ms}ms"), - ), - })?; + let read_index = read_index_within(&**gate, group_id, round).await?; wait_applied_through(state, group_id, read_index, deadline).await }); futures::future::try_join_all(reads).await.map(|_| ()) } +/// A read index of `group_id` from its leader, refused once `round` passes. +async fn read_index_within( + gate: &dyn crate::control::cluster::read_index::RaftReadGate, + group_id: u64, + round: ReadIndexDeadline, +) -> crate::Result { + let asked = gate.read_index(group_id, round.remaining()); + let answer = tokio::time::timeout_at(tokio::time::Instant::from_std(round.instant()), asked) + .await + .map_err(|_| refused(group_id, "no quorum confirmed a read index in time"))?; + answer.map_err(|refusal| match refusal { + ReadIndexRefusal::NotLeader => refused(group_id, "no leader confirmed a read index"), + ReadIndexRefusal::Timeout { waited_ms } => refused( + group_id, + &format!("no quorum confirmed a read index within {waited_ms}ms"), + ), + }) +} + /// Wait until this node has applied `group_id` through `read_index`, or /// refuse at `deadline`. +/// +/// A replica here that awaits a snapshot install applies nothing until one +/// installs, so its read is refused at once. pub(crate) async fn wait_applied_through( state: &SharedState, group_id: u64, read_index: u64, - deadline: Instant, + deadline: StatementDeadline, ) -> crate::Result<()> { let watcher = state.applied_index_watcher(group_id); if watcher.current() >= read_index { return Ok(()); } - let remaining = deadline.saturating_duration_since(Instant::now()); - // The watcher parks its caller on a condition variable, so the wait runs - // on the blocking pool. - let outcome = tokio::task::spawn_blocking(move || watcher.wait_for(read_index, remaining)) - .await - .map_err(|e| refused(group_id, &format!("the apply wait did not finish: {e}")))?; - match outcome { + if awaits_snapshot(state, group_id) { + return Err(refused( + group_id, + "this replica awaits a snapshot install and applies nothing until one installs", + )); + } + match watcher.wait_for_async(read_index, deadline.instant()).await { WaitOutcome::Reached => Ok(()), WaitOutcome::TimedOut => Err(refused( group_id, @@ -165,6 +164,18 @@ pub(crate) async fn wait_applied_through( } } +/// Whether this node's replica of `group_id` awaits a snapshot install: its +/// data diverged from the log. +fn awaits_snapshot(state: &SharedState, group_id: u64) -> bool { + state.multi_raft.get().is_some_and(|multi_raft| { + multi_raft + .lock() + .unwrap_or_else(|p| p.into_inner()) + .replica_state(group_id) + == Some(nodedb_raft::ReplicaState::AwaitingSnapshot) + }) +} + fn refused(group_id: u64, detail: &str) -> crate::Error { crate::Error::LinearizableReadRefused { group_id, @@ -174,20 +185,122 @@ fn refused(group_id: u64, detail: &str) -> crate::Error { #[cfg(test)] mod tests { - use std::time::Duration; + use std::sync::Arc; + use std::time::{Duration, Instant}; - use super::{LINEARIZABLE_READ_TIMEOUT, linearizable_read_deadline}; + use async_trait::async_trait; - #[test] - fn the_deadline_never_exceeds_the_statement_budget() { - let short = Duration::from_millis(10); - let before = std::time::Instant::now(); - let deadline = linearizable_read_deadline(short); - assert!(deadline <= std::time::Instant::now() + short); - assert!(deadline >= before + short); + use super::*; + use crate::bridge::dispatch::Dispatcher; + use crate::control::cluster::read_index::RaftReadGate; + use crate::wal::WalManager; + + const GROUP: u64 = 3; + + fn test_state(dir: &tempfile::TempDir) -> Arc { + let wal = Arc::new( + WalManager::open_for_testing(&dir.path().join("linearizable.wal")).expect("wal"), + ); + let (dispatcher, _sides) = Dispatcher::new(1, 64); + SharedState::new(dispatcher, wal).expect("shared state") + } + + /// A gate whose quorum rounds never answer. + struct SilentGate; + + #[async_trait] + impl RaftReadGate for SilentGate { + async fn confirm_leader( + &self, + _group_id: u64, + _timeout: Duration, + ) -> Result { + std::future::pending().await + } + + fn within_staleness_bound( + &self, + _group_id: u64, + _max_staleness: Duration, + _applied: u64, + ) -> bool { + false + } + + fn holds_leader_lease(&self, _group_id: u64) -> bool { + false + } + + fn leader_lease_term(&self, _group_id: u64) -> Option { + None + } + + fn lease_read_index(&self, _group_id: u64) -> Option { + None + } + } + + /// A lagging apply delays the read: the wait follows the statement + /// deadline, not a short fixed bound. + #[tokio::test] + async fn the_apply_wait_follows_the_statement_deadline() { + let dir = tempfile::tempdir().expect("tempdir"); + let state = test_state(&dir); + let applier = { + let state = Arc::clone(&state); + tokio::spawn(async move { + tokio::time::sleep(Duration::from_millis(1_500)).await; + state.applied_index_watcher(GROUP).bump(9); + }) + }; + let deadline = StatementDeadline::after(Duration::from_secs(5)); + let waited = wait_applied_through(&state, GROUP, 9, deadline).await; + applier.await.expect("join applier"); + waited.expect("the read waits out the lag inside the statement deadline"); + } + + /// An apply that never reaches the read index is refused at the + /// statement deadline. + #[tokio::test] + async fn an_apply_that_never_arrives_is_refused_at_the_deadline() { + let dir = tempfile::tempdir().expect("tempdir"); + let state = test_state(&dir); + let started = Instant::now(); + let budget = Duration::from_millis(300); + let waited = wait_applied_through(&state, GROUP, 9, StatementDeadline::after(budget)).await; + assert!( + matches!(waited, Err(crate::Error::LinearizableReadRefused { .. })), + "{waited:?}" + ); + assert!(started.elapsed() >= budget); + } - let long = LINEARIZABLE_READ_TIMEOUT * 10; - let deadline = linearizable_read_deadline(long); - assert!(deadline <= std::time::Instant::now() + LINEARIZABLE_READ_TIMEOUT); + /// A read index no quorum confirms is refused at the transport RPC + /// timeout, long before a 30 s statement deadline. + #[tokio::test] + async fn a_read_index_with_no_quorum_is_refused_at_the_rpc_bound() { + let dir = tempfile::tempdir().expect("tempdir"); + let mut state = test_state(&dir); + Arc::get_mut(&mut state) + .expect("sole owner of fresh state") + .tuning + .cluster_transport + .rpc_timeout_secs = 1; + assert!( + state + .raft_read_gate + .set(Arc::new(SilentGate) as Arc) + .is_ok() + ); + let started = Instant::now(); + let deadline = StatementDeadline::after(Duration::from_secs(30)); + let read = confirm_linearizable_read(&state, &[GROUP], deadline).await; + let waited = started.elapsed(); + assert!( + matches!(read, Err(crate::Error::LinearizableReadRefused { .. })), + "{read:?}" + ); + assert!(waited >= Duration::from_secs(1), "{waited:?}"); + assert!(waited < Duration::from_secs(10), "{waited:?}"); } } diff --git a/nodedb/src/control/cluster/metadata_applier/audit.rs b/nodedb/src/control/cluster/metadata_applier/audit.rs index bffc2e766..2b98bc492 100644 --- a/nodedb/src/control/cluster/metadata_applier/audit.rs +++ b/nodedb/src/control/cluster/metadata_applier/audit.rs @@ -121,9 +121,10 @@ pub(super) fn emit_ddl_audit( // mutex lets audit-log contention stall EVERY metadata apply, and with it // collection materialization, DDL, and any proposer waiting on the applied // index. That is a liveness bug rather than a slow path: one contended - // acquisition here was measured parking the loop for a full 5s propose - // timeout, so peer `CollectionSchema` announces never materialized and the - // engine writes that followed them were rejected as unknown collections. + // acquisition here was measured parking the loop past a proposer's whole + // apply wait, so peer `CollectionSchema` announces never materialized and + // the engine writes that followed them were rejected as unknown + // collections. // // The record's payload is fully built above and needs nothing further from // the apply loop, so deferring the emit keeps the compliance row (no silent diff --git a/nodedb/src/control/cluster/metadata_applier/catalog_ddl.rs b/nodedb/src/control/cluster/metadata_applier/catalog_ddl.rs index c71d79712..536642f21 100644 --- a/nodedb/src/control/cluster/metadata_applier/catalog_ddl.rs +++ b/nodedb/src/control/cluster/metadata_applier/catalog_ddl.rs @@ -81,6 +81,14 @@ impl MetadataCommitApplier { }; let shared = self.shared_state()?; + // Slows one collection's create on one node, as a long apply does. + #[cfg(feature = "failpoints")] + if let catalog_entry::CatalogEntry::PutCollection(stored) + | catalog_entry::CatalogEntry::PutCollectionIfAbsent(stored) = &stamped + { + crate::control::fail_gate::before_catalog_ddl_apply(shared.node_id, &stored.name).await; + } + // Holds back one node's apply of backup schedule marks with a // transient error, so a test can make that node's catalog lag the // metadata group. Raft re-delivers the entry once it is cleared. diff --git a/nodedb/src/control/cluster/metadata_applier/dispatch.rs b/nodedb/src/control/cluster/metadata_applier/dispatch.rs index fdf8343f5..d9b92ff80 100644 --- a/nodedb/src/control/cluster/metadata_applier/dispatch.rs +++ b/nodedb/src/control/cluster/metadata_applier/dispatch.rs @@ -275,14 +275,8 @@ impl MetadataCommitApplier { entries: &[MetadataEntry], raft_index: u64, ) -> Result<(), crate::Error> { - let shared = self.shared.get().and_then(std::sync::Weak::upgrade); for sub in entries { self.apply_host_side_effects(sub, raft_index).await?; - if let Some(shared) = &shared { - shared - .metadata_apply_progress - .fetch_add(1, std::sync::atomic::Ordering::Release); - } } Ok(()) } diff --git a/nodedb/src/control/cluster/metadata_applier/wedge.rs b/nodedb/src/control/cluster/metadata_applier/wedge.rs index 95d7ff076..021f502d7 100644 --- a/nodedb/src/control/cluster/metadata_applier/wedge.rs +++ b/nodedb/src/control/cluster/metadata_applier/wedge.rs @@ -132,6 +132,8 @@ pub fn classify(error: &crate::Error) -> ApplyFailureClass { | crate::Error::GroupMarksUnavailable { .. } | crate::Error::BackupCaptureMoved { .. } | crate::Error::MetadataLeaderUnavailable + | crate::Error::MetadataGroupGone { .. } + | crate::Error::MetadataDdlSuperseded { .. } | crate::Error::AuthorizationStateBehind { .. } | crate::Error::LinearizableReadRefused { .. } | crate::Error::ExecutionLimitExceeded { .. } diff --git a/nodedb/src/control/cluster/mod.rs b/nodedb/src/control/cluster/mod.rs index de6a5717e..079e5a11c 100644 --- a/nodedb/src/control/cluster/mod.rs +++ b/nodedb/src/control/cluster/mod.rs @@ -17,6 +17,7 @@ pub mod data_plane_error_wire; pub mod data_plane_fault_wire; pub mod decommission_bridge; pub mod execution_error_wire; +pub mod forwarded_budget; pub mod handle; pub mod init; pub mod leased_read; @@ -25,6 +26,7 @@ pub mod metadata_applier; pub mod metadata_image; pub mod metadata_stamp; pub mod pem_io; +pub mod read_deadline; pub mod read_index; pub mod recovery_check; pub mod redo_stream_membership; diff --git a/nodedb/src/control/cluster/read_deadline.rs b/nodedb/src/control/cluster/read_deadline.rs new file mode 100644 index 000000000..d244dfb17 --- /dev/null +++ b/nodedb/src/control/cluster/read_deadline.rs @@ -0,0 +1,67 @@ +// SPDX-License-Identifier: BUSL-1.1 + +//! The two bounds a linearizable read runs under. +//! +//! - [`StatementDeadline`] bounds the whole read. This node's wait to apply +//! through the read index follows it. +//! - [`ReadIndexDeadline`] bounds one quorum round for a read index: the +//! transport RPC timeout, never past the statement. A partitioned leader +//! is refused at this bound, not at the statement deadline. +//! +//! Each is its own type, so a caller cannot pass one where the other belongs. + +use std::time::{Duration, Instant}; + +pub use crate::control::server::shared::session::StatementDeadline; +use crate::control::state::SharedState; + +/// When one quorum round for a read index gives up. +#[derive(Debug, Clone, Copy, PartialEq, Eq, PartialOrd, Ord)] +pub struct ReadIndexDeadline(Instant); + +impl ReadIndexDeadline { + /// `rpc_timeout` from now, never past `statement`. + pub fn within(statement: StatementDeadline, rpc_timeout: Duration) -> Self { + Self(statement.instant().min(Instant::now() + rpc_timeout)) + } + + /// The bound `state`'s transport tuning gives a round started now, never + /// past `statement`. + pub fn of_round(state: &SharedState, statement: StatementDeadline) -> Self { + Self::within( + statement, + Duration::from_secs(state.tuning.cluster_transport.rpc_timeout_secs), + ) + } + + /// The deadline as an instant. + pub fn instant(self) -> Instant { + self.0 + } + + /// Time left until the deadline, zero once it passed. + pub fn remaining(self) -> Duration { + self.0.saturating_duration_since(Instant::now()) + } +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn the_round_bound_is_the_rpc_timeout_inside_a_long_statement() { + let statement = StatementDeadline::after(Duration::from_secs(30)); + let before = Instant::now(); + let round = ReadIndexDeadline::within(statement, Duration::from_secs(1)); + assert!(round.instant() >= before + Duration::from_secs(1)); + assert!(round.instant() < statement.instant()); + } + + #[test] + fn the_round_bound_never_passes_the_statement() { + let statement = StatementDeadline::after(Duration::from_millis(10)); + let round = ReadIndexDeadline::within(statement, Duration::from_secs(5)); + assert_eq!(round.instant(), statement.instant()); + } +} diff --git a/nodedb/src/control/cluster/read_index.rs b/nodedb/src/control/cluster/read_index.rs index 86493a307..711378ecb 100644 --- a/nodedb/src/control/cluster/read_index.rs +++ b/nodedb/src/control/cluster/read_index.rs @@ -40,8 +40,10 @@ pub trait RaftReadGate: Send + Sync { ) -> Result; /// Whether this node's replica of `group_id` is within `max_staleness` of - /// the leader. Local state only — no quorum round, so this does not block. - fn within_staleness_bound(&self, group_id: u64, max_staleness: Duration) -> bool; + /// the leader, with reads here seeing state through `applied`: the + /// group's host applied watermark. Local state only — no quorum round, so + /// this does not block. + fn within_staleness_bound(&self, group_id: u64, max_staleness: Duration, applied: u64) -> bool; /// Whether this node leads `group_id` under a leader lease that is valid /// now. Local state only, so this does not block. @@ -173,11 +175,11 @@ impl RaftReadGate for MultiRaftReadGate { .map_err(refusal_of) } - fn within_staleness_bound(&self, group_id: u64, max_staleness: Duration) -> bool { + fn within_staleness_bound(&self, group_id: u64, max_staleness: Duration, applied: u64) -> bool { self.multi_raft .lock() .unwrap_or_else(|p| p.into_inner()) - .within_staleness_bound(group_id, max_staleness) + .within_staleness_bound(group_id, max_staleness, applied) } fn holds_leader_lease(&self, group_id: u64) -> bool { diff --git a/nodedb/src/control/cluster/redo_stream_membership.rs b/nodedb/src/control/cluster/redo_stream_membership.rs index fdcc58ee0..c3d4dea3e 100644 --- a/nodedb/src/control/cluster/redo_stream_membership.rs +++ b/nodedb/src/control/cluster/redo_stream_membership.rs @@ -8,13 +8,14 @@ //! install before it applies here again. A remount resumes its old log, //! which can still name a dropped stream. //! - A mounted group that owes a snapshot refuses log entries until one -//! installs, and its leadership here moves to a replica that can lead. -//! This covers a group that mounted again, and a group whose apply here -//! held an entry of a dropped stream. +//! installs. Its replica here steps down from leadership and never +//! campaigns. This covers a group that mounted again, and a group whose +//! apply here holds an entry. use std::sync::Mutex; use nodedb_cluster::multi_raft::MultiRaft; +use nodedb_raft::ReplicaState; use crate::control::state::SharedState; @@ -56,8 +57,9 @@ fn drop_unmounted(shared: &SharedState, multi_raft: &Mutex) { } } -/// Make every mounted group that owes a snapshot refuse log entries, and -/// move a leadership of it this node holds: a leader takes no snapshot. +/// Make every mounted group that owes a snapshot refuse log entries. The +/// requirement steps down a leadership of it this node holds: a leader +/// takes no snapshot. /// /// The debt is read under the `MultiRaft` lock. An install settles the debt /// before it refreshes the group's requirement under the same lock, so this @@ -72,20 +74,12 @@ fn require_owed_snapshots(shared: &SharedState, multi_raft: &Mutex) { if !multi_raft.contains_group(group_id) || !shared.redo_chunks.owes_snapshot(group_id) { continue; } - if !multi_raft.snapshot_required(group_id) { - multi_raft.set_snapshot_required(group_id, true); + if multi_raft.replica_state(group_id) != Some(ReplicaState::AwaitingSnapshot) { + multi_raft.require_snapshot(group_id, ReplicaState::AwaitingSnapshot); tracing::warn!( group_id, "redo chunks: the group's replica here owes a snapshot install; it refuses \ - log entries until one installs" - ); - } - if let Some(target) = multi_raft.hand_off_leadership(group_id) { - tracing::info!( - group_id, - target, - "redo chunks: the group's replica here owes a snapshot install; its \ - leadership moves to a replica that can lead" + log entries and leads nothing until one installs" ); } } diff --git a/nodedb/src/control/cluster/snapshot_applier.rs b/nodedb/src/control/cluster/snapshot_applier.rs index 76d1def3e..399c4b147 100644 --- a/nodedb/src/control/cluster/snapshot_applier.rs +++ b/nodedb/src/control/cluster/snapshot_applier.rs @@ -283,16 +283,18 @@ impl DataPlaneSnapshotApplier { if floor == 0 || watcher.current() >= floor { return Ok(()); } - let waiting = Arc::clone(&watcher); - let outcome = tokio::task::spawn_blocking(move || waiting.wait_for(floor, limit)).await; + let outcome = watcher + .wait_for_async(floor, std::time::Instant::now() + limit) + .await; match outcome { - Ok(nodedb_cluster::WaitOutcome::Reached) => Ok(()), - Ok(nodedb_cluster::WaitOutcome::TimedOut | nodedb_cluster::WaitOutcome::GroupGone) - | Err(_) => Err(SnapshotInstallError::MetadataBehind { - group_id, - floor, - applied: watcher.current(), - }), + nodedb_cluster::WaitOutcome::Reached => Ok(()), + nodedb_cluster::WaitOutcome::TimedOut | nodedb_cluster::WaitOutcome::GroupGone => { + Err(SnapshotInstallError::MetadataBehind { + group_id, + floor, + applied: watcher.current(), + }) + } } } diff --git a/nodedb/src/control/cluster/snapshot_builder.rs b/nodedb/src/control/cluster/snapshot_builder.rs index c67b429ca..7beb0cc04 100644 --- a/nodedb/src/control/cluster/snapshot_builder.rs +++ b/nodedb/src/control/cluster/snapshot_builder.rs @@ -304,6 +304,8 @@ impl nodedb_cluster::SnapshotBuilder for DataPlaneSnapshotBuilder { last_included_index: u64, _last_included_term: u64, ) -> Result> { + #[cfg(feature = "failpoints")] + crate::control::fail_gate::before_snapshot_build(self.shared.node_id, group_id).await; // No routing table or an empty group ships nothing: the sender falls // back to the stub chunk, at the index asked for. let nothing = nodedb_cluster::BuiltGroupSnapshot { diff --git a/nodedb/src/control/cluster/snapshot_cut.rs b/nodedb/src/control/cluster/snapshot_cut.rs index c28fce096..946d02374 100644 --- a/nodedb/src/control/cluster/snapshot_cut.rs +++ b/nodedb/src/control/cluster/snapshot_cut.rs @@ -52,16 +52,9 @@ pub(crate) async fn settled_cut( let watcher = shared.applied_index_watcher(group_id); let cut = cut_of(tracker.started_through(group_id), watcher.current()); if watcher.current() < cut { - let waiting = std::sync::Arc::clone(&watcher); - let outcome = - tokio::task::spawn_blocking(move || waiting.wait_for(cut, CUT_SETTLE_TIMEOUT)) - .await - .map_err(|e| Error::Internal { - detail: format!( - "snapshot build: group {group_id}: the settle wait for cut {cut} \ - did not finish: {e}" - ), - })?; + let outcome = watcher + .wait_for_async(cut, std::time::Instant::now() + CUT_SETTLE_TIMEOUT) + .await; match outcome { WaitOutcome::Reached => {} WaitOutcome::TimedOut | WaitOutcome::GroupGone => { diff --git a/nodedb/src/control/cluster/start_raft/propose_error.rs b/nodedb/src/control/cluster/start_raft/propose_error.rs index 54001390e..ea053b6d6 100644 --- a/nodedb/src/control/cluster/start_raft/propose_error.rs +++ b/nodedb/src/control/cluster/start_raft/propose_error.rs @@ -34,11 +34,13 @@ pub(super) fn async_propose_error(vshard_id: u32, error: ClusterError) -> crate: ) | ClusterError::ReadIndexNotLeader { .. } | ClusterError::MigrationInProgress { .. } - | ClusterError::WrongOwner { .. } + | ClusterError::WrongOwner { .. } => crate::Error::NoLeader { + vshard_id: VShardId::new(vshard_id), + }, // The forward never reached the leader whole: the leader is not in // this node's topology, or the connect, stream open or write failed. // Nothing was proposed. A written forward ends as `Unanswered`. - | ClusterError::Transport { .. } => crate::Error::NoLeader { + unsent if unsent.is_unsent() => crate::Error::NoLeader { vshard_id: VShardId::new(vshard_id), }, // The forward did not answer before its timeout. The leader can have @@ -46,7 +48,7 @@ pub(super) fn async_propose_error(vshard_id: u32, error: ClusterError) -> crate: // deadline class, the same class the array fan-out gives it. A // forward whose stream failed after the request was written can // also have been proposed, so it takes the same class. - error @ (ClusterError::ShardTimeout { .. } | ClusterError::Unanswered { .. }) => { + error if error.is_lost_answer() => { tracing::warn!( vshard_id, %error, @@ -110,7 +112,11 @@ pub(super) fn async_propose_error(vshard_id: u32, error: ClusterError) -> crate: | ClusterError::Bm25Gather(_) | ClusterError::TsGather(_) | ClusterError::ShufflePush(_) - | ClusterError::RemoteUntyped { .. }) => crate::Error::Internal { + | ClusterError::RemoteUntyped { .. } + // The guards above take these. They stay named for exhaustiveness. + | ClusterError::Transport { .. } + | ClusterError::ShardTimeout { .. } + | ClusterError::Unanswered { .. }) => crate::Error::Internal { detail: format!("raft propose (async): {other}"), }, } diff --git a/nodedb/src/control/distributed_applier/apply_loop/context.rs b/nodedb/src/control/distributed_applier/apply_loop/context.rs index 3c7996901..6e2d65016 100644 --- a/nodedb/src/control/distributed_applier/apply_loop/context.rs +++ b/nodedb/src/control/distributed_applier/apply_loop/context.rs @@ -69,6 +69,12 @@ pub(super) enum LoopEvent<'a> { entry: StartedEntry<'a>, }, Finished(FinishedApply), + /// The entry's enqueue or apply panicked. `detail` is the panic payload. + Panicked { + group_id: u64, + log_index: u64, + detail: String, + }, } /// A future the loop collects. diff --git a/nodedb/src/control/distributed_applier/apply_loop/driver.rs b/nodedb/src/control/distributed_applier/apply_loop/driver.rs index abdba3b87..d74984f8b 100644 --- a/nodedb/src/control/distributed_applier/apply_loop/driver.rs +++ b/nodedb/src/control/distributed_applier/apply_loop/driver.rs @@ -5,8 +5,10 @@ //! and every started entry concluded. use std::sync::Arc; +use std::time::Duration; use tokio::sync::{mpsc, watch}; +use tokio::time::MissedTickBehavior; use crate::control::distributed_applier::applier::ApplyBatch; use crate::control::distributed_applier::proposal_ledger::{ @@ -55,7 +57,23 @@ pub async fn run_apply_loop( tracker: &tracker, }; let mut pipeline = Pipeline::new(ctx, ledger); + drive(&mut pipeline, &mut apply_rx, &state).await; +} + +/// Feed `pipeline` from `apply_rx` and its wakes until the channel closes and +/// every started entry concluded. +pub(super) async fn drive( + pipeline: &mut Pipeline<'_>, + apply_rx: &mut mpsc::Receiver, + state: &SharedState, +) { let mut install_released = state.raft_apply_gates.get().map(|g| g.subscribe_released()); + // A held group whose snapshot debt did not persist gets no batches. The + // loop wakes on each Raft tick to write the debt again. + let mut debt_retry = tokio::time::interval(Duration::from_millis( + state.tuning.cluster_transport.raft_tick_interval_ms, + )); + debt_retry.set_missed_tick_behavior(MissedTickBehavior::Delay); let mut accepting = true; loop { if !accepting && !pipeline.has_running() { @@ -64,13 +82,15 @@ pub async fn run_apply_loop( return; } let running = pipeline.has_running(); - let blocked = pipeline.install_blocked(); + let blocked = pipeline.awaits_install(); + let retry_debt = pipeline.retries_debt(); tokio::select! { biased; Some(event) = pipeline.next_event(), if running => { pipeline.handle(event); } () = wait_install_released(&mut install_released), if blocked => {} + _ = debt_retry.tick(), if retry_debt => {} batch = apply_rx.recv(), if accepting => match batch { Some(batch) => { pipeline.accept(batch); diff --git a/nodedb/src/control/distributed_applier/apply_loop/held_entry.rs b/nodedb/src/control/distributed_applier/apply_loop/held_entry.rs new file mode 100644 index 000000000..7cc642bbc --- /dev/null +++ b/nodedb/src/control/distributed_applier/apply_loop/held_entry.rs @@ -0,0 +1,292 @@ +// SPDX-License-Identifier: BUSL-1.1 + +//! A committed entry this replica cannot apply as its peers do holds its +//! group's lane (see [`super::lane`]). Later entries applied on state +//! without it would diverge from the other replicas, and reads would see +//! that state. +//! +//! While the lane holds, the group's replica here owes a snapshot install: +//! - The debt is durable, so a restart keeps it. +//! - The replica refuses log entries, steps down from a leadership it +//! holds, and never campaigns. Its leader sends a snapshot. +//! - The install that covers the held entry releases it and settles the +//! debt. Backlog entries at or below the install conclude as covered. +//! +//! The debt is kept once per install of the group. A debt write that +//! failed is written again on the apply loop's timed wake. + +use std::collections::{BTreeSet, HashMap}; + +use crate::control::state::SharedState; + +use super::context::ApplyContext; +use super::lane::Lane; +use super::proposal_gate::{HoldReason, ProposalGate}; +use super::snapshot_gate::{EntryAdmission, admit_entry}; + +/// Where a group's lane stands against a hold. +pub(super) enum HoldCheck { + /// No entry holds the lane: it can start entries. + Free, + /// An entry holds the lane and no install covers it yet. + Held, + /// An install holds or awaits the group's apply gate. + Installing, +} + +/// Note that the entry at `log_index` of `group_id` entered a hold. The +/// group's debt is recorded by [`keep_snapshot_owed`], under the group's +/// apply gate. +pub(super) fn note_hold(group_id: u64, log_index: u64, reason: HoldReason) { + tracing::error!( + group_id, + log_index, + reason = reason.as_str(), + "apply loop: this replica cannot apply the committed entry as its peers do; the \ + group's apply holds at it until a snapshot of the group installs" + ); +} + +/// The snapshot debt this loop kept for each held group. +#[derive(Default)] +pub(super) struct HeldDebts { + groups: HashMap, +} + +#[derive(Clone, Copy)] +struct KeptDebt { + /// The group's installed index when the debt was kept. + installed_through: u64, + /// Whether the debt write persisted. + recorded: bool, +} + +/// What a pass does for the debt of a held group. +#[derive(Debug, PartialEq, Eq)] +enum DebtStep { + /// The debt persisted after the group's last install. + Kept, + /// Write the debt. + Write { + /// Raise the replica's snapshot requirement too: no install lifted it + /// since this loop last raised it. + raise: bool, + /// The previous write failed: this write continues that failure run. + failing: bool, + }, +} + +impl HeldDebts { + /// An install settles the debt and can lift the requirement, so a hold + /// that outlives one keeps both again. + fn step(&self, group_id: u64, installed_through: u64) -> DebtStep { + match self.groups.get(&group_id) { + Some(kept) if kept.installed_through == installed_through && kept.recorded => { + DebtStep::Kept + } + Some(kept) => DebtStep::Write { + raise: kept.installed_through != installed_through, + failing: !kept.recorded, + }, + None => DebtStep::Write { + raise: true, + failing: false, + }, + } + } + + fn note(&mut self, group_id: u64, installed_through: u64, recorded: bool) { + self.groups.insert( + group_id, + KeptDebt { + installed_through, + recorded, + }, + ); + } + + /// Drop the record of `group_id`: no entry holds its lane. + fn forget(&mut self, group_id: u64) { + self.groups.remove(&group_id); + } + + /// Whether a held group's debt write failed. The apply loop wakes on a + /// timer to write it again. + pub fn retry_due(&self) -> bool { + self.groups.values().any(|kept| !kept.recorded) + } +} + +/// Record the group's snapshot debt and raise its replica's snapshot +/// requirement, once per install of the group. `installed_through` is the +/// group's installed index as the caller read it. +/// +/// The caller holds the group's apply gate or read it open below the held +/// entry, so no install that covers the entry settled the debt first. +pub(super) fn keep_snapshot_owed( + state: &SharedState, + debts: &mut HeldDebts, + group_id: u64, + log_index: u64, + installed_through: u64, +) { + let DebtStep::Write { raise, failing } = debts.step(group_id, installed_through) else { + return; + }; + let written = if state.redo_chunks.owes_snapshot(group_id) { + Ok(()) + } else { + state + .redo_chunks + .record_owed(state.credentials.catalog(), &BTreeSet::from([group_id])) + }; + let recorded = match written { + Ok(()) => { + if failing { + tracing::info!( + group_id, + log_index, + "apply loop: the snapshot debt of a held group persisted" + ); + } + true + } + // One report per failure run: the timed retries collapse into it. + Err(error) if failing => { + tracing::debug!( + group_id, + log_index, + %error, + "apply loop: the snapshot debt of a held group still did not persist" + ); + false + } + Err(error) => { + tracing::error!( + group_id, + log_index, + %error, + "apply loop: the snapshot debt of a held group did not persist; the loop \ + writes it again on each Raft tick" + ); + crate::diag::held_entry_debt_not_recorded(group_id, log_index, &error); + false + } + }; + debts.note(group_id, installed_through, recorded); + // Once the debt persists, the redo-stream membership pass holds the + // requirement up. Until then only this loop does. + if !raise && recorded { + return; + } + let Some(multi_raft) = state.multi_raft.get() else { + return; + }; + // The data here diverged from the log: the replica must not lead. + multi_raft + .lock() + .unwrap_or_else(|p| p.into_inner()) + .require_snapshot(group_id, nodedb_raft::ReplicaState::AwaitingSnapshot); +} + +/// Release the entry that holds `lane` once an installed snapshot covers +/// it. The install's index is read under the group's apply gate, as an +/// entry's admission reads it. An entry still held keeps its group owing a +/// snapshot. +pub(super) fn release_covered_hold( + ctx: ApplyContext<'_>, + lane: &mut Lane, + gate: &mut ProposalGate, + debts: &mut HeldDebts, + group_id: u64, +) -> HoldCheck { + let Some(held) = lane.held_index() else { + debts.forget(group_id); + return HoldCheck::Free; + }; + let gates = ctx.state.raft_apply_gates.get().map(|gates| &**gates); + let covered_through = ctx.tracker.covered_through(group_id); + let admission = admit_entry(gates, group_id, held, covered_through); + let installed_through = admission.installed_through; + // The permit lives to the end: no install starts before the debt is + // kept. + let (through, _permit) = match admission.decision { + EntryAdmission::Installing => return HoldCheck::Installing, + EntryAdmission::Start(permit) => (admission.installed_through, permit), + EntryAdmission::Covered => (admission.installed_through, None), + // Above the install's Raft index and at or below its cut: the + // installed state holds the entry. + EntryAdmission::CoveredByCut => (covered_through, None), + }; + match lane.cover_through(through) { + Some(proposal_key) => { + gate.note_covered(proposal_key); + debts.forget(group_id); + tracing::info!( + group_id, + log_index = held, + installed_through = through, + "apply loop: an installed snapshot covers the held entry; the group's apply \ + resumes" + ); + HoldCheck::Free + } + None => { + keep_snapshot_owed(ctx.state, debts, group_id, held, installed_through); + HoldCheck::Held + } + } +} + +#[cfg(test)] +mod tests { + use super::*; + + const GROUP: u64 = 4; + + /// The debt is written once per install of the group: an install + /// settles it and can lift the requirement. + #[test] + fn a_held_debt_is_kept_once_per_install() { + let mut debts = HeldDebts::default(); + let first = DebtStep::Write { + raise: true, + failing: false, + }; + assert_eq!(debts.step(GROUP, 10), first); + debts.note(GROUP, 10, true); + assert_eq!(debts.step(GROUP, 10), DebtStep::Kept); + assert!(!debts.retry_due()); + assert_eq!(debts.step(GROUP, 12), first, "an install settled the debt"); + } + + /// A failed write is retried as one failure run, across installs, until + /// a write persists or the hold ends. + #[test] + fn a_failed_debt_write_retries_as_one_failure_run() { + let mut debts = HeldDebts::default(); + debts.note(GROUP, 10, false); + assert!(debts.retry_due()); + assert_eq!( + debts.step(GROUP, 10), + DebtStep::Write { + raise: false, + failing: true, + } + ); + assert_eq!( + debts.step(GROUP, 12), + DebtStep::Write { + raise: true, + failing: true, + } + ); + + debts.note(GROUP, 10, true); + assert!(!debts.retry_due()); + + debts.note(GROUP, 10, false); + debts.forget(GROUP); + assert!(!debts.retry_due(), "a released hold retries nothing"); + } +} diff --git a/nodedb/src/control/distributed_applier/apply_loop/lane.rs b/nodedb/src/control/distributed_applier/apply_loop/lane.rs index 9cf667612..04b3a3e6f 100644 --- a/nodedb/src/control/distributed_applier/apply_loop/lane.rs +++ b/nodedb/src/control/distributed_applier/apply_loop/lane.rs @@ -9,6 +9,10 @@ //! order: the group's applied index is the highest index with every earlier //! entry finished, and its durable floor is the highest index with every //! earlier entry durable. +//! +//! An entry this replica cannot apply as its peers do holds the lane. It +//! never settles, the applied index stays below it, and no later entry +//! starts. Only a snapshot install that covers it releases it. use std::collections::VecDeque; @@ -21,7 +25,7 @@ use crate::control::state::tenant_marks::{MarkSite, TenantMarks}; use crate::control::wal_replication::types::CalvinRedoMeta; use crate::control::wal_replication::{ReplicatedEntry, ReplicatedWrite, decode_replicated_entry}; -use super::proposal_gate::PrefixStep; +use super::proposal_gate::{PrefixStep, Settlement}; /// A committed entry handed to the loop and not yet started. pub(super) struct QueuedEntry { @@ -146,6 +150,18 @@ pub(super) enum SlotState { Barrier, /// The entry concluded. Concluded(PrefixStep), + /// The entry holds the lane until a snapshot install covers it. Its + /// reason is reported when the hold starts. + Held, +} + +impl From for SlotState { + fn from(settlement: Settlement) -> Self { + match settlement { + Settlement::Settles(step) => Self::Concluded(step), + Settlement::Holds(_) => Self::Held, + } + } } /// A started entry not yet settled. @@ -199,6 +215,11 @@ impl Lane { } } + /// The group this lane applies. + pub fn group_id(&self) -> u64 { + self.group_id + } + /// The first started entry not yet settled. pub fn front_index(&self) -> Option { self.slots.front().map(|slot| slot.log_index) @@ -209,6 +230,31 @@ impl Lane { self.last_settled } + /// The entry that holds the lane, if any. No entry after it starts, so + /// it is the last started entry. + pub fn held_index(&self) -> Option { + self.slots.iter().find_map(|slot| match slot.state { + SlotState::Held => Some(slot.log_index), + SlotState::Starting + | SlotState::Running + | SlotState::Barrier + | SlotState::Concluded(_) => None, + }) + } + + /// Whether an entry holds the lane. + pub fn is_held(&self) -> bool { + self.held_index().is_some() + } + + /// The proposal key of the entry at `log_index` while it starts or runs. + pub fn running_key(&self, log_index: u64) -> Option { + self.slots.iter().find_map(|slot| { + let running = matches!(slot.state, SlotState::Starting | SlotState::Running); + (slot.log_index == log_index && running).then_some(slot.proposal_key) + }) + } + /// Whether any started entry of the group has not concluded. pub fn has_running(&self) -> bool { self.slots @@ -257,7 +303,7 @@ impl Lane { } /// Conclude the running entry at `log_index`: `conclude` receives its - /// proposal key and returns how it moves the prefix. `wrote_rows` says + /// proposal key and returns how it stands in the lane. `wrote_rows` says /// whether the apply wrote the entry's rows; an entry that wrote none /// raises no tenant write mark. Returns `false` when no running entry /// has that index. @@ -265,7 +311,7 @@ impl Lane { &mut self, log_index: u64, wrote_rows: bool, - conclude: impl FnOnce(u64) -> PrefixStep, + conclude: impl FnOnce(u64) -> Settlement, ) -> bool { let Some(slot) = self.slot_mut(log_index) else { return false; @@ -274,7 +320,7 @@ impl Lane { return false; } slot.user_write &= wrote_rows; - slot.state = SlotState::Concluded(conclude(slot.proposal_key)); + slot.state = SlotState::from(conclude(slot.proposal_key)); if self.blocking == Some(log_index) { self.blocking = None; } @@ -285,12 +331,13 @@ impl Lane { /// barrier's waiter, raise the tenant's write mark in this group, extend /// or break the durable prefix, and advance the applied watermark. The /// mark rises first, so a reader that waited for the applied index sees - /// it. Returns how many entries settled. + /// it. A held entry stops the pass: the applied index stays below it. + /// Returns how many entries settled. pub fn settle(&mut self, tracker: &ProposeTracker, marks: &TenantMarks) -> usize { let mut settled = 0; while let Some(front) = self.slots.front() { let step = match front.state { - SlotState::Starting | SlotState::Running => break, + SlotState::Starting | SlotState::Running | SlotState::Held => break, SlotState::Barrier => { // Every entry before the barrier finished; a waiting // backup can snapshot this group now. @@ -352,13 +399,19 @@ impl Lane { /// Count every entry of the group through `installed_through` durable. /// An installed snapshot holds their state, so a break at or below it - /// heals. + /// heals, and a held entry at or below it concludes. Returns the held + /// entry's proposal key when one concludes. /// /// The caller passes the index it read under the group's apply gate, /// before it admits an entry. Every entry above the index reaches the /// lane after the install, so none settles before this call. - pub fn cover_through(&mut self, installed_through: u64) { + pub fn cover_through(&mut self, installed_through: u64) -> Option { self.prefix.cover_through(installed_through); + let slot = self.slots.iter_mut().find(|slot| { + slot.log_index <= installed_through && matches!(slot.state, SlotState::Held) + })?; + slot.state = SlotState::Concluded(PrefixStep::Record(true)); + Some(slot.proposal_key) } /// Whether the durable floor moved past the floor last saved. @@ -381,8 +434,17 @@ impl Lane { #[cfg(test)] mod tests { + use super::super::proposal_gate::HoldReason; use super::*; + fn settles(durable: bool) -> Settlement { + Settlement::Settles(PrefixStep::Record(durable)) + } + + fn holds() -> Settlement { + Settlement::Holds(HoldReason::BodyMissing) + } + fn slot(log_index: u64, state: SlotState) -> Slot { Slot { log_index, @@ -402,10 +464,10 @@ mod tests { lane.push(slot(6, SlotState::Running)); lane.push(slot(7, SlotState::Running)); - assert!(lane.conclude(7, true, |_| PrefixStep::Record(true))); - assert!(lane.conclude(6, true, |_| PrefixStep::Record(true))); + assert!(lane.conclude(7, true, |_| settles(true))); + assert!(lane.conclude(6, true, |_| settles(true))); assert!( - !lane.conclude(6, true, |_| PrefixStep::Record(true)), + !lane.conclude(6, true, |_| settles(true)), "6 concluded already" ); assert_eq!( @@ -420,24 +482,86 @@ mod tests { "the timeout diagnostic names the entry the group waits behind" ); - assert!(lane.conclude(5, true, |_| PrefixStep::Record(true))); + assert!(lane.conclude(5, true, |_| settles(true))); assert_eq!(lane.settle(&tracker, &TenantMarks::default()), 3); assert_eq!(lane.take_floor_to_save(), Some(7)); assert_eq!(lane.take_floor_to_save(), None, "a saved floor saves once"); assert!(tracker.applying(1).is_none()); } + /// A held entry neither settles nor moves the applied index. Entries + /// before it settle. #[test] - fn a_non_durable_entry_holds_the_floor_across_later_settles() { + fn a_held_entry_neither_settles_nor_advances_the_applied_index() { + let watchers = std::sync::Arc::new(nodedb_cluster::GroupAppliedWatchers::new()); + let tracker = ProposeTracker::new().with_group_watchers(std::sync::Arc::clone(&watchers)); + let marks = TenantMarks::default(); + let mut lane = Lane::new(1); + lane.push(slot(1, SlotState::Running)); + lane.push(slot(2, SlotState::Running)); + assert!(lane.conclude(2, false, |_| holds())); + assert_eq!(lane.held_index(), Some(2)); + assert!(lane.has_running(), "entry 1 still runs"); + + assert!(lane.conclude(1, true, |_| settles(true))); + assert_eq!(lane.settle(&tracker, &marks), 1); + assert_eq!(watchers.get_or_create(1).current(), 1); + assert_eq!(lane.take_floor_to_save(), Some(1)); + + assert_eq!(lane.settle(&tracker, &marks), 0, "entry 2 holds the lane"); + assert_eq!( + watchers.get_or_create(1).current(), + 1, + "the applied index stays below the held entry" + ); + assert_eq!( + tracker.applying(1).map(|entry| entry.log_index), + Some(2), + "the group waits behind the held entry" + ); + assert_eq!(lane.take_floor_to_save(), None); + } + + /// A snapshot that installs below the held entry leaves it held. One + /// that covers it concludes it, and the floor passes it. + #[test] + fn only_a_covering_snapshot_releases_a_held_entry() { + let tracker = ProposeTracker::new(); + let marks = TenantMarks::default(); + let mut lane = Lane::new(1); + lane.push(slot(1, SlotState::Concluded(PrefixStep::Record(true)))); + let mut held = slot(2, SlotState::Running); + held.proposal_key = 77; + lane.push(held); + assert!(lane.conclude(2, false, |_| holds())); + lane.settle(&tracker, &marks); + assert_eq!(lane.take_floor_to_save(), Some(1)); + + assert_eq!(lane.cover_through(1), None, "the install is below entry 2"); + assert!(lane.is_held()); + assert_eq!(lane.settle(&tracker, &marks), 0); + + assert_eq!(lane.cover_through(4), Some(77)); + assert!(!lane.is_held()); + assert_eq!(lane.settle(&tracker, &marks), 1); + assert_eq!(lane.last_settled(), Some(2)); + assert_eq!(lane.take_floor_to_save(), Some(4)); + } + + /// A transient failure breaks the floor and the lane moves on. A later + /// entry still settles, and the floor holds below the break. + #[test] + fn a_non_durable_entry_holds_the_floor_but_not_the_lane() { let tracker = ProposeTracker::new(); let mut lane = Lane::new(1); lane.push(slot(1, SlotState::Concluded(PrefixStep::Record(true)))); lane.push(slot(2, SlotState::Concluded(PrefixStep::Record(false)))); - lane.settle(&tracker, &TenantMarks::default()); + assert_eq!(lane.settle(&tracker, &TenantMarks::default()), 2); assert_eq!(lane.take_floor_to_save(), Some(1)); lane.push(slot(3, SlotState::Concluded(PrefixStep::Record(true)))); - lane.settle(&tracker, &TenantMarks::default()); + assert_eq!(lane.settle(&tracker, &TenantMarks::default()), 1); + assert!(!lane.is_held()); assert_eq!( lane.take_floor_to_save(), None, @@ -446,7 +570,7 @@ mod tests { } #[test] - fn a_covering_snapshot_lets_the_floor_pass_a_held_entry() { + fn a_covering_snapshot_lets_the_floor_pass_a_broken_entry() { let tracker = ProposeTracker::new(); let marks = TenantMarks::default(); let mut lane = Lane::new(1); @@ -457,10 +581,10 @@ mod tests { assert_eq!(lane.take_floor_to_save(), Some(1)); // A snapshot at index 4 installs while entry 3 still runs. - lane.cover_through(4); + assert_eq!(lane.cover_through(4), None); assert_eq!(lane.take_floor_to_save(), Some(4)); - assert!(lane.conclude(3, false, |_| PrefixStep::Record(false))); + assert!(lane.conclude(3, false, |_| settles(false))); lane.push(slot(5, SlotState::Concluded(PrefixStep::Record(true)))); lane.settle(&tracker, &marks); assert_eq!( @@ -471,11 +595,11 @@ mod tests { } #[test] - fn a_held_entry_above_the_snapshot_still_holds_the_floor() { + fn a_broken_entry_above_the_snapshot_still_holds_the_floor() { let tracker = ProposeTracker::new(); let marks = TenantMarks::default(); let mut lane = Lane::new(1); - lane.cover_through(4); + assert_eq!(lane.cover_through(4), None); lane.push(slot(5, SlotState::Concluded(PrefixStep::Record(false)))); lane.push(slot(6, SlotState::Concluded(PrefixStep::Record(true)))); lane.settle(&tracker, &marks); @@ -559,7 +683,7 @@ mod tests { refused.collection = Some("docs".to_owned()); lane.push(refused); - assert!(lane.conclude(1, false, |_| PrefixStep::Record(true))); + assert!(lane.conclude(1, false, |_| settles(true))); lane.settle(&tracker, &marks); assert_eq!( marks.get(4, 7), @@ -579,7 +703,7 @@ mod tests { lane.settle(&tracker, &TenantMarks::default()); assert!(barrier.try_recv().is_err(), "entry 1 still runs"); - assert!(lane.conclude(1, true, |_| PrefixStep::Record(true))); + assert!(lane.conclude(1, true, |_| settles(true))); lane.settle(&tracker, &TenantMarks::default()); assert!(matches!(barrier.try_recv(), Ok(Ok(_)))); } diff --git a/nodedb/src/control/distributed_applier/apply_loop/metadata_floor.rs b/nodedb/src/control/distributed_applier/apply_loop/metadata_floor.rs index 9194ee802..180afd7c2 100644 --- a/nodedb/src/control/distributed_applier/apply_loop/metadata_floor.rs +++ b/nodedb/src/control/distributed_applier/apply_loop/metadata_floor.rs @@ -85,17 +85,11 @@ async fn await_metadata_floor(state: &SharedState, held: HeldEntry) -> crate::Re metadata_applied = watcher.current(), "a replicated write holds its group's lane for this node's metadata apply" ); + let floor = held.metadata_floor; loop { - let waiting = Arc::clone(&watcher); - let floor = held.metadata_floor; - let outcome = tokio::task::spawn_blocking(move || waiting.wait_for(floor, WAIT_SLICE)) - .await - .map_err(|e| crate::Error::Internal { - detail: format!( - "raft group {} entry {}: the metadata catch-up wait did not finish: {e}", - held.group_id, held.log_index - ), - })?; + let outcome = watcher + .wait_for_async(floor, std::time::Instant::now() + WAIT_SLICE) + .await; match outcome { WaitOutcome::Reached => return Ok(()), WaitOutcome::TimedOut => tracing::warn!( diff --git a/nodedb/src/control/distributed_applier/apply_loop/mod.rs b/nodedb/src/control/distributed_applier/apply_loop/mod.rs index 031510b6d..2f1c023f5 100644 --- a/nodedb/src/control/distributed_applier/apply_loop/mod.rs +++ b/nodedb/src/control/distributed_applier/apply_loop/mod.rs @@ -18,6 +18,9 @@ //! applies. //! - [`pipeline`]: every group's lane and the applies that run. //! - [`lane`]: one group's queued and started entries, settled in log order. +//! - [`settle`]: settles every group's concluded entries and saves the +//! floors that moved. +//! - [`panicked`]: an entry whose enqueue or apply panicked holds its group. //! - [`start`]: prepares one entry and routes it to its apply path. //! - [`cut_barrier`]: a backup cut's barrier: the group's cut floor, the //! restore point's place, and a database backup's capture. @@ -41,6 +44,8 @@ //! and records each applied proposal in the ledger. //! - [`snapshot_gate`]: skips an entry an installed snapshot covers, and //! orders each write before a later snapshot restore. +//! - [`held_entry`]: an entry this replica cannot apply as its peers do +//! holds its group's lane until a snapshot install covers it. //! - [`group_watch`]: per-group second-apply detection and backup cut floors. //! - [`bookkeeping`]: applied-floor persistence + Raft log compaction trigger. //! - [`helpers`]: shared response/result classification helpers. @@ -61,12 +66,15 @@ mod context; mod cut_barrier; mod driver; mod group_watch; +mod held_entry; mod helpers; mod lane; mod metadata_floor; +mod panicked; mod pipeline; mod proposal_gate; mod redo_chunk; +mod settle; mod snapshot_gate; mod start; mod surrogate_bind; diff --git a/nodedb/src/control/distributed_applier/apply_loop/panicked.rs b/nodedb/src/control/distributed_applier/apply_loop/panicked.rs new file mode 100644 index 000000000..4bedfec57 --- /dev/null +++ b/nodedb/src/control/distributed_applier/apply_loop/panicked.rs @@ -0,0 +1,96 @@ +// SPDX-License-Identifier: BUSL-1.1 + +//! An entry whose enqueue or apply panicked. +//! +//! The loop polls every running entry in one task, so an unguarded panic +//! ends the loop for every group and leaves each waiter to its deadline. +//! A guarded panic concludes its own entry instead: +//! - Its effect on this replica is unknown, so the entry holds its group's +//! lane like an entry this replica cannot apply as its peers do. No later +//! entry of the group starts until a snapshot install covers it. +//! - Its waiter receives an error at once. +//! - The panic files a report. +//! +//! Other groups keep applying. + +use std::panic::AssertUnwindSafe; + +use futures::FutureExt; + +use crate::data::panic_payload::panic_payload_to_string; + +use super::context::{ApplyContext, ApplyFuture, LoopEvent, LoopFuture}; +use super::lane::{Lane, SlotState}; +use super::proposal_gate::{EntryOutcome, HoldReason, ProposalGate}; + +/// Run `future`, the enqueue or apply of the entry at `log_index` of +/// `group_id`. A panic inside it becomes [`LoopEvent::Panicked`]. +pub(super) fn guarded(group_id: u64, log_index: u64, future: LoopFuture<'_>) -> LoopFuture<'_> { + Box::pin( + AssertUnwindSafe(future) + .catch_unwind() + .map(move |polled| match polled { + Ok(event) => event, + Err(payload) => LoopEvent::Panicked { + group_id, + log_index, + detail: panic_payload_to_string(payload.as_ref()), + }, + }), + ) +} + +/// The guarded event of an entry's running apply. +pub(super) fn finished_event( + group_id: u64, + log_index: u64, + apply: ApplyFuture<'_>, +) -> LoopFuture<'_> { + guarded( + group_id, + log_index, + Box::pin(apply.map(LoopEvent::Finished)), + ) +} + +/// Conclude the entry at `log_index` whose enqueue or apply panicked: report +/// it, answer its waiter with an error, and hold `lane`. Returns `false` when +/// no entry at that index starts or runs. +pub(super) fn conclude_panicked( + ctx: ApplyContext<'_>, + lane: &mut Lane, + gate: &mut ProposalGate, + log_index: u64, + detail: &str, +) -> bool { + let group_id = lane.group_id(); + let Some(proposal_key) = lane.running_key(log_index) else { + return false; + }; + tracing::error!( + group_id, + log_index, + "apply loop: the committed entry's apply panicked; the group's apply holds at it \ + until a snapshot of the group installs" + ); + crate::diag::apply_panicked(group_id, log_index, detail); + ctx.tracker.complete( + group_id, + log_index, + proposal_key, + Err(crate::Error::Internal { + detail: format!( + "the apply of group {group_id} entry {log_index} panicked on this replica; \ + the group's apply holds until a snapshot installs" + ), + }), + ); + ctx.tracker.note_concluded(group_id, log_index); + // A panicked enqueue leaves its entry starting. A panicked apply leaves + // it running, and this is a no-op. + lane.enqueued(log_index, SlotState::Running, None, false); + let outcome = EntryOutcome::Held { + reason: HoldReason::ApplyPanicked, + }; + lane.conclude(log_index, false, |key| gate.conclude(key, true, outcome)) +} diff --git a/nodedb/src/control/distributed_applier/apply_loop/pipeline.rs b/nodedb/src/control/distributed_applier/apply_loop/pipeline.rs index f99e02c5e..ad5dd1fc7 100644 --- a/nodedb/src/control/distributed_applier/apply_loop/pipeline.rs +++ b/nodedb/src/control/distributed_applier/apply_loop/pipeline.rs @@ -9,6 +9,10 @@ //! and no other: later entries of its group start and finish, and other //! groups never wait on it. Only the group's applied index and durable floor //! wait for it to finish. +//! +//! An entry this replica cannot apply as its peers do holds its group: no +//! later entry of the group starts until a snapshot install covers it (see +//! [`super::held_entry`]). use std::collections::HashMap; use std::sync::Arc; @@ -18,15 +22,17 @@ use futures::stream::{FuturesUnordered, StreamExt}; use crate::control::distributed_applier::applier::ApplyBatch; use crate::control::distributed_applier::proposal_ledger::ProposalLedger; -use crate::control::server::dispatch_utils::publish_settled_changes; use crate::control::state::SharedState; -use super::bookkeeping::record_durable_apply; -use super::context::{ApplyContext, ApplyFuture, LoopEvent, LoopFuture, Started, StartedEntry}; +use super::context::{ApplyContext, LoopEvent, LoopFuture, Started, StartedEntry}; use super::group_watch::GroupWatch; +use super::held_entry::{ + HeldDebts, HoldCheck, keep_snapshot_owed, note_hold, release_covered_hold, +}; use super::lane::{Lane, QueuedEntry, Slot, SlotState}; use super::metadata_floor::{HeldEntry, hold_for_metadata}; -use super::proposal_gate::{EntryOutcome, ProposalGate}; +use super::panicked::{conclude_panicked, finished_event, guarded}; +use super::proposal_gate::{EntryOutcome, HoldReason, ProposalGate}; use super::snapshot_gate::{EntryAdmission, admit_entry, hold_permit}; use super::start::{Prepared, prepare_entry}; @@ -37,6 +43,7 @@ pub(super) struct Pipeline<'a> { gate: ProposalGate, watch: GroupWatch, running: FuturesUnordered>, + debts: HeldDebts, /// Set by a pump that left an entry queued because a snapshot install /// held its group's apply gate. install_blocked: bool, @@ -50,14 +57,22 @@ impl<'a> Pipeline<'a> { gate: ProposalGate::new(ledger), watch: GroupWatch::with_cuts(ctx.state.pitr.recorded_cuts()), running: FuturesUnordered::new(), + debts: HeldDebts::default(), install_blocked: false, } } - /// Whether the last pump left an entry queued behind a snapshot install. - /// The loop pumps again once an install releases its gate. - pub fn install_blocked(&self) -> bool { - self.install_blocked + /// Whether a group waits for a snapshot install: the last pump left an + /// entry queued behind one, or an entry holds its group's lane. The loop + /// pumps again once an install releases its gate. + pub fn awaits_install(&self) -> bool { + self.install_blocked || self.lanes.values().any(Lane::is_held) + } + + /// Whether a held group's snapshot debt did not persist. The loop wakes + /// on a timer to write it again. + pub fn retries_debt(&self) -> bool { + self.debts.retry_due() } /// Queue a batch the applier handed off, behind its group's earlier @@ -116,13 +131,34 @@ impl<'a> Pipeline<'a> { .note_concluded(finished.group_id, finished.log_index); let gate = &mut self.gate; let wrote_rows = finished.outcome.wrote_rows(); + let hold = finished.outcome.hold_reason(); let handled = self.lanes.get_mut(&finished.group_id).is_some_and(|lane| { lane.conclude(finished.log_index, wrote_rows, |proposal_key| { gate.conclude(proposal_key, true, finished.outcome) }) }); + // The pump that follows records the group's debt under its + // apply gate. + if handled && let Some(reason) = hold { + note_hold(finished.group_id, finished.log_index, reason); + } (finished.group_id, finished.log_index, handled) } + LoopEvent::Panicked { + group_id, + log_index, + detail, + } => { + let gate = &mut self.gate; + let handled = self.lanes.get_mut(&group_id).is_some_and(|lane| { + conclude_panicked(self.ctx, lane, gate, log_index, &detail) + }); + // The pump that follows records the group's debt. + if handled { + note_hold(group_id, log_index, HoldReason::ApplyPanicked); + } + (group_id, log_index, handled) + } }; // The entry's enqueue returned, or its exclusive apply finished: // every entry of the group through it reached its core. @@ -150,7 +186,8 @@ impl<'a> Pipeline<'a> { }; match started { Started::Running(apply) => { - self.running.push(finished_event(apply)); + self.running + .push(finished_event(group_id, log_index, apply)); lane.enqueued(log_index, SlotState::Running, collection, user_write) } Started::Concluded(outcome) => { @@ -159,10 +196,15 @@ impl<'a> Pipeline<'a> { self.ctx.tracker.note_concluded(group_id, log_index); let gate = &mut self.gate; let wrote_rows = outcome.wrote_rows(); - lane.enqueued(log_index, SlotState::Running, collection, user_write) + let hold = outcome.hold_reason(); + let handled = lane.enqueued(log_index, SlotState::Running, collection, user_write) && lane.conclude(log_index, wrote_rows, |proposal_key| { gate.conclude(proposal_key, true, outcome) - }) + }); + if handled && let Some(reason) = hold { + note_hold(group_id, log_index, reason); + } + handled } } } @@ -178,7 +220,7 @@ impl<'a> Pipeline<'a> { let groups: Vec = self .lanes .iter() - .filter(|(_, lane)| !lane.backlog.is_empty()) + .filter(|(_, lane)| !lane.backlog.is_empty() || lane.is_held()) .map(|(group_id, _)| *group_id) .collect(); for group_id in groups { @@ -196,6 +238,17 @@ impl<'a> Pipeline<'a> { fn pump_group(&mut self, group_id: u64) { let shared: &'a Arc = self.ctx.state; let gates = shared.raft_apply_gates.get().map(|gates| &**gates); + let Some(lane) = self.lanes.get_mut(&group_id) else { + return; + }; + match release_covered_hold(self.ctx, lane, &mut self.gate, &mut self.debts, group_id) { + HoldCheck::Free => {} + HoldCheck::Held => return, + HoldCheck::Installing => { + self.install_blocked = true; + return; + } + } while let Some(queued) = self.next_startable(group_id) { let log_index = queued.entry.index; let proposal_key = queued.proposal_key(); @@ -203,8 +256,12 @@ impl<'a> Pipeline<'a> { let admission = admit_entry(gates, group_id, log_index, covered_through); // Before the entry joins the lane: no entry above the installed // index settles before the lane covers it. - if let Some(lane) = self.lanes.get_mut(&group_id) { - lane.cover_through(admission.installed_through); + if let Some(lane) = self.lanes.get_mut(&group_id) + && let Some(proposal_key) = lane.cover_through(admission.installed_through) + { + // The install holds every entry through its index: a held + // one concludes. + self.gate.note_covered(proposal_key); } let permit = match admission.decision { EntryAdmission::Installing => { @@ -282,12 +339,14 @@ impl<'a> Pipeline<'a> { held, prepare_entry(self.ctx, &mut self.watch, &self.gate, group_id, queued), ); + let mut hold = None; let (state, blocks, user_write) = match prepared { Prepared::Concluded(outcome) => { release_admission_holds(shared, group_id, log_index); let user_write = matches!(outcome, EntryOutcome::Repeat) && repeat_writes; + hold = outcome.hold_reason(); ( - SlotState::Concluded(self.gate.conclude(proposal_key, false, outcome)), + SlotState::from(self.gate.conclude(proposal_key, false, outcome)), false, user_write, ) @@ -300,12 +359,12 @@ impl<'a> Pipeline<'a> { self.gate.open(proposal_key); self.ctx.tracker.note_dispatched(group_id, log_index); let enqueue = hold_permit(permit, enqueue); - self.running - .push(Box::pin(enqueue.map(move |entry| LoopEvent::Enqueued { - group_id, - log_index, - entry, - }))); + let enqueued = Box::pin(enqueue.map(move |entry| LoopEvent::Enqueued { + group_id, + log_index, + entry, + })); + self.running.push(guarded(group_id, log_index, enqueued)); // The enqueue reports whether the plan writes user data. (SlotState::Starting, true, false) } @@ -313,8 +372,11 @@ impl<'a> Pipeline<'a> { // An array op or cell write: user data. self.gate.open(proposal_key); self.ctx.tracker.note_dispatched(group_id, log_index); - self.running - .push(finished_event(hold_permit(permit, apply))); + self.running.push(finished_event( + group_id, + log_index, + hold_permit(permit, apply), + )); (SlotState::Running, true, true) } }; @@ -332,6 +394,19 @@ impl<'a> Pipeline<'a> { user_write, state, }); + // The entry's admission permit is still held: no install that + // covers the entry settles the debt first. + if let Some(reason) = hold { + note_hold(group_id, log_index, reason); + let installed_through = admission.installed_through; + keep_snapshot_owed( + shared, + &mut self.debts, + group_id, + log_index, + installed_through, + ); + } } } @@ -350,7 +425,7 @@ impl<'a> Pipeline<'a> { self.ctx .tracker .complete_covered(group_id, log_index, proposal_key); - let concluded = SlotState::Concluded(self.gate.conclude(proposal_key, false, outcome)); + let concluded = SlotState::from(self.gate.conclude(proposal_key, false, outcome)); let Some(lane) = self.lanes.get_mut(&group_id) else { return false; }; @@ -367,13 +442,14 @@ impl<'a> Pipeline<'a> { /// Take the next entry of `group_id` when it can start now. /// - /// It waits while the group's previous write is in its enqueue or an - /// exclusive entry of the group runs, while it is exclusive and an - /// earlier entry of the group has not concluded, and while a copy of its - /// proposal runs: the ledger decides it once that copy concludes. + /// It waits while an entry holds the group's lane, while the group's + /// previous write is in its enqueue or an exclusive entry of the group + /// runs, while it is exclusive and an earlier entry of the group has not + /// concluded, and while a copy of its proposal runs: the ledger decides + /// it once that copy concludes. fn next_startable(&mut self, group_id: u64) -> Option { let lane = self.lanes.get_mut(&group_id)?; - if lane.blocking.is_some() { + if lane.blocking.is_some() || lane.is_held() { return None; } let front = lane.backlog.front()?; @@ -386,49 +462,10 @@ impl<'a> Pipeline<'a> { lane.backlog.pop_front() } - /// Settle every group's concluded entries in log order, release their - /// window, and save each durable floor that moved. - /// - /// The tenant write marks of the settled entries are persisted before any - /// floor that covers them. An entry above the saved floor is delivered - /// again after a restart and records its mark again, so every committed - /// write keeps a durable mark. + /// Settle every group's concluded entries in log order. See + /// [`super::settle::settle_lanes`]. pub fn settle(&mut self) { - let state = self.ctx.state; - let tracker = self.ctx.tracker; - for (group_id, lane) in &mut self.lanes { - let first = lane.front_index(); - let settled = lane.settle(tracker, &state.tenant_marks); - if settled > 0 { - tracker.window().release(*group_id, settled); - // The entries' change events publish in log order, at their - // log positions, on every replica. - if let (Some(first), Some(last)) = (first, lane.last_settled()) { - publish_settled_changes(state, *group_id, first, last); - } - } - if !lane.floor_pending() { - continue; - } - // One save per pass that moved the floor, never one per entry: - // each save is an fsync, and the pass coalesces every apply that - // finished before it. - if let Err(error) = state.tenant_marks.persist(state.credentials.catalog()) { - // The floor stays where it is, so every entry above it keeps - // its place in the log. The next pass persists and saves again. - tracing::error!( - group_id = *group_id, - %error, - "apply loop: tenant write marks did not persist; the applied floor waits" - ); - continue; - } - if let Some(floor) = lane.take_floor_to_save() { - // Every entry the floor covers keeps its changes on disk. - state.change_stream.journal_group(*group_id, floor); - record_durable_apply(state, *group_id, floor); - } - } + super::settle::settle_lanes(self.ctx, &mut self.lanes); } } @@ -441,6 +478,233 @@ fn release_admission_holds(state: &SharedState, group_id: u64, log_index: u64) { .release_through(group_id, log_index); } -fn finished_event(apply: ApplyFuture<'_>) -> LoopFuture<'_> { - Box::pin(apply.map(LoopEvent::Finished)) +#[cfg(test)] +mod tests { + use nodedb_cluster::{GroupAppliedWatchers, GroupApplyGates}; + use nodedb_raft::message::LogEntry; + + use super::*; + use crate::bridge::dispatch::Dispatcher; + use crate::control::distributed_applier::proposal_ledger::PROPOSAL_LEDGER_CAPACITY; + use crate::control::distributed_applier::propose_tracker::ProposeTracker; + use crate::wal::WalManager; + + use super::super::context::FinishedApply; + use super::super::proposal_gate::PrefixStep; + + const GROUP: u64 = 4; + + fn slot(log_index: u64, state: SlotState) -> Slot { + Slot { + log_index, + proposal_key: 0, + collection: None, + write_mark: None, + user_write: false, + state, + } + } + + /// Entry 5 holds its group with entry 6 queued behind it. Entry 6 never + /// starts and the applied index stays at 4 until an install covers 6. + /// Then 5 and 6 conclude covered and the applied index moves to 6. + #[tokio::test] + async fn a_held_entry_stops_its_group_until_an_install_covers_it() { + let dir = tempfile::tempdir().expect("tempdir"); + let wal = Arc::new(WalManager::open_for_testing(&dir.path().join("wal")).expect("wal")); + let (dispatcher, _sides) = Dispatcher::new(1, 64); + let state = SharedState::new(dispatcher, wal).expect("shared state"); + let gates = Arc::new(GroupApplyGates::new()); + gates.mount(GROUP); + assert!(state.raft_apply_gates.set(Arc::clone(&gates)).is_ok()); + let watchers = Arc::new(GroupAppliedWatchers::new()); + let tracker = Arc::new(ProposeTracker::new().with_group_watchers(Arc::clone(&watchers))); + let ctx = ApplyContext { + state: &state, + tracker: &tracker, + }; + let mut pipeline = Pipeline::new(ctx, ProposalLedger::new(PROPOSAL_LEDGER_CAPACITY)); + + let mut lane = Lane::new(GROUP); + lane.push(slot(4, SlotState::Concluded(PrefixStep::Record(true)))); + lane.push(slot(5, SlotState::Held)); + lane.backlog.push_back(QueuedEntry::new(LogEntry { + term: 1, + index: 6, + data: Vec::new(), + })); + pipeline.lanes.insert(GROUP, lane); + + pipeline.pump(); + pipeline.settle(); + assert_eq!( + pipeline.lanes[&GROUP].backlog.len(), + 1, + "entry 6 waits behind the held entry" + ); + assert!( + tracker.started_through(GROUP) < 6, + "entry 6 never starts while entry 5 holds" + ); + assert_eq!(watchers.get_or_create(GROUP).current(), 4); + assert!(pipeline.awaits_install()); + assert!( + state.redo_chunks.owes_snapshot(GROUP), + "a held group owes a snapshot install" + ); + assert!(!pipeline.retries_debt(), "the debt persisted"); + + // A snapshot through entry 4 installs: entry 5 still holds. + gates.install(GROUP).await.adopted(4); + pipeline.pump(); + pipeline.settle(); + assert!(pipeline.lanes[&GROUP].is_held()); + assert_eq!(watchers.get_or_create(GROUP).current(), 4); + + // A snapshot through entry 6 installs. + gates.install(GROUP).await.adopted(6); + pipeline.pump(); + pipeline.settle(); + let lane = &pipeline.lanes[&GROUP]; + assert!(!lane.is_held()); + assert!(lane.backlog.is_empty()); + assert_eq!(tracker.started_through(GROUP), 6); + assert_eq!( + watchers.get_or_create(GROUP).current(), + 6, + "entries 5 and 6 settle once the install covers them" + ); + assert!(!pipeline.awaits_install()); + } + + /// The apply of a running entry that never returns a result. + fn exploding_apply() -> FinishedApply { + panic!("the apply of entry 5 exploded") + } + + /// Entry 5's apply panics with entry 6 queued behind it. Its waiter + /// receives an error at once, entry 5 holds the group, entry 6 never + /// starts, and the applied index stays at 4. + #[tokio::test] + async fn a_panicking_apply_answers_its_waiter_and_holds_its_group() { + let dir = tempfile::tempdir().expect("tempdir"); + let wal = Arc::new(WalManager::open_for_testing(&dir.path().join("wal")).expect("wal")); + let (dispatcher, _sides) = Dispatcher::new(1, 64); + let state = SharedState::new(dispatcher, wal).expect("shared state"); + let gates = Arc::new(GroupApplyGates::new()); + gates.mount(GROUP); + assert!(state.raft_apply_gates.set(Arc::clone(&gates)).is_ok()); + let watchers = Arc::new(GroupAppliedWatchers::new()); + let tracker = Arc::new(ProposeTracker::new().with_group_watchers(Arc::clone(&watchers))); + let ctx = ApplyContext { + state: &state, + tracker: &tracker, + }; + let mut pipeline = Pipeline::new(ctx, ProposalLedger::new(PROPOSAL_LEDGER_CAPACITY)); + + const KEY: u64 = 77; + let waiter = tracker.register(GROUP, 5, KEY); + let mut lane = Lane::new(GROUP); + lane.push(slot(4, SlotState::Concluded(PrefixStep::Record(true)))); + lane.push(Slot { + proposal_key: KEY, + ..slot(5, SlotState::Running) + }); + lane.backlog.push_back(QueuedEntry::new(LogEntry { + term: 1, + index: 6, + data: Vec::new(), + })); + pipeline.lanes.insert(GROUP, lane); + pipeline.running.push(finished_event( + GROUP, + 5, + Box::pin(async { exploding_apply() }), + )); + + let event = pipeline.next_event().await.expect("the apply concludes"); + assert!(matches!(event, LoopEvent::Panicked { log_index: 5, .. })); + pipeline.handle(event); + pipeline.pump(); + pipeline.settle(); + + let answered = waiter.await.expect("the waiter is answered"); + assert!(answered.is_err(), "a panicked apply answers with an error"); + let lane = &pipeline.lanes[&GROUP]; + assert_eq!(lane.held_index(), Some(5), "entry 5 holds its group"); + assert_eq!(lane.backlog.len(), 1, "entry 6 waits behind entry 5"); + assert_eq!(watchers.get_or_create(GROUP).current(), 4); + assert!(pipeline.awaits_install()); + assert!( + state.redo_chunks.owes_snapshot(GROUP), + "a group held by a panic owes a snapshot install" + ); + } + + /// A held group's debt write fails twice and files one report. The + /// loop's Raft-tick wake writes it again, with no batch or install, and + /// the debt persists. + /// + /// The recorder is process-wide: nextest runs this test in its own + /// process. + #[tokio::test(start_paused = true)] + async fn a_failed_debt_write_persists_on_a_later_tick() { + let reports = tempfile::tempdir().expect("reports dir"); + assert!( + faultbox::init( + faultbox::Config::new("nodedb-test", "0.0.0", reports.path()) + .install_panic_hook(false), + ), + "this test must be the first initializer of the recorder" + ); + let dir = tempfile::tempdir().expect("tempdir"); + let wal = Arc::new(WalManager::open_for_testing(&dir.path().join("wal")).expect("wal")); + let (dispatcher, _sides) = Dispatcher::new(1, 64); + let state = SharedState::new(dispatcher, wal).expect("shared state"); + let gates = Arc::new(GroupApplyGates::new()); + gates.mount(GROUP); + assert!(state.raft_apply_gates.set(Arc::clone(&gates)).is_ok()); + let watchers = Arc::new(GroupAppliedWatchers::new()); + let tracker = Arc::new(ProposeTracker::new().with_group_watchers(watchers)); + let ctx = ApplyContext { + state: &state, + tracker: &tracker, + }; + let mut pipeline = Pipeline::new(ctx, ProposalLedger::new(PROPOSAL_LEDGER_CAPACITY)); + let mut lane = Lane::new(GROUP); + lane.push(slot(4, SlotState::Concluded(PrefixStep::Record(true)))); + lane.push(slot(5, SlotState::Held)); + pipeline.lanes.insert(GROUP, lane); + let catalog = state.credentials.catalog(); + + catalog.fail_next_redo_snapshot_owed_write_for_test(); + pipeline.pump(); + assert!(pipeline.retries_debt()); + catalog.fail_next_redo_snapshot_owed_write_for_test(); + pipeline.pump(); + assert!(pipeline.retries_debt(), "the second write failed too"); + assert!(!state.redo_chunks.owes_snapshot(GROUP)); + + // The sender stays open: only the timed wake can pump. + let (_tx, mut rx) = tokio::sync::mpsc::channel::(1); + let ran = tokio::time::timeout( + std::time::Duration::from_secs(1), + super::super::driver::drive(&mut pipeline, &mut rx, &state), + ) + .await; + assert!(ran.is_err(), "the loop waits for the held entry's install"); + assert!(!pipeline.retries_debt(), "a tick wrote the debt"); + assert!(state.redo_chunks.owes_snapshot(GROUP)); + assert!(pipeline.lanes[&GROUP].is_held()); + + let debt_reports: u64 = faultbox::reader::list(reports.path()) + .expect("list reports") + .iter() + .filter(|group| { + group.first.domain_kind.as_deref() == Some("nodedb.held_entry_debt_not_recorded") + }) + .map(faultbox::reader::Group::occurrences) + .sum(); + assert_eq!(debt_reports, 1, "one report for the failure run"); + } } diff --git a/nodedb/src/control/distributed_applier/apply_loop/proposal_gate.rs b/nodedb/src/control/distributed_applier/apply_loop/proposal_gate.rs index 7eda3df9d..f881d6d4e 100644 --- a/nodedb/src/control/distributed_applier/apply_loop/proposal_gate.rs +++ b/nodedb/src/control/distributed_applier/apply_loop/proposal_gate.rs @@ -34,16 +34,50 @@ pub(super) enum EntryOutcome { durable: bool, result: Option, }, + /// This replica cannot apply the entry as its peers do. The entry holds + /// its group's lane until a snapshot install covers it. + Held { reason: HoldReason }, +} + +/// Why this replica holds a committed entry it cannot apply as its peers do. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub(super) enum HoldReason { + /// The redo body is missing here: this replica dropped the chunk stream + /// the entry names. + BodyMissing, + /// The entry's bytes do not decode as the redo they name. + Malformed, + /// The entry's enqueue or apply panicked here, so its effect on this + /// replica is unknown. + ApplyPanicked, +} + +impl HoldReason { + pub fn as_str(self) -> &'static str { + match self { + Self::BodyMissing => "body_missing", + Self::Malformed => "malformed", + Self::ApplyPanicked => "apply_panicked", + } + } } impl EntryOutcome { + /// Why the entry holds its group's lane, when it does. + pub fn hold_reason(&self) -> Option { + match self { + Self::Held { reason } => Some(*reason), + Self::Skipped | Self::Repeat | Self::Covered | Self::Applied { .. } => None, + } + } + /// Whether the apply wrote the entry's rows. A refused or failed apply /// wrote none, so it raises no tenant write mark. A second copy writes /// nothing itself: its mark follows its first copy (see /// [`ProposalGate::prior_wrote_rows`]). pub fn wrote_rows(&self) -> bool { match self { - Self::Skipped | Self::Repeat | Self::Covered => false, + Self::Skipped | Self::Repeat | Self::Covered | Self::Held { .. } => false, Self::Applied { result: Some(outcome), .. @@ -66,6 +100,16 @@ pub(super) enum PrefixStep { Record(bool), } +/// How a concluded entry stands in its group's lane. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub(super) enum Settlement { + /// The entry settles in log order and moves the durable prefix by its + /// step. + Settles(PrefixStep), + /// The entry holds its group's lane until a snapshot install covers it. + Holds(HoldReason), +} + /// The outcome a waiter received, in the form the ledger keeps. A refusal is /// kept as its typed code; an error with no code keeps its message. pub(super) fn ledger_outcome(result: &crate::Result) -> AppliedOutcome { @@ -147,6 +191,14 @@ impl ProposalGate { self.ledger.note(proposal_key, None); } + /// Note a proposal an installed snapshot holds: a later copy of it is a + /// duplicate here too. + pub fn note_covered(&mut self, proposal_key: u64) { + if proposal_key != 0 { + self.ledger.note(proposal_key, None); + } + } + /// Note that a copy of `proposal_key` started and has not concluded. pub fn open(&mut self, proposal_key: u64) { if proposal_key != 0 { @@ -156,7 +208,7 @@ impl ProposalGate { /// Conclude an entry of `proposal_key` with `outcome`: note a durable /// apply in the ledger, close the copy `open` noted when `opened`, and - /// return how the entry moves its group's prefix. + /// return how the entry stands in its group's lane. /// /// No separate marker is written: the entry's own records carry its key, /// durable in the same write as its effect. @@ -165,7 +217,7 @@ impl ProposalGate { proposal_key: u64, opened: bool, outcome: EntryOutcome, - ) -> PrefixStep { + ) -> Settlement { if opened && proposal_key != 0 && let Some(copies) = self.in_flight.get_mut(&proposal_key) @@ -176,21 +228,21 @@ impl ProposalGate { } } match outcome { - EntryOutcome::Skipped => PrefixStep::Neutral, - EntryOutcome::Repeat => PrefixStep::Record(true), + EntryOutcome::Skipped => Settlement::Settles(PrefixStep::Neutral), + EntryOutcome::Repeat => Settlement::Settles(PrefixStep::Record(true)), EntryOutcome::Covered => { - // A later copy of the proposal is a duplicate here too. - if proposal_key != 0 { - self.ledger.note(proposal_key, None); - } - PrefixStep::Record(true) + self.note_covered(proposal_key); + Settlement::Settles(PrefixStep::Record(true)) } EntryOutcome::Applied { durable, result } => { if durable { self.ledger.note(proposal_key, result); } - PrefixStep::Record(durable) + Settlement::Settles(PrefixStep::Record(durable)) } + // No outcome is recorded: the install that releases the entry + // notes its proposal. + EntryOutcome::Held { reason } => Settlement::Holds(reason), } } } @@ -216,7 +268,7 @@ mod tests { result: Some(Ok(AppliedWrite::unversioned(Vec::new()))), }, ); - assert_eq!(step, PrefixStep::Record(true)); + assert_eq!(step, Settlement::Settles(PrefixStep::Record(true))); assert!(!gate.in_flight(42)); assert!(gate.skip_duplicate(&tracker, 1, 9, 42)); } @@ -234,7 +286,7 @@ mod tests { result: None, }, ); - assert_eq!(step, PrefixStep::Record(false)); + assert_eq!(step, Settlement::Settles(PrefixStep::Record(false))); assert!(!gate.in_flight(7)); assert!(!gate.skip_duplicate(&tracker, 1, 9, 7)); } @@ -246,8 +298,28 @@ mod tests { let tracker = ProposeTracker::new(); let mut gate = ProposalGate::new(ProposalLedger::new(PROPOSAL_LEDGER_CAPACITY)); let step = gate.conclude(11, false, EntryOutcome::Covered); - assert_eq!(step, PrefixStep::Record(true)); + assert_eq!(step, Settlement::Settles(PrefixStep::Record(true))); assert!(!EntryOutcome::Covered.wrote_rows()); assert!(gate.skip_duplicate(&tracker, 1, 14, 11)); } + + /// A held entry closes its in-flight copy and records no outcome, so a + /// later copy of its proposal is not skipped as applied. + #[test] + fn a_held_entry_holds_its_lane_and_records_no_outcome() { + let tracker = ProposeTracker::new(); + let mut gate = ProposalGate::new(ProposalLedger::new(PROPOSAL_LEDGER_CAPACITY)); + gate.open(9); + let held = EntryOutcome::Held { + reason: HoldReason::BodyMissing, + }; + assert!(!held.wrote_rows()); + let step = gate.conclude(9, true, held); + assert_eq!(step, Settlement::Holds(HoldReason::BodyMissing)); + assert!(!gate.in_flight(9)); + assert!(!gate.skip_duplicate(&tracker, 1, 12, 9)); + + gate.note_covered(9); + assert!(gate.skip_duplicate(&tracker, 1, 12, 9)); + } } diff --git a/nodedb/src/control/distributed_applier/apply_loop/redo_chunk.rs b/nodedb/src/control/distributed_applier/apply_loop/redo_chunk.rs index 335ed0c61..7e976e9c1 100644 --- a/nodedb/src/control/distributed_applier/apply_loop/redo_chunk.rs +++ b/nodedb/src/control/distributed_applier/apply_loop/redo_chunk.rs @@ -12,8 +12,8 @@ //! //! A group whose replica here owes a snapshot install is the exception (see //! `chunks::owed`). This node dropped streams of it that the other replicas -//! still hold. A refusal there can be this replica's alone, so it holds the -//! group's durable floor until the snapshot installs. +//! still hold. A refusal there can be this replica's alone: the entry's body +//! is missing here. It holds the group's lane until the snapshot installs. use crate::control::array_sync::raft_apply::AppliedPosition; use crate::control::distributed_applier::propose_tracker::{AppliedWrite, ProposeTracker}; @@ -28,7 +28,7 @@ use crate::wal::manager::RedoChunkPlacement; use super::calvin_redo::{ClaimOutcome, claim_chunked}; use super::context::{ApplyContext, FinishedApply}; -use super::proposal_gate::{EntryOutcome, ledger_outcome}; +use super::proposal_gate::{EntryOutcome, HoldReason, ledger_outcome}; use super::start::Prepared; use super::transaction_redo::{ RedoApply, conclude_held, conclude_held_claimed, conclude_installed_copy, enqueue_payload, @@ -65,6 +65,8 @@ pub(super) fn prepare_stream_entry<'a>( bytes, }; Prepared::Exclusive(Box::pin(async move { + #[cfg(feature = "failpoints")] + crate::control::fail_gate::before_chunk_apply(ctx.state.node_id, index).await; let applied = ctx.state.redo_chunks.apply_chunk(chunk).await; // The chunk's records are durable here, its entry not yet // concluded. @@ -86,6 +88,7 @@ pub(super) fn prepare_stream_entry<'a>( other => conclude_held( ctx, pos, + HoldReason::Malformed, crate::Error::Internal { detail: format!("redo stream apply called with another write: {other:?}"), }, @@ -139,13 +142,13 @@ fn concluded( outcome } -/// Hold a chunk entry this replica cannot decide as the other replicas do. -/// No ledger outcome records it, so a re-delivery applies it again. +/// Hold a chunk entry whose stream this replica dropped: the other +/// replicas decide it from a stream it lacks. No ledger outcome records it, +/// so a re-delivery applies it again. fn held(tracker: &ProposeTracker, pos: AppliedPosition, error: crate::Error) -> EntryOutcome { tracker.complete(pos.group_id, pos.log_index, pos.applied_key, Err(error)); - EntryOutcome::Applied { - durable: false, - result: None, + EntryOutcome::Held { + reason: HoldReason::BodyMissing, } } @@ -217,7 +220,9 @@ pub(super) fn prepare_chunked_final<'a>( // Another copy of the position installed, or installs now: this // copy's stream drops. ClaimOutcome::Refused => return conclude_installed_copy(ctx, pos, taken), - ClaimOutcome::Malformed(error) => return conclude_held(ctx, pos, error, taken), + ClaimOutcome::Malformed(error) => { + return conclude_held(ctx, pos, HoldReason::Malformed, error, taken); + } }; // Every chunk is durable and the final entry committed; nothing of the // final entry applied yet. @@ -235,9 +240,10 @@ pub(super) fn prepare_chunked_final<'a>( Ok(bytes) => bytes, Err(refusal) if ctx.state.redo_chunks.owes_snapshot(pos.group_id) => { let error = stream_lost_here(pos, chunked.stream, LostEntry::Final, &refusal); + let reason = HoldReason::BodyMissing; return match calvin { - Some(claim) => conclude_held_claimed(ctx, pos, error, taken, claim), - None => conclude_held(ctx, pos, error, taken), + Some(claim) => conclude_held_claimed(ctx, pos, reason, error, taken, claim), + None => conclude_held(ctx, pos, reason, error, taken), }; } Err(refusal) => { @@ -276,10 +282,13 @@ pub(super) fn prepare_chunked_final<'a>( calvin, }, )), - Err(error) => match calvin { - Some(claim) => conclude_held_claimed(ctx, pos, error, taken, claim), - None => conclude_held(ctx, pos, error, taken), - }, + Err(error) => { + let reason = HoldReason::Malformed; + match calvin { + Some(claim) => conclude_held_claimed(ctx, pos, reason, error, taken, claim), + None => conclude_held(ctx, pos, reason, error, taken), + } + } } } @@ -368,9 +377,8 @@ mod tests { fn holds(outcome: &EntryOutcome) -> bool { matches!( outcome, - EntryOutcome::Applied { - durable: false, - result: None + EntryOutcome::Held { + reason: HoldReason::BodyMissing } ) } @@ -378,7 +386,7 @@ mod tests { /// This node applied a stream's first chunk, left the group, and /// mounted it again on the group's old log. The next chunk and the /// final entry refuse here while every other replica installs the - /// redo. Each holds the group's durable floor instead of passing. + /// redo. Each holds the group's lane instead of passing. #[tokio::test] async fn a_final_entry_of_a_stream_this_replica_dropped_holds_the_apply() { let dir = tempfile::tempdir().expect("tempdir"); diff --git a/nodedb/src/control/distributed_applier/apply_loop/settle.rs b/nodedb/src/control/distributed_applier/apply_loop/settle.rs new file mode 100644 index 000000000..b7c07fc58 --- /dev/null +++ b/nodedb/src/control/distributed_applier/apply_loop/settle.rs @@ -0,0 +1,56 @@ +// SPDX-License-Identifier: BUSL-1.1 + +//! Settle every group's concluded entries and save the floors that moved. + +use std::collections::HashMap; + +use crate::control::server::dispatch_utils::publish_settled_changes; + +use super::bookkeeping::record_durable_apply; +use super::context::ApplyContext; +use super::lane::Lane; + +/// Settle every group's concluded entries in log order, release their +/// window, and save each durable floor that moved. +/// +/// The tenant write marks of the settled entries are persisted before any +/// floor that covers them. An entry above the saved floor is delivered +/// again after a restart and records its mark again, so every committed +/// write keeps a durable mark. +pub(super) fn settle_lanes(ctx: ApplyContext<'_>, lanes: &mut HashMap) { + let state = ctx.state; + let tracker = ctx.tracker; + for (group_id, lane) in lanes { + let first = lane.front_index(); + let settled = lane.settle(tracker, &state.tenant_marks); + if settled > 0 { + tracker.window().release(*group_id, settled); + // The entries' change events publish in log order, at their + // log positions, on every replica. + if let (Some(first), Some(last)) = (first, lane.last_settled()) { + publish_settled_changes(state, *group_id, first, last); + } + } + if !lane.floor_pending() { + continue; + } + // One save per pass that moved the floor, never one per entry: + // each save is an fsync, and the pass coalesces every apply that + // finished before it. + if let Err(error) = state.tenant_marks.persist(state.credentials.catalog()) { + // The floor stays where it is, so every entry above it keeps + // its place in the log. The next pass persists and saves again. + tracing::error!( + group_id = *group_id, + %error, + "apply loop: tenant write marks did not persist; the applied floor waits" + ); + continue; + } + if let Some(floor) = lane.take_floor_to_save() { + // Every entry the floor covers keeps its changes on disk. + state.change_stream.journal_group(*group_id, floor); + record_durable_apply(state, *group_id, floor); + } + } +} diff --git a/nodedb/src/control/distributed_applier/apply_loop/start.rs b/nodedb/src/control/distributed_applier/apply_loop/start.rs index f7563d87c..6356e560f 100644 --- a/nodedb/src/control/distributed_applier/apply_loop/start.rs +++ b/nodedb/src/control/distributed_applier/apply_loop/start.rs @@ -15,6 +15,7 @@ use crate::control::array_sync::raft_apply::{ use nodedb_raft::message::LogEntry; use crate::control::backup::cut_order::OrderedCut; +use crate::control::distributed_applier::propose_tracker::ProposeTracker; use crate::control::wal_replication::{ReplicatedEntry, ReplicatedWrite}; use crate::types::{DatabaseId, TenantId}; @@ -58,7 +59,8 @@ pub(super) fn prepare_entry<'a>( watch.note_apply(group_id, log_index); if entry.data.is_empty() { - return conclude_leader_change_noop(ctx, group_id, log_index); + conclude_leader_change_noop(ctx.tracker, group_id, log_index); + return Prepared::Concluded(EntryOutcome::Skipped); } // `0` for unparseable / pre-key entries; the tracker treats 0 as "no key" @@ -94,21 +96,15 @@ pub(super) fn prepare_entry<'a>( prepare_replicated(ctx, watch, pos, log_term, entry, scope, replicated) } -/// A leader-change no-op committed where a proposer can wait. The -/// proposer's data is gone; firing an empty success tells it the -/// write applied. `RetryableLeaderChange` makes the gateway re-propose. -fn conclude_leader_change_noop<'a>( - ctx: ApplyContext<'a>, - group_id: u64, - log_index: u64, -) -> Prepared<'a> { - tracing::error!( - group_id, - log_index, - "leader-change no-op committed at index where a proposer was waiting; \ - surfacing RetryableLeaderChange so the gateway re-proposes" - ); - ctx.tracker.complete( +/// Conclude the leader-change no-op at `log_index`. Every new leader commits +/// one. Returns whether a proposer waited at the index. +/// +/// A proposer whose entry held the index lost it to the leader change. It +/// gets `RetryableLeaderChange`, so the gateway re-proposes; an empty success +/// would tell it the write applied. A proposer that registers later finds +/// the same answer stored. Only an overwritten waiter is logged. +fn conclude_leader_change_noop(tracker: &ProposeTracker, group_id: u64, log_index: u64) -> bool { + let overwritten = tracker.complete( group_id, log_index, 0, @@ -117,7 +113,15 @@ fn conclude_leader_change_noop<'a>( log_index, }), ); - Prepared::Concluded(EntryOutcome::Skipped) + if overwritten { + tracing::warn!( + group_id, + log_index, + "a leader-change no-op committed where a proposer was waiting; surfacing \ + RetryableLeaderChange so the gateway re-proposes" + ); + } + overwritten } /// The scope a decoded entry applies under. The entry's incarnations move @@ -368,3 +372,36 @@ fn prepare_array_op<'a>( } })) } + +#[cfg(test)] +mod tests { + use super::*; + + /// A no-op at an index a proposer waits on answers it with + /// `RetryableLeaderChange`. + #[tokio::test] + async fn a_noop_over_a_waiter_surfaces_a_leader_change() { + let tracker = ProposeTracker::new(); + let waiter = tracker.register(4, 9, 77); + assert!(conclude_leader_change_noop(&tracker, 4, 9)); + let answer = waiter.await.expect("the waiter is answered"); + assert!( + matches!( + answer, + Err(crate::Error::RetryableLeaderChange { + group_id: 4, + log_index: 9 + }) + ), + "{answer:?}" + ); + } + + /// A no-op at an index no proposer waits on, as after every election, + /// finds no waiter and logs nothing. + #[test] + fn a_noop_with_no_waiter_finds_none() { + let tracker = ProposeTracker::new(); + assert!(!conclude_leader_change_noop(&tracker, 4, 1)); + } +} diff --git a/nodedb/src/control/distributed_applier/apply_loop/transaction_redo.rs b/nodedb/src/control/distributed_applier/apply_loop/transaction_redo.rs index 34e59d9de..0eec50834 100644 --- a/nodedb/src/control/distributed_applier/apply_loop/transaction_redo.rs +++ b/nodedb/src/control/distributed_applier/apply_loop/transaction_redo.rs @@ -38,7 +38,7 @@ use crate::types::{DatabaseId, TenantId, VShardId}; use super::calvin_redo::{CalvinClaim, ClaimOutcome, claim_inline}; use super::context::{ApplyContext, EnqueueFuture, FinishedApply, Started, StartedEntry}; use super::helpers::committed_response_result; -use super::proposal_gate::{EntryOutcome, ledger_outcome}; +use super::proposal_gate::{EntryOutcome, HoldReason, ledger_outcome}; use super::start::Prepared; /// Prepare one committed `TransactionRedo` entry. Its enqueue appends the @@ -57,7 +57,7 @@ pub(super) fn prepare_transaction_redo_entry<'a>( ) -> Prepared<'a> { let decoded = match decode_transaction_redo(&entry.write) { Ok(decoded) => decoded, - Err(error) => return conclude_held(ctx, pos, error, None), + Err(error) => return conclude_held(ctx, pos, HoldReason::Malformed, error, None), }; let target = RedoTarget { tenant_id: TenantId::new(entry.tenant_id), @@ -75,7 +75,9 @@ pub(super) fn prepare_transaction_redo_entry<'a>( ClaimOutcome::NotCalvin => None, ClaimOutcome::Claimed(claim) => Some(claim), ClaimOutcome::Refused => return conclude_installed_copy(ctx, pos, None), - ClaimOutcome::Malformed(error) => return conclude_held(ctx, pos, error, None), + ClaimOutcome::Malformed(error) => { + return conclude_held(ctx, pos, HoldReason::Malformed, error, None); + } }; Prepared::Enqueue(enqueue_payload( ctx, @@ -129,15 +131,16 @@ pub(super) struct RedoApply { /// Conclude an entry this replica cannot apply as the other replicas do: /// its own bytes are malformed, or its stream here is not the one the log -/// built. It holds the floor rather than skipping a committed transaction. -/// A taken stream stays on disk for it. +/// built. It holds its group's lane rather than skipping a committed +/// transaction. A taken stream stays on disk for it. pub(super) fn conclude_held<'a>( ctx: ApplyContext<'a>, pos: AppliedPosition, + reason: HoldReason, error: crate::Error, stream: Option, ) -> Prepared<'a> { - Prepared::Concluded(held_outcome(ctx, pos, error, stream)) + Prepared::Concluded(held_outcome(ctx, pos, reason, error, stream)) } /// [`conclude_held`] for an entry that claimed a Calvin position: the claim @@ -145,12 +148,13 @@ pub(super) fn conclude_held<'a>( pub(super) fn conclude_held_claimed<'a>( ctx: ApplyContext<'a>, pos: AppliedPosition, + reason: HoldReason, error: crate::Error, stream: Option, claim: CalvinClaim, ) -> Prepared<'a> { let event = claim.not_applied(error.to_string()); - let outcome = held_outcome(ctx, pos, error, stream); + let outcome = held_outcome(ctx, pos, reason, error, stream); report_then_conclude(ctx, pos, claim, event, outcome) } @@ -175,19 +179,32 @@ pub(super) fn report_then_conclude<'a>( } /// Resolve the waiter of an entry this replica holds, and settle its -/// stream. +/// stream. A malformed entry is reported here, where it is detected. A +/// missing body is reported where its stream refused. fn held_outcome( ctx: ApplyContext<'_>, pos: AppliedPosition, + reason: HoldReason, error: crate::Error, stream: Option, ) -> EntryOutcome { + match reason { + HoldReason::Malformed => { + tracing::error!( + group_id = pos.group_id, + log_index = pos.log_index, + %error, + "committed redo entry does not decode on this replica; the group's apply holds \ + at it until a snapshot of the group installs" + ); + crate::diag::redo_entry_malformed(pos.group_id, pos.log_index, &error); + } + // Each is reported at the site that detects it. + HoldReason::BodyMissing | HoldReason::ApplyPanicked => {} + } ctx.tracker .complete(pos.group_id, pos.log_index, pos.applied_key, Err(error)); - let outcome = EntryOutcome::Applied { - durable: false, - result: None, - }; + let outcome = EntryOutcome::Held { reason }; settle_stream(ctx, stream, &outcome); outcome } @@ -199,7 +216,10 @@ fn settle_stream(ctx: ApplyContext<'_>, stream: Option, outcome: &En let Some(stream) = stream else { return; }; - if matches!(outcome, EntryOutcome::Applied { durable: false, .. }) { + if matches!( + outcome, + EntryOutcome::Held { .. } | EntryOutcome::Applied { durable: false, .. } + ) { ctx.state.redo_chunks.park(stream); } } diff --git a/nodedb/src/control/exec_receiver/executor.rs b/nodedb/src/control/exec_receiver/executor.rs index 05ef7fed3..25501aabc 100644 --- a/nodedb/src/control/exec_receiver/executor.rs +++ b/nodedb/src/control/exec_receiver/executor.rs @@ -122,6 +122,8 @@ impl LocalPlanExecutor { ), TypedClusterError, > { + #[cfg(feature = "failpoints")] + crate::control::fail_gate::before_execute(self.state.node_id).await; let (deadline, database_id) = validate_request(&self.state, req)?; let plan = decode_plan(&self.state, database_id, req.tenant_id, &req.plan_bytes)?; // A backup's snapshot plan takes the backup's cut on this node first. diff --git a/nodedb/src/control/exec_receiver/metadata_applied.rs b/nodedb/src/control/exec_receiver/metadata_applied.rs index 56e2d99ea..19f88734e 100644 --- a/nodedb/src/control/exec_receiver/metadata_applied.rs +++ b/nodedb/src/control/exec_receiver/metadata_applied.rs @@ -25,17 +25,10 @@ pub(super) async fn answer_applied_plan( return None; }; let watcher = state.applied_index_watcher(METADATA_GROUP_ID); - let reached = match crate::control::metadata_proposer::wait::wait_applied( - watcher, *index, budget, - ) - .await - { - Ok(outcome) => outcome.is_reached(), - Err(error) => { - tracing::warn!(index = *index, %error, "metadata-applied wait did not finish"); - false - } - }; + let reached = watcher + .wait_for_async(*index, std::time::Instant::now() + budget) + .await + .is_reached(); Some(if reached { ExecuteResponse::ok(vec![Vec::new()], 0, Vec::new()) } else { diff --git a/nodedb/src/control/fail_gate.rs b/nodedb/src/control/fail_gate.rs index 264a84e1a..8688aa721 100644 --- a/nodedb/src/control/fail_gate.rs +++ b/nodedb/src/control/fail_gate.rs @@ -61,6 +61,13 @@ pub(crate) async fn before_dispatch(node_id: u64, plan: &PhysicalPlan, wal_lsn: } } +/// The execute gate of node `node_id`, `exec::before_execute`: a received +/// `ExecuteRequest` or `ExecuteStreamRequest` parks here before it is checked +/// or run. A test arms it so one node takes a request and never answers. +pub(crate) async fn before_execute(node_id: u64) { + wait(FailScope::Node(node_id), "exec::before_execute").await; +} + /// The Calvin submit gate of node `node_id`: after a transaction's collection /// incarnations are stamped and before the sequencer admits it. Named per /// collection, `calvin::after_stamp::`. A test parks a @@ -206,3 +213,39 @@ pub(crate) fn holds_metadata_apply(node_id: u64) -> bool { crate::control::cluster::metadata_applier::METADATA_APPLY_HOLD_POINT, ) } + +/// The redo chunk gate of node `node_id`: a committed `RedoChunk` entry +/// parks here before its chunk applies. Named per chunk index, +/// `redo_chunk::before_apply::`. The chunk's entry applies with +/// nothing else of its group in flight, so every later entry of the group +/// waits behind it. +pub(crate) async fn before_chunk_apply(node_id: u64, index: u32) { + wait( + FailScope::Node(node_id), + &format!("redo_chunk::before_apply::{index}"), + ) + .await; +} + +/// The data-group snapshot build gate of node `node_id`: a leader's build +/// of a snapshot of `group_id` parks here before it reads any state. Named +/// `snapshot::before_build::`. No snapshot of the group leaves +/// the node while it is parked. +pub(crate) async fn before_snapshot_build(node_id: u64, group_id: u64) { + wait( + FailScope::Node(node_id), + &format!("snapshot::before_build::{group_id}"), + ) + .await; +} + +/// The catalog DDL apply gate of node `node_id`: a `CatalogDdl` entry that +/// puts collection `collection` pauses here before it applies. Named +/// `catalog_ddl::before_apply::`, armed with `sleep()`. +/// The pause awaits a Tokio timer, as a slow apply awaits its effects. +pub(crate) async fn before_catalog_ddl_apply(node_id: u64, collection: &str) { + let name = format!("catalog_ddl::before_apply::{collection}"); + if let Some(FailAction::Sleep(pause)) = lookup(FailScope::Node(node_id), &name) { + tokio::time::sleep(pause).await; + } +} diff --git a/nodedb/src/control/gateway/cluster_error.rs b/nodedb/src/control/gateway/cluster_error.rs index 32c462e10..84f6790b1 100644 --- a/nodedb/src/control/gateway/cluster_error.rs +++ b/nodedb/src/control/gateway/cluster_error.rs @@ -1,7 +1,9 @@ // SPDX-License-Identifier: BUSL-1.1 -//! Map a remote node's [`TypedClusterError`] to the gateway's [`Error`]. +//! Map a remote node's [`TypedClusterError`], and a failed send to it, to +//! the gateway's [`Error`]. +use nodedb_cluster::ClusterError; use nodedb_cluster::rpc_codec::TypedClusterError; use crate::Error; @@ -74,10 +76,97 @@ pub(super) fn map_typed_cluster_error(err: TypedClusterError, vshard_id: u64) -> } } +/// The error of a failed `ExecuteRequest` or `ExecuteStreamRequest` send. +/// +/// A request the target never received, and a lost answer to a read, retry as +/// `NotLeader` with `leader_node = 0`. The unreachable node is then not kept as +/// leader, and the next try routes locally to the real leader. A lost answer +/// to any other plan is an unknown outcome: the plan can have run, and a +/// second send would run it again. +pub(super) fn send_error( + e: ClusterError, + resend_safe: bool, + node_id: u64, + vshard_id: u64, +) -> Error { + if e.is_lost_answer() && !resend_safe { + tracing::warn!( + node_id, + vshard_id, + error = %e, + "gateway: a remote execute reached its deadline with no answer" + ); + return Error::DeadlineExceeded { + request_id: crate::types::RequestId::new(0), + }; + } + Error::NotLeader { + vshard_id: VShardId::new((vshard_id % VShardId::COUNT as u64) as u32), + leader_node: 0, + leader_addr: format!("node-{node_id} (transport error: {e})"), + leader_term: 0, + } +} + #[cfg(test)] mod tests { use super::*; + fn unanswered() -> ClusterError { + ClusterError::Unanswered { + node_id: 2, + detail: "stream reset".into(), + } + } + + /// A write whose answer was lost can have applied: it is never sent again. + #[test] + fn a_lost_answer_to_a_write_is_an_unknown_outcome() { + assert!(matches!( + send_error(unanswered(), false, 2, 7), + Error::DeadlineExceeded { .. } + )); + } + + /// A read whose answer was lost retries through the leader lookup. + #[test] + fn a_lost_answer_to_a_read_retries() { + assert!(matches!( + send_error(unanswered(), true, 2, 7), + Error::NotLeader { leader_node: 0, .. } + )); + } + + /// A send that timed out is a lost answer: a read retries, a write is an + /// unknown outcome. + #[test] + fn a_timed_out_send_follows_the_lost_answer_rule() { + let timed_out = || ClusterError::ShardTimeout { + vshard_id: 7, + elapsed_ms: 1_000, + }; + assert!(matches!( + send_error(timed_out(), true, 2, 7), + Error::NotLeader { leader_node: 0, .. } + )); + assert!(matches!( + send_error(timed_out(), false, 2, 7), + Error::DeadlineExceeded { .. } + )); + } + + /// A request that never reached the target retries, read or write. + #[test] + fn an_unsent_request_retries() { + let unsent = ClusterError::Transport { + detail: "connect refused".into(), + }; + assert!(matches!( + send_error(unsent, false, 2, 7), + Error::NotLeader { leader_node: 0, .. } + )); + } + #[test] fn map_not_leader() { let err = TypedClusterError::NotLeader { diff --git a/nodedb/src/control/gateway/core.rs b/nodedb/src/control/gateway/core.rs index 64aaca526..5e776baf3 100644 --- a/nodedb/src/control/gateway/core.rs +++ b/nodedb/src/control/gateway/core.rs @@ -31,7 +31,7 @@ use crate::control::trace_export::EmitSpanParams; use crate::types::{DatabaseId, Lsn, TenantId, TraceId, TxnId, VShardId}; use nodedb_physical::physical_plan::PhysicalPlan; -use super::dispatcher::{DispatchRouteParams, dispatch_route, statement_deadline_ms}; +use super::dispatcher::{DispatchRouteParams, dispatch_route}; use super::fuser::fuse_payloads; use super::key_extractor::UnwiredKeyExtractor; use super::live_leaders::resolve_live_decision; @@ -41,6 +41,7 @@ use super::retry::retry_not_leader; use super::route::TaskRoute; use super::router::route_plan; use super::version_set::GatewayVersionSet; +use crate::control::server::shared::session::StatementDeadline; /// Context passed to [`Gateway::execute`]. pub struct QueryContext { @@ -248,7 +249,8 @@ impl Gateway { let single_route = routes.len() == 1; let mut not_found = false; - let deadline_ms = statement_deadline_ms(&shared); + // One deadline for every route and attempt of the statement. + let deadline = StatementDeadline::of_statement(&shared).tokio_instant(); // Gateway-level byte ceiling: per-route `dispatch_to_data_plane` // already caps each shard's payload; this additionally caps the // scatter-gather *sum* so an N-shard fan-out can't accumulate @@ -271,8 +273,8 @@ impl Gateway { let retry_counter = Arc::clone(&self.not_leader_retry_count); let version_set_for_route = version_set.clone(); let shared_for_route = Arc::clone(&shared); - let outcome = retry_not_leader(routing_ref, move |attempt| { - if attempt > 0 { + let outcome = retry_not_leader(routing_ref, deadline, move |attempt| { + if attempt.index > 0 { retry_counter.fetch_add(1, Ordering::Relaxed); } let plan = plan_for_retry.clone(); @@ -296,7 +298,8 @@ impl Gateway { tenant_id, database_id, trace_id, - deadline_ms, + deadline_ms: attempt.remaining_ms(), + attempts_left: attempt.attempts_left(), version_set: &version_set, txn_id, linearizable, diff --git a/nodedb/src/control/gateway/dispatch_local.rs b/nodedb/src/control/gateway/dispatch_local.rs index fc26c5325..a8e530dbe 100644 --- a/nodedb/src/control/gateway/dispatch_local.rs +++ b/nodedb/src/control/gateway/dispatch_local.rs @@ -98,7 +98,8 @@ pub(super) async fn dispatch_local( shared, proposer, entry, - crate::control::wal_replication::statement_propose_deadline(shared), + crate::control::server::shared::session::StatementDeadline::of_statement(shared) + .tokio_instant(), ) .await?; return Ok(DispatchOutcome { diff --git a/nodedb/src/control/gateway/dispatch_remote.rs b/nodedb/src/control/gateway/dispatch_remote.rs index 94b6de798..07cb3d112 100644 --- a/nodedb/src/control/gateway/dispatch_remote.rs +++ b/nodedb/src/control/gateway/dispatch_remote.rs @@ -18,7 +18,7 @@ use crate::control::state::SharedState; use crate::types::{DatabaseId, Lsn, TenantId, TraceId, TxnId, VShardId}; use nodedb_physical::physical_plan::wire as plan_wire; -use super::cluster_error::map_typed_cluster_error; +use super::cluster_error::{map_typed_cluster_error, send_error}; use super::dispatcher::DispatchOutcome; use super::version_set::GatewayVersionSet; @@ -32,7 +32,10 @@ pub(super) struct RemoteDispatchArgs<'a> { pub tenant_id: TenantId, pub database_id: DatabaseId, pub trace_id: TraceId, + /// Milliseconds left on the statement when this attempt starts. pub deadline_ms: u64, + /// The dispatch attempts this route has left, this one included. + pub attempts_left: usize, pub version_set: &'a GatewayVersionSet, /// Session-transaction context forwarded to the remote executor, or `None` /// for non-transactional dispatch. @@ -58,6 +61,7 @@ pub(super) async fn dispatch_remote( database_id, trace_id, deadline_ms, + attempts_left, version_set, txn_id, linearizable, @@ -133,6 +137,14 @@ pub(super) async fn dispatch_remote( // A read can be sent again after a lost answer. Any other plan can have // run on the target, so a lost answer is an unknown outcome. let resend_safe = required_permission(&plan) == Permission::Read; + // A read leaves time for the attempts after this one, so a node that + // never answers cannot spend the whole statement budget. + let attempt_ms = nodedb_cluster::rpc_codec::attempt_budget( + deadline_ms, + resend_safe, + attempts_left, + transport.rpc_timeout(), + ); // Encode the plan. let plan_bytes = plan_wire::encode(&plan).map_err(|e| Error::Internal { @@ -155,7 +167,7 @@ pub(super) async fn dispatch_remote( plan_bytes, tenant_id: tenant_id.as_u64(), database_id: database_id.as_u64(), - deadline_remaining_ms: deadline_ms, + deadline_remaining_ms: attempt_ms, trace_id: trace_id.0, descriptor_versions, txn_id, @@ -170,10 +182,10 @@ pub(super) async fn dispatch_remote( "gateway: dispatching ExecuteRequest to remote node" ); - // The remote handler works within `deadline_ms`: a write can wait at the + // The remote handler works within `attempt_ms`: a write can wait at the // leader's gate or in a Calvin queue for that long. The reply wait follows // that budget, never the transport's fixed RPC timeout. - let reply_wait = nodedb_cluster::rpc_codec::reply_wait(deadline_ms); + let reply_wait = nodedb_cluster::rpc_codec::reply_wait(attempt_ms); let resp_rpc = match tokio::time::timeout( reply_wait, transport.send_rpc_with_read_timeout(node_id, req, reply_wait), @@ -249,79 +261,11 @@ pub(super) fn scoped_vshard( Ok(Some(VShardId::new(raw))) } -/// The error of a failed `ExecuteRequest` send. -/// -/// A request the target never received, and a lost answer to a read, retry as -/// `NotLeader` with `leader_node = 0`. The unreachable node is then not kept as -/// leader, and the next try routes locally to the real leader. A lost answer -/// to any other plan is an unknown outcome: the plan can have run, and a -/// second send would run it again. -fn send_error(e: ClusterError, resend_safe: bool, node_id: u64, vshard_id: u64) -> Error { - let outcome_unknown = matches!( - e, - ClusterError::Unanswered { .. } | ClusterError::ShardTimeout { .. } - ); - if outcome_unknown && !resend_safe { - tracing::warn!( - node_id, - vshard_id, - error = %e, - "gateway: a remote execute reached its deadline with no answer" - ); - return Error::DeadlineExceeded { - request_id: crate::types::RequestId::new(0), - }; - } - Error::NotLeader { - vshard_id: VShardId::new((vshard_id % VShardId::COUNT as u64) as u32), - leader_node: 0, - leader_addr: format!("node-{node_id} (transport error: {e})"), - leader_term: 0, - } -} - #[cfg(test)] mod tests { use super::*; use nodedb_physical::physical_plan::{MetaOp, PhysicalPlan}; - fn unanswered() -> ClusterError { - ClusterError::Unanswered { - node_id: 2, - detail: "stream reset".into(), - } - } - - /// A write whose answer was lost can have applied: it is never sent again. - #[test] - fn a_lost_answer_to_a_write_is_an_unknown_outcome() { - assert!(matches!( - send_error(unanswered(), false, 2, 7), - Error::DeadlineExceeded { .. } - )); - } - - /// A read whose answer was lost retries through the leader lookup. - #[test] - fn a_lost_answer_to_a_read_retries() { - assert!(matches!( - send_error(unanswered(), true, 2, 7), - Error::NotLeader { leader_node: 0, .. } - )); - } - - /// A request that never reached the target retries, read or write. - #[test] - fn an_unsent_request_retries() { - let unsent = ClusterError::Transport { - detail: "connect refused".into(), - }; - assert!(matches!( - send_error(unsent, false, 2, 7), - Error::NotLeader { leader_node: 0, .. } - )); - } - #[test] fn a_transaction_meta_op_carries_its_route_vshard() { let plan = PhysicalPlan::Meta(MetaOp::DropTxnOverlay { diff --git a/nodedb/src/control/gateway/dispatch_remote_stream.rs b/nodedb/src/control/gateway/dispatch_remote_stream.rs index 3dd04f63a..51a87b7ed 100644 --- a/nodedb/src/control/gateway/dispatch_remote_stream.rs +++ b/nodedb/src/control/gateway/dispatch_remote_stream.rs @@ -11,10 +11,11 @@ use nodedb_physical::physical_plan::wire as plan_wire; use tracing::debug; use crate::Error; +use crate::control::security::identity::{Permission, required_permission}; use crate::control::server::result_stream::{ResultStream, RowBatch}; use crate::types::{Lsn, VShardId}; -use super::cluster_error::map_typed_cluster_error; +use super::cluster_error::{map_typed_cluster_error, send_error}; use super::dispatch_remote::{RemoteDispatchArgs, scoped_vshard}; /// Remote streaming dispatch via the multi-frame `ExecuteStreamRequest` RPC. @@ -35,6 +36,10 @@ use super::dispatch_remote::{RemoteDispatchArgs, scoped_vshard}; /// `Err` and never retried (re-running the plan duplicates the rows /// already streamed to the client). /// +/// The open and the first frame wait at most `reply_wait` of this attempt's +/// budget. A read leaves time there for the attempts after it. The remote +/// runs the stream within the statement's whole remaining budget. +/// /// The returned stream re-emits the buffered first batch followed by the rest. pub(super) async fn dispatch_remote_stream( args: RemoteDispatchArgs<'_>, @@ -48,6 +53,7 @@ pub(super) async fn dispatch_remote_stream( database_id, trace_id, deadline_ms, + attempts_left, version_set, txn_id, linearizable, @@ -91,6 +97,16 @@ pub(super) async fn dispatch_remote_stream( crate::control::server::exchange::Resolved::Stream(s) => return Ok(s), }; + // A read can be sent again after a lost answer. Any other plan can have + // run on the target, so a lost answer is an unknown outcome. + let resend_safe = required_permission(&plan) == Permission::Read; + let attempt_ms = nodedb_cluster::rpc_codec::attempt_budget( + deadline_ms, + resend_safe, + attempts_left, + transport.rpc_timeout(), + ); + let plan_bytes = plan_wire::encode(&plan).map_err(|e| Error::Internal { detail: format!("gateway: plan encode failed: {e}"), })?; @@ -110,6 +126,8 @@ pub(super) async fn dispatch_remote_stream( plan_bytes, tenant_id: tenant_id.as_u64(), database_id: database_id.as_u64(), + // The remote bounds the whole stream with this budget, so it keeps + // the statement's remainder. Only the first frame waits on `attempt_ms`. deadline_remaining_ms: deadline_ms, trace_id: trace_id.0, descriptor_versions, @@ -125,25 +143,37 @@ pub(super) async fn dispatch_remote_stream( "gateway: dispatching ExecuteStreamRequest to remote node" ); - // Open the stream eagerly. A failure to even open / send the request is a - // pre-row condition: map it like a transport failure in `dispatch_remote` - // so the retry loop routes elsewhere on the next attempt. - let stream = transport - .send_rpc_stream(node_id, req) - .await - .map_err(|e| Error::NotLeader { - vshard_id: VShardId::new((vshard_id % VShardId::COUNT as u64) as u32), - leader_node: 0, - leader_addr: format!("node-{node_id} (stream open error: {e})"), - leader_term: 0, - })?; - // The `async_stream` body is `!Unpin`; pin it on the heap so we can pull - // the eager first frame and then keep the tail around for `.chain`. - let mut stream = Box::pin(stream); - - // Eagerly pull the FIRST frame so a pre-row terminal error is catchable and + // Open the stream and pull the FIRST frame eagerly, within this attempt's + // reply wait, so a pre-row error or a silent node is catchable and // retryable. Any error here is a pre-row error. - let first = match stream.next().await { + let reply_wait = nodedb_cluster::rpc_codec::reply_wait(attempt_ms); + let opened = tokio::time::timeout(reply_wait, async move { + let stream = transport + .send_rpc_stream(node_id, req) + .await + .map_err(|e| send_error(e, resend_safe, node_id, vshard_id))?; + // The `async_stream` body is `!Unpin`; pin it on the heap so the tail + // stays usable for `.chain` after the first frame. + let mut stream = Box::pin(stream); + let first = stream.next().await; + Ok::<_, Error>((stream, first)) + }) + .await; + let (stream, first) = match opened { + Ok(opened) => opened?, + Err(_) => { + return Err(send_error( + ClusterError::ShardTimeout { + vshard_id: (vshard_id % VShardId::COUNT as u64) as u32, + elapsed_ms: u64::try_from(reply_wait.as_millis()).unwrap_or(u64::MAX), + }, + resend_safe, + node_id, + vshard_id, + )); + } + }; + let first = match first { Some(Ok((payload, lsn))) => RowBatch { payload, watermark_lsn: Lsn::new(lsn), @@ -151,7 +181,9 @@ pub(super) async fn dispatch_remote_stream( // read version is threaded on this remote path. read_versions: crate::types::ReadVersions::new(), }, - Some(Err(e)) => return Err(map_stream_cluster_error(e, vshard_id)), + Some(Err(e)) => { + return Err(map_stream_cluster_error(e, resend_safe, node_id, vshard_id)); + } // Clean EOF with zero rows: a valid empty result. Return an empty stream. None => return Ok(Box::pin(futures::stream::empty())), }; @@ -179,11 +211,17 @@ pub(super) async fn dispatch_remote_stream( /// /// A typed error (`StreamTerminal`, `ShardExecution`) maps through the same /// [`map_typed_cluster_error`] used by the one-shot path, so the gateway retry -/// loop handles it identically. A Data-Plane verdict keeps its code. Any other -/// cluster error becomes a transport-style `NotLeader` (leader_node = 0), so -/// the next attempt re-resolves routing rather than re-entrenching an -/// unreachable node. -fn map_stream_cluster_error(err: ClusterError, vshard_id: u64) -> Error { +/// loop handles it identically. A Data-Plane verdict keeps its code. A lost +/// answer goes through [`send_error`]: a read retries, any other plan is an +/// unknown outcome. Any other cluster error becomes a transport-style +/// `NotLeader` (leader_node = 0), so the next attempt re-resolves routing +/// rather than re-entrenching an unreachable node. +fn map_stream_cluster_error( + err: ClusterError, + resend_safe: bool, + node_id: u64, + vshard_id: u64, +) -> Error { match err { ClusterError::StreamTerminal { error, .. } | ClusterError::ShardExecution { error, .. } => { map_typed_cluster_error(*error, vshard_id) @@ -191,6 +229,8 @@ fn map_stream_cluster_error(err: ClusterError, vshard_id: u64) -> Error { // A verdict from a shard that answered. Retrying it on another route // repeats it, so it keeps its SQLSTATE. ClusterError::DataPlane { code } => Error::DataPlane(code.into()), + // The request can have run on the remote node. + lost if lost.is_lost_answer() => send_error(lost, resend_safe, node_id, vshard_id), other @ (ClusterError::Raft(_) | ClusterError::VShardNotMapped { .. } | ClusterError::GroupNotFound { .. } @@ -200,8 +240,6 @@ fn map_stream_cluster_error(err: ClusterError, vshard_id: u64) -> Error { | ClusterError::NodeUnreachable { .. } | ClusterError::GhostNotFound { .. } | ClusterError::Transport { .. } - | ClusterError::ShardTimeout { .. } - | ClusterError::Unanswered { .. } | ClusterError::Storage { .. } | ClusterError::Codec { .. } | ClusterError::UnsupportedWireVersion { .. } @@ -227,7 +265,10 @@ fn map_stream_cluster_error(err: ClusterError, vshard_id: u64) -> Error { | ClusterError::Bm25Gather(_) | ClusterError::TsGather(_) | ClusterError::ShufflePush(_) - | ClusterError::RemoteUntyped { .. }) => Error::NotLeader { + | ClusterError::RemoteUntyped { .. } + // The guards above take these. They stay named for exhaustiveness. + | ClusterError::ShardTimeout { .. } + | ClusterError::Unanswered { .. }) => Error::NotLeader { vshard_id: VShardId::new((vshard_id % VShardId::COUNT as u64) as u32), leader_node: 0, leader_addr: format!("stream dispatch error: {other}"), @@ -235,3 +276,54 @@ fn map_stream_cluster_error(err: ClusterError, vshard_id: u64) -> Error { }, } } + +#[cfg(test)] +mod tests { + use super::*; + + fn unanswered() -> ClusterError { + ClusterError::Unanswered { + node_id: 2, + detail: "stream reset".into(), + } + } + + /// A read whose stream answer was lost retries through the leader lookup. + #[test] + fn a_lost_stream_answer_to_a_read_retries() { + assert!(matches!( + map_stream_cluster_error(unanswered(), true, 2, 7), + Error::NotLeader { leader_node: 0, .. } + )); + } + + /// Any other plan can have run when its stream answer was lost, so it + /// is never sent again. + #[test] + fn a_lost_stream_answer_to_a_non_read_is_an_unknown_outcome() { + assert!(matches!( + map_stream_cluster_error(unanswered(), false, 2, 7), + Error::DeadlineExceeded { .. } + )); + let timed_out = ClusterError::ShardTimeout { + vshard_id: 7, + elapsed_ms: 1_000, + }; + assert!(matches!( + map_stream_cluster_error(timed_out, false, 2, 7), + Error::DeadlineExceeded { .. } + )); + } + + /// A transport error before any row retries, read or not. + #[test] + fn a_pre_row_transport_error_retries() { + let unsent = ClusterError::Transport { + detail: "connect refused".into(), + }; + assert!(matches!( + map_stream_cluster_error(unsent, false, 2, 7), + Error::NotLeader { leader_node: 0, .. } + )); + } +} diff --git a/nodedb/src/control/gateway/dispatcher.rs b/nodedb/src/control/gateway/dispatcher.rs index 79e396855..73e4ac17c 100644 --- a/nodedb/src/control/gateway/dispatcher.rs +++ b/nodedb/src/control/gateway/dispatcher.rs @@ -53,6 +53,9 @@ pub struct DispatchRouteParams<'a> { pub database_id: DatabaseId, pub trace_id: TraceId, pub deadline_ms: u64, + /// The dispatch attempts this route has left, this one included. A + /// caller that never retries passes 1. + pub attempts_left: usize, pub version_set: &'a GatewayVersionSet, pub txn_id: Option, /// The route is a leg of a linearizable read (see @@ -71,6 +74,7 @@ pub(crate) async fn dispatch_route( database_id, trace_id, deadline_ms, + attempts_left, version_set, txn_id, linearizable, @@ -103,6 +107,7 @@ pub(crate) async fn dispatch_route( database_id, trace_id, deadline_ms, + attempts_left, version_set, txn_id, linearizable, @@ -139,6 +144,8 @@ pub struct DispatchRouteStreamParams<'a> { pub database_id: DatabaseId, pub trace_id: TraceId, pub deadline_ms: u64, + /// The dispatch attempts this route has left, this one included. + pub attempts_left: usize, pub version_set: &'a GatewayVersionSet, /// The route is a leg of a linearizable read. pub linearizable: bool, @@ -172,6 +179,7 @@ pub(crate) async fn dispatch_route_stream( database_id, trace_id, deadline_ms, + attempts_left, version_set, linearizable, } = args; @@ -205,6 +213,7 @@ pub(crate) async fn dispatch_route_stream( database_id, trace_id, deadline_ms, + attempts_left, version_set, // No session-transaction context crosses the streaming gateway // boundary yet (see `resolve/exchange.rs`), so `None`. @@ -227,18 +236,6 @@ pub(crate) async fn dispatch_route_stream( } } -/// Milliseconds left on the running statement, for a remote hop's -/// `ExecuteRequest.deadline_remaining_ms`. -/// -/// The session's `statement_timeout` when one is installed on this connection, -/// else the node's configured default. A forwarded route must stop when the -/// statement that spawned it does, not on a budget of its own. -pub fn statement_deadline_ms(shared: &SharedState) -> u64 { - crate::control::server::shared::session::statement_deadline_ms( - shared.tuning.network.default_deadline_secs, - ) -} - #[cfg(test)] mod tests { use super::*; diff --git a/nodedb/src/control/gateway/error_map/class_parity/error_index.rs b/nodedb/src/control/gateway/error_map/class_parity/error_index.rs index d49285b9f..a28c9cfe5 100644 --- a/nodedb/src/control/gateway/error_map/class_parity/error_index.rs +++ b/nodedb/src/control/gateway/error_map/class_parity/error_index.rs @@ -3,7 +3,7 @@ //! A dense index per `crate::Error` variant. /// The number of `crate::Error` variants [`error_variant_index`] numbers. -pub(super) const ERROR_VARIANT_COUNT: usize = 121; +pub(super) const ERROR_VARIANT_COUNT: usize = 123; /// A dense index per `crate::Error` variant. Exhaustive, so a new variant /// fails to compile here until it gets an index, and @@ -72,6 +72,8 @@ pub(crate) fn error_variant_index(err: &crate::Error) -> usize { E::GroupQuorumUnavailable { .. } => 58, E::GroupMarksUnavailable { .. } => 59, E::MetadataLeaderUnavailable => 60, + E::MetadataGroupGone { .. } => 121, + E::MetadataDdlSuperseded { .. } => 122, E::AuthorizationStateBehind { .. } => 61, E::ExecutionLimitExceeded { .. } => 62, E::LimitExceeded { .. } => 63, diff --git a/nodedb/src/control/gateway/error_map/class_parity/error_samples.rs b/nodedb/src/control/gateway/error_map/class_parity/error_samples.rs index 0280a484d..ffedec6d4 100644 --- a/nodedb/src/control/gateway/error_map/class_parity/error_samples.rs +++ b/nodedb/src/control/gateway/error_map/class_parity/error_samples.rs @@ -248,6 +248,8 @@ pub(crate) fn error_samples() -> Vec { }, E::BackupCaptureMoved { group_id: 1 }, E::MetadataLeaderUnavailable, + E::MetadataGroupGone { log_index: 2 }, + E::MetadataDdlSuperseded { log_index: 2 }, E::AuthorizationStateBehind { detail: text() }, E::LinearizableReadRefused { group_id: 1, diff --git a/nodedb/src/control/gateway/error_map/resp.rs b/nodedb/src/control/gateway/error_map/resp.rs index 0da6aa8e4..e9eeb9fab 100644 --- a/nodedb/src/control/gateway/error_map/resp.rs +++ b/nodedb/src/control/gateway/error_map/resp.rs @@ -107,6 +107,8 @@ impl GatewayErrorMap { | Error::GroupMarksUnavailable { .. } | Error::BackupCaptureMoved { .. } | Error::MetadataLeaderUnavailable + | Error::MetadataGroupGone { .. } + | Error::MetadataDdlSuperseded { .. } | Error::AuthorizationStateBehind { .. } | Error::LinearizableReadRefused { .. } | Error::ExecutionLimitExceeded { .. } diff --git a/nodedb/src/control/gateway/live_leaders.rs b/nodedb/src/control/gateway/live_leaders.rs index 929cb8bec..74bbf543b 100644 --- a/nodedb/src/control/gateway/live_leaders.rs +++ b/nodedb/src/control/gateway/live_leaders.rs @@ -92,6 +92,7 @@ mod tests { last_applied: 0, last_log_index: 0, snapshot_index: 0, + replica_state: nodedb_raft::ReplicaState::Whole, member_count: 2, learner_count: 0, vshard_count: 0, diff --git a/nodedb/src/control/gateway/read_leg.rs b/nodedb/src/control/gateway/read_leg.rs index 94b7f29fe..3dc41a2f1 100644 --- a/nodedb/src/control/gateway/read_leg.rs +++ b/nodedb/src/control/gateway/read_leg.rs @@ -13,7 +13,7 @@ use crate::Error; use nodedb_physical::physical_plan::{PhysicalPlan, plan_contains_cluster_partitioned_leaf}; use crate::control::cluster::linearizable_read::{ - confirm_linearizable_read, groups_of_vshards, linearizable_read_deadline, + StatementDeadline, confirm_linearizable_read, groups_of_vshards, }; use crate::control::server::graph_dispatch::graph_read_groups; use crate::control::server::shared::write_admission::plan_is_write; @@ -52,7 +52,7 @@ pub(crate) async fn confirm_local_read( if read_groups.is_empty() { return Ok(()); } - let deadline = linearizable_read_deadline(Duration::from_millis(deadline_ms)); + let deadline = StatementDeadline::after(Duration::from_millis(deadline_ms)); if plan_contains_cluster_partitioned_leaf(plan) { let groups = graph_read_groups(shared, database_id, plan)?; return confirm_linearizable_read(shared, &groups, deadline).await; diff --git a/nodedb/src/control/gateway/retry.rs b/nodedb/src/control/gateway/retry.rs index 3b5880e91..4171a4b98 100644 --- a/nodedb/src/control/gateway/retry.rs +++ b/nodedb/src/control/gateway/retry.rs @@ -14,17 +14,23 @@ //! without updating the routing table — a subsequent routing lookup will //! re-read the table from the current routing state. //! +//! Every attempt shares the statement's deadline, so the attempts together +//! stay within one statement budget. No retry starts once it has passed. The +//! last refusal then ends the statement: it is definite, so it never becomes +//! `DeadlineExceeded`, which means a lost answer. +//! //! After `MAX_RETRIES` attempts the final `NotLeader` error is propagated. use std::future::Future; use std::sync::RwLock; -use tokio::time::{Duration, sleep}; +use tokio::time::{Duration, Instant, sleep}; use tracing::debug; use nodedb_cluster::RoutingTable; use crate::Error; +use crate::control::server::shared::session::deadline::hop_budget_ms; /// Maximum number of dispatch attempts (initial + 2 retries = 3 total). pub const MAX_RETRIES: usize = 3; @@ -32,24 +38,61 @@ pub const MAX_RETRIES: usize = 3; /// Backoff durations for each retry attempt. const BACKOFF_MS: [u64; MAX_RETRIES] = [50, 100, 200]; +/// One dispatch attempt of a statement. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub struct Attempt { + /// The attempt's 0-based index. + pub index: usize, + /// The statement's deadline. Every attempt carries the same instant. + pub deadline: Instant, +} + +impl Attempt { + /// The attempts left, this one included. + pub fn attempts_left(&self) -> usize { + MAX_RETRIES.saturating_sub(self.index) + } + + /// The milliseconds left on the statement now, at least 1. See + /// [`hop_budget_ms`]. + pub fn remaining_ms(&self) -> u64 { + hop_budget_ms(self.deadline.into_std()) + } + + /// The one attempt of a send that no retry follows. + pub fn sole(deadline: Instant) -> Self { + Self { + index: MAX_RETRIES - 1, + deadline, + } + } +} + /// Execute `f` up to `MAX_RETRIES` times, retrying on `Error::NotLeader`. /// -/// `f` receives the current attempt index (0-based). +/// `f` receives the current [`Attempt`]: its index and the statement's +/// `deadline`, which every attempt shares. /// /// On `NotLeader` with a hinted leader, the routing table is updated before /// the next retry so the caller's routing decision changes. On non-`NotLeader` /// errors the error is propagated immediately without retry. pub async fn retry_not_leader( routing: Option<&RwLock>, + deadline: Instant, f: F, ) -> Result where - F: Fn(usize) -> Fut, + F: Fn(Attempt) -> Fut, Fut: Future>, { - let mut last_err = None; - for (attempt, &backoff_ms) in BACKOFF_MS.iter().enumerate() { - match f(attempt).await { + let mut attempt = 0; + loop { + let refusal = match f(Attempt { + index: attempt, + deadline, + }) + .await + { Ok(v) => return Ok(v), Err(Error::NotLeader { vshard_id, @@ -89,16 +132,12 @@ where } } - if attempt + 1 < MAX_RETRIES { - sleep(Duration::from_millis(backoff_ms)).await; - } - - last_err = Some(Error::NotLeader { + Error::NotLeader { vshard_id, leader_node, leader_addr: String::new(), leader_term, - }); + } } Err(Error::RetryableLeaderChange { group_id, @@ -115,22 +154,31 @@ where group_id, log_index, "gateway: leader change overwrote entry — will re-propose" ); - if attempt + 1 < MAX_RETRIES { - sleep(Duration::from_millis(backoff_ms)).await; - } - - last_err = Some(Error::RetryableLeaderChange { + Error::RetryableLeaderChange { group_id, log_index, - }); + } } Err(other) => return Err(other), + }; + + // Both refusals mean nothing applied. When no retry follows, that + // definite refusal ends the statement, never an unknown outcome. + attempt += 1; + if attempt >= MAX_RETRIES { + return Err(refusal); + } + sleep(Duration::from_millis(BACKOFF_MS[attempt - 1])).await; + // A retry starts only while the statement has time left. That keeps + // every attempt's wait within the budget plus one reply margin. + if Instant::now() >= deadline { + debug!( + attempt, + "gateway: statement deadline passed — no further retry" + ); + return Err(refusal); } } - - Err(last_err.unwrap_or(Error::Internal { - detail: "retry_not_leader exhausted all attempts".into(), - })) } #[cfg(test)] @@ -143,9 +191,16 @@ mod tests { use super::*; use crate::types::VShardId; + fn far_deadline() -> Instant { + Instant::now() + Duration::from_secs(30) + } + #[tokio::test] async fn success_on_first_attempt() { - let result = retry_not_leader(None, |_attempt| async { Ok::(42) }).await; + let result = retry_not_leader(None, far_deadline(), |_attempt| async { + Ok::(42) + }) + .await; assert_eq!(result.unwrap(), 42); } @@ -153,7 +208,7 @@ mod tests { async fn success_on_second_attempt() { let call_count = Arc::new(AtomicUsize::new(0)); let count = Arc::clone(&call_count); - let result = retry_not_leader(None, move |_attempt| { + let result = retry_not_leader(None, far_deadline(), move |_attempt| { let c = Arc::clone(&count); async move { let n = c.fetch_add(1, Ordering::SeqCst); @@ -176,7 +231,7 @@ mod tests { #[tokio::test] async fn exhausts_retries_returns_not_leader() { - let result = retry_not_leader(None, |_| async { + let result = retry_not_leader(None, far_deadline(), |_| async { Err::(Error::NotLeader { vshard_id: VShardId::new(1), leader_node: 0, @@ -192,7 +247,7 @@ mod tests { async fn non_not_leader_error_propagates_immediately() { let call_count = Arc::new(AtomicUsize::new(0)); let count = Arc::clone(&call_count); - let result = retry_not_leader(None, move |_| { + let result = retry_not_leader(None, far_deadline(), move |_| { let c = Arc::clone(&count); async move { c.fetch_add(1, Ordering::SeqCst); @@ -215,7 +270,7 @@ mod tests { let call_count = Arc::new(AtomicUsize::new(0)); let count = Arc::clone(&call_count); - let _ = retry_not_leader(Some(&*rt_clone), move |_| { + let _ = retry_not_leader(Some(&*rt_clone), far_deadline(), move |_| { let c = Arc::clone(&count); async move { let n = c.fetch_add(1, Ordering::SeqCst); @@ -245,8 +300,8 @@ mod tests { assert!(table.observe_leader(group_id, 3, 7)); let rt = RwLock::new(table); - let _ = retry_not_leader(Some(&rt), |attempt| async move { - if attempt == 0 { + let _ = retry_not_leader(Some(&rt), far_deadline(), |attempt| async move { + if attempt.index == 0 { Err(Error::NotLeader { vshard_id: VShardId::new(0), leader_node: 2, @@ -268,8 +323,8 @@ mod tests { #[tokio::test] async fn a_term_less_redirect_never_replaces_a_termed_hint() { async fn redirect_to(rt: &RwLock, leader_node: u64) { - let _ = retry_not_leader(Some(rt), |attempt| async move { - if attempt == 0 { + let _ = retry_not_leader(Some(rt), far_deadline(), |attempt| async move { + if attempt.index == 0 { Err(Error::NotLeader { vshard_id: VShardId::new(0), leader_node, @@ -298,4 +353,118 @@ mod tests { (3, 7) ); } + + /// Every attempt carries the one statement deadline, so the budget left + /// shrinks across attempts and never restarts. + #[tokio::test] + async fn the_statement_deadline_is_shared_across_attempts() { + let deadline = far_deadline(); + let seen = Arc::new(std::sync::Mutex::new(Vec::new())); + let record = Arc::clone(&seen); + let result = retry_not_leader(None, deadline, move |attempt| { + let record = Arc::clone(&record); + async move { + record.lock().unwrap().push(( + attempt, + attempt.attempts_left(), + attempt.remaining_ms(), + )); + Err::<(), Error>(Error::NotLeader { + vshard_id: VShardId::new(0), + leader_node: 0, + leader_addr: String::new(), + leader_term: 0, + }) + } + }) + .await; + assert!(matches!(result, Err(Error::NotLeader { .. }))); + + let seen = seen.lock().unwrap(); + assert_eq!(seen.len(), MAX_RETRIES); + for (index, (attempt, attempts_left, _)) in seen.iter().enumerate() { + assert_eq!(attempt.index, index); + assert_eq!(attempt.deadline, deadline); + assert_eq!(*attempts_left, MAX_RETRIES - index); + } + let first_remaining = seen[0].2; + let last_remaining = seen[MAX_RETRIES - 1].2; + let backoff: u64 = BACKOFF_MS[..MAX_RETRIES - 1].iter().sum(); + assert!( + last_remaining <= first_remaining - backoff, + "the last attempt has {last_remaining} ms, the first had {first_remaining} ms" + ); + } + + /// A retry never starts once the statement's deadline has passed. The + /// refusal of the attempt that ran stays the answer: nothing applied. + #[tokio::test] + async fn no_retry_starts_past_the_deadline() { + let calls = Arc::new(AtomicUsize::new(0)); + let count = Arc::clone(&calls); + let result = retry_not_leader(None, Instant::now(), move |_| { + let c = Arc::clone(&count); + async move { + c.fetch_add(1, Ordering::SeqCst); + Err::<(), Error>(Error::NotLeader { + vshard_id: VShardId::new(0), + leader_node: 7, + leader_addr: String::new(), + leader_term: 3, + }) + } + }) + .await; + assert!( + matches!( + result, + Err(Error::NotLeader { + leader_node: 7, + leader_term: 3, + .. + }) + ), + "got {result:?}" + ); + assert_eq!(calls.load(Ordering::SeqCst), 1); + } + + /// An overwritten entry was never applied. Past the deadline it stays a + /// definite refusal, never an unknown outcome. + #[tokio::test] + async fn an_overwritten_entry_past_the_deadline_stays_definite() { + let result = retry_not_leader(None, Instant::now(), |_| async { + Err::<(), Error>(Error::RetryableLeaderChange { + group_id: 4, + log_index: 9, + }) + }) + .await; + assert!( + matches!( + result, + Err(Error::RetryableLeaderChange { + group_id: 4, + log_index: 9 + }) + ), + "got {result:?}" + ); + } + + /// A sole attempt leaves no retry behind it. + #[test] + fn a_sole_attempt_has_one_attempt_left() { + let deadline = far_deadline(); + let attempt = Attempt::sole(deadline); + assert_eq!(attempt.attempts_left(), 1); + assert_eq!(attempt.deadline, deadline); + } + + /// A spent deadline still sends a 1 ms hop, so the receiver checks it. + #[tokio::test] + async fn remaining_ms_never_reaches_zero() { + let attempt = Attempt::sole(Instant::now()); + assert_eq!(attempt.remaining_ms(), 1); + } } diff --git a/nodedb/src/control/gateway/stream.rs b/nodedb/src/control/gateway/stream.rs index ea90ad3cd..03d81642e 100644 --- a/nodedb/src/control/gateway/stream.rs +++ b/nodedb/src/control/gateway/stream.rs @@ -15,10 +15,11 @@ use crate::control::server::shared::clone_write::CloneCheckedTask; use nodedb_physical::physical_plan::PhysicalPlan; use super::core::{Gateway, QueryContext, authorized_plan_for_context}; -use super::dispatcher::{DispatchRouteStreamParams, dispatch_route_stream, statement_deadline_ms}; +use super::dispatcher::{DispatchRouteStreamParams, dispatch_route_stream}; use super::live_leaders::resolve_live_decision; use super::retry::retry_not_leader; use super::route::TaskRoute; +use crate::control::server::shared::session::StatementDeadline; impl Gateway { /// Streaming sibling of [`execute`](Gateway::execute). @@ -63,7 +64,8 @@ impl Gateway { let routes = self.compute_routes(plan, ctx)?; - let deadline_ms = statement_deadline_ms(&shared); + // One deadline for every route and attempt of the statement. + let deadline = StatementDeadline::of_statement(&shared).tokio_instant(); let mut per_route: Vec = Vec::with_capacity(routes.len()); for route in routes { @@ -78,8 +80,8 @@ impl Gateway { // attempt re-resolves the routing decision and re-opens the stream; // a pre-row NotLeader / DescriptorMismatch is retryable, anything // after the first frame is terminal (handled inside the stream). - let stream = retry_not_leader(routing_ref, move |attempt| { - if attempt > 0 { + let stream = retry_not_leader(routing_ref, deadline, move |attempt| { + if attempt.index > 0 { retry_counter.fetch_add(1, Ordering::Relaxed); } let plan = plan_for_retry.clone(); @@ -102,7 +104,8 @@ impl Gateway { tenant_id, database_id, trace_id, - deadline_ms, + deadline_ms: attempt.remaining_ms(), + attempts_left: attempt.attempts_left(), version_set: &version_set, linearizable, }) diff --git a/nodedb/src/control/lease/drain_propose.rs b/nodedb/src/control/lease/drain_propose.rs index 4b9c39b71..808169f1e 100644 --- a/nodedb/src/control/lease/drain_propose.rs +++ b/nodedb/src/control/lease/drain_propose.rs @@ -194,9 +194,6 @@ fn lease_holder_is_member(shared: &SharedState, node_id: u64) -> bool { } } -/// How long a drain variant's propose waits for its local apply. -pub(super) const DRAIN_PROPOSE_TIMEOUT: Duration = Duration::from_secs(5); - /// Encode a drain variant for the metadata group. pub(super) fn encode_drain(entry: &MetadataEntry, operation: &str) -> Result, Error> { encode_entry(entry).map_err(|e| Error::Config { @@ -204,24 +201,6 @@ pub(super) fn encode_drain(entry: &MetadataEntry, operation: &str) -> Result Result<(), Error> { - if outcome.is_reached() { - return Ok(()); - } - Err(Error::Config { - detail: format!( - "descriptor drain {operation} did not apply within {DRAIN_PROPOSE_TIMEOUT:?} \ - (log index {log_index}, current: {current}, outcome: {outcome:?})" - ), - }) -} - #[cfg(test)] mod tests { use std::sync::Arc; diff --git a/nodedb/src/control/lease/drain_propose_async.rs b/nodedb/src/control/lease/drain_propose_async.rs index 3246ccaf3..3d3d8334f 100644 --- a/nodedb/src/control/lease/drain_propose_async.rs +++ b/nodedb/src/control/lease/drain_propose_async.rs @@ -11,14 +11,11 @@ use std::time::{Duration, Instant}; use nodedb_cluster::{DescriptorId, DrainOwner, MetadataEntry}; -use crate::control::metadata_proposer::wait::wait_applied; +use crate::control::metadata_proposer::wait::await_metadata_apply; use crate::control::state::SharedState; use crate::error::Error; -use super::drain_propose::{ - DRAIN_PROPOSE_TIMEOUT, POLL_INTERVAL, drain_applied_or_error, drain_start_entry, - drained_or_timed_out, encode_drain, -}; +use super::drain_propose::{POLL_INTERVAL, drain_start_entry, drained_or_timed_out, encode_drain}; /// Drain every lease on `id` at `version <= up_to_version` for a `Put*` DDL. /// @@ -140,7 +137,8 @@ pub async fn end_drain_async( .await } -/// Propose a drain variant and await its apply on this node. +/// Propose a drain variant and await its apply on this node, until the +/// statement deadline. async fn propose_drain_async( shared: &SharedState, entry: MetadataEntry, @@ -150,12 +148,5 @@ async fn propose_drain_async( let log_index = handle .propose_async(encode_drain(&entry, operation)?) .await?; - let watcher = shared.applied_index_watcher(nodedb_cluster::METADATA_GROUP_ID); - let outcome = wait_applied( - std::sync::Arc::clone(&watcher), - log_index, - DRAIN_PROPOSE_TIMEOUT, - ) - .await?; - drain_applied_or_error(outcome, operation, log_index, watcher.current()) + await_metadata_apply(shared, log_index).await.map(|_| ()) } diff --git a/nodedb/src/control/lease/leader_wait.rs b/nodedb/src/control/lease/leader_wait.rs index 52c7aca70..f3321d75b 100644 --- a/nodedb/src/control/lease/leader_wait.rs +++ b/nodedb/src/control/lease/leader_wait.rs @@ -10,9 +10,6 @@ use nodedb_cluster::{MetadataEntry, encode_entry}; use crate::control::state::SharedState; use crate::error::Error; -/// Same propose-and-wait timeout the catalog DDL path uses. -pub(in crate::control) const PROPOSE_TIMEOUT: Duration = Duration::from_secs(5); - /// Backoff schedule for re-proposing while the metadata group elects a leader. /// /// Reads take a descriptor lease, so a lease proposal issued in the first @@ -21,7 +18,7 @@ pub(in crate::control) const PROPOSE_TIMEOUT: Duration = Duration::from_secs(5); /// failed proposal, so it is waited out here rather than surfaced as a failed /// statement. Bounded at ~1.6s total: long enough for a single-node or healthy /// multi-node election, short enough that a genuinely leaderless group still -/// fails well inside [`PROPOSE_TIMEOUT`]. +/// fails well inside the statement deadline. const LEADER_ELECTION_BACKOFF_MS: [u64; 7] = [10, 25, 50, 100, 200, 400, 800]; /// Propose `raw`, waiting out an in-progress metadata election. @@ -55,10 +52,10 @@ async fn propose_once_leader_is_elected( /// Encode `entry`, propose it through the metadata raft handle, and await /// the local applied watermark until the proposed log index applies or the -/// timeout fires. +/// statement deadline passes. /// /// Shared by the lease grant and release paths. `operation` is a short label -/// for the encode-failure and timeout errors. +/// for the encode-failure error. pub(in crate::control) async fn propose_and_wait( shared: &SharedState, entry: &MetadataEntry, @@ -69,22 +66,5 @@ pub(in crate::control) async fn propose_and_wait( detail: format!("descriptor lease {operation} encode: {e}"), })?; let log_index = propose_once_leader_is_elected(handle.as_ref(), raw, operation).await?; - - let watcher = shared.applied_index_watcher(nodedb_cluster::METADATA_GROUP_ID); - let outcome = crate::control::metadata_proposer::wait::wait_applied( - std::sync::Arc::clone(&watcher), - log_index, - PROPOSE_TIMEOUT, - ) - .await?; - if !outcome.is_reached() { - return Err(Error::Config { - detail: format!( - "descriptor lease {operation} did not apply within {PROPOSE_TIMEOUT:?} \ - (log index {log_index}, current: {}, outcome: {outcome:?})", - watcher.current() - ), - }); - } - Ok(log_index) + crate::control::metadata_proposer::wait::await_metadata_apply(shared, log_index).await } diff --git a/nodedb/src/control/lease/mod.rs b/nodedb/src/control/lease/mod.rs index 4b398ee51..e248e1a86 100644 --- a/nodedb/src/control/lease/mod.rs +++ b/nodedb/src/control/lease/mod.rs @@ -51,7 +51,7 @@ pub use drain::{DescriptorDrainTracker, DrainEntry}; pub use drain_apply::{apply_drain_ends, apply_drain_start}; pub use drain_propose_async::{drain_for_ddl_async, drain_for_owner_async, end_drain_async}; pub use holders::{LeaseHolders, LeaseRevocation, revoke_if_fenced, revoke_on_release}; -pub(super) use leader_wait::{PROPOSE_TIMEOUT, propose_and_wait}; +pub(super) use leader_wait::propose_and_wait; pub use propose::{DEFAULT_LEASE_DURATION, acquire_lease, compute_expires_at, force_refresh_lease}; pub(crate) use propose::{acquire_lease_after_admission, drain_owner_list, ensure_not_draining}; pub use refcount::{LeaseRefCount, QueryLeaseScope}; diff --git a/nodedb/src/control/lease/release.rs b/nodedb/src/control/lease/release.rs index 805e958a4..383691e3b 100644 --- a/nodedb/src/control/lease/release.rs +++ b/nodedb/src/control/lease/release.rs @@ -17,6 +17,9 @@ pub(crate) struct LeaseReleaseHandle { node_id: u64, metadata_raft: Arc, applied_watcher: Arc, + /// `tuning.network.default_deadline_secs`: the apply wait's budget + /// outside a statement. + default_deadline_secs: u64, grant_gate: Arc>, refcounts: Arc, } @@ -29,6 +32,7 @@ impl LeaseReleaseHandle { node_id: shared.node_id, metadata_raft: Arc::clone(shared.metadata_raft_handle()?), applied_watcher: shared.applied_index_watcher(nodedb_cluster::METADATA_GROUP_ID), + default_deadline_secs: shared.tuning.network.default_deadline_secs, grant_gate: Arc::clone(&shared.lease_grant_gate), refcounts: Arc::clone(&shared.lease_refcount), }) @@ -90,22 +94,16 @@ impl LeaseReleaseHandle { detail: format!("descriptor lease release encode: {error}"), })?; let log_index = self.metadata_raft.propose_async(raw).await?; - let outcome = crate::control::metadata_proposer::wait::wait_applied( - Arc::clone(&self.applied_watcher), + let deadline = crate::control::server::shared::session::StatementDeadline::with_default( + self.default_deadline_secs, + ) + .instant(); + crate::control::metadata_proposer::wait::await_applied( + &self.applied_watcher, log_index, - super::PROPOSE_TIMEOUT, + deadline, ) .await?; - if !outcome.is_reached() { - return Err(Error::Config { - detail: format!( - "descriptor lease release did not apply within {:?} \ - (log index {log_index}, current: {}, outcome: {outcome:?})", - super::PROPOSE_TIMEOUT, - self.applied_watcher.current() - ), - }); - } Ok(()) } } diff --git a/nodedb/src/control/lease/self_fence.rs b/nodedb/src/control/lease/self_fence.rs index a4a16eb3c..9ce2f504c 100644 --- a/nodedb/src/control/lease/self_fence.rs +++ b/nodedb/src/control/lease/self_fence.rs @@ -24,7 +24,11 @@ use crate::control::state::SharedState; /// exists to drain or release this node's leases, so the node never fences. pub(crate) fn lease_use_is_fenced(shared: &SharedState) -> bool { match shared.raft_read_gate.get() { - Some(gate) => !gate.within_staleness_bound(METADATA_GROUP_ID, LEASE_SELF_FENCE_WINDOW), + Some(gate) => !gate.within_staleness_bound( + METADATA_GROUP_ID, + LEASE_SELF_FENCE_WINDOW, + shared.applied_index_watcher(METADATA_GROUP_ID).current(), + ), None => false, } } @@ -66,7 +70,12 @@ mod tests { Err(ReadIndexRefusal::NotLeader) } - fn within_staleness_bound(&self, group_id: u64, max_staleness: Duration) -> bool { + fn within_staleness_bound( + &self, + group_id: u64, + max_staleness: Duration, + _applied: u64, + ) -> bool { group_id == METADATA_GROUP_ID && Duration::from_millis(self.contact_age_ms.load(Ordering::SeqCst)) <= max_staleness diff --git a/nodedb/src/control/maintenance/clone_materializer/chained.rs b/nodedb/src/control/maintenance/clone_materializer/chained.rs index 1a3286d82..d125cf24e 100644 --- a/nodedb/src/control/maintenance/clone_materializer/chained.rs +++ b/nodedb/src/control/maintenance/clone_materializer/chained.rs @@ -259,7 +259,8 @@ async fn commit_relink( target: RedoTarget, payload: &TransactionRedoPayload, ) -> crate::Result<()> { - let deadline = crate::control::wal_replication::statement_propose_deadline(state); + let deadline = crate::control::server::shared::session::StatementDeadline::of_statement(state) + .tokio_instant(); propose_transaction_redo(state, target, payload, deadline).await?; Ok(()) } diff --git a/nodedb/src/control/metadata_proposer/catalog.rs b/nodedb/src/control/metadata_proposer/catalog.rs index d71f2408c..58aeed322 100644 --- a/nodedb/src/control/metadata_proposer/catalog.rs +++ b/nodedb/src/control/metadata_proposer/catalog.rs @@ -3,9 +3,9 @@ //! Propose a catalog entry and wait until this node applied it. use std::sync::atomic::Ordering; -use std::time::Duration; +use std::time::Instant; -use nodedb_cluster::{METADATA_GROUP_ID, MetadataEntry, WaitOutcome, encode_entry}; +use nodedb_cluster::{METADATA_GROUP_ID, MetadataEntry, encode_entry}; use crate::control::catalog_entry::{self, CatalogEntry}; use crate::control::propose_outcome::ProposeOutcome; @@ -14,8 +14,9 @@ use crate::error::Error; use super::ddl_prepare::{acquire_ddl_prepare_lease_async, lock_ddl_preparation_async}; use super::handle::MetadataRaftHandle; -use super::timeouts::{DEFAULT_DRAIN_TIMEOUT, DEFAULT_PROPOSE_TIMEOUT}; -use super::wait::wait_tracking_progress_async; +use super::timeouts::DEFAULT_DRAIN_TIMEOUT; +use super::wait::await_applied; +use crate::control::server::shared::session::StatementDeadline; /// Propose a `CatalogEntry` and wait until this node applied it, on any /// runtime flavor. @@ -80,7 +81,7 @@ async fn propose_and_apply_locally( handle.as_ref(), lease.token(), catalog_ddl_entry(&stamped)?, - DEFAULT_PROPOSE_TIMEOUT, + StatementDeadline::of_statement(shared).instant(), ) .await } @@ -205,17 +206,18 @@ pub(super) fn catalog_ddl_entry_with( } /// Propose `entry` under the preparation lease `token` and wait until this -/// node applied it. Returns the log index. Fails when another lease owner -/// superseded `token` before the apply. +/// node applied it, until `deadline`. Returns the log index. /// -/// `timeout` bounds a stall, not the whole wait, as -/// [`super::wait::wait_tracking_progress_async`] describes. +/// - Past `deadline`: `Error::DeadlineExceeded`. The entry was proposed, so +/// its outcome is unknown and no caller resends it. +/// - Another lease owner superseded `token` before the apply: +/// `Error::MetadataDdlSuperseded`. The apply skipped the entry. pub(super) async fn propose_prepared( shared: &SharedState, handle: &dyn MetadataRaftHandle, token: u64, entry: MetadataEntry, - timeout: Duration, + deadline: Instant, ) -> Result { let raw = encode_entry(&MetadataEntry::DdlPrepared { token, @@ -226,79 +228,100 @@ pub(super) async fn propose_prepared( })?; let log_index = handle.propose_async(raw).await?; let watcher = shared.applied_index_watcher(METADATA_GROUP_ID); - let outcome = - wait_tracking_progress_async(std::sync::Arc::clone(&watcher), log_index, timeout, || { - shared.metadata_apply_progress.load(Ordering::Acquire) - }) - .await?; - match outcome { - WaitOutcome::Reached - if shared.metadata_ddl.applied_token.load(Ordering::Acquire) == token => - { - Ok(log_index) - } - WaitOutcome::Reached => Err(Error::Config { - detail: "metadata DDL preparation ownership was superseded before apply".into(), - }), - WaitOutcome::TimedOut => Err(Error::Config { - detail: format!( - "metadata propose timed out after {timeout:?} without apply progress waiting \ - for log index {log_index} (current: {})", - watcher.current() - ), - }), - WaitOutcome::GroupGone => Err(Error::Config { - detail: "metadata group no longer hosted on this node".into(), - }), + await_applied(&watcher, log_index, deadline).await?; + if shared.metadata_ddl.applied_token.load(Ordering::Acquire) == token { + Ok(log_index) + } else { + Err(Error::MetadataDdlSuperseded { log_index }) } } #[cfg(test)] mod tests { use std::sync::Arc; - use std::sync::atomic::AtomicU64; - - use nodedb_cluster::AppliedIndexWatcher; + use std::time::Duration; use super::*; - const WINDOW: Duration = Duration::from_millis(60); + /// A handle that assigns every proposal log index 5. + struct AtFive; + + impl MetadataRaftHandle for AtFive { + fn propose_async<'a>( + &'a self, + _bytes: Vec, + ) -> crate::control::metadata_proposer::ProposeFuture<'a> { + Box::pin(async { Ok::(5) }) + } + } + + fn test_state() -> (Arc, tempfile::TempDir) { + let dir = tempfile::tempdir().expect("tempdir"); + let wal = Arc::new( + crate::wal::WalManager::open_for_testing(&dir.path().join("propose.wal")).expect("wal"), + ); + let (dispatcher, _sides) = crate::bridge::dispatch::Dispatcher::new(1, 64); + let state = SharedState::new(dispatcher, wal).expect("shared state"); + (state, dir) + } - /// Progress that keeps moving for many windows never times out; the wait - /// ends when the entry applies. + fn ddl() -> MetadataEntry { + MetadataEntry::CatalogDdl { + payload: Vec::new(), + } + } + + /// An apply that lands after a 1 s quiet gap, inside a 2 s deadline, + /// answers the log index. #[tokio::test] - async fn steady_progress_never_times_out() { - let watcher = Arc::new(AppliedIndexWatcher::new()); - let progress = Arc::new(AtomicU64::new(0)); - let driver = { - let (watcher, progress) = (Arc::clone(&watcher), Arc::clone(&progress)); - std::thread::spawn(move || { - for _ in 0..20 { - std::thread::sleep(Duration::from_millis(20)); - progress.fetch_add(1, Ordering::Release); - } - watcher.bump(5); + async fn a_slow_apply_inside_the_deadline_succeeds() { + const TOKEN: u64 = 77; + let (state, _dir) = test_state(); + let applier = { + let state = Arc::clone(&state); + tokio::spawn(async move { + tokio::time::sleep(Duration::from_secs(1)).await; + state + .metadata_ddl + .applied_token + .store(TOKEN, Ordering::Release); + state.applied_index_watcher(METADATA_GROUP_ID).bump(5); }) }; - let outcome = wait_tracking_progress_async(Arc::clone(&watcher), 5, WINDOW, || { - progress.load(Ordering::Acquire) - }) - .await - .expect("the wait finishes"); - driver.join().unwrap(); - assert!(outcome.is_reached(), "{outcome:?}"); + let deadline = Instant::now() + Duration::from_secs(2); + let applied = propose_prepared(&state, &AtFive, TOKEN, ddl(), deadline).await; + applier.await.expect("join applier"); + assert_eq!(applied.expect("the entry applied"), 5); } - /// A stall of one window times the wait out. + /// An entry that never applies ends at the deadline as `DeadlineExceeded`, + /// never as a config error: it was proposed, so its outcome is unknown. #[tokio::test] - async fn a_stall_times_out() { - let watcher = Arc::new(AppliedIndexWatcher::new()); - let started = std::time::Instant::now(); - let outcome = wait_tracking_progress_async(watcher, 5, WINDOW, || 7) - .await - .expect("the wait finishes"); - assert!(matches!(outcome, WaitOutcome::TimedOut), "{outcome:?}"); - assert!(started.elapsed() < WINDOW * 4); + async fn an_entry_that_never_applies_ends_at_the_deadline() { + let (state, _dir) = test_state(); + let budget = Duration::from_millis(300); + let started = Instant::now(); + let result = propose_prepared(&state, &AtFive, 1, ddl(), started + budget).await; + assert!( + matches!(result, Err(Error::DeadlineExceeded { .. })), + "{result:?}" + ); + assert!(started.elapsed() >= budget); + } + + /// An entry whose apply skipped it under another lease owner is + /// superseded, not a success. + #[tokio::test] + async fn a_superseded_token_is_a_typed_refusal() { + let (state, _dir) = test_state(); + state.metadata_ddl.applied_token.store(9, Ordering::Release); + state.applied_index_watcher(METADATA_GROUP_ID).bump(5); + let deadline = Instant::now() + Duration::from_secs(2); + let result = propose_prepared(&state, &AtFive, 1, ddl(), deadline).await; + assert!( + matches!(result, Err(Error::MetadataDdlSuperseded { log_index: 5 })), + "{result:?}" + ); } /// A one-node cluster stamps a create, commits it through its metadata diff --git a/nodedb/src/control/metadata_proposer/catalog_batch.rs b/nodedb/src/control/metadata_proposer/catalog_batch.rs index 4e2dc3122..dff28c3b7 100644 --- a/nodedb/src/control/metadata_proposer/catalog_batch.rs +++ b/nodedb/src/control/metadata_proposer/catalog_batch.rs @@ -13,7 +13,7 @@ use crate::error::Error; use super::catalog::{catalog_ddl_entry, propose_prepared}; use super::ddl_prepare::{acquire_ddl_prepare_lease_async, lock_ddl_preparation_async}; use super::handle::MetadataRaftHandle; -use super::timeouts::{DEFAULT_DRAIN_TIMEOUT, DEFAULT_PROPOSE_TIMEOUT}; +use super::timeouts::DEFAULT_DRAIN_TIMEOUT; /// What happened to the entries a batch plan built. #[derive(Debug, Clone, Copy, PartialEq, Eq)] @@ -113,14 +113,14 @@ async fn propose_replicated( for entry in &entries { wrapped.push(catalog_ddl_entry(entry)?); } - // The wait follows apply progress: a batch of slow purges keeps it - // alive, and only a stall of `DEFAULT_PROPOSE_TIMEOUT` ends it. + // A batch of slow purges applies for as long as the statement lets it. let log_index = propose_prepared( shared, handle, lease.token(), MetadataEntry::Batch { entries: wrapped }, - DEFAULT_PROPOSE_TIMEOUT, + crate::control::server::shared::session::StatementDeadline::of_statement(shared) + .instant(), ) .await?; Ok::<_, Error>(Some((log_index, entries))) diff --git a/nodedb/src/control/metadata_proposer/ddl_prepare.rs b/nodedb/src/control/metadata_proposer/ddl_prepare.rs index 2e03fcc77..a9eb2bffe 100644 --- a/nodedb/src/control/metadata_proposer/ddl_prepare.rs +++ b/nodedb/src/control/metadata_proposer/ddl_prepare.rs @@ -3,18 +3,16 @@ //! The DDL preparation lease: the local lock and the replicated lease that //! serialize descriptor preparation across the cluster. -use std::sync::Arc; use std::sync::atomic::Ordering; use std::time::{Duration, Instant, SystemTime, UNIX_EPOCH}; -use nodedb_cluster::{METADATA_GROUP_ID, MetadataEntry, WaitOutcome, encode_entry}; +use nodedb_cluster::{MetadataEntry, encode_entry}; use crate::control::state::SharedState; use crate::error::Error; use super::handle::MetadataRaftHandle; -use super::timeouts::DEFAULT_PROPOSE_TIMEOUT; -use super::wait::wait_applied; +use super::wait::await_metadata_apply; /// How long a proposer waits for the lease. It outlasts the leader's /// stuck-owner fallback, so a reclaim always frees the lease first. @@ -47,37 +45,15 @@ fn encode_metadata(entry: &MetadataEntry) -> Result, Error> { }) } -/// The result of waiting `timeout` for log index `index` to apply. -fn applied_or_error( - outcome: WaitOutcome, - index: u64, - timeout: Duration, - current: u64, -) -> Result { - match outcome { - WaitOutcome::Reached => Ok(index), - WaitOutcome::TimedOut => Err(Error::Config { - detail: format!( - "metadata propose timed out after {timeout:?} waiting for log index {index} \ - (current: {current})" - ), - }), - WaitOutcome::GroupGone => Err(Error::Config { - detail: "metadata group no longer hosted on this node".into(), - }), - } -} - +/// Propose `entry` and wait until this node applied it, until the statement +/// deadline. Returns its log index. pub(super) async fn propose_metadata_and_wait_async( shared: &SharedState, handle: &dyn MetadataRaftHandle, entry: &MetadataEntry, - timeout: Duration, ) -> Result { let index = handle.propose_async(encode_metadata(entry)?).await?; - let watcher = shared.applied_index_watcher(METADATA_GROUP_ID); - let outcome = wait_applied(Arc::clone(&watcher), index, timeout).await?; - applied_or_error(outcome, index, timeout, watcher.current()) + await_metadata_apply(shared, index).await } /// What the acquire loop does next, from the current lease owner. @@ -117,7 +93,6 @@ pub(crate) async fn release_ddl_prepare_token( shared, handle.as_ref(), &MetadataEntry::DdlPrepareRelease { token }, - DEFAULT_PROPOSE_TIMEOUT, ) .await .map(|_| ()) @@ -145,20 +120,29 @@ impl DdlPrepareLease<'_> { self.token } - /// Release the lease and wait until the release applied here. A failed - /// release is logged: the metadata leader reclaims the lease once - /// [`super::ddl_owner::DDL_PREPARE_LEASE`] passed. + /// Release the lease and wait until the release applied here, until the + /// statement deadline. A failed release is logged: the metadata leader + /// reclaims the lease once [`super::ddl_owner::DDL_PREPARE_LEASE`] passed. pub(crate) async fn release(mut self) { self.released = true; - if let Err(error) = propose_metadata_and_wait_async( + match propose_metadata_and_wait_async( self.shared, self.handle, &MetadataEntry::DdlPrepareRelease { token: self.token }, - DEFAULT_PROPOSE_TIMEOUT, ) .await { - tracing::error!(token = self.token, %error, "metadata DDL lease release failed"); + Ok(_) => {} + // Proposed: it applies in log order, ahead of this node's next + // acquire. + Err(error @ Error::DeadlineExceeded { .. }) => tracing::warn!( + token = self.token, + %error, + "metadata DDL lease release did not apply here before the statement deadline" + ), + Err(error) => { + tracing::error!(token = self.token, %error, "metadata DDL lease release failed"); + } } } } @@ -189,7 +173,6 @@ pub(crate) async fn acquire_ddl_prepare_lease_async<'a>( token, node_id: shared.node_id, }, - DEFAULT_PROPOSE_TIMEOUT, ) .await?; @@ -220,6 +203,8 @@ pub(crate) async fn lock_ddl_preparation_async( #[cfg(test)] mod tests { + use std::sync::Arc; + use super::*; fn owner(shared: &SharedState) -> Option { diff --git a/nodedb/src/control/metadata_proposer/ddl_reclaim.rs b/nodedb/src/control/metadata_proposer/ddl_reclaim.rs index 76ed82c13..dd9b07c0c 100644 --- a/nodedb/src/control/metadata_proposer/ddl_reclaim.rs +++ b/nodedb/src/control/metadata_proposer/ddl_reclaim.rs @@ -20,7 +20,6 @@ use crate::error::Error; use super::ddl_owner::{current_owner, reclaim_cause}; use super::ddl_prepare::propose_metadata_and_wait_async; use super::handle::MetadataRaftHandle; -use super::timeouts::DEFAULT_PROPOSE_TIMEOUT; /// How often the leader checks the lease owner. const RECLAIM_POLL: Duration = Duration::from_millis(250); @@ -99,20 +98,10 @@ pub(crate) async fn reclaim_ddl_prepare_lease( token: u64, ) -> Result<(), Error> { if shared.pending_ddl.contains(token) { - propose_metadata_and_wait_async( - shared, - handle, - &MetadataEntry::DdlPendingCancel { token }, - DEFAULT_PROPOSE_TIMEOUT, - ) - .await?; + propose_metadata_and_wait_async(shared, handle, &MetadataEntry::DdlPendingCancel { token }) + .await?; } - propose_metadata_and_wait_async( - shared, - handle, - &MetadataEntry::DdlPrepareRelease { token }, - DEFAULT_PROPOSE_TIMEOUT, - ) - .await - .map(|_| ()) + propose_metadata_and_wait_async(shared, handle, &MetadataEntry::DdlPrepareRelease { token }) + .await + .map(|_| ()) } diff --git a/nodedb/src/control/metadata_proposer/handle.rs b/nodedb/src/control/metadata_proposer/handle.rs index 175799028..810220f6f 100644 --- a/nodedb/src/control/metadata_proposer/handle.rs +++ b/nodedb/src/control/metadata_proposer/handle.rs @@ -8,6 +8,7 @@ use nodedb_cluster::ClusterError; use nodedb_raft::RaftError; use crate::error::Error; +use crate::types::RequestId; /// Type-erased handle for proposing to the metadata raft group. /// @@ -95,31 +96,41 @@ impl MetadataRaftHandle for RaftLoopProposerHandle { } /// The error a metadata proposal returns for a cluster error. +/// +/// A group with no leader to take the proposal proposed nothing, so the +/// caller can re-propose: [`Error::MetadataLeaderUnavailable`]. A forward +/// with no answer can have been proposed, so its outcome is unknown: +/// [`Error::DeadlineExceeded`]. A typed verdict keeps its class. fn metadata_propose_error(error: ClusterError) -> Error { match error { - // An election in progress is transient, not a failure of this - // proposal. Keep it typed rather than flattening it into a generic - // config error, so callers can wait the election out instead of - // failing the statement — a node that has recently restarted answers - // every metadata proposal this way for a moment. - ClusterError::Raft(RaftError::NotLeader { - leader_hint: None, .. - }) => Error::MetadataLeaderUnavailable, + // An election, a transfer in flight, or a leader that stepped down. + // Nothing was proposed. + ClusterError::Raft( + RaftError::NotLeader { .. } | RaftError::LeadershipTransferInProgress, + ) => Error::MetadataLeaderUnavailable, + // A forward that never reached the leader whole proposed nothing. + error if error.is_unsent() => Error::MetadataLeaderUnavailable, + // The forward is not resend-safe and got no answer. The leader can + // have appended the entry, so the outcome is unknown. + error if error.is_lost_answer() => { + tracing::warn!( + %error, + "metadata proposal forward got no answer from the leader" + ); + Error::DeadlineExceeded { + request_id: RequestId::new(0), + } + } // A typed verdict keeps its class. ClusterError::DataPlane { code } => Error::DataPlane(code.into()), ClusterError::ShardExecution { error, .. } | ClusterError::StreamTerminal { error, .. } => { Error::from(*error) } other @ (ClusterError::Raft( - RaftError::NotLeader { - leader_hint: Some(_), - .. - } - | RaftError::LogCompacted { .. } + RaftError::LogCompacted { .. } | RaftError::CompactionAheadOfApplied { .. } | RaftError::ProposalRejected { .. } | RaftError::InvalidTransferTarget { .. } - | RaftError::LeadershipTransferInProgress | RaftError::GroupNotFound { .. } | RaftError::Transport { .. } | RaftError::Storage { .. } @@ -134,9 +145,6 @@ fn metadata_propose_error(error: ClusterError) -> Error { | ClusterError::MigrationPauseBudgetExceeded { .. } | ClusterError::NodeUnreachable { .. } | ClusterError::GhostNotFound { .. } - | ClusterError::Transport { .. } - | ClusterError::ShardTimeout { .. } - | ClusterError::Unanswered { .. } | ClusterError::Storage { .. } | ClusterError::Codec { .. } | ClusterError::UnsupportedWireVersion { .. } @@ -162,8 +170,54 @@ fn metadata_propose_error(error: ClusterError) -> Error { | ClusterError::Bm25Gather(_) | ClusterError::TsGather(_) | ClusterError::ShufflePush(_) - | ClusterError::RemoteUntyped { .. }) => Error::Config { + | ClusterError::RemoteUntyped { .. } + // The guards above take these three. They stay named for exhaustiveness. + | ClusterError::Transport { .. } + | ClusterError::ShardTimeout { .. } + | ClusterError::Unanswered { .. }) => Error::Config { detail: format!("metadata propose: {other}"), }, } } + +#[cfg(test)] +mod tests { + use super::*; + + /// A forward with no answer can have appended the entry. It is an + /// unknown outcome, never a config error and never retried. + #[test] + fn a_lost_metadata_answer_is_an_unknown_outcome() { + let unanswered = metadata_propose_error(ClusterError::Unanswered { + node_id: 2, + detail: "connection lost".into(), + }); + assert!(matches!(unanswered, Error::DeadlineExceeded { .. })); + + let timed_out = metadata_propose_error(ClusterError::ShardTimeout { + vshard_id: 0, + elapsed_ms: 50, + }); + assert!(matches!(timed_out, Error::DeadlineExceeded { .. })); + } + + /// A forward that never went out proposed nothing. The caller waits for + /// a leader and re-proposes. + #[test] + fn an_unsent_metadata_forward_waits_for_a_leader() { + let unsent = metadata_propose_error(ClusterError::Transport { + detail: "connect to node 2 refused".into(), + }); + assert!(matches!(unsent, Error::MetadataLeaderUnavailable)); + + let stale = metadata_propose_error(ClusterError::Raft(RaftError::NotLeader { + leader_hint: Some(2), + term: 3, + })); + assert!(matches!(stale, Error::MetadataLeaderUnavailable)); + + let moving = + metadata_propose_error(ClusterError::Raft(RaftError::LeadershipTransferInProgress)); + assert!(matches!(moving, Error::MetadataLeaderUnavailable)); + } +} diff --git a/nodedb/src/control/metadata_proposer/mod.rs b/nodedb/src/control/metadata_proposer/mod.rs index c06c8a0e2..c6d65e9ff 100644 --- a/nodedb/src/control/metadata_proposer/mod.rs +++ b/nodedb/src/control/metadata_proposer/mod.rs @@ -23,12 +23,14 @@ //! Semantics: //! //! 1. If this node is the metadata-group leader, proposes the entry, waits -//! until its local applied watermark reaches the assigned log index for -//! as long as the apply moves (a 5s stall ends the wait), and returns the -//! log index on success. -//! 2. If this node is NOT the leader, returns -//! `Error::Config { detail: "metadata propose: not leader ..." }`. -//! Gateway-side redirection will make this transparent. +//! until its local applied watermark reaches the assigned log index, and +//! returns the log index. The wait ends at the statement deadline with +//! `Error::DeadlineExceeded`: the entry was proposed, so its outcome is +//! unknown. +//! 2. If this node is NOT the leader, forwards the entry to the leader. +//! With no leader to take it, returns `Error::MetadataLeaderUnavailable`: +//! nothing was proposed. A forward with no answer returns +//! `Error::DeadlineExceeded`: the outcome is unknown. pub mod catalog; pub mod catalog_batch; @@ -53,4 +55,4 @@ pub use replicated_entries::{ propose_restore_point, propose_surrogate_hwm, propose_surrogate_reserve, propose_sync_peer_bind, propose_sync_producer_fence, propose_sync_producer_register, }; -pub use timeouts::{DEFAULT_DRAIN_TIMEOUT, DEFAULT_PROPOSE_TIMEOUT}; +pub use timeouts::DEFAULT_DRAIN_TIMEOUT; diff --git a/nodedb/src/control/metadata_proposer/replicated_entries.rs b/nodedb/src/control/metadata_proposer/replicated_entries.rs index 0e2cce34b..96d2c6b7f 100644 --- a/nodedb/src/control/metadata_proposer/replicated_entries.rs +++ b/nodedb/src/control/metadata_proposer/replicated_entries.rs @@ -8,19 +8,17 @@ //! runtime flavor. Every node runs a metadata group, a one-node cluster //! included. -use std::sync::Arc; - -use nodedb_cluster::{METADATA_GROUP_ID, MetadataEntry, encode_entry}; +use nodedb_cluster::{MetadataEntry, encode_entry}; use crate::control::catalog_entry::CatalogEntry; use crate::control::state::SharedState; use crate::error::Error; use crate::event::cdc::consumer_group::OffsetCommit; -use super::timeouts::DEFAULT_PROPOSE_TIMEOUT; +use super::wait::await_metadata_apply; -/// Propose `entry` and wait until this node reaches its log index. `label` -/// names the entry in the errors. +/// Propose `entry` and wait until this node reaches its log index, until the +/// statement deadline. `label` names the entry in the errors. async fn propose_and_wait( shared: &SharedState, entry: &MetadataEntry, @@ -32,17 +30,7 @@ async fn propose_and_wait( })?; let log_index = handle.propose_async(raw).await?; - - let watcher = shared.applied_index_watcher(METADATA_GROUP_ID); - let outcome = - super::wait::wait_applied(Arc::clone(&watcher), log_index, DEFAULT_PROPOSE_TIMEOUT).await?; - if !outcome.is_reached() { - return Err(Error::Config { - detail: format!("{label} propose timed out waiting for log index {log_index}"), - }); - } - - Ok(log_index) + await_metadata_apply(shared, log_index).await } /// Propose a cluster restore point at watermark `hlc` and wait until this @@ -86,7 +74,7 @@ pub async fn propose_surrogate_hwm(shared: &SharedState, hwm: u32) -> Result, - index: u64, - timeout: Duration, -) -> Result { - tokio::task::spawn_blocking(move || watcher.wait_for(index, timeout)) - .await - .map_err(|error| Error::Config { - detail: format!("metadata apply wait for log index {index} did not finish: {error}"), - }) +/// Wait until `watcher` reaches `log_index`, the index of a metadata entry +/// this node proposed, or `deadline` passes. Returns `log_index`. +pub(crate) async fn await_applied( + watcher: &AppliedIndexWatcher, + log_index: u64, + deadline: Instant, +) -> Result { + let outcome = watcher.wait_for_async(log_index, deadline).await; + applied_or_error(outcome, log_index, watcher.current()) } -/// Wait until `watcher` reaches `log_index`, for as long as the apply moves. -/// Each window parks on the blocking pool. -/// -/// Each window of `stall` that ends with neither the applied index nor -/// `progress()` changed ends the wait with `TimedOut`. Any change inside a -/// window opens another one, so a slow apply that keeps moving never times -/// out, and a stuck one times out after one quiet window. -pub(super) async fn wait_tracking_progress_async( - watcher: Arc, +/// Wait until this node applied the metadata entry it proposed at +/// `log_index`, until the statement deadline. Returns `log_index`. +pub(crate) async fn await_metadata_apply( + shared: &SharedState, log_index: u64, - stall: Duration, - mut progress: impl FnMut() -> u64, -) -> Result { - let mut seen = (watcher.current(), progress()); - loop { - let outcome = wait_applied(Arc::clone(&watcher), log_index, stall).await?; - if !matches!(outcome, WaitOutcome::TimedOut) { - return Ok(outcome); - } - let now = (watcher.current(), progress()); - if now == seen { - return Ok(outcome); +) -> Result { + let watcher = shared.applied_index_watcher(METADATA_GROUP_ID); + let deadline = StatementDeadline::of_statement(shared).instant(); + await_applied(&watcher, log_index, deadline).await +} + +/// The verdict on a proposed metadata entry from its apply wait. `applied` +/// is the watermark when the wait ended. +fn applied_or_error(outcome: WaitOutcome, log_index: u64, applied: u64) -> Result { + match outcome { + WaitOutcome::Reached => Ok(log_index), + WaitOutcome::TimedOut => { + tracing::warn!( + log_index, + applied, + "a proposed metadata entry did not apply here before the statement deadline; \ + its outcome is unknown" + ); + Err(Error::DeadlineExceeded { + request_id: RequestId::new(0), + }) } - seen = now; + WaitOutcome::GroupGone => Err(Error::MetadataGroupGone { log_index }), + } +} + +#[cfg(test)] +mod tests { + use std::time::Duration; + + use super::*; + + /// A metadata group that left this node ends the wait with its own error. + #[tokio::test] + async fn a_gone_group_ends_the_wait_with_a_typed_error() { + let watcher = AppliedIndexWatcher::new(); + watcher.close(); + let result = await_applied(&watcher, 5, Instant::now() + Duration::from_secs(5)).await; + assert!( + matches!(result, Err(Error::MetadataGroupGone { log_index: 5 })), + "{result:?}" + ); } } diff --git a/nodedb/src/control/orchestrated_write.rs b/nodedb/src/control/orchestrated_write.rs index dd4568dbf..55681ef45 100644 --- a/nodedb/src/control/orchestrated_write.rs +++ b/nodedb/src/control/orchestrated_write.rs @@ -49,7 +49,8 @@ pub(crate) async fn apply_orchestrated_write( } })?; let request_id = RequestId::new(state.request_id_counter.fetch_add(1, Ordering::Relaxed)); - let deadline = crate::control::wal_replication::statement_propose_deadline(state); + let deadline = crate::control::server::shared::session::StatementDeadline::of_statement(state) + .tokio_instant(); match propose_replicated_entry(state, proposer, entry, deadline).await { Ok((payload, write_versions)) => { let response = Response { diff --git a/nodedb/src/control/planner/calvin/dependent_recon.rs b/nodedb/src/control/planner/calvin/dependent_recon.rs index 8bbbb277b..df7221560 100644 --- a/nodedb/src/control/planner/calvin/dependent_recon.rs +++ b/nodedb/src/control/planner/calvin/dependent_recon.rs @@ -30,6 +30,7 @@ use crate::control::planner::calvin::{ use crate::control::planner::implicit_edges::{ EdgeUpdateCtx, append_implicit_edge_delete_tasks, append_implicit_edge_update_tasks, }; +use crate::control::server::shared::session::StatementDeadline; use crate::control::state::SharedState; use crate::types::{DatabaseId, TenantId, TraceId}; use nodedb_physical::physical_plan::OllpPredictedEdge; @@ -241,7 +242,8 @@ async fn dispatch_dependent_edge_recon_inner( ) .await?; - let timeout = std::time::Duration::from_secs(state.tuning.network.default_deadline_secs); + // One deadline for every attempt: the statement's. + let deadline = StatementDeadline::of_statement(state).tokio_instant(); let ollp_max_retries = orc.ollp_max_retries() as u32; // `submit`: build the TxClass with the loop-supplied prediction (NOT a @@ -422,7 +424,7 @@ async fn dispatch_dependent_edge_recon_inner( registry, orchestrator: orc, predicate_class_hash: pred_class, - timeout, + deadline, ollp_max_retries, initial_predicted, submit, diff --git a/nodedb/src/control/planner/calvin/dependent_recon_crdt.rs b/nodedb/src/control/planner/calvin/dependent_recon_crdt.rs index 238961314..439930ab2 100644 --- a/nodedb/src/control/planner/calvin/dependent_recon_crdt.rs +++ b/nodedb/src/control/planner/calvin/dependent_recon_crdt.rs @@ -34,6 +34,7 @@ use crate::Error; use crate::control::cluster::calvin::executor::ollp::error::OllpError; use crate::control::security::identity::AuthenticatedIdentity; use crate::control::server::graph_dispatch::{VShardRead, read_on_vshard}; +use crate::control::server::shared::session::StatementDeadline; use crate::control::state::SharedState; use crate::types::VShardId; use crate::types::{DatabaseId, TenantId}; @@ -162,7 +163,7 @@ pub(super) async fn dispatch_crdt_doc_deletes( registry, orchestrator: orc, predicate_class_hash: pred_class, - timeout: std::time::Duration::from_secs(state.tuning.network.default_deadline_secs), + deadline: StatementDeadline::of_statement(state).tokio_instant(), ollp_max_retries: orc.ollp_max_retries() as u32, initial_predicted, submit, diff --git a/nodedb/src/control/planner/calvin/mod.rs b/nodedb/src/control/planner/calvin/mod.rs index b0feb9ac1..e1018a3d9 100644 --- a/nodedb/src/control/planner/calvin/mod.rs +++ b/nodedb/src/control/planner/calvin/mod.rs @@ -43,8 +43,8 @@ pub use explain::calvin_explain_preamble; pub use predicate::predicate_class_for_filters; pub use retry_loop::{DependentOutcome, DependentRetryArgs, run_dependent_with_retry}; pub use submit::{ - RoutedAssignment, submit_and_await_calvin, submit_and_await_calvin_with_timeout, - submit_calvin_routed, submit_calvin_routed_assign, submit_calvin_routed_write, + RoutedAssignment, submit_and_await_calvin, submit_and_await_calvin_until, submit_calvin_routed, + submit_calvin_routed_assign, submit_calvin_routed_write, }; pub use tx_class::{ PassiveReads, build_predicted_tx_class, build_read_dependent_tx_class, diff --git a/nodedb/src/control/planner/calvin/read_dependent/item_move.rs b/nodedb/src/control/planner/calvin/read_dependent/item_move.rs index 05ef4a5e0..4bea8b730 100644 --- a/nodedb/src/control/planner/calvin/read_dependent/item_move.rs +++ b/nodedb/src/control/planner/calvin/read_dependent/item_move.rs @@ -40,6 +40,7 @@ use crate::control::security::audit::ArcAuditEmitter; use crate::control::security::identity::AuthenticatedIdentity; use crate::control::security::request_scope::RequestAuthScope; use crate::control::server::shared::authorization::authorize_task_set; +use crate::control::server::shared::session::StatementDeadline; use crate::control::state::SharedState; use crate::types::{DatabaseId, TenantId}; @@ -196,7 +197,7 @@ pub async fn move_item_across_shards( registry, orchestrator, predicate_class_hash: class_hash, - timeout: std::time::Duration::from_secs(state.tuning.network.default_deadline_secs), + deadline: StatementDeadline::of_statement(state).tokio_instant(), ollp_max_retries: u32::from(orchestrator.ollp_max_retries()), initial_predicted: initial_item, submit, diff --git a/nodedb/src/control/planner/calvin/reservation.rs b/nodedb/src/control/planner/calvin/reservation.rs index e37c6b174..799f349e4 100644 --- a/nodedb/src/control/planner/calvin/reservation.rs +++ b/nodedb/src/control/planner/calvin/reservation.rs @@ -17,21 +17,26 @@ //! storage I/O or io_uring directly. use std::collections::BTreeSet; -use std::time::Duration; use nodedb_cluster::calvin::SEQUENCER_GROUP_ID; use nodedb_cluster::calvin::types::{LockKeyWire, ReleaseReason, TxnIdWire}; +use nodedb_cluster::rpc_codec::reply_wait; use nodedb_cluster::{ ClusterError, RaftRpc, ReleaseReservationRequest, ReleaseReservationResponse, - ReserveReadRequest, ReserveReadResponse, + ReserveReadRequest, ReserveReadResponse, TypedClusterError, }; use crate::Error; use crate::control::cluster::warm_peers::register_peers_from_topology; +use crate::control::server::shared::session::StatementDeadline; use crate::control::state::SharedState; /// Submit a reserve-read to THIS node's reservation inbox and await the -/// assigned owner, bounded by `timeout`. +/// assigned owner until `deadline`. +/// +/// A deadline spent before the submit refuses it: nothing was submitted. A +/// wait that reaches the deadline is an unknown outcome: the service can +/// still grant the reservation, and lease GC reaps a lost grant. /// /// PRECONDITION: this node is the sequencer-group leader (its service /// assigns). Callers that are not the leader MUST route via @@ -44,27 +49,29 @@ pub(crate) async fn submit_local_reserve_read( key: LockKeyWire, vshard: u32, owner: Option, - timeout: Duration, + deadline: tokio::time::Instant, ) -> crate::Result { let inbox = state .reservation_inbox .get() .ok_or(Error::SequencerUnavailable)?; + super::submit::budget::budget_ms_until(deadline)?; let rx = inbox .submit_reserve(key, vshard, owner) .map_err(|e| Error::BadRequest { detail: format!("reservation service rejected reserve-read: {e}"), })?; - tokio::time::timeout(timeout, rx) - .await - .map_err(|_| Error::Internal { - detail: "timed out waiting for reservation assignment".to_owned(), - })? - .map_err(|_| Error::Internal { + match tokio::time::timeout_at(deadline, rx).await { + Ok(Ok(assigned)) => Ok(assigned), + Ok(Err(_)) => Err(Error::Internal { detail: "reservation assignment channel closed".to_owned(), - }) + }), + Err(_) => Err(Error::DeadlineExceeded { + request_id: crate::types::RequestId::new(0), + }), + } } /// Submit a reserve-read, routing it to the sequencer-group leader. @@ -111,11 +118,12 @@ pub(crate) async fn submit_reserve_read( }); } + // Every wait of this reserve-read ends at the statement's deadline. + let deadline = StatementDeadline::of_statement(state).tokio_instant(); // Leader is self: submit locally (a self-RPC will be a pointless extra // hop and the local inbox is the one that gets the assignment). if leader == state.node_id { - let local_timeout = super::submit::budget::statement_budget(state)?; - return submit_local_reserve_read(state, key, vshard, owner, local_timeout).await; + return submit_local_reserve_read(state, key, vshard, owner, deadline).await; } // Remote leader: ensure its address is registered before dispatch, then @@ -138,7 +146,7 @@ pub(crate) async fn submit_reserve_read( .transpose()?; // The leader works on the reservation only as long as the statement waits. - let deadline_remaining_ms = super::submit::budget::statement_budget_ms(state)?; + let deadline_remaining_ms = super::submit::budget::budget_ms_until(deadline)?; let req = ReserveReadRequest { lock_key_bytes, vshard, @@ -151,7 +159,7 @@ pub(crate) async fn submit_reserve_read( // assigned (up to `deadline_remaining_ms`). The generic short `rpc_timeout` // will abort the call long before that, so bound the response read by the // forwarded deadline plus a margin for the round-trip itself. - let read_timeout = Duration::from_millis(deadline_remaining_ms.saturating_add(2_000)); + let read_timeout = reply_wait(deadline_remaining_ms); match transport .send_rpc_with_read_timeout(leader, RaftRpc::ReserveReadRequest(req), read_timeout) .await @@ -163,10 +171,17 @@ pub(crate) async fn submit_reserve_read( format: "msgpack".to_owned(), detail: format!("failed to decode TxnIdWire owner from reserve-read reply: {e}"), }), + // An error with no class keeps the leader in its message. + Ok(RaftRpc::ReserveReadResponse(ReserveReadResponse { + error: Some(TypedClusterError::Internal { code: 0, message }), + .. + })) => Err(Error::Internal { + detail: format!("reserve-read failed on sequencer leader node {leader}: {message}"), + }), + // Every other error keeps its class. A leader-side deadline stays an + // unknown outcome, and an expired budget stays a definite refusal. Ok(RaftRpc::ReserveReadResponse(ReserveReadResponse { error: Some(e), .. })) => { - Err(Error::Internal { - detail: format!("reserve-read failed on sequencer leader node {leader}: {e:?}"), - }) + Err(Error::from(e)) } Ok(other) => Err(Error::Internal { detail: format!("reserve-read: unexpected reply from node {leader}: {other:?}"), @@ -298,7 +313,7 @@ pub(crate) async fn release_reservation( deadline_remaining_ms, trace_id: [0u8; 16], }; - let read_timeout = Duration::from_millis(deadline_remaining_ms.saturating_add(2_000)); + let read_timeout = reply_wait(deadline_remaining_ms); match transport .send_rpc_with_read_timeout( diff --git a/nodedb/src/control/planner/calvin/retry_loop.rs b/nodedb/src/control/planner/calvin/retry_loop.rs index fbe58d25c..63ef39a8c 100644 --- a/nodedb/src/control/planner/calvin/retry_loop.rs +++ b/nodedb/src/control/planner/calvin/retry_loop.rs @@ -15,6 +15,8 @@ use crate::control::cluster::calvin::executor::ollp::error::OllpError; use crate::control::cluster::calvin::executor::ollp::orchestrator::OllpOrchestrator; use crate::control::planner::calvin::abort_error::calvin_abort_error; use crate::control::planner::calvin::submit::RoutedAssignment; +use crate::control::planner::calvin::submit::budget::budget_ms_until; +use crate::control::planner::calvin::submit::wait::await_completion; use crate::{Error, OllpExhaustedCause}; // ── run_dependent_with_retry ────────────────────────────────────────────────── @@ -64,11 +66,16 @@ pub enum DependentOutcome { /// `submit` and `rescan` are injected so this loop is unit-testable WITHOUT a /// live server/executor: a fake scheduler driving the real /// [`CalvinCompletionRegistry`] suffices. +/// +/// Every attempt shares the statement's `deadline`. A spent deadline before a +/// submit refuses it, since nothing of that attempt was submitted. A +/// completion wait that reaches the deadline is an unknown outcome, and so is +/// a submit that answers [`Error::DeadlineExceeded`]: neither is resubmitted. pub struct DependentRetryArgs<'a, P, SF, RF> { pub registry: &'a CalvinCompletionRegistry, pub orchestrator: &'a OllpOrchestrator, pub predicate_class_hash: u64, - pub timeout: std::time::Duration, + pub deadline: tokio::time::Instant, pub ollp_max_retries: u32, pub initial_predicted: P, pub submit: SF, @@ -88,7 +95,7 @@ where registry, orchestrator, predicate_class_hash, - timeout, + deadline, ollp_max_retries, initial_predicted, mut submit, @@ -97,6 +104,7 @@ where let mut predicted = initial_predicted; let mut retry: u32 = 0; loop { + budget_ms_until(deadline)?; let assignment = match submit(&predicted).await { Ok(Some(assignment)) => assignment, // Nothing to write: the prediction decided an empty write set, so @@ -107,6 +115,12 @@ where // it, so surface the real error instead of retrying and reporting // predicate drift that never happened. Err(OllpError::Terminal(error)) => return Err(*error), + // The submit can have been sequenced. A resubmit can apply it twice. + Err(OllpError::Retryable(error)) + if matches!(*error, Error::DeadlineExceeded { .. }) => + { + return Err(*error); + } Err(ollp_err) => { // PRE-ADMISSION failure (routed submit / circuit / budget). // Nothing executed, so there is no aborted attempt to re-scan @@ -132,15 +146,7 @@ where // registry, which receives the replicated completion ack on every // sequencer-group member. let txn_id = TxnId::new(assignment.epoch, assignment.position); - let completion_rx = registry.register_completion_report(txn_id, assignment.participants); - let report = tokio::time::timeout(timeout, completion_rx) - .await - .map_err(|_| Error::Internal { - detail: "timed out waiting for Calvin completion".into(), - })? - .map_err(|_| Error::Internal { - detail: "Calvin completion channel closed".into(), - })?; + let report = await_completion(registry, txn_id, assignment.participants, deadline).await?; match report.outcome { // Return the completed txn's id so the caller can drain the applied @@ -223,6 +229,11 @@ mod tests { use nodedb_cluster::calvin::VerdictOutcome; use nodedb_cluster::calvin::sequencer::error::SequencerError; + /// A deadline no test reaches. + fn far_deadline() -> tokio::time::Instant { + tokio::time::Instant::now() + std::time::Duration::from_secs(5) + } + /// Build an orchestrator with zero backoff so the retry loop runs instantly. fn zero_backoff_orchestrator() -> OllpOrchestrator { OllpOrchestrator::new(OllpConfig { @@ -296,7 +307,7 @@ mod tests { registry: ®istry, orchestrator: &orchestrator, predicate_class_hash: 0xABCD, - timeout: std::time::Duration::from_secs(5), + deadline: far_deadline(), ollp_max_retries: 5, initial_predicted: vec![1, 2, 3], submit: move |_predicted: &Vec| { @@ -367,7 +378,7 @@ mod tests { registry: ®istry, orchestrator: &orchestrator, predicate_class_hash: 0xABCD, - timeout: std::time::Duration::from_secs(5), + deadline: far_deadline(), ollp_max_retries: 3, initial_predicted: vec![1], submit: move |_predicted: &Vec| { @@ -436,7 +447,7 @@ mod tests { registry: ®istry, orchestrator: &orchestrator, predicate_class_hash: 0xABCD, - timeout: std::time::Duration::from_secs(5), + deadline: far_deadline(), ollp_max_retries: 5, initial_predicted: vec![1], submit: move |_predicted: &Vec| { @@ -503,7 +514,7 @@ mod tests { registry: ®istry, orchestrator: &orchestrator, predicate_class_hash: 0xABCD, - timeout: std::time::Duration::from_secs(5), + deadline: far_deadline(), ollp_max_retries: 5, initial_predicted: Vec::::new(), // A zero-match prediction builds no TxClass, so nothing is @@ -551,7 +562,7 @@ mod tests { registry: ®istry, orchestrator: &orchestrator, predicate_class_hash: 0xABCD, - timeout: std::time::Duration::from_secs(5), + deadline: far_deadline(), ollp_max_retries: 5, initial_predicted: vec![1], submit: move |_predicted: &Vec| { @@ -597,7 +608,7 @@ mod tests { registry: ®istry, orchestrator: &orchestrator, predicate_class_hash: 0xABCD, - timeout: std::time::Duration::from_secs(5), + deadline: far_deadline(), ollp_max_retries: 2, initial_predicted: vec![1], // Retryable every time: the loop exhausts, and the carried cause @@ -668,7 +679,7 @@ mod tests { registry: ®istry, orchestrator: &orchestrator, predicate_class_hash: 0xABCD, - timeout: std::time::Duration::from_secs(5), + deadline: far_deadline(), ollp_max_retries: 5, initial_predicted: vec![1], submit: move |_predicted: &Vec| { @@ -735,4 +746,112 @@ mod tests { assert_eq!(submits, 1); assert_eq!(rescans, 0); } + + /// A submit whose assignment the fake never completes. The completion + /// wait ends at the statement's deadline as an unknown outcome. + #[tokio::test(start_paused = true)] + async fn the_completion_wait_ends_at_the_statement_deadline() { + let registry = CalvinCompletionRegistry::new_detached(); + let orchestrator = zero_backoff_orchestrator(); + let start = tokio::time::Instant::now(); + let deadline = start + std::time::Duration::from_millis(100); + let submit_calls = Arc::new(AtomicU64::new(0)); + + let result = { + let submit_calls = Arc::clone(&submit_calls); + run_dependent_with_retry(DependentRetryArgs { + registry: ®istry, + orchestrator: &orchestrator, + predicate_class_hash: 0xABCD, + deadline, + ollp_max_retries: 5, + initial_predicted: vec![1], + submit: move |_predicted: &Vec| { + let submit_calls = Arc::clone(&submit_calls); + async move { + let n = submit_calls.fetch_add(1, Ordering::SeqCst); + Ok::, OllpError>(Some(fake_assignment(n + 1))) + } + }, + rescan: move || async move { Ok(vec![1]) }, + }) + .await + }; + + assert!( + matches!(result, Err(Error::DeadlineExceeded { .. })), + "expected an unknown outcome, got {result:?}" + ); + let waited = start.elapsed(); + assert!( + waited >= std::time::Duration::from_millis(100) + && waited < std::time::Duration::from_millis(200), + "the wait ends at the statement deadline, not later: {waited:?}" + ); + assert_eq!( + submit_calls.load(Ordering::SeqCst), + 1, + "an unknown outcome is never resubmitted" + ); + } + + /// A drift abort whose re-read ends past the deadline. The next attempt + /// is refused before its submit: nothing of it was submitted. + #[tokio::test(start_paused = true)] + async fn a_retry_after_the_deadline_is_refused_before_submit() { + let registry = CalvinCompletionRegistry::new_detached(); + let orchestrator = zero_backoff_orchestrator(); + let deadline = tokio::time::Instant::now() + std::time::Duration::from_millis(100); + + let (tx, rx) = tokio::sync::mpsc::channel::(16); + let fake = spawn_fake_scheduler(Arc::clone(®istry), rx, u32::MAX); + let seq = Arc::new(AtomicU64::new(1)); + let submit_calls = Arc::new(AtomicU64::new(0)); + + let result = { + let seq = Arc::clone(&seq); + let submit_calls = Arc::clone(&submit_calls); + let tx = tx.clone(); + run_dependent_with_retry(DependentRetryArgs { + registry: ®istry, + orchestrator: &orchestrator, + predicate_class_hash: 0xABCD, + deadline, + ollp_max_retries: 5, + initial_predicted: vec![1], + submit: move |_predicted: &Vec| { + let seq = Arc::clone(&seq); + let submit_calls = Arc::clone(&submit_calls); + let tx = tx.clone(); + async move { + submit_calls.fetch_add(1, Ordering::SeqCst); + let assignment = fake_assignment(seq.fetch_add(1, Ordering::SeqCst)); + let txn = TxnId::new(assignment.epoch, assignment.position); + tx.send(txn).await.expect("fake recv alive"); + Ok::, OllpError>(Some(assignment)) + } + }, + rescan: move || async move { + tokio::time::sleep(std::time::Duration::from_millis(150)).await; + Ok(vec![2]) + }, + }) + .await + }; + + let Err(Error::DataPlane(code)) = &result else { + panic!("expected a definite refusal, got {result:?}"); + }; + assert_eq!( + code, + &crate::bridge::envelope::ErrorCode::ExpiredBeforeExecution + ); + assert_eq!( + submit_calls.load(Ordering::SeqCst), + 1, + "the attempt after the deadline is never submitted" + ); + drop(tx); + let _ = fake.await; + } } diff --git a/nodedb/src/control/planner/calvin/submit/assign.rs b/nodedb/src/control/planner/calvin/submit/assign.rs index f2781ac71..a02a14a83 100644 --- a/nodedb/src/control/planner/calvin/submit/assign.rs +++ b/nodedb/src/control/planner/calvin/submit/assign.rs @@ -6,22 +6,25 @@ //! the OLLP coordinator loop drives the transaction to completion itself. use std::collections::BTreeSet; -use std::time::Duration; use nodedb_cluster::calvin::SEQUENCER_GROUP_ID; use nodedb_cluster::calvin::types::TxClass; -use nodedb_cluster::{ClusterError, RaftRpc, SubmitCalvinInboxRequest, SubmitCalvinInboxResponse}; +use nodedb_cluster::rpc_codec::reply_wait; +use nodedb_cluster::{ + ClusterError, RaftRpc, SubmitCalvinInboxRequest, SubmitCalvinInboxResponse, TypedClusterError, +}; use super::stream::{PartStream, StreamTarget, stream_parts}; use crate::Error; use crate::control::cluster::warm_peers::register_peers_from_topology; +use crate::control::server::shared::session::StatementDeadline; use crate::control::state::SharedState; /// The sequencer ASSIGNMENT for a submitted dependent (OLLP) `TxClass`. /// /// Returned by [`submit_calvin_routed_assign`] AS SOON AS the sequencer assigns /// the transaction — completion is NOT awaited (the OLLP coordinator loop drives -/// the dependent transaction to completion itself in a later unit). +/// the dependent transaction to completion itself). #[derive(Debug, Clone, Copy, PartialEq, Eq)] pub struct RoutedAssignment { pub inbox_seq: u64, @@ -31,38 +34,39 @@ pub struct RoutedAssignment { } /// Submit `tx_class` to THIS node's Calvin sequencer inbox and await only its -/// ASSIGNMENT (NOT completion), bounded by `timeout`. +/// ASSIGNMENT (NOT completion), until `deadline`. /// /// PRECONDITION: this node is the sequencer-group leader (its service assigns). /// Callers that are not the leader MUST route via /// [`submit_calvin_routed_assign`]. The local primitive for the OLLP dependent -/// path — the sibling of [`super::local::submit_and_await_calvin_with_timeout`] that stops at -/// the assignment phase. +/// path — the sibling of [`super::local::submit_and_await_calvin_until`] that +/// stops at the assignment phase. /// /// `pub(crate)` so [`crate::control::server::calvin_submit::inbox_hook`] can /// call it after decoding the wire bytes, mirroring how `hook.rs` delegates to -/// `submit_and_await_calvin_with_timeout`. +/// `submit_and_await_calvin_until`. pub(crate) async fn submit_local_assign( state: &SharedState, mut tx_class: TxClass, - timeout: Duration, + deadline: tokio::time::Instant, ) -> crate::Result { super::local::raise_metadata_floor(state, &mut tx_class); super::local::stamp_incarnations(state, &mut tx_class)?; super::unique_claims::stamp_unique_claims(state, &mut tx_class)?; let stream = super::parts::split_into_parts(state, &mut tx_class)?; - assign_prepared(state, tx_class, stream, timeout).await + assign_prepared(state, tx_class, stream, deadline).await } /// Submit a stamped `tx_class` to this node's sequencer, await its /// assignment, then stream its parts when it carries them as parts. /// -/// PRECONDITION: this node is the sequencer-group leader. +/// PRECONDITION: this node is the sequencer-group leader. A deadline spent +/// before the submit refuses it: nothing was submitted. async fn assign_prepared( state: &SharedState, tx_class: TxClass, stream: Option, - timeout: Duration, + deadline: tokio::time::Instant, ) -> crate::Result { let inbox = state .sequencer_inbox @@ -73,6 +77,7 @@ async fn assign_prepared( .get() .ok_or(Error::SequencerUnavailable)?; + super::budget::budget_ms_until(deadline)?; let (inbox_seq, assignment_rx) = inbox .submit_with(tx_class, registry) @@ -80,11 +85,11 @@ async fn assign_prepared( detail: format!("Calvin sequencer rejected transaction: {e}"), })?; let (epoch, position, participants) = - super::local::await_assignment(registry, inbox_seq, assignment_rx, timeout).await?; + super::wait::await_assignment(registry, inbox_seq, assignment_rx, deadline).await?; // A lost stream aborts the transaction, and the caller's completion // wait reports it. if let Some(stream) = &stream { - stream_parts(state, StreamTarget::Local, stream, true) + stream_parts(state, StreamTarget::Local, stream, true, deadline) .await .into_result()?; } @@ -149,11 +154,12 @@ pub async fn submit_calvin_routed_assign( }); } + // Every wait of this submit ends at the statement's deadline. + let deadline = StatementDeadline::of_statement(state).tokio_instant(); // Leader is self: submit-and-assign locally (a self-RPC will be a pointless // extra hop and the local registry is the one that gets the assignment). if leader == state.node_id { - let local_timeout = super::budget::statement_budget(state)?; - return assign_prepared(state, tx_class, stream, local_timeout).await; + return assign_prepared(state, tx_class, stream, deadline).await; } // Remote leader: ensure its address is registered before dispatch, then send @@ -168,7 +174,7 @@ pub async fn submit_calvin_routed_assign( })?; // The leader works on the transaction only as long as the statement waits. - let deadline_remaining_ms = super::budget::statement_budget_ms(state)?; + let deadline_remaining_ms = super::budget::budget_ms_until(deadline)?; let req = SubmitCalvinInboxRequest { tx_class_bytes, deadline_remaining_ms, @@ -179,7 +185,7 @@ pub async fn submit_calvin_routed_assign( // assigned (up to `deadline_remaining_ms`). The generic short `rpc_timeout` // will abort the call long before that, so bound the response read by the // forwarded deadline plus a margin for the round-trip itself. - let read_timeout = Duration::from_millis(deadline_remaining_ms.saturating_add(2_000)); + let read_timeout = reply_wait(deadline_remaining_ms); let assignment = match transport .send_rpc_with_read_timeout(leader, RaftRpc::SubmitCalvinInboxRequest(req), read_timeout) .await @@ -196,14 +202,21 @@ pub async fn submit_calvin_routed_assign( position, participants: participants as usize, }, + // An error with no class keeps the leader in its message. Ok(RaftRpc::SubmitCalvinInboxResponse(SubmitCalvinInboxResponse { - error: Some(e), + error: Some(TypedClusterError::Internal { code: 0, message }), .. })) => { return Err(Error::Internal { - detail: format!("calvin-inbox failed on sequencer leader node {leader}: {e:?}"), + detail: format!("calvin-inbox failed on sequencer leader node {leader}: {message}"), }); } + // Every other error keeps its class. A leader-side deadline stays an + // unknown outcome, and an expired budget stays a definite refusal. + Ok(RaftRpc::SubmitCalvinInboxResponse(SubmitCalvinInboxResponse { + error: Some(e), + .. + })) => return Err(Error::from(e)), Ok(other) => { return Err(Error::Internal { detail: format!("calvin-inbox: unexpected reply from node {leader}: {other:?}"), @@ -232,7 +245,7 @@ pub async fn submit_calvin_routed_assign( // stream aborts the transaction, and the caller's completion wait // reports it. if let Some(stream) = &stream { - stream_parts(state, StreamTarget::Remote(leader), stream, true) + stream_parts(state, StreamTarget::Remote(leader), stream, true, deadline) .await .into_result()?; } diff --git a/nodedb/src/control/planner/calvin/submit/budget.rs b/nodedb/src/control/planner/calvin/submit/budget.rs index 7d50b8dae..a4336fbe9 100644 --- a/nodedb/src/control/planner/calvin/submit/budget.rs +++ b/nodedb/src/control/planner/calvin/submit/budget.rs @@ -6,11 +6,7 @@ //! leader works on it only as long as the statement waits. The budget is //! what is left of the statement's deadline, never the node default. -use std::time::Duration; - use crate::bridge::envelope::ErrorCode; -use crate::control::state::SharedState; -use crate::control::wal_replication::statement_propose_deadline; /// The milliseconds left to `deadline`, for a request about to go out. /// @@ -23,19 +19,10 @@ pub(crate) fn budget_ms_until(deadline: tokio::time::Instant) -> crate::Result crate::Result { - budget_ms_until(statement_propose_deadline(state)) -} - -/// [`statement_budget_ms`] as a wait bound for the local half of the request. -pub(crate) fn statement_budget(state: &SharedState) -> crate::Result { - statement_budget_ms(state).map(Duration::from_millis) -} - #[cfg(test)] mod tests { + use std::time::Duration; + use super::*; use crate::control::server::shared::session::{conn_scope, deadline, statement_deadline}; diff --git a/nodedb/src/control/planner/calvin/submit/local.rs b/nodedb/src/control/planner/calvin/submit/local.rs index 043dac370..4ed9979aa 100644 --- a/nodedb/src/control/planner/calvin/submit/local.rs +++ b/nodedb/src/control/planner/calvin/submit/local.rs @@ -7,17 +7,15 @@ //! completion ack. A submit on a non-leader is drained and discarded, so //! non-leader callers must route through [`super::routed`]. -use std::time::Duration; - use nodedb_cluster::calvin::types::TxClass; -use nodedb_cluster::calvin::{ - Assignment, AssignmentReceiver, AttemptOutcome, CalvinCompletionRegistry, TxnId, -}; +use nodedb_cluster::calvin::{AttemptOutcome, TxnId}; use super::stream::{PartStream, StreamTarget, stream_parts}; +use super::wait::{await_assignment, await_completion}; use crate::Error; use crate::bridge::envelope::{PhysicalPlan, Response}; use crate::control::planner::calvin::abort_error::calvin_abort_error; +use crate::control::server::shared::session::StatementDeadline; use crate::control::state::{CalvinApplyResult, SharedState}; /// Build a minimal Control-Plane [`Response`] carrying only the RETURNING @@ -95,78 +93,50 @@ pub(crate) fn stamp_incarnations(state: &SharedState, tx_class: &mut TxClass) -> Ok(()) } -/// Await the sequencer assignment of submission `inbox_seq`, bounded by -/// `timeout`. -/// -/// A closed channel means the sequencer rejected or discarded the submission -/// without sequencing it: a read/write cycle in its epoch, a leadership -/// change, or a halted sequencer. Nothing applied, so the caller gets a -/// retryable refusal. On timeout the registration is dropped, so no sender -/// stays behind. A timed-out submission can still be sequenced later. -pub(crate) async fn await_assignment( - registry: &CalvinCompletionRegistry, - inbox_seq: u64, - assignment_rx: AssignmentReceiver, - timeout: Duration, -) -> crate::Result { - match tokio::time::timeout(timeout, assignment_rx).await { - Ok(Ok(assignment)) => Ok(assignment), - Ok(Err(_)) => Err(Error::RetryableRefusal { - reason: "the Calvin sequencer did not sequence the transaction; nothing was \ - applied" - .to_owned(), - }), - Err(_) => { - registry.drop_assignment(inbox_seq); - Err(Error::Internal { - detail: "timed out waiting for Calvin sequencer assignment".to_owned(), - }) - } - } -} - /// Submit `tx_class` to THIS node's Calvin sequencer inbox and await completion. /// /// PRECONDITION: this node is the sequencer-group leader (its service assigns; /// its registry receives the replicated completion ack). Callers that are not /// the leader MUST route via [`super::routed::submit_calvin_routed`]. /// -/// The assignment + completion waits are bounded by -/// `state.tuning.network.default_deadline_secs`. +/// The assignment and completion waits end at the running statement's +/// deadline. pub async fn submit_and_await_calvin( state: &SharedState, tx_class: TxClass, ) -> crate::Result> { - let timeout = Duration::from_secs(state.tuning.network.default_deadline_secs); - submit_and_await_calvin_with_timeout(state, tx_class, timeout).await + let deadline = StatementDeadline::of_statement(state).tokio_instant(); + submit_and_await_calvin_until(state, tx_class, deadline).await } -/// [`submit_and_await_calvin`] with an explicit deadline budget. +/// [`submit_and_await_calvin`] with an explicit deadline. /// -/// Used by the leader-side RPC handler so the forwarded submit-and-await is -/// bounded by the coordinator's remaining deadline rather than this node's full -/// default deadline. -pub async fn submit_and_await_calvin_with_timeout( +/// The leader-side RPC handler passes the coordinator's remaining deadline, +/// so the forwarded submit-and-await never outlives the statement. +pub async fn submit_and_await_calvin_until( state: &SharedState, mut tx_class: TxClass, - timeout: Duration, + deadline: tokio::time::Instant, ) -> crate::Result> { raise_metadata_floor(state, &mut tx_class); stamp_incarnations(state, &mut tx_class)?; super::unique_claims::stamp_unique_claims(state, &mut tx_class)?; let stream = super::parts::split_into_parts(state, &mut tx_class)?; - submit_prepared_and_await(state, tx_class, stream, timeout).await + submit_prepared_and_await(state, tx_class, stream, deadline).await } /// Submit a stamped `tx_class` to this node's sequencer, stream its parts /// when it carries them as parts, and await its completion. /// /// PRECONDITION: this node is the sequencer-group leader. +/// +/// A deadline spent before the submit refuses it: nothing was submitted. A +/// wait that reaches the deadline after the submit is an unknown outcome. pub(crate) async fn submit_prepared_and_await( state: &SharedState, tx_class: TxClass, stream: Option, - timeout: Duration, + deadline: tokio::time::Instant, ) -> crate::Result> { #[cfg(feature = "failpoints")] crate::control::fail_gate::after_calvin_stamp(state.node_id, &tx_class).await; @@ -196,6 +166,7 @@ pub(crate) async fn submit_prepared_and_await( .is_source_vshard(vshard.as_u32()) }); + super::budget::budget_ms_until(deadline)?; let (inbox_seq, assignment_rx) = inbox .submit_with(tx_class, registry) @@ -203,40 +174,23 @@ pub(crate) async fn submit_prepared_and_await( detail: format!("Calvin sequencer rejected transaction: {e}"), })?; let (epoch, position, participants) = - await_assignment(registry, inbox_seq, assignment_rx, timeout).await?; + await_assignment(registry, inbox_seq, assignment_rx, deadline).await?; // The header holds its locks on every participant until the parts // arrive. A lost stream aborts the transaction, and the completion // below reports it. if let Some(stream) = &stream { - stream_parts(state, StreamTarget::Local, stream, true) + stream_parts(state, StreamTarget::Local, stream, true, deadline) .await .into_result()?; } - let completion_rx = - registry.register_completion_report(TxnId::new(epoch, position), participants); - let report = tokio::time::timeout(timeout, completion_rx) - .await - .map_err(|_| { - let err = Error::Internal { - detail: "timed out waiting for Calvin transaction completion".to_owned(), - }; - // This timeout is the only signal a silently-never-completed - // Calvin write ever produces; file it as a structured report at - // the one site that detects it, since the error alone gives an - // operator no clue which transaction or participant stalled. - crate::diag::calvin_completion_timeout( - &err, - epoch, - position, - participants, - timeout.as_secs(), - ); - err - })? - .map_err(|_| Error::Internal { - detail: "Calvin completion channel closed".to_owned(), - })?; + let report = await_completion( + registry, + TxnId::new(epoch, position), + participants, + deadline, + ) + .await?; // The global cross-shard verdict was ABORT and the writes were dropped. // The verdict's reason picks the error the client sees. match report.outcome { @@ -546,43 +500,6 @@ mod tests { TsInstallCounts::from_payload(response.payload.as_bytes()).expect("install counts") } - /// The sequencer dropped the assignment of a submission it did not - /// sequence. The caller fails at once with a retryable refusal, long - /// before its timeout. - #[tokio::test] - async fn an_unsequenced_submission_is_a_retryable_refusal() { - let registry = CalvinCompletionRegistry::new_detached(); - let rx = registry.register_submission(7); - registry.drop_assignment(7); - match await_assignment(®istry, 7, rx, Duration::from_secs(3600)).await { - Err(Error::RetryableRefusal { .. }) => {} - other => panic!("expected a retryable refusal, got {other:?}"), - } - } - - #[tokio::test] - async fn an_assigned_submission_returns_its_assignment() { - let registry = CalvinCompletionRegistry::new_detached(); - let rx = registry.register_submission(3); - registry.note_assigned(3, TxnId::new(5, 1), 2); - let assignment = await_assignment(®istry, 3, rx, Duration::from_secs(3600)) - .await - .expect("assigned"); - assert_eq!(assignment, (5, 1, 2)); - } - - /// A wait that times out reports the timeout, not a refusal: the - /// submission can still be sequenced. - #[tokio::test] - async fn a_timed_out_wait_is_not_a_refusal() { - let registry = CalvinCompletionRegistry::new_detached(); - let rx = registry.register_submission(8); - match await_assignment(®istry, 8, rx, Duration::from_millis(1)).await { - Err(Error::Internal { detail }) => assert!(detail.contains("timed out"), "{detail}"), - other => panic!("expected the timeout error, got {other:?}"), - } - } - /// With one replica per vShard, the coordinator's node hosts no replica of /// either participant, so its sidecar holds nothing. The counts both /// participants' acks carried still reach the statement. diff --git a/nodedb/src/control/planner/calvin/submit/mod.rs b/nodedb/src/control/planner/calvin/submit/mod.rs index 20bec2a59..cce77da35 100644 --- a/nodedb/src/control/planner/calvin/submit/mod.rs +++ b/nodedb/src/control/planner/calvin/submit/mod.rs @@ -1,6 +1,6 @@ // SPDX-License-Identifier: BUSL-1.1 -//! Calvin submit-and-await primitive and sequencer-leader routing (Cv1). +//! Calvin submit-and-await primitive and sequencer-leader routing. //! //! NodeDB's Calvin cross-shard write path only completes when the transaction is //! submitted on the SEQUENCER-GROUP leader: @@ -38,8 +38,9 @@ pub mod parts; pub mod routed; pub mod stream; pub mod unique_claims; +pub mod wait; pub(crate) use assign::submit_local_assign; pub use assign::{RoutedAssignment, submit_calvin_routed_assign}; -pub use local::{submit_and_await_calvin, submit_and_await_calvin_with_timeout}; +pub use local::{submit_and_await_calvin, submit_and_await_calvin_until}; pub use routed::{submit_calvin_routed, submit_calvin_routed_write}; diff --git a/nodedb/src/control/planner/calvin/submit/routed.rs b/nodedb/src/control/planner/calvin/submit/routed.rs index 7e71b6e80..91e89a3f9 100644 --- a/nodedb/src/control/planner/calvin/submit/routed.rs +++ b/nodedb/src/control/planner/calvin/submit/routed.rs @@ -7,10 +7,10 @@ //! leader over one `SubmitCalvinTxn` RPC. use std::collections::BTreeSet; -use std::time::Duration; use nodedb_cluster::calvin::SEQUENCER_GROUP_ID; use nodedb_cluster::calvin::types::TxClass; +use nodedb_cluster::rpc_codec::reply_wait; use nodedb_cluster::{ ClusterError, RaftRpc, SubmitCalvinTxnRequest, SubmitCalvinTxnResponse, TypedClusterError, }; @@ -18,6 +18,7 @@ use nodedb_cluster::{ use crate::Error; use crate::bridge::envelope::Response; use crate::control::cluster::warm_peers::register_peers_from_topology; +use crate::control::server::shared::session::StatementDeadline; use crate::control::state::SharedState; use super::local::{submit_prepared_and_await, synthetic_returning_response}; @@ -119,11 +120,12 @@ pub async fn submit_calvin_routed( }); } + // Every wait of this submit ends at the statement's deadline. + let deadline = StatementDeadline::of_statement(state).tokio_instant(); // Leader is self: submit-and-await locally (a self-RPC will be a pointless // extra hop and the local registry is the one that completes). if leader == state.node_id { - let timeout = super::budget::statement_budget(state)?; - return submit_prepared_and_await(state, tx_class, stream, timeout).await; + return submit_prepared_and_await(state, tx_class, stream, deadline).await; } // Remote leader: ensure its address is registered before dispatch, then send @@ -138,7 +140,7 @@ pub async fn submit_calvin_routed( })?; // The leader works on the transaction only as long as the statement waits. - let deadline_remaining_ms = super::budget::statement_budget_ms(state)?; + let deadline_remaining_ms = super::budget::budget_ms_until(deadline)?; let req = SubmitCalvinTxnRequest { tx_class_bytes, deadline_remaining_ms, @@ -150,7 +152,7 @@ pub async fn submit_calvin_routed( // short `rpc_timeout` (a normal request/response round-trip budget) will // abort the call long before that, so bound the response read by the // forwarded deadline plus a margin for the round-trip itself. - let read_timeout = Duration::from_millis(deadline_remaining_ms.saturating_add(2_000)); + let read_timeout = reply_wait(deadline_remaining_ms); let header = transport.send_rpc_with_read_timeout( leader, RaftRpc::SubmitCalvinTxnRequest(req), @@ -163,7 +165,8 @@ pub async fn submit_calvin_routed( None => header.await, Some(stream) => { tokio::pin!(header); - let streaming = stream_parts(state, StreamTarget::Remote(leader), stream, false); + let streaming = + stream_parts(state, StreamTarget::Remote(leader), stream, false, deadline); tokio::pin!(streaming); tokio::select! { reply = &mut header => reply, diff --git a/nodedb/src/control/planner/calvin/submit/stream.rs b/nodedb/src/control/planner/calvin/submit/stream.rs index d000a2b68..64132b11b 100644 --- a/nodedb/src/control/planner/calvin/submit/stream.rs +++ b/nodedb/src/control/planner/calvin/submit/stream.rs @@ -19,7 +19,8 @@ //! //! A `Full` answer comes from a live leader that holds the stream, so it //! keeps the stream alive. A stream with no answer from such a leader for -//! the network deadline stops as lost. +//! the stall window stops as lost. A stream also stops as lost at the +//! statement's deadline: the statement no longer waits for its outcome. use std::collections::BTreeSet; use std::time::{Duration, Instant}; @@ -80,24 +81,37 @@ impl StreamEnd { } } -/// Stream `stream` to `target`. `assigned` says the caller holds the -/// header's assignment, so the leader has opened the stream. +/// Stream `stream` to `target` until the statement's `deadline`. +/// `assigned` says the caller holds the header's assignment, so the leader +/// has opened the stream. pub(crate) async fn stream_parts( state: &SharedState, target: StreamTarget, stream: &PartStream, assigned: bool, + deadline: tokio::time::Instant, ) -> StreamEnd { if stream.parts.is_empty() { return StreamEnd::Done; } - let deadline = Duration::from_secs(state.tuning.network.default_deadline_secs.max(1)); + let stall_window = stall_window(state); // no-determinism: coordinator-side liveness timer, never in the log. - let mut progress = StreamProgress::new(stream.parts.len(), assigned, deadline, Instant::now()); + let mut progress = StreamProgress::new( + stream.parts.len(), + assigned, + stall_window, + deadline.into_std(), + Instant::now(), + ); let mut backoff = FIRST_BACKOFF; loop { + let left = deadline.saturating_duration_since(tokio::time::Instant::now()); + if left.is_zero() { + return StreamEnd::Lost; + } let batch = batch_from(&stream.parts, progress.next); - let reply = match offer(state, target, stream.id, batch).await { + let read_timeout = stall_window.min(left); + let reply = match offer(state, target, stream.id, batch, read_timeout).await { Ok(reply) => reply, Err(error) => { warn!(%error, "calvin part stream: offer failed; retrying"); @@ -136,6 +150,11 @@ enum Step { End(StreamEnd), } +/// The longest a stream waits with no answer from a leader that holds it. +fn stall_window(state: &SharedState) -> Duration { + Duration::from_secs(state.tuning.network.default_deadline_secs.max(1)) +} + /// Where a stream stands, and when the leader last showed it alive. #[derive(Debug)] struct StreamProgress { @@ -146,19 +165,28 @@ struct StreamProgress { taken_any: bool, /// Whether the caller holds the header's assignment. assigned: bool, - deadline: Duration, + stall_window: Duration, + /// The statement's deadline. The stream stops there. + ends_at: Instant, /// The last answer from a leader that holds the stream. last_alive: Instant, } impl StreamProgress { - fn new(total: usize, assigned: bool, deadline: Duration, now: Instant) -> Self { + fn new( + total: usize, + assigned: bool, + stall_window: Duration, + ends_at: Instant, + now: Instant, + ) -> Self { Self { total, next: 0, taken_any: false, assigned, - deadline, + stall_window, + ends_at, last_alive: now, } } @@ -168,8 +196,9 @@ impl StreamProgress { /// An `Accepted` or `Full` answer comes from a leader that holds the /// stream, so it counts as liveness: a full queue drains as the leader /// proposes. Only an `Unknown` answer, or no answer, runs down the - /// deadline. A leader that stops proposing closes the stalled stream, - /// and its next answer is `Unknown`. + /// stall window. A leader that stops proposing closes the stalled + /// stream, and its next answer is `Unknown`. The statement's deadline + /// ends any stream that still owes parts. fn on_reply(&mut self, reply: &PartsOfferReply, now: Instant) -> Step { let owed = usize::try_from(reply.next_index).unwrap_or(usize::MAX); match reply.status { @@ -196,7 +225,8 @@ impl StreamProgress { if reply.status == PartsOfferStatus::Accepted { return Step::Offer; } - if now.saturating_duration_since(self.last_alive) > self.deadline { + if now >= self.ends_at || now.saturating_duration_since(self.last_alive) > self.stall_window + { return Step::End(StreamEnd::Lost); } Step::Wait @@ -231,6 +261,7 @@ async fn offer( target: StreamTarget, stream: PartStreamId, batch: Vec, + read_timeout: Duration, ) -> crate::Result { match target { StreamTarget::Local => { @@ -246,7 +277,9 @@ async fn offer( transport_error: false, }) } - StreamTarget::Remote(leader) => offer_remote(state, leader, stream, batch).await, + StreamTarget::Remote(leader) => { + offer_remote(state, leader, stream, batch, read_timeout).await + } } } @@ -255,6 +288,7 @@ async fn offer_remote( leader: u64, stream: PartStreamId, batch: Vec, + read_timeout: Duration, ) -> crate::Result { let transport = state .cluster_transport @@ -270,7 +304,6 @@ async fn offer_remote( stream_seq: stream.seq, parts_bytes, }); - let read_timeout = Duration::from_secs(state.tuning.network.default_deadline_secs.max(1)); let reply = transport .send_rpc_with_read_timeout(leader, request, read_timeout) .await @@ -338,15 +371,16 @@ mod tests { } } - /// A leader that holds the stream and answers `Full` past the deadline - /// keeps it alive: its queue drains as it proposes. Only no answer runs - /// the deadline down. + /// A leader that holds the stream and answers `Full` past the stall + /// window keeps it alive: its queue drains as it proposes. Only no answer + /// runs the stall window down. #[test] - fn full_answers_keep_a_stream_alive_past_the_deadline() { - let deadline = Duration::from_secs(1); + fn full_answers_keep_a_stream_alive_past_the_stall_window() { + let stall_window = Duration::from_secs(1); // no-determinism: test-only synthetic timeline. let start = Instant::now(); - let mut progress = StreamProgress::new(4, true, deadline, start); + let ends_at = start + Duration::from_secs(3600); + let mut progress = StreamProgress::new(4, true, stall_window, ends_at, start); assert_eq!( progress.on_reply(&reply(PartsOfferStatus::Accepted, 1, false), start), Step::Offer @@ -366,7 +400,7 @@ mod tests { alive + Duration::from_millis(500) ), Step::Wait, - "one missed answer within the deadline waits" + "one missed answer within the stall window waits" ); assert_eq!( progress.on_reply( @@ -374,7 +408,29 @@ mod tests { alive + Duration::from_secs(2) ), Step::End(StreamEnd::Lost), - "no answer for the deadline loses the stream" + "no answer for the stall window loses the stream" + ); + } + + /// A leader that keeps answering `Full` still loses the stream at the + /// statement's deadline: nobody waits for the outcome past it. + #[test] + fn the_statement_deadline_ends_a_live_stream() { + let stall_window = Duration::from_secs(10); + // no-determinism: test-only synthetic timeline. + let start = Instant::now(); + let ends_at = start + Duration::from_secs(2); + let mut progress = StreamProgress::new(4, true, stall_window, ends_at, start); + assert_eq!( + progress.on_reply( + &reply(PartsOfferStatus::Full, 0, false), + start + Duration::from_secs(1) + ), + Step::Wait + ); + assert_eq!( + progress.on_reply(&reply(PartsOfferStatus::Full, 0, false), ends_at), + Step::End(StreamEnd::Lost) ); } } diff --git a/nodedb/src/control/planner/calvin/submit/wait.rs b/nodedb/src/control/planner/calvin/submit/wait.rs new file mode 100644 index 000000000..d04468117 --- /dev/null +++ b/nodedb/src/control/planner/calvin/submit/wait.rs @@ -0,0 +1,138 @@ +// SPDX-License-Identifier: BUSL-1.1 + +//! The two waits of a submitted Calvin transaction: its sequencer assignment +//! and its completion. +//! +//! Both end at the statement's deadline. A submission that reached the +//! sequencer inbox can still be sequenced after its wait ends, so an elapsed +//! wait is an unknown outcome, never a definite refusal. + +use nodedb_cluster::calvin::{ + Assignment, AssignmentReceiver, CalvinCompletionRegistry, CompletionReport, TxnId, +}; + +use crate::Error; +use crate::types::RequestId; + +/// The error of a wait that ended at the statement's deadline: the outcome +/// is unknown. +fn unknown_outcome() -> Error { + Error::DeadlineExceeded { + request_id: RequestId::new(0), + } +} + +/// Await the sequencer assignment of submission `inbox_seq` until `deadline`. +/// +/// A closed channel means the sequencer rejected or discarded the submission +/// without sequencing it: a read/write cycle in its epoch, a leadership +/// change, or a halted sequencer. Nothing applied, so the caller gets a +/// retryable refusal. On the deadline the registration is dropped, so no +/// sender stays behind. +pub(crate) async fn await_assignment( + registry: &CalvinCompletionRegistry, + inbox_seq: u64, + assignment_rx: AssignmentReceiver, + deadline: tokio::time::Instant, +) -> crate::Result { + match tokio::time::timeout_at(deadline, assignment_rx).await { + Ok(Ok(assignment)) => Ok(assignment), + Ok(Err(_)) => Err(Error::RetryableRefusal { + reason: "the Calvin sequencer did not sequence the transaction; nothing was \ + applied" + .to_owned(), + }), + Err(_) => { + registry.drop_assignment(inbox_seq); + tracing::warn!( + inbox_seq, + "Calvin submission reached the statement deadline before its assignment" + ); + Err(unknown_outcome()) + } + } +} + +/// Await the completion report of `txn_id` until `deadline`. +/// +/// An elapsed wait files a `calvin_completion_timeout` report: it is the only +/// signal a never-completed Calvin write produces. +pub(crate) async fn await_completion( + registry: &CalvinCompletionRegistry, + txn_id: TxnId, + participants: usize, + deadline: tokio::time::Instant, +) -> crate::Result { + let waiting_since = tokio::time::Instant::now(); + let completion_rx = registry.register_completion_report(txn_id, participants); + match tokio::time::timeout_at(deadline, completion_rx).await { + Ok(Ok(report)) => Ok(report), + Ok(Err(_)) => Err(Error::Internal { + detail: "Calvin completion channel closed".to_owned(), + }), + Err(_) => { + let error = unknown_outcome(); + let waited_ms = u64::try_from( + deadline + .saturating_duration_since(waiting_since) + .as_millis(), + ) + .unwrap_or(u64::MAX); + crate::diag::calvin_completion_timeout( + &error, + txn_id.epoch, + txn_id.position, + participants, + waited_ms, + ); + Err(error) + } + } +} + +#[cfg(test)] +mod tests { + use std::time::Duration; + + use super::*; + + /// The sequencer dropped the assignment of a submission it did not + /// sequence. The caller fails at once with a retryable refusal, long + /// before its deadline. + #[tokio::test] + async fn an_unsequenced_submission_is_a_retryable_refusal() { + let registry = CalvinCompletionRegistry::new_detached(); + let rx = registry.register_submission(7); + registry.drop_assignment(7); + let deadline = tokio::time::Instant::now() + Duration::from_secs(3600); + match await_assignment(®istry, 7, rx, deadline).await { + Err(Error::RetryableRefusal { .. }) => {} + other => panic!("expected a retryable refusal, got {other:?}"), + } + } + + #[tokio::test] + async fn an_assigned_submission_returns_its_assignment() { + let registry = CalvinCompletionRegistry::new_detached(); + let rx = registry.register_submission(3); + registry.note_assigned(3, TxnId::new(5, 1), 2); + let deadline = tokio::time::Instant::now() + Duration::from_secs(3600); + let assignment = await_assignment(®istry, 3, rx, deadline) + .await + .expect("assigned"); + assert_eq!(assignment, (5, 1, 2)); + } + + /// A wait that reaches the deadline is an unknown outcome, never a + /// refusal: the submission can still be sequenced. + #[tokio::test] + async fn a_timed_out_wait_is_not_a_refusal() { + let registry = CalvinCompletionRegistry::new_detached(); + let rx = registry.register_submission(8); + let deadline = tokio::time::Instant::now() + Duration::from_millis(1); + match await_assignment(®istry, 8, rx, deadline).await { + Err(Error::DeadlineExceeded { .. }) => {} + other => panic!("expected an unknown outcome, got {other:?}"), + } + } +} diff --git a/nodedb/src/control/planner/calvin/tx_class/static_builder.rs b/nodedb/src/control/planner/calvin/tx_class/static_builder.rs index 00178eb3f..920175898 100644 --- a/nodedb/src/control/planner/calvin/tx_class/static_builder.rs +++ b/nodedb/src/control/planner/calvin/tx_class/static_builder.rs @@ -379,7 +379,8 @@ mod tests { /// A predicate read locks its whole collection `Shared`. A later row /// writer of that collection takes `Intent` on it, so it waits until the - /// reader releases. A point read locks only its row. + /// reader releases. A point read locks its row `Shared` and its + /// collection `IntentShared`. #[test] fn a_predicate_read_blocks_a_later_writer_until_released() { use crate::control::cluster::calvin::scheduler::LockMode; @@ -414,11 +415,12 @@ mod tests { }), Some(&LockMode::Shared) ); - assert!( - !reader_locks.contains_key(&LockKey::Collection { + assert_eq!( + reader_locks.get(&LockKey::Collection { collection: std::sync::Arc::from("read_col"), }), - "a point read locks nothing collection-wide" + Some(&LockMode::IntentShared), + "a point read announces itself on its collection" ); let writer = build_single_vshard_tx_class( diff --git a/nodedb/src/control/security/auth_fence/cluster.rs b/nodedb/src/control/security/auth_fence/cluster.rs index a99dfb790..f704a7c24 100644 --- a/nodedb/src/control/security/auth_fence/cluster.rs +++ b/nodedb/src/control/security/auth_fence/cluster.rs @@ -87,14 +87,10 @@ pub(crate) async fn wait_applied( if watcher.current() >= target { return Ok(()); } - // The watcher parks its caller on a condition variable, so the wait runs - // on the blocking pool. - let outcome = tokio::task::spawn_blocking(move || watcher.wait_for(target, timeout)) + match watcher + .wait_for_async(target, Instant::now() + timeout) .await - .map_err(|e| crate::Error::Internal { - detail: format!("applied-index wait for raft group {group_id} did not finish: {e}"), - })?; - match outcome { + { WaitOutcome::Reached => Ok(()), WaitOutcome::TimedOut => Err(behind(format!( "raft group {group_id} was not applied through index {target} in time" diff --git a/nodedb/src/control/security/catalog/redo_snapshot_owed.rs b/nodedb/src/control/security/catalog/redo_snapshot_owed.rs index 6ae86066a..46a537e0f 100644 --- a/nodedb/src/control/security/catalog/redo_snapshot_owed.rs +++ b/nodedb/src/control/security/catalog/redo_snapshot_owed.rs @@ -58,6 +58,16 @@ impl SystemCatalog { .map_err(|e| catalog_err("insert redo_snapshot_owed", e))?; } } + #[cfg(test)] + if self + .fail_next_redo_snapshot_owed_write + .swap(false, std::sync::atomic::Ordering::SeqCst) + { + return Err(crate::Error::Storage { + engine: "catalog".into(), + detail: "injected redo_snapshot_owed write failure".into(), + }); + } txn.commit() .map_err(|e| catalog_err("redo_snapshot_owed commit", e)) } diff --git a/nodedb/src/control/security/catalog/system_catalog.rs b/nodedb/src/control/security/catalog/system_catalog.rs index 80efd3e78..e91e91ddf 100644 --- a/nodedb/src/control/security/catalog/system_catalog.rs +++ b/nodedb/src/control/security/catalog/system_catalog.rs @@ -32,6 +32,8 @@ pub struct SystemCatalog { pub(super) fail_next_collection_write: Arc, #[cfg(test)] pub(super) fail_next_surrogate_write: Arc, + #[cfg(test)] + pub(super) fail_next_redo_snapshot_owed_write: Arc, } impl crate::storage::RedbBacked for SystemCatalog { @@ -67,6 +69,8 @@ impl SystemCatalog { fail_next_collection_write: Arc::new(std::sync::atomic::AtomicBool::new(false)), #[cfg(test)] fail_next_surrogate_write: Arc::new(std::sync::atomic::AtomicBool::new(false)), + #[cfg(test)] + fail_next_redo_snapshot_owed_write: Arc::new(std::sync::atomic::AtomicBool::new(false)), }; catalog.bootstrap_default_database()?; catalog.reload_event_definitions()?; @@ -93,6 +97,8 @@ impl SystemCatalog { fail_next_collection_write: Arc::new(std::sync::atomic::AtomicBool::new(false)), #[cfg(test)] fail_next_surrogate_write: Arc::new(std::sync::atomic::AtomicBool::new(false)), + #[cfg(test)] + fail_next_redo_snapshot_owed_write: Arc::new(std::sync::atomic::AtomicBool::new(false)), }; catalog.bootstrap_default_database()?; catalog.reload_event_definitions()?; @@ -126,6 +132,14 @@ impl SystemCatalog { .store(true, std::sync::atomic::Ordering::SeqCst); } + /// Make the next `save_redo_snapshot_owed` raise, so a held group's debt + /// write fails once. + #[cfg(test)] + pub(crate) fn fail_next_redo_snapshot_owed_write_for_test(&self) { + self.fail_next_redo_snapshot_owed_write + .store(true, std::sync::atomic::Ordering::SeqCst); + } + /// Bootstrap every `_system.*` table from the canonical registry — /// but only if at least one is actually missing. Probing read-only /// first keeps `open` byte-idempotent on an already-bootstrapped diff --git a/nodedb/src/control/server/calvin_submit/hook.rs b/nodedb/src/control/server/calvin_submit/hook.rs index 57a061d0a..a3ef71c61 100644 --- a/nodedb/src/control/server/calvin_submit/hook.rs +++ b/nodedb/src/control/server/calvin_submit/hook.rs @@ -1,7 +1,7 @@ // SPDX-License-Identifier: BUSL-1.1 //! `RegistryCalvinSubmit` — bridges the cluster `SubmitCalvinTxn` trigger to a -//! node-local Calvin submit-and-await (Cv1). +//! node-local Calvin submit-and-await. //! //! `nodedb-cluster` cannot depend on `nodedb` (circular), so the submit logic //! lives here and is exposed to the transport via the @@ -17,9 +17,10 @@ //! 1. decodes `tx_class_bytes` (msgpack) into a `TxClass` and re-derives its //! cached participating-vshard set via `restore_derived`; //! 2. submits it to this node's Calvin sequencer inbox and awaits assignment + -//! completion through `submit_and_await_calvin_with_timeout`, bounded by the -//! forwarded `deadline_remaining_ms` so it cannot outlive the coordinator's -//! remaining budget; +//! completion through `submit_and_await_calvin_until`, ending at the +//! deadline the forwarded `deadline_remaining_ms` names, so it cannot +//! outlive the coordinator's remaining budget. A spent budget is refused +//! unsubmitted, a definite refusal; //! 3. maps `Ok(())` → `error: None` and `Err` → a typed cluster error that //! keeps the Data-Plane verdict's code — never a silent drop. //! @@ -32,13 +33,13 @@ //! io_uring directly. use std::sync::Arc; -use std::time::Duration; use nodedb_cluster::calvin::types::TxClass; use nodedb_cluster::{SubmitCalvinTxnRequest, SubmitCalvinTxnResponse, TypedClusterError}; use crate::control::cluster::data_plane_error_wire::execution_error_to_typed; -use crate::control::planner::calvin::submit_and_await_calvin_with_timeout; +use crate::control::cluster::forwarded_budget::hook_deadline; +use crate::control::planner::calvin::submit_and_await_calvin_until; use crate::control::state::SharedState; /// `nodedb`-side implementation of [`nodedb_cluster::CalvinSubmit`]. @@ -62,6 +63,17 @@ impl RegistryCalvinSubmit { #[async_trait::async_trait] impl nodedb_cluster::CalvinSubmit for RegistryCalvinSubmit { async fn on_submit_calvin_txn(&self, req: SubmitCalvinTxnRequest) -> SubmitCalvinTxnResponse { + // A spent budget is refused before any work: nothing is submitted. + let deadline = match hook_deadline(req.deadline_remaining_ms) { + Ok(deadline) => deadline, + Err(refusal) => { + return SubmitCalvinTxnResponse { + error: Some(refusal), + payload_bytes: None, + }; + } + }; + let mut tx_class: TxClass = match zerompk::from_msgpack(&req.tx_class_bytes) { Ok(tc) => tc, Err(e) => { @@ -86,8 +98,7 @@ impl nodedb_cluster::CalvinSubmit for RegistryCalvinSubmit { }; } - let timeout = Duration::from_millis(req.deadline_remaining_ms.max(1)); - match submit_and_await_calvin_with_timeout(&self.state, tx_class, timeout).await { + match submit_and_await_calvin_until(&self.state, tx_class, deadline).await { // Forward the applied RETURNING payload (drained from this leader's // local sidecar) back to the remote coordinator over the non-Raft // RPC response; `None` for a plain write with no rows to surface. @@ -105,3 +116,40 @@ impl nodedb_cluster::CalvinSubmit for RegistryCalvinSubmit { } } } + +#[cfg(test)] +mod tests { + use nodedb_cluster::CalvinSubmit; + + use super::*; + use crate::bridge::dispatch::Dispatcher; + use crate::bridge::envelope::ErrorCode; + use crate::wal::WalManager; + + /// A forward with a spent budget is refused unsubmitted. The coordinator + /// sees a definite refusal, never a 1 ms wait that ends unknown. + #[tokio::test] + async fn a_spent_budget_is_refused_before_the_submit() { + let dir = tempfile::tempdir().unwrap(); + let wal = Arc::new(WalManager::open_for_testing(&dir.path().join("test.wal")).unwrap()); + let (dispatcher, _data_sides) = Dispatcher::new(1, 64); + let state = SharedState::new(dispatcher, wal).unwrap(); + + let reply = RegistryCalvinSubmit::new(state) + .on_submit_calvin_txn(SubmitCalvinTxnRequest { + tx_class_bytes: Vec::new(), + deadline_remaining_ms: 0, + trace_id: [0u8; 16], + }) + .await; + let refusal = reply.error.expect("a spent budget is refused"); + assert!( + matches!( + crate::Error::from(refusal), + crate::Error::DataPlane(ErrorCode::ExpiredBeforeExecution) + ), + "a spent budget must come back as ExpiredBeforeExecution" + ); + assert!(reply.payload_bytes.is_none()); + } +} diff --git a/nodedb/src/control/server/calvin_submit/inbox_hook.rs b/nodedb/src/control/server/calvin_submit/inbox_hook.rs index cf4578d59..e903f6b29 100644 --- a/nodedb/src/control/server/calvin_submit/inbox_hook.rs +++ b/nodedb/src/control/server/calvin_submit/inbox_hook.rs @@ -1,7 +1,7 @@ // SPDX-License-Identifier: BUSL-1.1 //! `RegistryCalvinSubmitInbox` — bridges the cluster `SubmitCalvinInbox` trigger -//! to a node-local Calvin submit-and-ASSIGN (Cv1). +//! to a node-local Calvin submit-and-ASSIGN. //! //! OLLP dependent sibling of [`RegistryCalvinSubmit`](super::hook::RegistryCalvinSubmit): //! `nodedb-cluster` cannot depend on `nodedb` (circular), so the submit logic @@ -19,14 +19,15 @@ //! 2. submits it to this node's Calvin sequencer inbox and awaits only the //! ASSIGNMENT (NOT completion) through the local `CalvinCompletionRegistry`, //! bounded by the forwarded `deadline_remaining_ms` so it cannot outlive the -//! coordinator's remaining budget; +//! coordinator's remaining budget. A spent budget is refused unsubmitted; //! 3. maps `Ok(RoutedAssignment { .. })` → a success response carrying -//! the assignment and `Err` → a typed [`TypedClusterError::Internal`] — never -//! a silent drop. +//! the assignment and `Err` → a typed cluster error that keeps the error's +//! class — never a silent drop. A deadline stays an unknown outcome, so +//! the coordinator never resubmits a transaction that can be sequenced. //! //! It must NOT await completion: the OLLP coordinator loop drives the dependent -//! transaction to completion itself in a later unit; this hook returns as soon as -//! the assignment is observed. +//! transaction to completion itself. This hook returns as soon as the +//! assignment is observed. //! //! # Plane discipline //! @@ -37,7 +38,6 @@ //! io_uring directly. use std::sync::Arc; -use std::time::Duration; use nodedb_cluster::calvin::PartsOfferStatus; use nodedb_cluster::calvin::types::{PartStreamId, StreamedPart, TxClass}; @@ -46,6 +46,8 @@ use nodedb_cluster::{ TypedClusterError, }; +use crate::control::cluster::data_plane_error_wire::execution_error_to_typed; +use crate::control::cluster::forwarded_budget::hook_deadline; use crate::control::planner::calvin::submit::submit_local_assign; use crate::control::state::SharedState; @@ -73,6 +75,20 @@ impl nodedb_cluster::CalvinSubmitInbox for RegistryCalvinSubmitInbox { &self, req: SubmitCalvinInboxRequest, ) -> SubmitCalvinInboxResponse { + // A spent budget is refused before any work: nothing is submitted. + let deadline = match hook_deadline(req.deadline_remaining_ms) { + Ok(deadline) => deadline, + Err(refusal) => { + return SubmitCalvinInboxResponse { + inbox_seq: 0, + epoch: 0, + position: 0, + participants: 0, + error: Some(refusal), + }; + } + }; + let mut tx_class: TxClass = match zerompk::from_msgpack(&req.tx_class_bytes) { Ok(tc) => tc, Err(e) => { @@ -103,8 +119,7 @@ impl nodedb_cluster::CalvinSubmitInbox for RegistryCalvinSubmitInbox { }; } - let timeout = Duration::from_millis(req.deadline_remaining_ms.max(1)); - match submit_local_assign(&self.state, tx_class, timeout).await { + match submit_local_assign(&self.state, tx_class, deadline).await { Ok(a) => SubmitCalvinInboxResponse { inbox_seq: a.inbox_seq, epoch: a.epoch, @@ -117,10 +132,7 @@ impl nodedb_cluster::CalvinSubmitInbox for RegistryCalvinSubmitInbox { epoch: 0, position: 0, participants: 0, - error: Some(TypedClusterError::Internal { - code: 0, - message: format!("calvin-inbox local submit-and-assign failed: {e}"), - }), + error: Some(execution_error_to_typed(e)), }, } } @@ -154,3 +166,40 @@ impl nodedb_cluster::CalvinSubmitInbox for RegistryCalvinSubmitInbox { } } } + +#[cfg(test)] +mod tests { + use nodedb_cluster::CalvinSubmitInbox; + + use super::*; + use crate::bridge::dispatch::Dispatcher; + use crate::bridge::envelope::ErrorCode; + use crate::wal::WalManager; + + /// A forward with a spent budget is refused unsubmitted. The coordinator + /// sees a definite refusal, never a 1 ms wait that ends unknown. + #[tokio::test] + async fn a_spent_budget_is_refused_before_the_submit() { + let dir = tempfile::tempdir().unwrap(); + let wal = Arc::new(WalManager::open_for_testing(&dir.path().join("test.wal")).unwrap()); + let (dispatcher, _data_sides) = Dispatcher::new(1, 64); + let state = SharedState::new(dispatcher, wal).unwrap(); + + let reply = RegistryCalvinSubmitInbox::new(state) + .on_submit_calvin_inbox(SubmitCalvinInboxRequest { + tx_class_bytes: Vec::new(), + deadline_remaining_ms: 0, + trace_id: [0u8; 16], + }) + .await; + let refusal = reply.error.expect("a spent budget is refused"); + assert!( + matches!( + crate::Error::from(refusal), + crate::Error::DataPlane(ErrorCode::ExpiredBeforeExecution) + ), + "a spent budget must come back as ExpiredBeforeExecution" + ); + assert_eq!(reply.inbox_seq, 0); + } +} diff --git a/nodedb/src/control/server/calvin_submit/mod.rs b/nodedb/src/control/server/calvin_submit/mod.rs index 149e72b2a..c6f68c919 100644 --- a/nodedb/src/control/server/calvin_submit/mod.rs +++ b/nodedb/src/control/server/calvin_submit/mod.rs @@ -1,6 +1,6 @@ // SPDX-License-Identifier: BUSL-1.1 -//! Routed Calvin-submit (Cv1): the sequencer-leader-side cluster hook that +//! Routed Calvin-submit: the sequencer-leader-side cluster hook that //! submits a forwarded `TxClass` to the local Calvin sequencer inbox and awaits //! its completion. The coordinator-side routing helper lives in //! `crate::control::planner::calvin::submit::submit_calvin_routed`. diff --git a/nodedb/src/control/server/dispatch_utils/change_events/extract.rs b/nodedb/src/control/server/dispatch_utils/change_events/extract.rs index aa7fd992b..ecf0452b3 100644 --- a/nodedb/src/control/server/dispatch_utils/change_events/extract.rs +++ b/nodedb/src/control/server/dispatch_utils/change_events/extract.rs @@ -659,7 +659,7 @@ mod tests { slice_msgpack: Vec::new(), attr_projection: Vec::new(), limit: 0, - slice_hilbert_ranges: Vec::new(), + slice_tile_keys: Vec::new(), prefix_bits: 8, system_time: nodedb_types::SystemTimeScope::Current, valid_at_ms: None, @@ -671,7 +671,7 @@ mod tests { attr_idx: 0, reducer_msgpack: Vec::new(), group_by_dim: -1, - slice_hilbert_ranges: Vec::new(), + slice_tile_keys: Vec::new(), prefix_bits: 8, system_as_of: None, valid_at_ms: None, diff --git a/nodedb/src/control/server/dispatch_utils/durability_barrier.rs b/nodedb/src/control/server/dispatch_utils/durability_barrier.rs index 6f5a94006..ff9958523 100644 --- a/nodedb/src/control/server/dispatch_utils/durability_barrier.rs +++ b/nodedb/src/control/server/dispatch_utils/durability_barrier.rs @@ -26,7 +26,7 @@ use std::sync::atomic::{AtomicU64, Ordering}; use crate::bridge::envelope::PhysicalPlan; use crate::control::server::shared::write_admission::plan_is_write; -use nodedb_physical::physical_plan::{GraphOp, MetaOp}; +use nodedb_physical::physical_plan::{GraphOp, KvOp, MetaOp}; /// Count of writes acknowledged with no durable redo record despite belonging /// to an engine whose every write-class op mints one on this path. @@ -63,7 +63,9 @@ pub fn writes_acked_without_durability() -> u64 { /// * `ClusterArray` — a coordinator-side routing wrapper: each owning shard's /// apply mints the redo for the cells it actually holds; /// * an empty `EdgePutBatch` / `EdgeDeleteBatch`, which has no edge to make -/// durable and appends nothing on purpose. +/// durable and appends nothing on purpose; +/// * a KV `ResolvedWrite` with no mutation, such as a predicate UPDATE / +/// DELETE that matched no row. It appends one record per mutation, so none. /// /// Exchange / gather plans never reach the barrier at all: they are resolved /// into a merged response before `submit_write` is entered. @@ -75,6 +77,7 @@ pub(super) fn funnel_minted_redo_engine(plan: &PhysicalPlan) -> Option<&'static // `PhysicalPlan` variant must be classified here by name rather than // silently inheriting "no durability expected". match plan { + PhysicalPlan::Kv(KvOp::ResolvedWrite { mutations, .. }) if mutations.is_empty() => None, PhysicalPlan::Kv(_) => Some("kv"), PhysicalPlan::Vector(_) => Some("vector"), PhysicalPlan::Columnar(_) => Some("columnar"), @@ -174,6 +177,19 @@ mod tests { assert_eq!(funnel_minted_redo_engine(&plan), Some("transaction")); } + /// A resolved write with no mutation appends no record, so its + /// acknowledgement waits on nothing. A predicate write that matched no + /// row resolves to it. + #[test] + fn an_empty_kv_resolved_write_is_not_held_to_the_barrier() { + let empty = PhysicalPlan::Kv(KvOp::ResolvedWrite { + mutations: Vec::new(), + response_payload: Vec::new(), + rls_write_check: nodedb_types::RlsWriteCheck::decided_earlier_in_request(), + }); + assert_eq!(funnel_minted_redo_engine(&empty), None); + } + /// A read carries no durability obligation at all. #[test] fn read_requires_no_redo() { diff --git a/nodedb/src/control/server/dispatch_utils/durable_write.rs b/nodedb/src/control/server/dispatch_utils/durable_write.rs index 3458f1439..3e3205d17 100644 --- a/nodedb/src/control/server/dispatch_utils/durable_write.rs +++ b/nodedb/src/control/server/dispatch_utils/durable_write.rs @@ -215,7 +215,8 @@ async fn propose_if_replicable( return Ok(None); }; let entry = entry.with_event_source(event_source); - let deadline = crate::control::wal_replication::statement_propose_deadline(shared); + let deadline = crate::control::server::shared::session::StatementDeadline::of_statement(shared) + .tokio_instant(); let (payload, write_versions) = propose_replicated_entry(shared, proposer, entry, deadline).await?; Ok(Some(replicated_write_response( diff --git a/nodedb/src/control/server/dispatch_utils/owner_read.rs b/nodedb/src/control/server/dispatch_utils/owner_read.rs index 5050277c6..4f9880986 100644 --- a/nodedb/src/control/server/dispatch_utils/owner_read.rs +++ b/nodedb/src/control/server/dispatch_utils/owner_read.rs @@ -22,7 +22,7 @@ use futures::future::{BoxFuture, Either, Ready, ready}; use crate::bridge::envelope::{ErrorCode, Payload, PhysicalPlan, Response, Status}; use crate::control::cluster::linearizable_read::{ - confirm_linearizable_read, hosted_groups_of_vshards, statement_read_deadline, + StatementDeadline, confirm_linearizable_read, hosted_groups_of_vshards, }; use crate::control::gateway::RouteDecision; use crate::control::gateway::core::QueryContext; @@ -78,7 +78,7 @@ async fn route_cluster_read( scope: OwnedReadScope, plan: PhysicalPlan, ) -> crate::Result { - let deadline = statement_read_deadline(shared); + let deadline = StatementDeadline::of_statement(shared); if nodedb_physical::physical_plan::plan_contains_cluster_partitioned_leaf(&plan) { if scope.linearizable { let groups = crate::control::server::graph_dispatch::graph_read_groups( @@ -200,7 +200,8 @@ pub(crate) async fn prepare_local_pass( ) -> crate::Result<()> { match read_placement(shared, vshard_id, txn_id)? { ReadPlacement::Here(hosted) => { - confirm_linearizable_read(shared, &hosted, statement_read_deadline(shared)).await + confirm_linearizable_read(shared, &hosted, StatementDeadline::of_statement(shared)) + .await } ReadPlacement::Owner => { let (leader_node, leader_term) = shared diff --git a/nodedb/src/control/server/exchange/all_cores/home_versions.rs b/nodedb/src/control/server/exchange/all_cores/home_versions.rs index 950b81069..3bc7cb3a5 100644 --- a/nodedb/src/control/server/exchange/all_cores/home_versions.rs +++ b/nodedb/src/control/server/exchange/all_cores/home_versions.rs @@ -22,7 +22,7 @@ use nodedb_physical::physical_plan::{ }; use crate::control::cluster::leased_read::{LeaseRefusal, confirm_leased_read}; -use crate::control::cluster::linearizable_read::statement_read_deadline; +use crate::control::cluster::linearizable_read::StatementDeadline; use crate::control::server::exchange::owning_core::dispatch_single_owning_core; use crate::control::state::SharedState; use crate::types::{DatabaseId, Lsn, TenantId, TraceId, VShardId}; @@ -49,7 +49,7 @@ pub(super) async fn fan_home_versions( groups.sort_unstable(); groups.dedup(); let refusals: HashMap = - confirm_leased_read(state, &groups, statement_read_deadline(state)) + confirm_leased_read(state, &groups, StatementDeadline::of_statement(state)) .await? .into_iter() .map(|refusal| (refusal.group_id, refusal)) diff --git a/nodedb/src/control/server/exchange/cluster_leaf.rs b/nodedb/src/control/server/exchange/cluster_leaf.rs index 7b53b7b9f..d2c70d237 100644 --- a/nodedb/src/control/server/exchange/cluster_leaf.rs +++ b/nodedb/src/control/server/exchange/cluster_leaf.rs @@ -213,7 +213,7 @@ mod tests { slice_msgpack: Vec::new(), attr_projection: Vec::new(), limit: 0, - slice_hilbert_ranges: Vec::new(), + slice_tile_keys: Vec::new(), prefix_bits: 0, system_time: nodedb_types::SystemTimeScope::Current, valid_at_ms: None, @@ -226,7 +226,7 @@ mod tests { attr_idx: 0, reducer_msgpack: Vec::new(), group_by_dim: -1, - slice_hilbert_ranges: Vec::new(), + slice_tile_keys: Vec::new(), prefix_bits: 0, system_as_of: None, valid_at_ms: None, diff --git a/nodedb/src/control/server/graph_dispatch/cluster_resolve.rs b/nodedb/src/control/server/graph_dispatch/cluster_resolve.rs index b75c79d91..da452516a 100644 --- a/nodedb/src/control/server/graph_dispatch/cluster_resolve.rs +++ b/nodedb/src/control/server/graph_dispatch/cluster_resolve.rs @@ -100,6 +100,7 @@ pub(in crate::control::server::graph_dispatch) async fn dispatch_superstep_to_no database_id, trace_id: TraceId::ZERO, deadline_ms, + attempts_left: 1, version_set, // This resolve path carries no session-transaction context. txn_id: None, diff --git a/nodedb/src/control/server/graph_dispatch/hop.rs b/nodedb/src/control/server/graph_dispatch/hop.rs index 19ad04d0a..2c621bdf3 100644 --- a/nodedb/src/control/server/graph_dispatch/hop.rs +++ b/nodedb/src/control/server/graph_dispatch/hop.rs @@ -55,12 +55,11 @@ use std::collections::BTreeMap; use futures::future::join_all; use crate::bridge::envelope::PhysicalPlan; -use crate::control::gateway::dispatcher::{ - DispatchRouteParams, dispatch_route, statement_deadline_ms, -}; +use crate::control::gateway::dispatcher::{DispatchRouteParams, dispatch_route}; use crate::control::gateway::live_leaders::LiveLeaders; use crate::control::gateway::version_set::GatewayVersionSet; use crate::control::gateway::{RouteDecision, TaskRoute}; +use crate::control::server::shared::session::StatementDeadline; use crate::control::state::SharedState; use crate::engine::graph::edge_store::Direction; use crate::engine::graph::traversal_options::GraphTraversalOptions; @@ -356,7 +355,7 @@ async fn expand_remote( // owning `SharedState`. let shared_arc = super::cluster_resolve::gateway_shared(shared)?; - let deadline_ms = statement_deadline_ms(&shared_arc); + let deadline_ms = StatementDeadline::of_statement(&shared_arc).hop_budget_ms(); // Graph structural ops touch no named collection, so the version set is // empty (descriptor-version checks do not apply to node-id-keyed edges). let version_set = GatewayVersionSet::from_pairs(Vec::new()); @@ -386,6 +385,7 @@ async fn expand_remote( database_id, trace_id: TraceId::ZERO, deadline_ms, + attempts_left: 1, version_set: &version_set, txn_id, linearizable, diff --git a/nodedb/src/control/server/graph_dispatch/keyed_read.rs b/nodedb/src/control/server/graph_dispatch/keyed_read.rs index d54974b01..f0f12e041 100644 --- a/nodedb/src/control/server/graph_dispatch/keyed_read.rs +++ b/nodedb/src/control/server/graph_dispatch/keyed_read.rs @@ -107,7 +107,10 @@ pub async fn read_on_vshard( tenant_id, database_id, trace_id: TraceId::ZERO, - deadline_ms: crate::control::gateway::dispatcher::statement_deadline_ms(state), + deadline_ms: + crate::control::server::shared::session::StatementDeadline::of_statement(state) + .hop_budget_ms(), + attempts_left: 1, version_set: &version_set, txn_id, linearizable, diff --git a/nodedb/src/control/server/graph_dispatch/match_scatter/round_loop.rs b/nodedb/src/control/server/graph_dispatch/match_scatter/round_loop.rs index 0d70e3a77..634606d6a 100644 --- a/nodedb/src/control/server/graph_dispatch/match_scatter/round_loop.rs +++ b/nodedb/src/control/server/graph_dispatch/match_scatter/round_loop.rs @@ -106,6 +106,7 @@ fn push_dispatch_fut<'f>( database_id, trace_id: TraceId::ZERO, deadline_ms, + attempts_left: 1, version_set: &version_set, txn_id: read.txn_id, linearizable: read.linearizable, diff --git a/nodedb/src/control/server/graph_dispatch/match_scatter/round_zero.rs b/nodedb/src/control/server/graph_dispatch/match_scatter/round_zero.rs index e134bf6fe..286b3a640 100644 --- a/nodedb/src/control/server/graph_dispatch/match_scatter/round_zero.rs +++ b/nodedb/src/control/server/graph_dispatch/match_scatter/round_zero.rs @@ -114,6 +114,7 @@ pub(super) async fn scatter_round_zero( database_id, trace_id: TraceId::ZERO, deadline_ms, + attempts_left: 1, version_set: &version_set, txn_id, linearizable, diff --git a/nodedb/src/control/server/graph_dispatch/rag_fusion/stages.rs b/nodedb/src/control/server/graph_dispatch/rag_fusion/stages.rs index 1c08c512d..50c52f31e 100644 --- a/nodedb/src/control/server/graph_dispatch/rag_fusion/stages.rs +++ b/nodedb/src/control/server/graph_dispatch/rag_fusion/stages.rs @@ -9,10 +9,9 @@ use nodedb_physical::physical_plan::{GraphOp, RagBindingRow, RagLegs, RagStage}; use crate::bridge::envelope::PhysicalPlan; use crate::control::gateway::TaskRoute; -use crate::control::gateway::dispatcher::{ - DispatchRouteParams, dispatch_route, statement_deadline_ms, -}; +use crate::control::gateway::dispatcher::{DispatchRouteParams, dispatch_route}; use crate::control::gateway::version_set::GatewayVersionSet; +use crate::control::server::shared::session::StatementDeadline; use crate::control::state::SharedState; use crate::types::{DatabaseId, Lsn, TenantId, TraceId}; @@ -73,7 +72,8 @@ pub(super) async fn export_legs( tenant_id, database_id, trace_id: TraceId::ZERO, - deadline_ms: statement_deadline_ms(state), + deadline_ms: StatementDeadline::of_statement(state).hop_budget_ms(), + attempts_left: 1, version_set: &GatewayVersionSet::from_pairs(Vec::new()), txn_id: None, linearizable, diff --git a/nodedb/src/control/server/graph_dispatch/read_groups.rs b/nodedb/src/control/server/graph_dispatch/read_groups.rs index 4ae02cb5f..40fbc05f6 100644 --- a/nodedb/src/control/server/graph_dispatch/read_groups.rs +++ b/nodedb/src/control/server/graph_dispatch/read_groups.rs @@ -26,8 +26,7 @@ use nodedb_types::{CollectionKey, QualifiedCollection}; use crate::bridge::envelope::PhysicalPlan; use crate::control::cluster::linearizable_read::{ - confirm_linearizable_read, groups_hosted_here, hosted_groups_of_vshards, - statement_read_deadline, + StatementDeadline, confirm_linearizable_read, groups_hosted_here, hosted_groups_of_vshards, }; use crate::control::state::SharedState; use crate::types::{DatabaseId, VShardId}; @@ -94,7 +93,7 @@ pub async fn confirm_graph_read( plan: &PhysicalPlan, ) -> crate::Result<()> { let groups = graph_read_groups(state, database_id, plan)?; - confirm_linearizable_read(state, &groups, statement_read_deadline(state)).await + confirm_linearizable_read(state, &groups, StatementDeadline::of_statement(state)).await } fn keyed_groups<'a>( diff --git a/nodedb/src/control/server/graph_dispatch/whole_graph.rs b/nodedb/src/control/server/graph_dispatch/whole_graph.rs index 75f68861f..0052aad94 100644 --- a/nodedb/src/control/server/graph_dispatch/whole_graph.rs +++ b/nodedb/src/control/server/graph_dispatch/whole_graph.rs @@ -37,9 +37,9 @@ use std::collections::BTreeMap; use futures::future::join_all; use crate::bridge::envelope::{Payload, PhysicalPlan}; -use crate::control::gateway::dispatcher::statement_deadline_ms; use crate::control::gateway::version_set::GatewayVersionSet; use crate::control::server::exchange::execute_plan_all_local_cores; +use crate::control::server::shared::session::StatementDeadline; use crate::control::state::SharedState; use crate::types::{DatabaseId, TenantId, TraceId, VShardId}; use nodedb_graph::{AlgoParams, GraphAlgorithm}; @@ -76,7 +76,7 @@ pub async fn scatter_to_graph_owners( let targets = enumerate_shards(state)?.targets; let shared_arc = gateway_shared(state)?; let version_set = GatewayVersionSet::from_pairs(Vec::new()); - let deadline_ms = statement_deadline_ms(state); + let deadline_ms = StatementDeadline::of_statement(state).hop_budget_ms(); let legs = targets.into_iter().map(|target| { let plan = plan.clone(); let shared_arc = shared_arc.clone(); @@ -123,7 +123,7 @@ pub async fn run_graph_algo( system_as_of_ms: Option, linearizable: bool, ) -> crate::Result { - let deadline_ms = statement_deadline_ms(state); + let deadline_ms = StatementDeadline::of_statement(state).hop_budget_ms(); // A current-state BSP run pins its own read cut from a Calvin cut marker // (`run_cut`). A historical run gathers the edges live at its system time. if state.cluster_routing.is_some() && system_as_of_ms.is_none() { diff --git a/nodedb/src/control/server/http/routes/health.rs b/nodedb/src/control/server/http/routes/health.rs index 85519e9e8..33998b111 100644 --- a/nodedb/src/control/server/http/routes/health.rs +++ b/nodedb/src/control/server/http/routes/health.rs @@ -247,8 +247,8 @@ fn outcome_floor_stuck_body( /// How long a write window can hold the outcome floor before readiness reports /// it: the longest path from a window's open to its final outcome. /// -/// - A statement write reaches its outcome by the longest statement deadline, -/// plus the wait the node gives a committed entry to apply. +/// - A statement write reaches its outcome by the longest statement deadline. +/// Its waits for a committed entry to apply end at that deadline too. /// - A vector index install waits two core dispatch deadlines. /// /// A window older than both is stuck. @@ -258,8 +258,7 @@ fn outcome_floor_bound(state: &AppState) -> std::time::Duration { network .default_deadline_secs .max(network.copy_deadline_secs), - ) - .saturating_add(crate::control::metadata_proposer::DEFAULT_PROPOSE_TIMEOUT); + ); statement.max(crate::control::catalog_entry::post_apply::vector_install_longest_core_wait()) } @@ -486,8 +485,8 @@ mod tests { assert_ne!(body["reason"], "calvin_apply_halted"); } - /// The bound covers the longest statement deadline plus the apply wait, - /// and the vector install's two core dispatch deadlines. + /// The bound covers the longest statement deadline, and the vector + /// install's two core dispatch deadlines. #[tokio::test] async fn the_outcome_floor_bound_covers_every_path_to_a_final_outcome() { let dir = tempfile::tempdir().expect("tempdir"); @@ -497,7 +496,7 @@ mod tests { network .default_deadline_secs .max(network.copy_deadline_secs), - ) + crate::control::metadata_proposer::DEFAULT_PROPOSE_TIMEOUT; + ); let vector = crate::control::catalog_entry::post_apply::vector_install_longest_core_wait(); let bound = outcome_floor_bound(&state); diff --git a/nodedb/src/control/server/native/dispatch/graph_match.rs b/nodedb/src/control/server/native/dispatch/graph_match.rs index b1ba9fbfd..7e7ad061d 100644 --- a/nodedb/src/control/server/native/dispatch/graph_match.rs +++ b/nodedb/src/control/server/native/dispatch/graph_match.rs @@ -199,7 +199,8 @@ async fn match_across_shards( ctx.tenant_id(), ctx.database_id(), query.clone(), - crate::control::gateway::dispatcher::statement_deadline_ms(ctx.state), + crate::control::server::shared::session::StatementDeadline::of_statement(ctx.state) + .hop_budget_ms(), crate::control::server::graph_dispatch::GraphRead { txn_id, // The native protocol has no read-consistency setting. diff --git a/nodedb/src/control/server/pgwire/handler/dispatch/replicated.rs b/nodedb/src/control/server/pgwire/handler/dispatch/replicated.rs index 2c9237cdd..05192caf7 100644 --- a/nodedb/src/control/server/pgwire/handler/dispatch/replicated.rs +++ b/nodedb/src/control/server/pgwire/handler/dispatch/replicated.rs @@ -34,7 +34,8 @@ impl NodeDbPgHandler { &self.state, proposer, entry, - crate::control::wal_replication::statement_propose_deadline(&self.state), + crate::control::server::shared::session::StatementDeadline::of_statement(&self.state) + .tokio_instant(), ) .await?; diff --git a/nodedb/src/control/server/pgwire/handler/dispatch/routing.rs b/nodedb/src/control/server/pgwire/handler/dispatch/routing.rs index 5cd4c48da..9d7efa4e1 100644 --- a/nodedb/src/control/server/pgwire/handler/dispatch/routing.rs +++ b/nodedb/src/control/server/pgwire/handler/dispatch/routing.rs @@ -6,9 +6,7 @@ use std::sync::Arc; use crate::bridge::envelope::Response; -use crate::control::cluster::linearizable_read::{ - confirm_linearizable_read, statement_read_deadline, -}; +use crate::control::cluster::linearizable_read::{StatementDeadline, confirm_linearizable_read}; use crate::control::security::identity::AuthenticatedIdentity; use crate::control::server::exchange::ReadScope; use crate::control::server::exchange::resolve::{ @@ -228,7 +226,7 @@ impl NodeDbPgHandler { match read_placement(&self.state, checked.vshard_id(), checked.txn_id())? { ReadPlacement::Here(groups) => { if linearizable { - let deadline = statement_read_deadline(&self.state); + let deadline = StatementDeadline::of_statement(&self.state); confirm_linearizable_read(&self.state, &groups, deadline).await?; } } diff --git a/nodedb/src/control/server/pgwire/handler/routing/placement.rs b/nodedb/src/control/server/pgwire/handler/routing/placement.rs index 2ed7cae99..a8c78bd31 100644 --- a/nodedb/src/control/server/pgwire/handler/routing/placement.rs +++ b/nodedb/src/control/server/pgwire/handler/routing/placement.rs @@ -273,15 +273,20 @@ impl NodeDbPgHandler { /// Whether this node's replica of `group_id` meets `consistency`. /// /// `Eventual` asks for no freshness at all, so any replica satisfies it. - /// `BoundedStaleness` asks how far behind the leader this replica is, - /// which only Raft can answer. With no gate installed there is no cluster, - /// and the local copy is the only copy. + /// `BoundedStaleness` asks how far behind the leader this replica is: + /// Raft's last leader contact against this node's applied watermark. With + /// no gate installed there is no cluster, and the local copy is the only + /// copy. fn replica_satisfies(&self, group_id: u64, consistency: ReadConsistency) -> bool { let Some(max_staleness) = consistency.max_staleness() else { return true; }; match self.state.raft_read_gate.get() { - Some(gate) => gate.within_staleness_bound(group_id, max_staleness), + Some(gate) => gate.within_staleness_bound( + group_id, + max_staleness, + self.state.applied_index_watcher(group_id).current(), + ), None => true, } } diff --git a/nodedb/src/control/server/pgwire/types/error_map.rs b/nodedb/src/control/server/pgwire/types/error_map.rs index c393d9e39..994fa5eb3 100644 --- a/nodedb/src/control/server/pgwire/types/error_map.rs +++ b/nodedb/src/control/server/pgwire/types/error_map.rs @@ -411,7 +411,8 @@ pub fn error_to_sqlstate(err: &crate::Error) -> (&'static str, &'static str, Str // Class `XX` is never treated as transient, so no client re-proposes // a write that can have committed. crate::Error::CommittedResultUnavailable { .. } - | crate::Error::ProposalOutcomeUnknown { .. } => { + | crate::Error::ProposalOutcomeUnknown { .. } + | crate::Error::MetadataGroupGone { .. } => { ("ERROR", sqlstate::INTERNAL_ERROR, err.to_string()) } // Server-side faults and system defects. The client can act on none @@ -419,6 +420,7 @@ pub fn error_to_sqlstate(err: &crate::Error) -> (&'static str, &'static str, Str crate::Error::MaterializedSumResolutionMissing { .. } | crate::Error::RetryableLeaderChange { .. } | crate::Error::MetadataLeaderUnavailable + | crate::Error::MetadataDdlSuperseded { .. } | crate::Error::Wal(_) | crate::Error::Dispatch { .. } | crate::Error::Storage { .. } diff --git a/nodedb/src/control/server/reservation/hooks.rs b/nodedb/src/control/server/reservation/hooks.rs index d928b85f4..095124edf 100644 --- a/nodedb/src/control/server/reservation/hooks.rs +++ b/nodedb/src/control/server/reservation/hooks.rs @@ -23,7 +23,6 @@ //! directly. use std::sync::Arc; -use std::time::Duration; use nodedb_cluster::calvin::types::{LockKeyWire, ReleaseReason, TxnIdWire}; use nodedb_cluster::{ @@ -31,6 +30,8 @@ use nodedb_cluster::{ TypedClusterError, }; +use crate::control::cluster::data_plane_error_wire::execution_error_to_typed; +use crate::control::cluster::forwarded_budget::hook_deadline; use crate::control::planner::calvin::reservation::{ submit_local_release, submit_local_reserve_read, }; @@ -57,6 +58,16 @@ impl RegistryReserveRead { #[async_trait::async_trait] impl nodedb_cluster::ReserveRead for RegistryReserveRead { async fn on_reserve_read(&self, req: ReserveReadRequest) -> ReserveReadResponse { + // A spent budget is refused before any work: nothing is reserved. + let deadline = match hook_deadline(req.deadline_remaining_ms) { + Ok(deadline) => deadline, + Err(refusal) => { + return ReserveReadResponse { + owner_bytes: None, + error: Some(refusal), + }; + } + }; let key: LockKeyWire = match zerompk::from_msgpack(&req.lock_key_bytes) { Ok(k) => k, Err(e) => { @@ -85,8 +96,7 @@ impl nodedb_cluster::ReserveRead for RegistryReserveRead { None => None, }; - let timeout = Duration::from_millis(req.deadline_remaining_ms.max(1)); - match submit_local_reserve_read(&self.state, key, req.vshard, owner, timeout).await { + match submit_local_reserve_read(&self.state, key, req.vshard, owner, deadline).await { Ok(assigned) => match zerompk::to_msgpack_vec(&assigned) { Ok(bytes) => ReserveReadResponse { owner_bytes: Some(bytes), @@ -100,12 +110,11 @@ impl nodedb_cluster::ReserveRead for RegistryReserveRead { }), }, }, + // The error keeps its class: a deadline stays an unknown + // outcome, an expired budget a definite refusal. Err(e) => ReserveReadResponse { owner_bytes: None, - error: Some(TypedClusterError::Internal { - code: 0, - message: format!("reserve-read local submit failed: {e}"), - }), + error: Some(execution_error_to_typed(e)), }, } } @@ -172,3 +181,42 @@ impl nodedb_cluster::ReleaseReservation for RegistryReleaseReservation { } } } + +#[cfg(test)] +mod tests { + use nodedb_cluster::ReserveRead; + + use super::*; + use crate::bridge::dispatch::Dispatcher; + use crate::bridge::envelope::ErrorCode; + use crate::wal::WalManager; + + /// A reserve-read with a spent budget is refused unsubmitted, a definite + /// refusal the coordinator keeps. + #[tokio::test] + async fn a_spent_budget_is_refused_before_the_reserve() { + let dir = tempfile::tempdir().unwrap(); + let wal = Arc::new(WalManager::open_for_testing(&dir.path().join("test.wal")).unwrap()); + let (dispatcher, _data_sides) = Dispatcher::new(1, 64); + let state = SharedState::new(dispatcher, wal).unwrap(); + + let reply = RegistryReserveRead::new(state) + .on_reserve_read(ReserveReadRequest { + lock_key_bytes: Vec::new(), + vshard: 0, + owner_bytes: None, + deadline_remaining_ms: 0, + trace_id: [0u8; 16], + }) + .await; + let refusal = reply.error.expect("a spent budget is refused"); + assert!( + matches!( + crate::Error::from(refusal), + crate::Error::DataPlane(ErrorCode::ExpiredBeforeExecution) + ), + "a spent budget must come back as ExpiredBeforeExecution" + ); + assert!(reply.owner_bytes.is_none()); + } +} diff --git a/nodedb/src/control/server/shared/ddl/neutral/collection/purge/dispatch.rs b/nodedb/src/control/server/shared/ddl/neutral/collection/purge/dispatch.rs index e07ee1859..e95990d1c 100644 --- a/nodedb/src/control/server/shared/ddl/neutral/collection/purge/dispatch.rs +++ b/nodedb/src/control/server/shared/ddl/neutral/collection/purge/dispatch.rs @@ -102,8 +102,8 @@ pub async fn dispatch_unregister_collection( } } - // Each core's acknowledgement counts as apply progress as it arrives, so - // a proposer waiting on this purge sees a slow reclaim move. + // Every core is awaited, so the first error never cuts a core's reclaim + // short. let mut pending: FuturesUnordered<_> = receivers .into_iter() .map(|(_request_id, core_id, mut receiver)| async move { @@ -134,15 +134,8 @@ pub async fn dispatch_unregister_collection( let mut first_error = None; while let Some(response) = pending.next().await { - match response { - Ok(()) => { - state - .metadata_apply_progress - .fetch_add(1, std::sync::atomic::Ordering::Release); - } - Err(error) => { - first_error.get_or_insert(error); - } + if let Err(error) = response { + first_error.get_or_insert(error); } } first_error.map_or(Ok(()), Err) diff --git a/nodedb/src/control/server/shared/ddl/neutral/match_ops.rs b/nodedb/src/control/server/shared/ddl/neutral/match_ops.rs index ead880675..84fea4c2a 100644 --- a/nodedb/src/control/server/shared/ddl/neutral/match_ops.rs +++ b/nodedb/src/control/server/shared/ddl/neutral/match_ops.rs @@ -200,7 +200,9 @@ pub async fn match_query( // LOCAL scatter/resume leg so this node's cores merge the transaction's // staged edge overlay (read-your-own-writes); remote legs read committed CSR // (multi-node overlay forwarding is a separate unit). - let deadline_ms = crate::control::gateway::dispatcher::statement_deadline_ms(state); + let deadline_ms = + crate::control::server::shared::session::StatementDeadline::of_statement(state) + .hop_budget_ms(); match graph_dispatch::scatter_match( state, tenant_id, diff --git a/nodedb/src/control/server/shared/retry.rs b/nodedb/src/control/server/shared/retry.rs index 9897ce4a0..afe191e0d 100644 --- a/nodedb/src/control/server/shared/retry.rs +++ b/nodedb/src/control/server/shared/retry.rs @@ -134,6 +134,8 @@ impl RetryableSchemaChange for Error { | Error::GroupMarksUnavailable { .. } | Error::BackupCaptureMoved { .. } | Error::MetadataLeaderUnavailable + | Error::MetadataGroupGone { .. } + | Error::MetadataDdlSuperseded { .. } | Error::AuthorizationStateBehind { .. } | Error::LinearizableReadRefused { .. } | Error::ExecutionLimitExceeded { .. } diff --git a/nodedb/src/control/server/shared/session/commit/read_validation.rs b/nodedb/src/control/server/shared/session/commit/read_validation.rs index 6397554f7..de0b3e946 100644 --- a/nodedb/src/control/server/shared/session/commit/read_validation.rs +++ b/nodedb/src/control/server/shared/session/commit/read_validation.rs @@ -35,14 +35,13 @@ use std::collections::BTreeMap; use nodedb_physical::physical_plan::{HomeAnswer, HomeVersion, HomeVersionProbe, MetaOp}; use crate::bridge::envelope::PhysicalPlan; -use crate::control::gateway::dispatcher::{ - DispatchRouteParams, dispatch_route, statement_deadline_ms, -}; +use crate::control::gateway::dispatcher::{DispatchRouteParams, dispatch_route}; use crate::control::gateway::live_leaders::resolve_live_decision; use crate::control::gateway::version_set::GatewayVersionSet; use crate::control::gateway::{RouteDecision, TaskRoute}; use crate::control::server::exchange::all_cores::execute_plan_all_local_cores; use crate::control::server::graph_dispatch::cluster_resolve::gateway_shared; +use crate::control::server::shared::session::StatementDeadline; use crate::control::state::SharedState; use crate::types::{DatabaseId, TenantId, TraceId, VShardId}; use nodedb_types::WriteVersion; @@ -294,7 +293,8 @@ async fn current_versions( tenant_id, database_id, trace_id: TraceId::ZERO, - deadline_ms: statement_deadline_ms(shared), + deadline_ms: StatementDeadline::of_statement(shared).hop_budget_ms(), + attempts_left: 1, version_set: &GatewayVersionSet::from_pairs(Vec::new()), txn_id: None, // The leader proves its leadership itself: it answers each diff --git a/nodedb/src/control/server/shared/session/commit/single_shard.rs b/nodedb/src/control/server/shared/session/commit/single_shard.rs index e61fe5d36..461652d33 100644 --- a/nodedb/src/control/server/shared/session/commit/single_shard.rs +++ b/nodedb/src/control/server/shared/session/commit/single_shard.rs @@ -207,7 +207,8 @@ async fn commit_redo( // The proposal forwards to the group leader and returns once the redo is // committed and applied on this node by the apply loop. A redo past the // entry limit travels as chunks, then its final entry. - let deadline = crate::control::wal_replication::statement_propose_deadline(state); + let deadline = crate::control::server::shared::session::StatementDeadline::of_statement(state) + .tokio_instant(); match propose_transaction_redo(state, target, payload, deadline).await { Ok((applied, _)) => Ok(super::ts_rejections::applied_rejected_by_collection( &applied, diff --git a/nodedb/src/control/server/shared/session/ddl_compensate.rs b/nodedb/src/control/server/shared/session/ddl_compensate.rs index e698bc197..b6b1b0cc8 100644 --- a/nodedb/src/control/server/shared/session/ddl_compensate.rs +++ b/nodedb/src/control/server/shared/session/ddl_compensate.rs @@ -74,10 +74,7 @@ async fn propose_reversals( }; let log_index = propose_and_await(state, handle, &prepared).await?; if state.metadata_ddl.applied_token.load(Ordering::Acquire) != token { - return Err(crate::Error::Config { - detail: "commit compensation: DDL preparation ownership was superseded before apply" - .into(), - }); + return Err(crate::Error::MetadataDdlSuperseded { log_index }); } for entry in &stamped { verify_applied(entry, catalog)?; diff --git a/nodedb/src/control/server/shared/session/ddl_flush.rs b/nodedb/src/control/server/shared/session/ddl_flush.rs index 296a862cf..dcba93250 100644 --- a/nodedb/src/control/server/shared/session/ddl_flush.rs +++ b/nodedb/src/control/server/shared/session/ddl_flush.rs @@ -13,7 +13,7 @@ use std::sync::Arc; -use nodedb_cluster::{METADATA_GROUP_ID, MetadataEntry, PendingDdlObject, encode_entry}; +use nodedb_cluster::{MetadataEntry, PendingDdlObject, encode_entry}; use crate::control::catalog_entry::{self, CatalogEntry}; use crate::control::metadata_proposer::{DdlPrepareLease, MetadataRaftHandle}; @@ -123,7 +123,7 @@ impl PendingDdlHandle<'_> { } /// Propose `entry` to the metadata group and wait until this node's applied -/// watermark reaches its log index. +/// watermark reaches its log index, until the statement deadline. pub(super) async fn propose_and_await( state: &SharedState, handle: &dyn MetadataRaftHandle, @@ -133,22 +133,7 @@ pub(super) async fn propose_and_await( detail: format!("metadata entry encode: {e}"), })?; let log_index = handle.propose_async(raw).await?; - let watcher = state.applied_index_watcher(METADATA_GROUP_ID); - let outcome = crate::control::metadata_proposer::wait::wait_applied( - Arc::clone(&watcher), - log_index, - crate::control::metadata_proposer::DEFAULT_PROPOSE_TIMEOUT, - ) - .await?; - if !outcome.is_reached() { - return Err(crate::Error::Internal { - detail: format!( - "metadata propose timed out waiting for log index {log_index} (current: {})", - watcher.current() - ), - }); - } - Ok(log_index) + crate::control::metadata_proposer::wait::await_metadata_apply(state, log_index).await } /// The committed prior value for `entry`, encoded the same shape its @@ -307,22 +292,11 @@ async fn reserve_buffered( // The propose reserves only while `token` owns the lease. A lease the // metadata leader reclaimed meanwhile left no record. if !state.pending_ddl.contains(token) { - return Err(superseded(token, "reserved")); + return Err(crate::Error::MetadataDdlSuperseded { log_index }); } Ok((log_index, objects)) } -/// The error for a pending DDL whose preparation lease the metadata leader -/// reclaimed before its `step` applied. -fn superseded(token: u64, step: &str) -> crate::Error { - crate::Error::Config { - detail: format!( - "metadata DDL preparation ownership of token {token} was reclaimed before the \ - buffered DDL was {step}; nothing was applied" - ), - } -} - /// Commit the objects `handle` reserved: propose `DdlPendingFinalize`, wait /// for local apply, then release the preparation lease `handle` has held /// since [`begin_commit`] constructed it via `propose_pending_buffered`. @@ -378,7 +352,7 @@ async fn finalize_reserved( .load(std::sync::atomic::Ordering::Acquire) != token { - return Err(superseded(token, "finalized")); + return Err(crate::Error::MetadataDdlSuperseded { log_index }); } Ok((log_index, bears_authorization)) } diff --git a/nodedb/src/control/server/shared/session/deadline.rs b/nodedb/src/control/server/shared/session/deadline.rs index cbe6447db..eedd1d3d9 100644 --- a/nodedb/src/control/server/shared/session/deadline.rs +++ b/nodedb/src/control/server/shared/session/deadline.rs @@ -89,15 +89,79 @@ pub fn statement_deadline(default_deadline_secs: u64) -> Instant { } /// Milliseconds left on the current statement, for the remote-hop -/// `ExecuteRequest.deadline_remaining_ms` field. -/// -/// Saturates at 1: a hop dispatched with `0` is refused up front by the -/// receiver's deadline check, which will report a deadline the local half has -/// not actually reached yet. +/// `ExecuteRequest.deadline_remaining_ms` field. See [`hop_budget_ms`]. pub fn statement_deadline_ms(default_deadline_secs: u64) -> u64 { - let remaining = - statement_deadline(default_deadline_secs).saturating_duration_since(Instant::now()); - (remaining.as_millis() as u64).max(1) + hop_budget_ms(statement_deadline(default_deadline_secs)) +} + +/// The budget a remote hop carries for `deadline`: the milliseconds left now, +/// at least 1. +/// +/// A receiver refuses a hop sent with `0` before it checks anything else. It +/// reports a deadline the local half has not reached yet. +pub fn hop_budget_ms(deadline: Instant) -> u64 { + let remaining = deadline.saturating_duration_since(Instant::now()); + u64::try_from(remaining.as_millis()) + .unwrap_or(u64::MAX) + .max(1) +} + +/// The deadline of the statement running on this task, as a value. +/// +/// The one place a caller with `SharedState` reads the statement deadline: +/// the installed deadline, or now plus `tuning.network.default_deadline_secs` +/// outside a statement. Waits, retry loops and proposals that share the +/// statement's budget all take it from here. +#[derive(Debug, Clone, Copy, PartialEq, Eq, PartialOrd, Ord)] +pub struct StatementDeadline(Instant); + +impl StatementDeadline { + /// The running statement's deadline, or the node default outside one. + pub fn of_statement(state: &crate::control::state::SharedState) -> Self { + Self::with_default(state.tuning.network.default_deadline_secs) + } + + /// The running statement's deadline, or `default_deadline_secs` from now + /// outside one. + pub fn with_default(default_deadline_secs: u64) -> Self { + Self(statement_deadline(default_deadline_secs)) + } + + /// A deadline at `instant`. + pub fn at(instant: Instant) -> Self { + Self(instant) + } + + /// A deadline `budget` from now. + pub fn after(budget: Duration) -> Self { + Self(Instant::now() + budget) + } + + /// The deadline as a std instant. + pub fn instant(self) -> Instant { + self.0 + } + + /// The deadline as a Tokio instant, for `tokio::time` waits. + pub fn tokio_instant(self) -> tokio::time::Instant { + tokio::time::Instant::from_std(self.0) + } + + /// The milliseconds a remote hop carries for this deadline. See + /// [`hop_budget_ms`]. + pub fn hop_budget_ms(self) -> u64 { + hop_budget_ms(self.0) + } +} + +/// Install a fresh node-default deadline for cleanup work, such as an overlay +/// drop. +/// +/// Cleanup often runs after its statement ran out of time. Under that dead +/// deadline every request it sends is refused before it runs. Dropping the +/// guard restores the statement's deadline. +pub fn enter_cleanup(default_deadline_secs: u64) -> StatementScope { + enter(None, default_deadline_secs) } fn store(deadline: Option) { @@ -179,6 +243,33 @@ mod tests { .await; } + #[test] + fn a_spent_deadline_still_sends_a_one_millisecond_hop() { + assert_eq!(hop_budget_ms(Instant::now()), 1); + let live = hop_budget_ms(Instant::now() + Duration::from_secs(10)); + assert!(live > 9_000 && live <= 10_000, "budget {live} ms"); + } + + /// Cleanup after a timed-out statement runs on the node default, and the + /// statement's deadline returns once cleanup ends. + #[tokio::test] + async fn cleanup_gets_a_fresh_deadline_after_its_statement_timed_out() { + conn_scope::scoped(async { + let _statement = enter(Some(Duration::from_millis(1)), 30); + let dead = current().expect("the statement installs a deadline"); + tokio::time::sleep(Duration::from_millis(5)).await; + assert!(dead <= Instant::now()); + { + let _cleanup = enter_cleanup(30); + let fresh = statement_deadline(30); + assert!(fresh >= Instant::now() + Duration::from_secs(29)); + assert!(statement_deadline_ms(30) > 29_000); + } + assert_eq!(current(), Some(dead)); + }) + .await; + } + #[tokio::test] async fn outside_a_connection_scope_the_node_default_applies() { assert!(current().is_none()); diff --git a/nodedb/src/control/server/shared/session/leader_forward.rs b/nodedb/src/control/server/shared/session/leader_forward.rs index 843a3c180..848e7f30a 100644 --- a/nodedb/src/control/server/shared/session/leader_forward.rs +++ b/nodedb/src/control/server/shared/session/leader_forward.rs @@ -34,10 +34,9 @@ //! on a non-leader. use crate::bridge::envelope::{Payload, PhysicalPlan, Response}; -use crate::control::gateway::dispatcher::{ - DispatchRouteParams, dispatch_route, statement_deadline_ms, -}; +use crate::control::gateway::dispatcher::{DispatchRouteParams, dispatch_route}; use crate::control::gateway::live_leaders::resolve_live_decision; +use crate::control::gateway::retry::Attempt; use crate::control::gateway::{GatewayVersionSet, RouteDecision, TaskRoute}; use crate::control::server::graph_dispatch::cluster_resolve::gateway_shared; use crate::control::server::shared::write_admission::bare_ok_response; @@ -68,11 +67,16 @@ pub(crate) fn resolve_leader(task: &PhysicalTask, state: &SharedState) -> RouteD /// write, so the forwarded stage carries the same descriptor versions a normal /// remote write does and is not spuriously rejected; a `DropTxnOverlay` touches /// no user collection, so its version set is empty. +/// +/// `attempt` carries the deadline and the attempts left. A caller inside a +/// retry loop passes the loop's attempt. A one-shot caller passes +/// [`Attempt::sole`]. pub(crate) async fn forward_to_leader( state: &SharedState, decision: RouteDecision, forward_task: PhysicalTask, version_plan: &PhysicalPlan, + attempt: Attempt, ) -> crate::Result { // The remote dispatch primitive needs an owned `Arc`; upgrade // the gateway's back-reference (loud typed error if the gateway is not wired @@ -99,7 +103,8 @@ pub(crate) async fn forward_to_leader( tenant_id: forward_task.tenant_id, database_id: forward_task.database_id, trace_id: TraceId::ZERO, - deadline_ms: statement_deadline_ms(&shared), + deadline_ms: attempt.remaining_ms(), + attempts_left: attempt.attempts_left(), version_set: &version_set, // The leader's session-less staging handler reads THIS to key the // per-transaction overlay; forwarding it is the whole point. diff --git a/nodedb/src/control/server/shared/session/mod.rs b/nodedb/src/control/server/shared/session/mod.rs index 45b8465a4..9881f09ab 100644 --- a/nodedb/src/control/server/shared/session/mod.rs +++ b/nodedb/src/control/server/shared/session/mod.rs @@ -54,7 +54,9 @@ pub use self::connection::{ ConnectionId, ConnectionIdError, ConnectionMetadata, ConnectionRegistrationError, SessionId, }; pub use self::cross_shard_mode::{CrossShardTxnMode, parse_value as parse_cross_shard_value}; -pub use self::deadline::{statement_budget, statement_deadline, statement_deadline_ms}; +pub use self::deadline::{ + StatementDeadline, statement_budget, statement_deadline, statement_deadline_ms, +}; pub use self::outcome::{AbortReason, CommitOutcome, TxnDataPlane}; pub use self::params::{parse_set_command, parse_show_command}; pub use self::read_set::{ diff --git a/nodedb/src/control/server/shared/session/overlay_drop.rs b/nodedb/src/control/server/shared/session/overlay_drop.rs index b65914c27..0b32ae45e 100644 --- a/nodedb/src/control/server/shared/session/overlay_drop.rs +++ b/nodedb/src/control/server/shared/session/overlay_drop.rs @@ -10,11 +10,15 @@ use crate::control::gateway::RouteDecision; use crate::control::gateway::retry::retry_not_leader; use crate::control::state::SharedState; +use super::deadline::{self, StatementDeadline}; use super::leader_forward::{forward_to_leader, resolve_leader}; use super::outcome::TxnDataPlane; use crate::control::server::graph_dispatch::cluster_resolve::gateway_shared; /// Release a transaction's staging overlay on the vShard's current leader. +/// +/// The drop runs on its own node-default deadline. Its statement can have +/// timed out, and that dead deadline refuses every send before it runs. pub(super) async fn drop_txn_overlay( state: &SharedState, dp: &impl TxnDataPlane, @@ -22,6 +26,7 @@ pub(super) async fn drop_txn_overlay( vshard_id: crate::types::VShardId, txn_id: crate::types::TxnId, ) -> crate::Result<()> { + let _cleanup = deadline::enter_cleanup(state.tuning.network.default_deadline_secs); let drop_plan = PhysicalPlan::Meta(MetaOp::DropTxnOverlay { txn_id }); let task = PhysicalTask { tenant_id, @@ -39,13 +44,15 @@ pub(super) async fn drop_txn_overlay( _ => { let shared = gateway_shared(state)?; let routing_ref = shared.cluster_routing.as_deref(); - retry_not_leader(routing_ref, |_attempt| { + // Reads the cleanup deadline installed above. + let cleanup_deadline = StatementDeadline::of_statement(&shared).tokio_instant(); + retry_not_leader(routing_ref, cleanup_deadline, |attempt| { let task = task.clone(); let drop_plan = drop_plan.clone(); async move { match resolve_leader(&task, state) { RouteDecision::Local => dp.dispatch_no_wal(task).await, - remote => forward_to_leader(state, remote, task, &drop_plan).await, + remote => forward_to_leader(state, remote, task, &drop_plan, attempt).await, } } }) @@ -54,3 +61,91 @@ pub(super) async fn drop_txn_overlay( } } } + +#[cfg(test)] +mod tests { + use std::future::Future; + use std::pin::Pin; + use std::sync::Mutex; + use std::time::{Duration, Instant}; + + use super::*; + use crate::bridge::dispatch::Dispatcher; + use crate::bridge::envelope::{Payload, Response, Status}; + use crate::control::server::shared::session::conn_scope; + use crate::types::{Lsn, RequestId, TenantId, TxnId, VShardId}; + use crate::wal::WalManager; + + /// Records the statement deadline each dispatch runs under. + #[derive(Default)] + struct DeadlineDp { + seen: Mutex>>, + } + + impl TxnDataPlane for DeadlineDp { + fn dispatch_no_wal<'a>( + &'a self, + _task: PhysicalTask, + ) -> Pin> + Send + 'a>> { + self.seen.lock().unwrap().push(deadline::current()); + Box::pin(async move { + Ok(Response { + request_id: RequestId::new(1), + status: Status::Ok, + attempt: 1, + partial: false, + payload: Payload::empty(), + watermark_lsn: Lsn::ZERO, + error_code: None, + stage_vote: None, + read_versions: crate::types::ReadVersions::new(), + write_set: Vec::new(), + }) + }) + } + + fn event_source(&self) -> crate::event::EventSource { + crate::event::EventSource::User + } + } + + /// A drop after its statement timed out runs on a fresh node-default + /// deadline, and the statement's deadline returns afterwards. + #[tokio::test] + async fn an_overlay_drop_outlives_its_timed_out_statement() { + let dir = tempfile::tempdir().unwrap(); + let wal = std::sync::Arc::new( + WalManager::open_for_testing(&dir.path().join("test.wal")).unwrap(), + ); + let (dispatcher, _data_sides) = Dispatcher::new(1, 64); + let state = SharedState::new(dispatcher, wal).unwrap(); + let default_secs = state.tuning.network.default_deadline_secs; + let dp = DeadlineDp::default(); + + conn_scope::scoped(async { + let _statement = deadline::enter(Some(Duration::from_millis(1)), default_secs); + let dead = deadline::current().expect("the statement installs a deadline"); + tokio::time::sleep(Duration::from_millis(5)).await; + + drop_txn_overlay( + &state, + &dp, + TenantId::new(1), + VShardId::new(3), + TxnId::new(7), + ) + .await + .unwrap(); + + let seen = dp.seen.lock().unwrap(); + assert_eq!(seen.len(), 1); + let cleanup = seen[0].expect("the drop runs under a deadline"); + assert!( + cleanup > Instant::now() + Duration::from_secs(default_secs.saturating_sub(1)), + "the drop ran under the statement's dead deadline" + ); + assert_eq!(deadline::current(), Some(dead)); + }) + .await; + } +} diff --git a/nodedb/src/control/server/shared/session/staging_gate.rs b/nodedb/src/control/server/shared/session/staging_gate.rs index 301f01784..3758aa460 100644 --- a/nodedb/src/control/server/shared/session/staging_gate.rs +++ b/nodedb/src/control/server/shared/session/staging_gate.rs @@ -25,9 +25,11 @@ use std::future::Future; use crate::bridge::envelope::{ErrorCode, PhysicalPlan, Response, Status}; use crate::control::gateway::RouteDecision; +use crate::control::gateway::retry::Attempt; use crate::control::security::request_scope::RequestAuthScope; use crate::control::server::shared::metering::{PlanMeteringInfo, meter_staged_write}; use crate::control::server::shared::quota_admission::admit_quota_for_dispatch; +use crate::control::server::shared::session::StatementDeadline; use crate::control::server::shared::sql::staging_predicates::{ is_stageable_write, require_affected_count, stageable_write_shape, stages_returning, }; @@ -371,9 +373,15 @@ where RouteDecision::Local => dispatch(stage_task) .await .map_err(StagingGateError::Dispatch)?, - remote => forward_to_leader(state, remote, stage_task, &task.plan) - .await - .map_err(StagingGateError::Dispatch)?, + remote => forward_to_leader( + state, + remote, + stage_task, + &task.plan, + Attempt::sole(StatementDeadline::of_statement(state).tokio_instant()), + ) + .await + .map_err(StagingGateError::Dispatch)?, }; if resp.status == Status::Error { diff --git a/nodedb/src/control/server/shared/write_admission/admission_keys.rs b/nodedb/src/control/server/shared/write_admission/admission_keys.rs index 469be6e1f..d48c7e299 100644 --- a/nodedb/src/control/server/shared/write_admission/admission_keys.rs +++ b/nodedb/src/control/server/shared/write_admission/admission_keys.rs @@ -20,9 +20,8 @@ //! alike on every replica. //! - Every Calvin writer of a columnar or timeseries collection locks it //! `Exclusive`. So an append still orders against each Calvin writer. -//! - A contended append waits and never routes. A routed write applies -//! outside the data-group log. Each replica then installs it against its -//! own schema at its own point. +//! - A contended append waits and never routes. The data-group log already +//! orders appends, so routing one to Calvin gains nothing. //! //! A write no Calvin transaction sequences still takes keys: //! @@ -354,6 +353,27 @@ mod tests { } } + /// A held TRUNCATE locks its collection `Exclusive`. A fast-path point + /// write announces itself `Intent` on that collection, so it cannot take + /// its keys while the TRUNCATE holds. + #[test] + fn a_held_truncate_fences_a_point_write() { + let mut table = LockManager::new(); + let truncate = PhysicalPlan::Document(DocumentOp::Truncate { + collection: qualified("docs"), + restart_identity: false, + resolved_sum_targets: Vec::new(), + declared_primary_key: None, + }); + let (truncate_keys, _) = keys_of(&truncate); + assert!(table.try_acquire(TxnId::new(1, 0), truncate_keys)); + let (point_keys, _) = keys_of(&doc_insert("x", 17)); + assert!( + !table.try_acquire(TxnId::new(TxnId::AUTOCOMMIT_EPOCH, 0), point_keys), + "the point write must wait for the truncate" + ); + } + /// A resolved write no Calvin transaction sequences still locks the rows /// it names, and waits rather than routes when contended. #[test] diff --git a/nodedb/src/control/server/shared/write_admission/leader_gate.rs b/nodedb/src/control/server/shared/write_admission/leader_gate.rs index 27a8c5c3a..a90dd4ea3 100644 --- a/nodedb/src/control/server/shared/write_admission/leader_gate.rs +++ b/nodedb/src/control/server/shared/write_admission/leader_gate.rs @@ -280,8 +280,6 @@ fn entry_scope(write: &ReplicatedWrite, database_id: crate::types::DatabaseId) - | ReplicatedWrite::ColumnarTruncate { .. } | ReplicatedWrite::TimeseriesTruncate { .. } | ReplicatedWrite::KvResolvedWrite { .. } - | ReplicatedWrite::KvPredicateUpdate { .. } - | ReplicatedWrite::KvPredicateDelete { .. } | ReplicatedWrite::DocumentResolvedWrite { .. } | ReplicatedWrite::VectorDirectDelete { .. } | ReplicatedWrite::VectorDirectTruncate { .. } diff --git a/nodedb/src/control/server/shared/write_admission/predicate/txn_buffering/classify.rs b/nodedb/src/control/server/shared/write_admission/predicate/txn_buffering/classify.rs index 2a878ca9e..0476aefbf 100644 --- a/nodedb/src/control/server/shared/write_admission/predicate/txn_buffering/classify.rs +++ b/nodedb/src/control/server/shared/write_admission/predicate/txn_buffering/classify.rs @@ -466,11 +466,42 @@ mod tests { ) } + /// A write whose rows a predicate names. It resolves to rows before it is + /// logged, so the log encoder refuses its raw form. + fn resolves_before_log(plan: &PhysicalPlan) -> bool { + matches!( + plan, + PhysicalPlan::Kv(KvOp::PredicateUpdate { .. } | KvOp::PredicateDelete { .. }) + | PhysicalPlan::Vector( + VectorOp::DirectDelete { + targets: nodedb_physical::physical_plan::VectorWriteTargets::Predicate(_), + .. + } | VectorOp::DirectUpdate { + targets: nodedb_physical::physical_plan::VectorWriteTargets::Predicate(_), + .. + } + ) + ) + } + + /// The classifier agrees with the encoder. A predicate write is refused + /// at the log boundary with a typed error and still buffers in a + /// transaction, where staging resolves it. fn assert_matches_oracle(plan: &PhysicalPlan) { - let expected = to_replicated_entry(tenant(), db(), vshard(), plan) + let encoded = to_replicated_entry(tenant(), db(), vshard(), plan); + let actual = plan_requires_txn_buffering(plan); + if resolves_before_log(plan) { + assert!( + matches!(encoded, Err(crate::Error::Internal { .. })), + "expected {plan:?} to be refused at the log boundary, got {:?}", + encoded.as_ref().err() + ); + assert!(actual, "expected {plan:?} to require txn buffering"); + return; + } + let expected = encoded .expect("encode must not error for this fixture") .is_some(); - let actual = plan_requires_txn_buffering(plan); assert_eq!( actual, expected, "plan_requires_txn_buffering disagrees with to_replicated_entry for {plan:?}" @@ -2001,7 +2032,7 @@ mod tests { attr_projection: Vec::new(), limit: 0, cell_filter: None, - hilbert_range: None, + shard_bucket: None, system_time: SystemTimeScope::Current, valid_at_ms: None, }), @@ -2016,7 +2047,7 @@ mod tests { group_by_dim: -1, cell_filter: None, return_partial: false, - hilbert_range: None, + shard_bucket: None, system_as_of: None, valid_at_ms: None, }), @@ -2063,7 +2094,7 @@ mod tests { slice_msgpack: Vec::new(), attr_projection: Vec::new(), limit: 0, - slice_hilbert_ranges: Vec::new(), + slice_tile_keys: Vec::new(), prefix_bits: 0, system_time: SystemTimeScope::Current, valid_at_ms: None, @@ -2073,7 +2104,7 @@ mod tests { attr_idx: 0, reducer_msgpack: Vec::new(), group_by_dim: -1, - slice_hilbert_ranges: Vec::new(), + slice_tile_keys: Vec::new(), prefix_bits: 0, system_as_of: None, valid_at_ms: None, diff --git a/nodedb/src/control/server/surrogate_exchange/authority.rs b/nodedb/src/control/server/surrogate_exchange/authority.rs index fc0420d10..d162eddbe 100644 --- a/nodedb/src/control/server/surrogate_exchange/authority.rs +++ b/nodedb/src/control/server/surrogate_exchange/authority.rs @@ -78,7 +78,9 @@ pub(crate) async fn assign_many_at_home( home.as_u32(), ReplicatedWrite::SurrogateBind { identities }, ); - let deadline = crate::control::wal_replication::statement_propose_deadline(state); + let deadline = + crate::control::server::shared::session::StatementDeadline::of_statement(state) + .tokio_instant(); propose_replicated_entry(state, proposer, entry, deadline).await?; } pks.iter() diff --git a/nodedb/src/control/server/sync/async_dispatch/delta/compensation.rs b/nodedb/src/control/server/sync/async_dispatch/delta/compensation.rs index 6029411a7..4ac641779 100644 --- a/nodedb/src/control/server/sync/async_dispatch/delta/compensation.rs +++ b/nodedb/src/control/server/sync/async_dispatch/delta/compensation.rs @@ -54,6 +54,7 @@ pub(super) fn compensation_hint_for_dispatch_error(e: &crate::Error) -> Compensa | crate::Error::GroupMarksUnavailable { .. } | crate::Error::BackupCaptureMoved { .. } | crate::Error::MetadataLeaderUnavailable + | crate::Error::MetadataDdlSuperseded { .. } | crate::Error::AuthorizationStateBehind { .. } | crate::Error::LinearizableReadRefused { .. } | crate::Error::DispatchCapacity { .. } @@ -69,7 +70,8 @@ pub(super) fn compensation_hint_for_dispatch_error(e: &crate::Error) -> Compensa }, // The write committed or was overwritten, and nothing here says which. // Never re-pushed: a committed write will apply twice. - other @ crate::Error::ProposalOutcomeUnknown { .. } => CompensationHint::Custom { + other @ (crate::Error::ProposalOutcomeUnknown { .. } + | crate::Error::MetadataGroupGone { .. }) => CompensationHint::Custom { constraint: "proposal_outcome_unknown".into(), detail: other.to_string(), }, diff --git a/nodedb/src/control/server/sync/refusal.rs b/nodedb/src/control/server/sync/refusal.rs index e97ccf6d8..593148db5 100644 --- a/nodedb/src/control/server/sync/refusal.rs +++ b/nodedb/src/control/server/sync/refusal.rs @@ -146,6 +146,8 @@ pub(super) fn retryable_refusal_reason(error: &crate::Error) -> Option<&str> { | crate::Error::GroupMarksUnavailable { .. } | crate::Error::BackupCaptureMoved { .. } | crate::Error::MetadataLeaderUnavailable + | crate::Error::MetadataGroupGone { .. } + | crate::Error::MetadataDdlSuperseded { .. } | crate::Error::AuthorizationStateBehind { .. } | crate::Error::LinearizableReadRefused { .. } | crate::Error::ExecutionLimitExceeded { .. } @@ -234,6 +236,7 @@ fn is_indeterminate(error: &crate::Error) -> bool { | crate::Error::GroupMarksUnavailable { .. } | crate::Error::BackupCaptureMoved { .. } | crate::Error::MetadataLeaderUnavailable + | crate::Error::MetadataDdlSuperseded { .. } | crate::Error::AuthorizationStateBehind { .. } | crate::Error::LinearizableReadRefused { .. } // Shed by a resource or rate gate: the class `53`. @@ -248,7 +251,8 @@ fn is_indeterminate(error: &crate::Error) -> bool { }, // The write committed: pushing the same bytes again applies it twice. crate::Error::CommittedResultUnavailable { .. } - | crate::Error::ProposalOutcomeUnknown { .. } => false, + | crate::Error::ProposalOutcomeUnknown { .. } + | crate::Error::MetadataGroupGone { .. } => false, // Refused on its merits, or a fault the same bytes reproduce. crate::Error::RejectedConstraint { .. } | crate::Error::TxnOverlayMemoryExceeded { .. } diff --git a/nodedb/src/control/state/fields.rs b/nodedb/src/control/state/fields.rs index 29891b91a..8e93257ec 100644 --- a/nodedb/src/control/state/fields.rs +++ b/nodedb/src/control/state/fields.rs @@ -119,10 +119,6 @@ pub struct SharedState { pub group_watchers: Arc, /// This node's state of the replicated descriptor preparation lease. pub metadata_ddl: super::metadata_ddl::MetadataDdlState, - /// Apply-progress steps this node made on metadata entries: each batch - /// sub-entry and each purge-reclaim step (a core acknowledgement, a closed - /// scan). A proposer on a long apply reads it to tell slow from stalled. - pub metadata_apply_progress: AtomicU64, /// Node-local table of in-flight `DdlPendingPropose` records. Persisted in /// `SystemCatalog` and seeded at boot. pub pending_ddl: crate::control::pending_ddl::PendingDdlTable, diff --git a/nodedb/src/control/state/init.rs b/nodedb/src/control/state/init.rs index f1fbf5f35..6b20f3f09 100644 --- a/nodedb/src/control/state/init.rs +++ b/nodedb/src/control/state/init.rs @@ -139,7 +139,6 @@ impl SharedState { .0, group_watchers: Arc::new(nodedb_cluster::GroupAppliedWatchers::new()), metadata_ddl: super::MetadataDdlState::new(), - metadata_apply_progress: std::sync::atomic::AtomicU64::new(0), pending_ddl: crate::control::pending_ddl::PendingDdlTable::new(), metadata_apply_wedge: std::sync::Arc::default(), sequencer_halt: std::sync::Arc::default(), diff --git a/nodedb/src/control/state/init_prod/open.rs b/nodedb/src/control/state/init_prod/open.rs index 256000b9c..d418778b0 100644 --- a/nodedb/src/control/state/init_prod/open.rs +++ b/nodedb/src/control/state/init_prod/open.rs @@ -210,7 +210,6 @@ impl SharedState { .0, group_watchers: Arc::new(nodedb_cluster::GroupAppliedWatchers::new()), metadata_ddl: crate::control::state::MetadataDdlState::new(), - metadata_apply_progress: std::sync::atomic::AtomicU64::new(0), pending_ddl: crate::control::pending_ddl::PendingDdlTable::new(), metadata_apply_wedge: std::sync::Arc::default(), sequencer_halt: std::sync::Arc::default(), diff --git a/nodedb/src/control/surrogate/assign/core/flush.rs b/nodedb/src/control/surrogate/assign/core/flush.rs index d4c9d52d6..f0476a7dd 100644 --- a/nodedb/src/control/surrogate/assign/core/flush.rs +++ b/nodedb/src/control/surrogate/assign/core/flush.rs @@ -76,10 +76,10 @@ impl SurrogateHwmPersist for CombinedPersist<'_> { // view of the watermark. // // Awaiting it inline stalls the writer. `propose_surrogate_hwm` - // blocks for `DEFAULT_PROPOSE_TIMEOUT` (5s) waiting for the entry - // to commit. Surrogate assignment runs on the Raft apply loop as - // well as on the coordinator. A flush from the apply path parks - // the very loop that must commit the entry until the timeout fires. + // waits up to the statement deadline for the entry to apply. + // Surrogate assignment runs on the Raft apply loop as well as on the + // coordinator. A flush from the apply path parks the very loop that + // must commit the entry until the deadline passes. // // Out-of-order or duplicate delivery is safe: `apply_surrogate_alloc` // advances the watermark through `restore_hwm`, which is idempotent diff --git a/nodedb/src/control/vshard_admission.rs b/nodedb/src/control/vshard_admission.rs index 6092b2fbd..27750af0a 100644 --- a/nodedb/src/control/vshard_admission.rs +++ b/nodedb/src/control/vshard_admission.rs @@ -296,7 +296,7 @@ pub(crate) async fn await_admitted_applies( state, at.group_id, at.log_index, - deadline, + crate::control::cluster::linearizable_read::StatementDeadline::at(deadline), ) .await?; } diff --git a/nodedb/src/control/wal_replication/mod.rs b/nodedb/src/control/wal_replication/mod.rs index bf0e1cd61..46d9e8d7e 100644 --- a/nodedb/src/control/wal_replication/mod.rs +++ b/nodedb/src/control/wal_replication/mod.rs @@ -19,7 +19,6 @@ pub use decode::{decode_replicated_entry, from_replicated_entry}; pub use encode::to_replicated_entry; pub(crate) use propose::{ propose_replicated_entry, stamp_collection_incarnations, stamp_metadata_floor, - statement_propose_deadline, }; pub use replicable_write::ReplicableWrite; pub use types::{ diff --git a/nodedb/src/control/wal_replication/propose.rs b/nodedb/src/control/wal_replication/propose.rs index 6f85d6503..d7185ab3c 100644 --- a/nodedb/src/control/wal_replication/propose.rs +++ b/nodedb/src/control/wal_replication/propose.rs @@ -19,14 +19,6 @@ const FIRST_BACKOFF: std::time::Duration = std::time::Duration::from_millis(10); /// Longest wait between two re-proposals. const MAX_BACKOFF: std::time::Duration = std::time::Duration::from_millis(200); -/// The deadline a proposal made now carries: the running statement's -/// deadline, or the node default outside a statement. -pub(crate) fn statement_propose_deadline(state: &SharedState) -> tokio::time::Instant { - tokio::time::Instant::from_std(crate::control::server::shared::session::statement_deadline( - state.tuning.network.default_deadline_secs, - )) -} - /// Stamp this node's metadata floor on `entry`: the catalog the write was /// planned against, including the batch being applied now (see /// `AppliedIndexWatcher::floor`). Every replica applies the entry only once diff --git a/nodedb/src/control/write_resolve/propose.rs b/nodedb/src/control/write_resolve/propose.rs index 16e7b697a..79522cd93 100644 --- a/nodedb/src/control/write_resolve/propose.rs +++ b/nodedb/src/control/write_resolve/propose.rs @@ -42,7 +42,8 @@ pub(super) async fn propose_resolved( ), })?; - let deadline = crate::control::wal_replication::statement_propose_deadline(state); + let deadline = crate::control::server::shared::session::StatementDeadline::of_statement(state) + .tokio_instant(); match propose_replicated_entry(state, proposer, entry, deadline).await { Ok((payload, write_versions)) => { let request_id = diff --git a/nodedb/src/data/executor/core_loop/write_index/index.rs b/nodedb/src/data/executor/core_loop/write_index/index.rs index 7fd5d7136..743dfe800 100644 --- a/nodedb/src/data/executor/core_loop/write_index/index.rs +++ b/nodedb/src/data/executor/core_loop/write_index/index.rs @@ -50,6 +50,10 @@ pub struct WriteVersionIndex { /// floor: a predicate reader validates against it). Never GC'd: bounded /// by the live collections of the core's vShards. coll_write: HashMap, + /// Last write that removed every row of a collection, such as a TRUNCATE + /// that records no per-row version. A row read validates against it. + /// Never GC'd: bounded like `coll_write`. + coll_wipe: HashMap, /// Per-secondary-index-dimension write-VALUE versions, the finer-grained /// sibling of `coll_write` an index-range read validates against. pub(in crate::data::executor) index_values: IndexValueVersionIndex, @@ -103,6 +107,28 @@ impl WriteVersionIndex { self.raise_latest(vshard, version); } + /// Record a committed write at `version` on `vshard` that removed every + /// row of `collection`. Advances the collection's version and its wipe + /// version, so every row read older than `version` fails validation. + pub fn note_wipe( + &mut self, + db: DatabaseId, + tenant: TenantId, + vshard: VShardId, + collection: &str, + version: WriteVersion, + ) { + self.note_write(db, tenant, vshard, collection, None, version); + let coll_key = CollKey { + vshard, + db, + tenant, + collection: Box::from(collection), + }; + let slot = self.coll_wipe.entry(coll_key).or_default(); + *slot = (*slot).max(version); + } + /// Raise `vshard`'s latest version to `version` when it is higher. pub(in crate::data::executor) fn raise_latest( &mut self, @@ -120,6 +146,7 @@ impl WriteVersionIndex { pub fn install(&mut self, vshard: VShardId, version: WriteVersion) { self.last_write.retain(|key, _| key.vshard != vshard); self.coll_write.retain(|key, _| key.vshard != vshard); + self.coll_wipe.retain(|key, _| key.vshard != vshard); self.index_values.drop_vshard(vshard); let bounds = self.vshards.entry(vshard).or_default(); bounds.installed = bounds.installed.max(version); @@ -203,6 +230,24 @@ impl WriteVersionIndex { .unwrap_or_else(|| self.bounds(vshard).installed) } + /// The version of the last write that removed every row of + /// `collection` on `vshard`, `ZERO` when none is recorded. + fn collection_wipe( + &self, + db: DatabaseId, + tenant: TenantId, + vshard: VShardId, + collection: &str, + ) -> WriteVersion { + let key = CollKey { + vshard, + db, + tenant, + collection: Box::from(collection), + }; + self.coll_wipe.get(&key).copied().unwrap_or_default() + } + fn bounds(&self, vshard: VShardId) -> VShardBounds { self.vshards.get(&vshard).copied().unwrap_or_default() } @@ -218,6 +263,9 @@ impl WriteVersionIndex { /// A key or value with no recorded version was not written since the /// vShard was installed, or GC evicted it. Its version is then at most /// the installed or evicted bound, and never above the collection's. + /// + /// A write that removed every row of the collection also removed the + /// key or value read, so its version bounds every point and index read. pub(crate) fn read_is_valid( &self, db: DatabaseId, @@ -230,6 +278,7 @@ impl WriteVersionIndex { let collection_current = self.collection_current(db, tenant, vshard, collection); let bounds = self.bounds(vshard); let unrecorded = collection_current.min(bounds.installed.max(bounds.evicted)); + let wipe = self.collection_wipe(db, tenant, vshard, collection); match key { ReadKeyIdent::Point(repr) => { let write_key = WriteKey { @@ -243,6 +292,7 @@ impl WriteVersionIndex { .get(&write_key) .copied() .unwrap_or(unrecorded) + .max(wipe) <= read } ReadKeyIdent::Predicate => collection_current <= read, @@ -256,7 +306,9 @@ impl WriteVersionIndex { }; match self.index_values.eq_version(dim, value) { IndexVersion::Untracked => collection_current <= read, - IndexVersion::Tracked(recorded) => recorded.unwrap_or(unrecorded) <= read, + IndexVersion::Tracked(recorded) => { + recorded.unwrap_or(unrecorded).max(wipe) <= read + } } } ReadKeyIdent::IndexRange { field, lo, hi } => { @@ -275,7 +327,7 @@ impl WriteVersionIndex { // A value GC evicted from the range is unknown, so the // range takes the unrecorded bound as well. IndexVersion::Tracked(recorded) => { - recorded.unwrap_or_default().max(unrecorded) <= read + recorded.unwrap_or_default().max(unrecorded).max(wipe) <= read } } } @@ -288,7 +340,7 @@ impl WriteVersionIndex { /// latest version in the same epoch, then, if more than /// [`MAX_KEY_ENTRIES`] per-key entries remain, drops the lowest versions /// until back within bound. Each vShard's evicted bound rises to the - /// highest version it lost. The per-collection map is left untouched. + /// highest version it lost. The per-collection and wipe maps are left untouched. pub fn gc(&mut self) { let latest: HashMap = self.latest_by_vshard().collect(); let mut evicted: HashMap = HashMap::new(); @@ -417,6 +469,55 @@ mod tests { assert!(!valid(&index, &ReadKeyIdent::Predicate, 10)); } + #[test] + fn a_collection_write_leaves_a_point_read_valid() { + let mut index = WriteVersionIndex::new(); + write(&mut index, Some(KeyRepr::Surrogate(7)), 5); + write(&mut index, None, 20); + assert!(valid(&index, &point(7), 10)); + } + + #[test] + fn a_wipe_after_the_read_invalidates_a_point_read() { + let mut index = WriteVersionIndex::new(); + write(&mut index, Some(KeyRepr::Surrogate(7)), 5); + index.note_wipe(db(), tenant(), vshard(), "orders", at(20)); + assert!(!valid(&index, &point(7), 10), "the wipe removed row 7"); + assert!( + !valid(&index, &point(8), 10), + "the wipe bounds a row with no recorded version" + ); + assert!(!valid(&index, &ReadKeyIdent::Predicate, 10)); + assert!(valid(&index, &point(7), 20)); + assert!(valid(&index, &point(8), 20)); + } + + #[test] + fn a_wipe_invalidates_a_tracked_index_read() { + let mut index = WriteVersionIndex::new(); + record_value(&mut index, "email", "a@b.c", 5); + index.note_wipe(db(), tenant(), vshard(), "orders", at(20)); + let eq = ReadKeyIdent::IndexEq { + field: "email".to_string(), + value: "a@b.c".to_string(), + }; + let range = ReadKeyIdent::IndexRange { + field: "email".to_string(), + lo: Some("a".to_string()), + hi: Some("b".to_string()), + }; + assert!(!valid(&index, &eq, 10)); + assert!(!valid(&index, &range, 10)); + assert!(valid(&index, &eq, 20)); + } + + #[test] + fn a_wipe_on_another_collection_leaves_a_point_read_valid() { + let mut index = WriteVersionIndex::new(); + index.note_wipe(db(), tenant(), vshard(), "items", at(20)); + assert!(valid(&index, &point(7), 10)); + } + #[test] fn a_write_on_another_vshard_never_moves_a_read() { let mut index = WriteVersionIndex::new(); diff --git a/nodedb/src/data/executor/core_loop/write_index/record.rs b/nodedb/src/data/executor/core_loop/write_index/record.rs index 6b54575a7..c1a1c859f 100644 --- a/nodedb/src/data/executor/core_loop/write_index/record.rs +++ b/nodedb/src/data/executor/core_loop/write_index/record.rs @@ -94,6 +94,61 @@ impl CoreLoop { } } + /// Record a committed write that removed every row of `collection`: a + /// TRUNCATE. Every point and index read of the collection older than the + /// write then fails validation. + /// + /// In the install pass of a committed-redo apply the version waits in the + /// apply scope, exactly like [`Self::note_write`]. + pub(in crate::data::executor) fn note_collection_wipe( + &mut self, + db: DatabaseId, + tenant: TenantId, + collection: &str, + stamp: WriteStamp, + ) { + if let Some(scope) = self.redo_apply.scope.as_mut() { + scope.defer_collection_wipe(db, tenant, collection, stamp); + return; + } + self.publish_collection_wipe(db, tenant, collection, stamp); + } + + /// Record a wipe version and advance the core watermark monotonically. + pub(in crate::data::executor) fn publish_collection_wipe( + &mut self, + db: DatabaseId, + tenant: TenantId, + collection: &str, + stamp: WriteStamp, + ) { + self.floors.applied_prefix.note_applied(stamp.lsn); + let version = self.write_version_of(stamp); + self.write_index + .note_wipe(db, tenant, stamp.vshard, collection, version); + if stamp.lsn > self.watermark { + self.watermark = stamp.lsn; + } + } + + /// Record a committed whole-collection removal, if `task` carries a WAL + /// record. See [`Self::note_collection_wipe`]. + pub(in crate::data::executor) fn note_task_collection_wipe( + &mut self, + task: &ExecutionTask, + tid: u64, + collection: &str, + ) { + if let Some(stamp) = self.task_write_stamp(task) { + self.note_collection_wipe( + task.request.database_id, + TenantId::new(tid), + collection, + stamp, + ); + } + } + /// Record a committed write's collection version only (no per-key /// entry), if `task` carries a WAL record. Shared by the columnar-family /// write handlers (columnar / timeseries / array / spatial / FTS) whose @@ -155,11 +210,39 @@ impl CoreLoop { key: Option, record_lsn: u64, ) { - if record_lsn == 0 { - return; + let db = DatabaseId::new(database_id); + if let Some(stamp) = self.replay_stamp(db, collection, record_lsn) { + self.note_write(db, TenantId::new(tenant_id), collection, key, stamp); } + } + + /// Record a whole-collection removal a replay arm applied from the WAL + /// record at `record_lsn`, stamped like [`Self::note_replay_write`]. + pub(in crate::data::executor) fn note_replay_wipe( + &mut self, + database_id: u64, + tenant_id: u64, + collection: &str, + record_lsn: u64, + ) { let db = DatabaseId::new(database_id); - let stamp = match self.write_index.record_stamps.get(record_lsn) { + if let Some(stamp) = self.replay_stamp(db, collection, record_lsn) { + self.note_collection_wipe(db, TenantId::new(tenant_id), collection, stamp); + } + } + + /// The stamp of a write a replay arm applied from the WAL record at + /// `record_lsn`. `None` when `record_lsn == 0` (no durable record). + fn replay_stamp( + &self, + db: DatabaseId, + collection: &str, + record_lsn: u64, + ) -> Option { + if record_lsn == 0 { + return None; + } + Some(match self.write_index.record_stamps.get(record_lsn) { Some(stamp) => stamp, None => WriteStamp { // A name without its database's qualifier is a bare name. @@ -169,8 +252,7 @@ impl CoreLoop { lsn: Lsn::new(record_lsn), entry: None, }, - }; - self.note_write(db, TenantId::new(tenant_id), collection, key, stamp); + }) } /// Run horizon GC on the per-core version index. Invoked from the periodic @@ -411,6 +493,103 @@ mod tests { assert_eq!(core.watermark, Lsn::new(42)); } + #[test] + fn a_kv_truncate_invalidates_a_point_read_of_a_removed_key() { + let (mut core, _, _, _dir) = make_core(); + let resp = core.execute_kv_put( + &wal_task(42), + KvWriteParams { + did: DatabaseId::DEFAULT.as_u64(), + tid: 1, + collection: "kv", + key: b"k1".as_slice(), + value: b"v1".as_slice(), + ttl_ms: 0, + surrogate: Surrogate::new(3), + returning: None, + rls_filters: &[], + }, + ); + assert_eq!(resp.status, Status::Ok); + let read = core.write_index.latest(vshard0()); + let point = + nodedb_types::calvin::ReadKeyIdent::Point(KeyRepr::KvKey(Box::from(b"k1".as_slice()))); + let current = |core: &CoreLoop| { + core.write_index.read_is_valid( + DatabaseId::DEFAULT, + TenantId::new(1), + vshard0(), + "kv", + &point, + read, + ) + }; + assert!(current(&core), "the read saw the put"); + + let resp = core.execute_kv_truncate(&wal_task(50), DatabaseId::DEFAULT.as_u64(), 1, "kv"); + assert_eq!(resp.status, Status::Ok); + assert!( + !current(&core), + "the truncate removed k1 after the read observed it" + ); + assert_eq!(core.watermark, Lsn::new(50)); + } + + #[test] + fn a_document_truncate_invalidates_an_index_read_of_a_removed_row() { + let (mut core, _, _, _dir) = make_core(); + let put = wal_task(42); + let resp = point_put(&mut core, &put, "docs", "d1", 3, "open"); + assert_eq!(resp.status, Status::Ok, "{:?}", resp.error_code); + // The index value the put of an indexed collection records. + let stamp = core.task_write_stamp(&put).expect("a WAL task has a stamp"); + core.note_index_write_values( + DatabaseId::DEFAULT, + TenantId::new(1), + "docs", + &[("a".to_string(), "open".to_string())], + stamp, + ); + let read = core.write_index.latest(vshard0()); + let index_eq = nodedb_types::calvin::ReadKeyIdent::IndexEq { + field: "a".to_string(), + value: "open".to_string(), + }; + let point = nodedb_types::calvin::ReadKeyIdent::Point(KeyRepr::Surrogate(3)); + let current = |core: &CoreLoop, key: &nodedb_types::calvin::ReadKeyIdent| { + core.write_index.read_is_valid( + DatabaseId::DEFAULT, + TenantId::new(1), + vshard0(), + "docs", + key, + read, + ) + }; + assert!(current(&core, &index_eq), "the index read saw the put"); + assert!(current(&core, &point), "the point read saw the put"); + + let resp = core.execute_truncate( + &wal_task(50), + 1, + crate::data::executor::handlers::truncate::TruncateParams { + collection: "docs", + resolved_sum_targets: &[], + declared_primary_key: None, + }, + ); + assert_eq!(resp.status, Status::Ok, "{:?}", resp.error_code); + assert!( + !current(&core, &index_eq), + "the truncate removed the indexed row after the read observed it" + ); + assert!( + !current(&core, &point), + "the truncate removed d1 after the read observed it" + ); + assert_eq!(core.watermark, Lsn::new(50)); + } + #[test] fn an_autocommit_apply_is_named_by_the_next_replay_stamp() { let (mut core, _, _, _dir) = make_core(); diff --git a/nodedb/src/data/executor/core_loop/write_index/validate.rs b/nodedb/src/data/executor/core_loop/write_index/validate.rs index 51991a501..be11d6da6 100644 --- a/nodedb/src/data/executor/core_loop/write_index/validate.rs +++ b/nodedb/src/data/executor/core_loop/write_index/validate.rs @@ -255,7 +255,7 @@ mod tests { attr_projection: Vec::new(), limit: 0, cell_filter: None, - hilbert_range: None, + shard_bucket: None, system_time: nodedb_types::SystemTimeScope::Current, valid_at_ms: None, }, diff --git a/nodedb/src/data/executor/handlers/columnar_truncate.rs b/nodedb/src/data/executor/handlers/columnar_truncate.rs index 51c4bc833..1077aa7d6 100644 --- a/nodedb/src/data/executor/handlers/columnar_truncate.rs +++ b/nodedb/src/data/executor/handlers/columnar_truncate.rs @@ -78,6 +78,8 @@ impl CoreLoop { self.checkpoint_coordinator .mark_dirty("columnar", truncated); + // No wipe: columnar and spatial reads record only `Predicate` keys, + // which the collection version bounds. self.note_collection_write(task, collection); self.invalidate_aggregate_cache_for_collection(db.as_u64(), tid.as_u64(), collection); diff --git a/nodedb/src/data/executor/handlers/control/calvin_txn_id.rs b/nodedb/src/data/executor/handlers/control/calvin_txn_id.rs index a104cb2d8..d0e5c9a0b 100644 --- a/nodedb/src/data/executor/handlers/control/calvin_txn_id.rs +++ b/nodedb/src/data/executor/handlers/control/calvin_txn_id.rs @@ -27,13 +27,16 @@ //! distinct `(epoch, position, vshard)` triples collide on the same //! synthetic `TxnId` and corrupt each other's staged overlay. -use crate::types::TxnId; +use crate::types::{TxnId, VShardId}; -const VSHARD_BITS: u32 = 10; +const VSHARD_BITS: u32 = VShardId::BITS as u32; const POSITION_BITS: u32 = 20; const EPOCH_BITS: u32 = 33; -const VSHARD_MAX: u32 = 1 << VSHARD_BITS; +// The marker bit and the three fields fill the id exactly. +const _: () = assert!(1 + EPOCH_BITS + POSITION_BITS + VSHARD_BITS == 64); + +const VSHARD_MAX: u32 = VShardId::COUNT; const POSITION_MAX: u32 = 1 << POSITION_BITS; const EPOCH_MAX: u64 = 1 << EPOCH_BITS; diff --git a/nodedb/src/data/executor/handlers/mod.rs b/nodedb/src/data/executor/handlers/mod.rs index a30ee2c1c..b0f9c5926 100644 --- a/nodedb/src/data/executor/handlers/mod.rs +++ b/nodedb/src/data/executor/handlers/mod.rs @@ -99,6 +99,8 @@ pub mod vector_direct_delete; pub mod vector_direct_resolve; pub mod vector_direct_row; pub mod vector_direct_targets; +#[cfg(test)] +pub mod vector_direct_test_rows; pub mod vector_direct_truncate; pub mod vector_direct_update; pub mod vector_index_drop; diff --git a/nodedb/src/data/executor/handlers/timeseries/truncate.rs b/nodedb/src/data/executor/handlers/timeseries/truncate.rs index 05aa0eb8f..92c610aab 100644 --- a/nodedb/src/data/executor/handlers/timeseries/truncate.rs +++ b/nodedb/src/data/executor/handlers/timeseries/truncate.rs @@ -165,6 +165,8 @@ impl CoreLoop { self.checkpoint_coordinator .mark_dirty("timeseries", truncated as usize); + // No wipe: timeseries reads record only `Predicate` keys, which the + // collection version bounds. self.note_collection_write(task, collection); self.invalidate_aggregate_cache_for_collection(db.as_u64(), tid.as_u64(), collection); diff --git a/nodedb/src/data/executor/handlers/transaction/redo_apply/entry.rs b/nodedb/src/data/executor/handlers/transaction/redo_apply/entry.rs index edd5d42f2..bb5333dd8 100644 --- a/nodedb/src/data/executor/handlers/transaction/redo_apply/entry.rs +++ b/nodedb/src/data/executor/handlers/transaction/redo_apply/entry.rs @@ -45,7 +45,7 @@ use crate::types::TenantId; use crate::wal::RedoRecord; use super::passes::{RedoTarget, final_refusal}; -use super::state::{RedoApplyPass, RedoApplyScope, TsInstalled}; +use super::state::{DeferredWriteKind, RedoApplyPass, RedoApplyScope, TsInstalled}; use super::sub_ops::{document_ops, kv_ops, label_ops}; use crate::engine::timeseries::install_counts::TsInstallCounts; @@ -198,13 +198,21 @@ impl CoreLoop { // Write versions and the watermark move only once the record is // settled. for version in std::mem::take(&mut scope.write_versions) { - self.publish_write_version( - version.db, - version.tenant, - &version.collection, - version.key, - version.stamp, - ); + match version.kind { + DeferredWriteKind::Wipe => self.publish_collection_wipe( + version.db, + version.tenant, + &version.collection, + version.stamp, + ), + DeferredWriteKind::Row(key) => self.publish_write_version( + version.db, + version.tenant, + &version.collection, + key, + version.stamp, + ), + } } // Events leave only once the record is settled. Every // one names the record's LSN: the install held the watermark back. diff --git a/nodedb/src/data/executor/handlers/transaction/redo_apply/state.rs b/nodedb/src/data/executor/handlers/transaction/redo_apply/state.rs index 6de4a417d..5d4a80e5f 100644 --- a/nodedb/src/data/executor/handlers/transaction/redo_apply/state.rs +++ b/nodedb/src/data/executor/handlers/transaction/redo_apply/state.rs @@ -121,8 +121,16 @@ pub(in crate::data::executor) struct DeferredWriteVersion { pub db: crate::types::DatabaseId, pub tenant: crate::types::TenantId, pub collection: String, - pub key: Option, pub stamp: WriteStamp, + pub kind: DeferredWriteKind, +} + +/// What a held-back write version records. +pub(in crate::data::executor) enum DeferredWriteKind { + /// A write to one key (`Some`), or to the collection only (`None`). + Row(Option), + /// A write that removed every row of the collection. + Wipe, } /// `(database, tenant, collection)`. @@ -179,8 +187,28 @@ impl RedoApplyScope { db, tenant, collection: collection.to_string(), - key, stamp, + kind: DeferredWriteKind::Row(key), + }); + } + } + + /// Hold back one whole-collection removal until the record settles. The + /// validate pass writes nothing, so it holds nothing. + pub(in crate::data::executor) fn defer_collection_wipe( + &mut self, + db: crate::types::DatabaseId, + tenant: crate::types::TenantId, + collection: &str, + stamp: WriteStamp, + ) { + if self.pass == RedoApplyPass::Install { + self.write_versions.push(DeferredWriteVersion { + db, + tenant, + collection: collection.to_string(), + stamp, + kind: DeferredWriteKind::Wipe, }); } } diff --git a/nodedb/src/data/executor/handlers/truncate.rs b/nodedb/src/data/executor/handlers/truncate.rs index 84d3afaf8..f0a408bc1 100644 --- a/nodedb/src/data/executor/handlers/truncate.rs +++ b/nodedb/src/data/executor/handlers/truncate.rs @@ -229,6 +229,11 @@ impl CoreLoop { declared_primary_key, *storage_key, ); + // The removed rows' index values record no version, so the + // first removal records a wipe that bounds every index read. + if truncated == 0 { + self.note_task_collection_wipe(task, tid, collection); + } // Record the removal's version against the row's surrogate // and collection, as a point delete does. self.note_surrogate_write(task, tid, collection, surrogate.as_u32()); diff --git a/nodedb/src/data/executor/mod.rs b/nodedb/src/data/executor/mod.rs index 648850612..ae25b41f6 100644 --- a/nodedb/src/data/executor/mod.rs +++ b/nodedb/src/data/executor/mod.rs @@ -53,7 +53,6 @@ mod wal_replay_kv_field; mod wal_replay_kv_incr; mod wal_replay_kv_index; mod wal_replay_kv_insert_conflict; -mod wal_replay_kv_predicate; mod wal_replay_kv_sorted_index; mod wal_replay_kv_transfer; mod wal_replay_kv_ttl; diff --git a/nodedb/src/diag/context/data_plane.rs b/nodedb/src/diag/context/data_plane.rs index f1859a5ee..2d64dc04d 100644 --- a/nodedb/src/diag/context/data_plane.rs +++ b/nodedb/src/diag/context/data_plane.rs @@ -69,7 +69,8 @@ pub(in crate::diag) struct CalvinCompletionTimeout { pub epoch: u64, pub position: u32, pub participants: usize, - pub timeout_secs: u64, + /// How long the completion wait ran before the statement deadline. + pub waited_ms: u64, } impl DomainContext for CalvinCompletionTimeout { @@ -88,10 +89,10 @@ impl DomainContext for CalvinCompletionTimeout { "epoch": self.epoch, "position": self.position, "participants": self.participants, - "timeout_secs": self.timeout_secs, + "waited_ms": self.waited_ms, "why_fatal": "this timeout is the only signal a Calvin-routed write ever \ produces for a completion ack that never arrived; the caller \ - sees a generic internal error with no indication of which \ + sees a deadline error with no indication of which \ participant or stage stalled, and the write's outcome is \ unknown to the client", "operator_action": "check the sequencer-group leader and the listed \ diff --git a/nodedb/src/diag/context/held_entry.rs b/nodedb/src/diag/context/held_entry.rs new file mode 100644 index 000000000..77c294acd --- /dev/null +++ b/nodedb/src/diag/context/held_entry.rs @@ -0,0 +1,113 @@ +// SPDX-License-Identifier: BUSL-1.1 + +//! Forensic payloads for a committed entry a replica holds: its redo bytes +//! do not decode, its apply panicked, or its group's snapshot debt did not +//! persist. + +use faultbox::DomainContext; +use faultbox::serde_json::{Value, json}; + +/// A committed redo entry whose own bytes do not decode as the redo they +/// name on this replica. +pub(in crate::diag) struct RedoEntryMalformed { + pub group_id: u64, + pub log_index: u64, + /// The decode error. + pub error: String, +} + +impl DomainContext for RedoEntryMalformed { + fn domain_kind(&self) -> &'static str { + "nodedb.redo_entry_malformed" + } + + fn grouping_key(&self) -> String { + // One bug class: a committed redo that does not decode. Group and + // index are the occurrence. + "redo_entry_malformed".to_string() + } + + fn to_json(&self) -> Value { + json!({ + "group_id": self.group_id, + "log_index": self.log_index, + "error": self.error, + "impact": "the group's apply on this replica holds at the entry. Its applied \ + index stays below it and reads wait. The replica refuses log \ + entries, steps down from leadership, and never campaigns until a \ + snapshot of the group installs", + "operator_action": "the group's leader sends a snapshot. When every replica \ + holds the same entry, no replica can lead the group: the \ + proposer encoded a redo no replica decodes, and the error \ + names the field", + }) + } +} + +/// The snapshot debt of a group whose apply holds an entry did not persist. +pub(in crate::diag) struct HeldEntryDebtNotRecorded { + pub group_id: u64, + pub log_index: u64, + /// The catalog error. + pub error: String, +} + +impl DomainContext for HeldEntryDebtNotRecorded { + fn domain_kind(&self) -> &'static str { + "nodedb.held_entry_debt_not_recorded" + } + + fn grouping_key(&self) -> String { + // The apply loop retries on each Raft tick while the write fails: one + // report per root cause. + "held_entry_debt_not_recorded".to_string() + } + + fn to_json(&self) -> Value { + json!({ + "group_id": self.group_id, + "log_index": self.log_index, + "error": self.error, + "impact": "the replica refuses log entries until a snapshot installs, but a \ + sequencer snapshot install can lift that before the held entry is \ + covered. The apply loop writes the debt again on each Raft tick \ + and holds the requirement up until a write persists", + "operator_action": "the error names why the system catalog refused the write", + }) + } +} + +/// A committed entry whose enqueue or apply panicked on this replica. +pub(in crate::diag) struct ApplyPanicked { + pub group_id: u64, + pub log_index: u64, + /// The panic payload. + pub detail: String, +} + +impl DomainContext for ApplyPanicked { + fn domain_kind(&self) -> &'static str { + "nodedb.apply_panicked" + } + + fn grouping_key(&self) -> String { + // One bug class: an apply that panicked. Group and index are the + // occurrence. + "apply_panicked".to_string() + } + + fn to_json(&self) -> Value { + json!({ + "group_id": self.group_id, + "log_index": self.log_index, + "detail": self.detail, + "impact": "the entry's effect on this replica is unknown. Its proposer \ + receives an error. The group's apply holds at the entry, its \ + applied index stays below it, and the replica refuses log entries \ + until a snapshot of the group installs. Other groups keep applying", + "operator_action": "the panic names the code that failed. The group's leader \ + sends a snapshot. When every replica panics on the entry, \ + no replica can lead the group", + }) + } +} diff --git a/nodedb/src/diag/context/mod.rs b/nodedb/src/diag/context/mod.rs index 695e302c4..a1e6b2c64 100644 --- a/nodedb/src/diag/context/mod.rs +++ b/nodedb/src/diag/context/mod.rs @@ -14,6 +14,7 @@ mod crdt_dead_letter_store; mod cut_barrier; mod data_plane; mod event_image; +mod held_entry; mod index_rebuild; mod ingest; mod lease; @@ -43,6 +44,7 @@ pub(in crate::diag) use data_plane::{ CalvinApplyHalted, CalvinCompletionTimeout, CoreFailStopped, DataPlaneResponseLost, }; pub(in crate::diag) use event_image::{StrictRowImageUnrendered, TimeseriesRowImageUndecodable}; +pub(in crate::diag) use held_entry::{ApplyPanicked, HeldEntryDebtNotRecorded, RedoEntryMalformed}; pub(in crate::diag) use index_rebuild::IndexRebuildNotInstalled; pub(in crate::diag) use ingest::IlpAcceptedLinesDropped; pub use ingest::IlpFlushOutcome; diff --git a/nodedb/src/diag/context/redo_stream.rs b/nodedb/src/diag/context/redo_stream.rs index c3ae03284..db7752d7d 100644 --- a/nodedb/src/diag/context/redo_stream.rs +++ b/nodedb/src/diag/context/redo_stream.rs @@ -89,13 +89,13 @@ impl DomainContext for RedoStreamLostHere { "entry": self.entry, "refusal": self.refusal, "impact": "the other replicas can install the stream's redo, which this replica \ - lacks. Its durable floor holds at the entry, so a restart applies the \ - entry again. Later entries of the group apply here on the state \ - without the redo until a snapshot of the group installs", - "operator_action": "the node's membership pass makes the group's replica here \ - refuse log entries, and moves a leadership it holds. The \ - group's leader then sends a snapshot. When the group has no \ - other replica, no snapshot comes: restore the group from a \ + lacks. The group's apply here holds at the entry: no later entry \ + applies, its applied index stays below the entry, and reads wait \ + until a snapshot of the group installs", + "operator_action": "the group's replica here refuses log entries, steps down \ + from leadership, and never campaigns. The group's leader \ + then sends a snapshot. When the group has no other \ + replica, no snapshot comes: restore the group from a \ backup", }) } diff --git a/nodedb/src/diag/mod.rs b/nodedb/src/diag/mod.rs index 82573cd7b..cebbc6ede 100644 --- a/nodedb/src/diag/mod.rs +++ b/nodedb/src/diag/mod.rs @@ -10,17 +10,18 @@ mod recording; pub use context::{DATABASE_SCOPE, IlpFlushOutcome, LostResponseWrite, TENANT_SCOPE}; pub use recording::{ - IndexRebuildTarget, VectorBuildTarget, batch_insert_without_surrogates, calvin_apply_halted, - calvin_barrier_log_store_failed, calvin_completion_timeout, catalog_apply_orphan_row, - collection_purge_row_missing, columnar_segment_corrupt, consumer_group_offsets_retained, - continuous_aggregate_not_applied, crdt_dead_letter_not_enqueued, crdt_dead_letter_not_restored, - crdt_dead_letter_not_stored, cut_barrier_not_placed, data_plane_core_fail_stopped, - data_plane_response_lost, data_plane_responses_lost, descriptor_lease_not_renewed, entry_kind, - fts_index_update_failed, history_compaction_not_applied, ilp_invalid_utf8_drop, + IndexRebuildTarget, VectorBuildTarget, apply_panicked, batch_insert_without_surrogates, + calvin_apply_halted, calvin_barrier_log_store_failed, calvin_completion_timeout, + catalog_apply_orphan_row, collection_purge_row_missing, columnar_segment_corrupt, + consumer_group_offsets_retained, continuous_aggregate_not_applied, + crdt_dead_letter_not_enqueued, crdt_dead_letter_not_restored, crdt_dead_letter_not_stored, + cut_barrier_not_placed, data_plane_core_fail_stopped, data_plane_response_lost, + data_plane_responses_lost, descriptor_lease_not_renewed, entry_kind, fts_index_update_failed, + held_entry_debt_not_recorded, history_compaction_not_applied, ilp_invalid_utf8_drop, ilp_line_read_drop, index_rebuild_not_installed, metadata_apply_wedged, orphaned_index_entry_after_delete, quota_row_invalid, quota_row_undecodable, quota_row_write_failed, quota_scope_purge_incomplete, quota_scope_replay_aborted, - raft_entries_reapplied, raft_entry_reapplied, redo_abandon_given_up, + raft_entries_reapplied, raft_entry_reapplied, redo_abandon_given_up, redo_entry_malformed, redo_snapshot_debt_not_recorded, redo_stream_lost_here, replay_record_unapplied, retention_autowire_orphaned, scope_quota_not_installed, strict_row_image_unrendered, strict_row_undecodable, synonym_group_not_applied, timeseries_partition_unreadable, diff --git a/nodedb/src/diag/recording/data_plane.rs b/nodedb/src/diag/recording/data_plane.rs index d7c128c82..aad0d0552 100644 --- a/nodedb/src/diag/recording/data_plane.rs +++ b/nodedb/src/diag/recording/data_plane.rs @@ -34,22 +34,22 @@ pub fn data_plane_response_lost(core_id: usize, write: context::LostResponseWrit .emit(); } -/// Report a Calvin cross-shard transaction whose completion wait timed out. -/// Called from the completion-timeout arm of -/// `submit_and_await_calvin_with_timeout`; the sibling "channel closed" arm -/// is a different root cause and is not reported here. +/// Report a Calvin cross-shard transaction whose completion wait reached the +/// statement deadline. Called from the deadline arm of the coordinator's +/// completion wait. The "channel closed" arm is a different root cause and +/// is not reported here. pub fn calvin_completion_timeout( err: &crate::Error, epoch: u64, position: u32, participants: usize, - timeout_secs: u64, + waited_ms: u64, ) { let ctx = context::CalvinCompletionTimeout { epoch, position, participants, - timeout_secs, + waited_ms, }; let _ = Capture::new( EventKind::Error, diff --git a/nodedb/src/diag/recording/held_entry.rs b/nodedb/src/diag/recording/held_entry.rs new file mode 100644 index 000000000..4d855fa8f --- /dev/null +++ b/nodedb/src/diag/recording/held_entry.rs @@ -0,0 +1,62 @@ +// SPDX-License-Identifier: BUSL-1.1 + +//! Capture sites for a committed entry the data-group apply loop holds. + +use faultbox::{Capture, EventKind, error_chain_of}; + +use crate::diag::context; + +/// Report a committed redo entry of `group_id` at `log_index` whose bytes do +/// not decode on this replica. Called from the apply loop, the one site that +/// holds the entry. +pub fn redo_entry_malformed(group_id: u64, log_index: u64, error: &crate::Error) { + let ctx = context::RedoEntryMalformed { + group_id, + log_index, + error: error.to_string(), + }; + let _ = Capture::new( + EventKind::InvariantViolation, + "committed redo entry does not decode: the group's apply holds until a snapshot \ + installs", + ) + .domain(&ctx) + .error_chain(error_chain_of(error)) + .emit(); +} + +/// Report that the snapshot debt of `group_id`, whose apply holds the entry +/// at `log_index`, did not persist. Called from the apply loop, the one site +/// that records the debt of a held entry, once per failure run. +pub fn held_entry_debt_not_recorded(group_id: u64, log_index: u64, error: &crate::Error) { + let ctx = context::HeldEntryDebtNotRecorded { + group_id, + log_index, + error: error.to_string(), + }; + let _ = Capture::new( + EventKind::Error, + "snapshot debt of a group whose apply holds an entry did not persist", + ) + .domain(&ctx) + .error_chain(error_chain_of(error)) + .emit(); +} + +/// Report that the enqueue or apply of the committed entry of `group_id` at +/// `log_index` panicked on this replica. Called from the apply loop, the one +/// site that collects the entry's apply. +pub fn apply_panicked(group_id: u64, log_index: u64, detail: &str) { + let ctx = context::ApplyPanicked { + group_id, + log_index, + detail: detail.to_string(), + }; + let _ = Capture::new( + EventKind::InvariantViolation, + "a committed entry's apply panicked: the group's apply holds until a snapshot \ + installs", + ) + .domain(&ctx) + .emit(); +} diff --git a/nodedb/src/diag/recording/mod.rs b/nodedb/src/diag/recording/mod.rs index 4a9e24f17..8b8ba65c1 100644 --- a/nodedb/src/diag/recording/mod.rs +++ b/nodedb/src/diag/recording/mod.rs @@ -15,6 +15,7 @@ mod crdt; mod cut_barrier; mod data_plane; mod event_image; +mod held_entry; mod index_rebuild; mod ingest; mod lease; @@ -44,6 +45,7 @@ pub use data_plane::{ data_plane_response_lost, data_plane_responses_lost, }; pub use event_image::{strict_row_image_unrendered, timeseries_row_image_undecodable}; +pub use held_entry::{apply_panicked, held_entry_debt_not_recorded, redo_entry_malformed}; pub use index_rebuild::{IndexRebuildTarget, index_rebuild_not_installed}; pub use ingest::{ilp_invalid_utf8_drop, ilp_line_read_drop}; pub use lease::descriptor_lease_not_renewed; diff --git a/nodedb/src/error/types.rs b/nodedb/src/error/types.rs index b023ac085..1e8eeb3cc 100644 --- a/nodedb/src/error/types.rs +++ b/nodedb/src/error/types.rs @@ -470,11 +470,30 @@ pub enum Error { )] BackupCaptureMoved { group_id: u64 }, - /// No leader elected on the metadata group yet. Transient — an election - /// is in progress — so callers can wait it out instead of failing. - #[error("metadata raft group has no elected leader yet; retry needed")] + /// The metadata group has no leader to take a proposal right now. Nothing + /// was proposed. An election, a leadership transfer, a stale leader or an + /// unreachable leader causes it. Callers can wait it out and re-propose. + #[error("metadata raft group has no reachable leader; retry needed")] MetadataLeaderUnavailable, + /// The metadata group left this node before it applied the entry this + /// node proposed at `log_index`. The entry's outcome is unknown, so no + /// caller re-proposes it blindly. + #[error( + "metadata entry at log index {log_index}: the metadata raft group left this node before \ + it applied the entry; its outcome is unknown. Check the catalog before any retry" + )] + MetadataGroupGone { log_index: u64 }, + + /// Another owner took the DDL preparation lease before the DDL this node + /// proposed at `log_index` applied. Its apply skipped the DDL, so + /// nothing changed and the client can retry the statement. + #[error( + "metadata DDL at log index {log_index} was superseded: the DDL preparation lease moved to \ + another owner before it applied; nothing changed. Retry the statement" + )] + MetadataDdlSuperseded { log_index: u64 }, + /// A statement cannot be planned yet: this node's roles, grants, policies /// or permission trees may be behind a change already acknowledged, and /// did not catch up before the deadline. Nothing ran; the client retries. diff --git a/nodedb/src/error_classify/public.rs b/nodedb/src/error_classify/public.rs index a3b0753e5..59907b08c 100644 --- a/nodedb/src/error_classify/public.rs +++ b/nodedb/src/error_classify/public.rs @@ -219,12 +219,13 @@ pub(crate) fn classify(e: &Error) -> NodeDbError { )), // Internal class `XX`: no driver or pool treats it as transient, so // nothing re-proposes a write that already committed. - Error::CommittedResultUnavailable { .. } | Error::ProposalOutcomeUnknown { .. } => { - NodeDbError::internal(e.to_string()) - } + Error::CommittedResultUnavailable { .. } + | Error::ProposalOutcomeUnknown { .. } + | Error::MetadataGroupGone { .. } => NodeDbError::internal(e.to_string()), Error::MetadataLeaderUnavailable => NodeDbError::dispatch( - "metadata raft group has no elected leader yet; retry exhausted".to_string(), + "metadata raft group has no reachable leader; retry exhausted".to_string(), ), + Error::MetadataDdlSuperseded { .. } => NodeDbError::dispatch(e.to_string()), // The code renders `55P03`, the SQLSTATE pgwire gives the variant, so // the class survives a node hop as a bare numeric code. Error::AuthorizationStateBehind { .. } | Error::LinearizableReadRefused { .. } => { diff --git a/nodedb/src/error_classify/unclassified.rs b/nodedb/src/error_classify/unclassified.rs index 65ecd6bfd..fa36d7d6e 100644 --- a/nodedb/src/error_classify/unclassified.rs +++ b/nodedb/src/error_classify/unclassified.rs @@ -105,6 +105,8 @@ pub(crate) fn is_unclassified_failure(e: &Error) -> bool { | Error::GroupMarksUnavailable { .. } | Error::BackupCaptureMoved { .. } | Error::MetadataLeaderUnavailable + | Error::MetadataGroupGone { .. } + | Error::MetadataDdlSuperseded { .. } | Error::AuthorizationStateBehind { .. } | Error::LinearizableReadRefused { .. } | Error::ExecutionLimitExceeded { .. } diff --git a/nodedb/src/error_from.rs b/nodedb/src/error_from.rs index 2cc32a820..e68e35546 100644 --- a/nodedb/src/error_from.rs +++ b/nodedb/src/error_from.rs @@ -435,6 +435,8 @@ impl From for nodedb_cluster::rpc_codec::TypedClusterError { | Error::GroupMarksUnavailable { .. } | Error::BackupCaptureMoved { .. } | Error::MetadataLeaderUnavailable + | Error::MetadataGroupGone { .. } + | Error::MetadataDdlSuperseded { .. } | Error::AuthorizationStateBehind { .. } | Error::LinearizableReadRefused { .. } | Error::ExecutionLimitExceeded { .. } diff --git a/nodedb/src/event/topic/publish.rs b/nodedb/src/event/topic/publish.rs index 04d18d751..a128c80b2 100644 --- a/nodedb/src/event/topic/publish.rs +++ b/nodedb/src/event/topic/publish.rs @@ -104,7 +104,8 @@ async fn publish_replicated( state, proposer, entry, - crate::control::wal_replication::statement_propose_deadline(state), + crate::control::server::shared::session::StatementDeadline::of_statement(state) + .tokio_instant(), ) .await .map_err(|error| match error { diff --git a/nodedb/src/types/record_home.rs b/nodedb/src/types/record_home.rs index c971a775e..777c7638b 100644 --- a/nodedb/src/types/record_home.rs +++ b/nodedb/src/types/record_home.rs @@ -13,7 +13,7 @@ //! that holds edges it also lives on the key vShard of its key: a live edge //! write carries each endpoint's surrogate from the collection home, and //! every replica of each endpoint home binds it on apply. -//! - An array cell lives on the vShard its Hilbert prefix routes to. +//! - An array cell lives on the vShard its tile key routes to. //! //! Every collection is collection-homed. Key-homed data is graph data, whose //! partition key is the node id. @@ -38,7 +38,7 @@ pub enum HomedRecord<'a> { key: &'a [u8], holds_edges: bool, }, - /// Array cells whose Hilbert prefixes route to `vshard`. + /// Array cells whose tile keys route to `vshard`. ArrayCells { vshard: VShardId }, } diff --git a/nodedb/src/types/snapshot.rs b/nodedb/src/types/snapshot.rs index 6220622a8..8792557f0 100644 --- a/nodedb/src/types/snapshot.rs +++ b/nodedb/src/types/snapshot.rs @@ -335,7 +335,7 @@ pub struct GroupEventLane { pub publish_cursors: Vec, } -/// One array's cell versions whose Hilbert prefixes route to one vShard. +/// One array's cell versions whose tile keys route to one vShard. #[derive( Debug, Clone, diff --git a/nodedb/tests/crash_harness/vshards.rs b/nodedb/tests/crash_harness/vshards.rs index f9187388f..8f8776799 100644 --- a/nodedb/tests/crash_harness/vshards.rs +++ b/nodedb/tests/crash_harness/vshards.rs @@ -41,8 +41,8 @@ pub fn single_node_data_group(name: &str) -> u32 { /// `INT64` dimension `[lo..hi]` array in. /// /// An array cell write routes by its tile, never by the array's name. The -/// tile's key picks a bucket from its top `prefix_bits` bits, and the bucket -/// owns vShard `bucket * (1024 >> prefix_bits)`. At the default of 8 bits +/// tile's key picks a bucket from the top `prefix_bits` bits of +/// `fmix64(key)`, and the bucket owns vShard `bucket * (1024 >> prefix_bits)`. At the default of 8 bits /// every array vShard is a multiple of 4, so every cell of every array /// applies in data group 1. Only `prefix_bits` of 9 or more spreads cells /// over more than one group. `tile_extent` must match the array's diff --git a/nodedb/tests/crash_redo_chunk_stream.rs b/nodedb/tests/crash_redo_chunk_stream.rs index 2eb760a61..fe487aa18 100644 --- a/nodedb/tests/crash_redo_chunk_stream.rs +++ b/nodedb/tests/crash_redo_chunk_stream.rs @@ -9,7 +9,8 @@ //! //! - after every chunk is durable and before the final entry applies: boot //! rebuilds the stream from its WAL records, and the re-delivered final -//! entry installs every row once; +//! entry installs every row once. The restart slows that install, and the +//! node reports ready only after it, so the first read sees every row; //! - after the first chunk is durable: no final entry exists, so no row //! installs, and the stream drops at the group's next term. A later //! chunked commit installs as usual. @@ -37,6 +38,9 @@ const STATEMENTS: usize = 6; /// Bounded wait for the injected abort. const CRASH_TIMEOUT: Duration = Duration::from_secs(60); +/// How long the restart holds the re-delivered final entry's install. +const SLOW_INSTALL_MS: u64 = 3_000; + /// A harness whose server reads a config file with the lowered entry limit. fn harness() -> CrashHarness { let h = CrashHarness::new(); @@ -91,8 +95,12 @@ async fn commit_large_transaction(h: &CrashHarness, coll: &str, prefix: &str) -> } /// Arm `fail_point`, crash the server inside a chunked commit, and restart it -/// without the fail point. -async fn crash_inside_a_chunked_commit(h: &mut CrashHarness, fail_point: &str) { +/// with `restart_failpoints` armed, or with none. +async fn crash_inside_a_chunked_commit( + h: &mut CrashHarness, + fail_point: &str, + restart_failpoints: Option<&str>, +) { h.set_env("NODEDB_FAILPOINTS", &format!("{fail_point}=abort")); h.spawn(); h.wait_ready(); @@ -115,19 +123,30 @@ async fn crash_inside_a_chunked_commit(h: &mut CrashHarness, fail_point: &str) { h.keep_data_dir_note(), diagnostics::log_tail_section(&log) ); - h.clear_env("NODEDB_FAILPOINTS"); + match restart_failpoints { + Some(spec) => h.set_env("NODEDB_FAILPOINTS", spec), + None => h.clear_env("NODEDB_FAILPOINTS"), + } h.reopen(); } #[tokio::test(flavor = "multi_thread")] async fn a_crash_before_the_final_entry_applies_installs_the_whole_redo_once() { let mut h = harness(); - crash_inside_a_chunked_commit(&mut h, "redo_chunk::before_final_install").await; + let slow_install = format!("redo_chunk::before_final_install=sleep({SLOW_INSTALL_MS})"); + crash_inside_a_chunked_commit( + &mut h, + "redo_chunk::before_final_install", + Some(&slow_install), + ) + .await; + // The first read after ready: readiness waits for the slowed install. let expected = STATEMENTS * ROWS_PER_STATEMENT; assert_eq!( count(&h, "chunk_crash").await, expected, - "the committed final entry must install every row of its rebuilt stream once\n{}", + "the node reported ready before the re-delivered final entry installed every row of \ + its rebuilt stream once\n{}", diagnostics::log_tail_section(&h.server_log()) ); } @@ -135,7 +154,7 @@ async fn a_crash_before_the_final_entry_applies_installs_the_whole_redo_once() { #[tokio::test(flavor = "multi_thread")] async fn a_crash_between_chunk_entries_installs_nothing_and_frees_the_stream() { let mut h = harness(); - crash_inside_a_chunked_commit(&mut h, "redo_chunk::after_chunk_durable").await; + crash_inside_a_chunked_commit(&mut h, "redo_chunk::after_chunk_durable", None).await; assert_eq!( count(&h, "chunk_crash").await, 0, diff --git a/nodedb/tests/crash_replay_stamp_calvin.rs b/nodedb/tests/crash_replay_stamp_calvin.rs index d15ef09bf..a8d7def09 100644 --- a/nodedb/tests/crash_replay_stamp_calvin.rs +++ b/nodedb/tests/crash_replay_stamp_calvin.rs @@ -47,9 +47,10 @@ const CRASH_TIMEOUT: Duration = Duration::from_secs(60); /// How long the Calvin sequencer may take to elect its leader after a boot. const CALVIN_READY_TIMEOUT: Duration = Duration::from_secs(30); -/// The error a COMMIT returns once its completion deadline passes with no -/// acknowledgement. The redo record stays appended and the install stays held. -const COMMIT_DEADLINE: &str = "timed out waiting for Calvin transaction completion"; +/// The error a COMMIT returns once its statement deadline passes with no +/// acknowledgement: an unknown outcome. The redo record stays appended and +/// the install stays held. +const COMMIT_DEADLINE: &str = "57014: request req:0 exceeded deadline"; const LOG_DIRECTIVES: &str = "warn,nodedb::data::executor::kv_checkpoint=info"; diff --git a/nodedb/tests/inproc/cases/surrogate_round_trip.rs b/nodedb/tests/inproc/cases/surrogate_round_trip.rs index fe0b72849..b9d4cedee 100644 --- a/nodedb/tests/inproc/cases/surrogate_round_trip.rs +++ b/nodedb/tests/inproc/cases/surrogate_round_trip.rs @@ -681,7 +681,7 @@ fn surrogate_round_trip_all_engines() { attr_projection: vec![], limit: 100, cell_filter: None, - hilbert_range: None, + shard_bucket: None, system_time: nodedb_types::SystemTimeScope::Current, valid_at_ms: None, }), @@ -703,7 +703,7 @@ fn surrogate_round_trip_all_engines() { attr_projection: vec![], limit: 100, cell_filter: Some(intersection.clone()), - hilbert_range: None, + shard_bucket: None, system_time: nodedb_types::SystemTimeScope::Current, valid_at_ms: None, }), diff --git a/nodedb/tests/wire/cases/mod.rs b/nodedb/tests/wire/cases/mod.rs index 96e339a60..8173f040f 100644 --- a/nodedb/tests/wire/cases/mod.rs +++ b/nodedb/tests/wire/cases/mod.rs @@ -191,6 +191,7 @@ mod session_handle_security; mod session_plan_cache_permission_tree_revoke; mod session_plan_cache_rls_revoke; mod session_statement_timeout; +mod slow_catalog_ddl_apply; mod sorted_index_authorization; mod spatial_cp_dp_query; mod sql_aggregate_functions; diff --git a/nodedb/tests/wire/cases/slow_catalog_ddl_apply.rs b/nodedb/tests/wire/cases/slow_catalog_ddl_apply.rs new file mode 100644 index 000000000..ff131bb0d --- /dev/null +++ b/nodedb/tests/wire/cases/slow_catalog_ddl_apply.rs @@ -0,0 +1,47 @@ +// SPDX-License-Identifier: BUSL-1.1 + +//! Compiled only with `--features failpoints`. +//! +//! A catalog DDL whose apply runs longer than a few seconds still succeeds. +//! The proposer waits for its apply until the statement deadline, so a slow +//! apply is never reported as an error while the deadline holds. + +#[cfg(feature = "failpoints")] +use crate::harness::TestServer; + +/// How long the armed fail point holds the collection's apply. +#[cfg(feature = "failpoints")] +const APPLY_PAUSE_MS: u64 = 6_000; + +#[cfg(feature = "failpoints")] +#[tokio::test(flavor = "multi_thread", worker_threads = 4)] +async fn a_create_collection_with_a_six_second_apply_succeeds() { + let server = TestServer::start_with_failpoints(&format!( + "catalog_ddl::before_apply::slow_apply=sleep({APPLY_PAUSE_MS})" + )) + .await; + + let started = std::time::Instant::now(); + server + .exec( + "CREATE COLLECTION slow_apply (id TEXT PRIMARY KEY, val INT) \ + WITH (engine='document_strict')", + ) + .await + .expect("CREATE COLLECTION must succeed while its apply runs inside the deadline"); + assert!( + started.elapsed() >= std::time::Duration::from_millis(APPLY_PAUSE_MS), + "the fail point did not hold the apply, so this test proves nothing: {:?}", + started.elapsed() + ); + + server + .exec("INSERT INTO slow_apply (id, val) VALUES ('a', 1)") + .await + .expect("the created collection takes writes"); + let rows = server + .query_rows("SELECT val FROM slow_apply WHERE id = 'a'") + .await + .expect("the created collection serves reads"); + assert_eq!(rows, vec![vec!["1".to_string()]]); +} From 46021b05ab19f2b9365da6f81a320d399244aba7 Mon Sep 17 00:00:00 2001 From: Farhan Syah Date: Sun, 11 Oct 2026 15:00:37 +0800 Subject: [PATCH 4/5] feat(calvin): carry backup cut barriers in data-group snapshots A data-group snapshot now covers the backup cut barriers and cut floors the builder applied in the group, so a receiver that never replays those log entries still holds the state a waiting cut needs. Install restores them alongside the Calvin applied positions and dependent-read barrier logs. Forgetting the Calvin state of vShards a node left now reports its error and stays pending for retry on the next reconcile pass, and is held back until boot recovery has replayed every data group. Add a cluster test that installs a snapshot across a backup cut. --- .../calvin_backup_cut_snapshot_install.rs | 164 +++++++ .../cases/calvin_multi_shard_redo_restart.rs | 54 +++ .../tests/common_suite/cases/mod.rs | 1 + .../node/lifecycle/spawn_full.rs | 6 + nodedb/src/bootstrap/data_group_recovery.rs | 30 ++ nodedb/src/control/backup/cut.rs | 2 +- .../control/backup/cut_order/ordered_cut.rs | 36 ++ .../src/control/backup/cut_order/registry.rs | 29 ++ .../cluster/calvin/scheduler/barrier_store.rs | 29 +- .../scheduler/driver/core/cut_marker.rs | 34 ++ .../scheduler/driver/core/read_result.rs | 6 +- .../cluster/calvin_snapshot/capture.rs | 18 +- .../cluster/calvin_snapshot/cut_barriers.rs | 195 +++++++++ .../cluster/calvin_snapshot/install.rs | 12 +- .../control/cluster/calvin_snapshot/mod.rs | 1 + .../cluster/calvin_snapshot/requirement.rs | 10 +- .../src/control/cluster/left_vshard_forget.rs | 413 ++++++++++++++++++ nodedb/src/control/cluster/mod.rs | 1 + .../src/control/cluster/snapshot_builder.rs | 10 +- .../control/cluster/snapshot_install/error.rs | 3 + .../src/control/cluster/start_raft/hooks.rs | 6 +- .../src/control/cluster/start_raft_helpers.rs | 35 +- .../apply_loop/calvin_read_result.rs | 15 +- .../apply_loop/group_watch.rs | 89 +++- .../distributed_applier/apply_loop/settle.rs | 106 +++++ nodedb/src/control/pitr/restore_point/mod.rs | 7 +- .../pitr/restore_point/recorded_cuts.rs | 116 +++++ nodedb/src/control/pitr/restore_point/seed.rs | 54 +++ nodedb/src/control/pitr/state.rs | 18 +- .../security/catalog/calvin_barrier_logs.rs | 137 +++++- .../control/security/catalog/cut_floors.rs | 61 ++- .../security/catalog/system_catalog.rs | 32 ++ nodedb/src/control/state/calvin_bases.rs | 5 + nodedb/src/control/state/calvin_local.rs | 6 +- nodedb/src/control/state/fields.rs | 4 + nodedb/src/control/state/init.rs | 1 + nodedb/src/control/state/init_prod/open.rs | 1 + nodedb/src/diag/context/mod.rs | 4 +- nodedb/src/diag/context/raft_apply.rs | 62 ++- nodedb/src/diag/mod.rs | 4 +- nodedb/src/diag/recording/mod.rs | 3 +- nodedb/src/diag/recording/raft_apply.rs | 24 +- nodedb/src/types/mod.rs | 4 +- nodedb/src/types/snapshot_calvin.rs | 50 +++ 44 files changed, 1806 insertions(+), 92 deletions(-) create mode 100644 nodedb-cluster-tests/tests/common_suite/cases/calvin_backup_cut_snapshot_install.rs create mode 100644 nodedb/src/control/cluster/calvin_snapshot/cut_barriers.rs create mode 100644 nodedb/src/control/cluster/left_vshard_forget.rs create mode 100644 nodedb/src/control/pitr/restore_point/recorded_cuts.rs diff --git a/nodedb-cluster-tests/tests/common_suite/cases/calvin_backup_cut_snapshot_install.rs b/nodedb-cluster-tests/tests/common_suite/cases/calvin_backup_cut_snapshot_install.rs new file mode 100644 index 000000000..eee85303e --- /dev/null +++ b/nodedb-cluster-tests/tests/common_suite/cases/calvin_backup_cut_snapshot_install.rs @@ -0,0 +1,164 @@ +// SPDX-License-Identifier: BUSL-1.1 + +//! A replica that catches up by `InstallSnapshot` past a backup cut's +//! barrier takes that cut at once. +//! +//! 1. A 3-node cluster with a low log-compaction threshold holds one +//! collection in data group `G`. The replica `F` neither leads `G` by +//! preference nor leads the metadata group. +//! 2. `F` stops. A live node takes a backup cut at watermark `W`: every data +//! group applies the cut's barrier on the live nodes. +//! 3. Writes compact `G`'s log past the barrier. `F` restarts and can catch +//! up only by a snapshot install, so it never applies the barrier entry. +//! 4. `F` takes the same cut at `W`, as a remote source node of a backup +//! does. The snapshot carried the barrier, so the cut finishes at once. +//! Without it, `F` waits for a barrier no leader places again and the cut +//! fails with "Retry the backup". + +use std::time::{Duration, Instant}; + +use crate::common::cluster_harness::{TestCluster, TestClusterNode, wait_for}; + +/// Low enough that a few dozen single-row inserts compact `G`'s log. +const COMPACTION_THRESHOLD: u64 = 4; + +const COLLECTION: &str = "cut_install_docs"; + +/// Most rows the test writes to compact `G` past the barrier. +const MAX_ROWS: usize = 200; + +const STEP_DEADLINE: Duration = Duration::from_secs(30); +const STEP_POLL: Duration = Duration::from_millis(50); + +/// Well inside the cut's 30 s statement deadline: a cut that waits for a +/// barrier no leader places runs to that deadline. +const CUT_BUDGET: Duration = Duration::from_secs(20); + +fn node_by_id(cluster: &TestCluster, node_id: u64) -> &TestClusterNode { + cluster + .nodes + .iter() + .find(|node| node.node_id == node_id) + .unwrap_or_else(|| panic!("node {node_id} is live")) +} + +/// The preferred leader of `group_id`, from `node`'s routing table. +fn preferred_leader(node: &TestClusterNode, group_id: u64) -> u64 { + let routing = node + .shared + .cluster_routing + .as_ref() + .expect("cluster routing") + .read() + .unwrap_or_else(|p| p.into_inner()); + nodedb_cluster::rebalancer::preferred_leaders(&routing) + .get(&group_id) + .copied() + .unwrap_or_else(|| panic!("group {group_id} has a preferred leader")) +} + +#[tokio::test(flavor = "multi_thread", worker_threads = 6)] +async fn a_snapshot_installed_replica_takes_a_cut_whose_barrier_it_skipped() { + let mut cluster = TestCluster::spawn_three_with_compaction_threshold(COMPACTION_THRESHOLD) + .await + .expect("3-node cluster with a low compaction threshold"); + cluster + .exec_ddl_on_any_leader(&format!( + "CREATE COLLECTION {COLLECTION} (id TEXT PRIMARY KEY, payload TEXT) \ + WITH (engine='document_strict')" + )) + .await + .expect("create the collection"); + cluster.wait_for_preferred_leaders(STEP_DEADLINE).await; + cluster.wait_for_full_apply_convergence(STEP_DEADLINE).await; + + let probe = &cluster.nodes[0]; + let group_id = probe + .group_id_for_collection(COLLECTION) + .expect("the collection maps to a data group"); + let group_leader = preferred_leader(probe, group_id); + let metadata_leader = probe.metadata_group_leader(); + let lagging_index = cluster + .nodes + .iter() + .position(|node| node.node_id != group_leader && node.node_id != metadata_leader) + .expect("a third node leads neither group"); + let lagging_id = cluster.nodes[lagging_index].node_id; + assert!( + cluster.nodes[lagging_index].hosts_data_group(group_id), + "node {lagging_id} replicates group {group_id}" + ); + + let stopped = cluster + .stop_member(lagging_index) + .await + .expect("stop the lagging replica"); + + // A live node takes the cut. Every group applies its barrier there. + let cutter = node_by_id(&cluster, group_leader); + let watermark = cutter.shared.hlc_clock.now().wall_ns; + nodedb::control::backup::cut::cut_at(&cutter.shared, watermark) + .await + .unwrap_or_else(|e| panic!("the cut on node {group_leader}: {e}")); + let barrier_floor = cutter.shared.applied_index_watcher(group_id).current(); + + // Compact the group's log past the barrier. + let mut written = 0; + while cutter.group_snapshot_index(group_id) <= barrier_floor { + assert!( + written < MAX_ROWS, + "group {group_id} did not compact past index {barrier_floor} after {written} \ + writes: {}", + cutter.group_status_line(group_id) + ); + cutter + .client + .simple_query(&format!( + "INSERT INTO {COLLECTION} (id, payload) VALUES ('r{written}', 'v')" + )) + .await + .unwrap_or_else(|e| panic!("insert r{written}: {e}")); + written += 1; + if written % 10 == 0 { + tokio::time::sleep(STEP_POLL).await; + } + } + + cluster + .restart_member(stopped) + .await + .expect("restart the lagging replica"); + let lagging = node_by_id(&cluster, lagging_id); + wait_for( + "the lagging replica installs a snapshot past the barrier", + STEP_DEADLINE, + STEP_POLL, + || lagging.local_snapshot_index_for_group(group_id) > barrier_floor, + ) + .await; + cluster.wait_for_preferred_leaders(STEP_DEADLINE).await; + assert!( + !lagging.leads_group(group_id), + "node {lagging_id} leads group {group_id}: its own driver could place the barrier" + ); + + let started = Instant::now(); + let cut = tokio::time::timeout( + CUT_BUDGET, + nodedb::control::backup::cut::cut_at(&lagging.shared, watermark), + ) + .await + .unwrap_or_else(|_| { + panic!( + "node {lagging_id}: the cut at {watermark} still waits after {CUT_BUDGET:?}: {}", + lagging.group_status_line(group_id) + ) + }); + cut.unwrap_or_else(|e| panic!("node {lagging_id}: the cut at {watermark}: {e}")); + assert!( + !lagging.leads_group(group_id), + "node {lagging_id} led group {group_id} during the cut after {:?}", + started.elapsed() + ); + cluster.shutdown().await; +} diff --git a/nodedb-cluster-tests/tests/common_suite/cases/calvin_multi_shard_redo_restart.rs b/nodedb-cluster-tests/tests/common_suite/cases/calvin_multi_shard_redo_restart.rs index c0a212ac2..21c3a69fc 100644 --- a/nodedb-cluster-tests/tests/common_suite/cases/calvin_multi_shard_redo_restart.rs +++ b/nodedb-cluster-tests/tests/common_suite/cases/calvin_multi_shard_redo_restart.rs @@ -44,6 +44,18 @@ fn sequencer_leader(node: &TestClusterNode) -> u64 { .unwrap_or(0) } +/// Whether `vshard_id`'s applied ledger on `node` holds an applied position. +fn ledger_holds_a_position(node: &TestClusterNode, vshard_id: u32) -> bool { + node.shared + .calvin + .applied + .get(vshard_id) + .is_some_and(|ledger| { + ledger.max_applied_epoch() + != nodedb::control::cluster::calvin::scheduler::NOT_YET_APPLIED_EPOCH + }) +} + /// Count of transactions the single-node sequencer has admitted to an epoch, /// or `0` if the sequencer metrics handle is not installed yet. fn admitted_total(node: &TestClusterNode) -> u64 { @@ -206,6 +218,19 @@ async fn calvin_multi_shard_write_in_explicit_block_commits_and_survives_restart tokio::time::sleep(Duration::from_millis(25)).await; } + // The txn applied on the KV collection's vShard, so this node holds + // stored Calvin state for it. + let kv_vshard = nodedb_types::CollectionKey::from_bare(nodedb::types::DatabaseId::DEFAULT, &kv) + .vshard() + .as_u32(); + wait_for( + "the KV vShard's applied ledger holds the txn", + Duration::from_secs(10), + Duration::from_millis(25), + || ledger_holds_a_position(&node, kv_vshard), + ) + .await; + // WAL-only restart: shut down cleanly (flushing the WAL, releasing every // redb handle) and reopen against the SAME directory — no vector // checkpoint, so the HNSW must be rebuilt purely from the committed @@ -264,5 +289,34 @@ async fn calvin_multi_shard_write_in_explicit_block_commits_and_survives_restart tokio::time::sleep(Duration::from_millis(25)).await; } + // Boot queued the stored Calvin state of every vShard for a forget + // check. Once the data groups replayed, the reconcile pass keeps the + // state of each vShard this node still replicates. + wait_for( + "boot recovery replays the data groups", + Duration::from_secs(30), + Duration::from_millis(50), + || node.shared.data_groups_replayed.load(Ordering::Acquire), + ) + .await; + wait_for( + "the reconcile pass settles every vShard queued at boot", + Duration::from_secs(10), + Duration::from_millis(50), + || { + node.shared + .calvin + .forget_pending + .lock() + .unwrap_or_else(|p| p.into_inner()) + .is_empty() + }, + ) + .await; + assert!( + ledger_holds_a_position(&node, kv_vshard), + "a vShard this node replicates keeps its applied ledger across a restart" + ); + node.shutdown().await; } diff --git a/nodedb-cluster-tests/tests/common_suite/cases/mod.rs b/nodedb-cluster-tests/tests/common_suite/cases/mod.rs index 3f2f80a5b..be7dd1fbf 100644 --- a/nodedb-cluster-tests/tests/common_suite/cases/mod.rs +++ b/nodedb-cluster-tests/tests/common_suite/cases/mod.rs @@ -12,6 +12,7 @@ mod assign_surrogate_cross_node; mod backup_schedule_failover; mod bitemporal_array_cluster; mod calvin_backup_cut_order; +mod calvin_backup_cut_snapshot_install; mod calvin_cdc_net_kinds; mod calvin_cluster_array_read_set; mod calvin_cluster_pgwire_e2e; diff --git a/nodedb-test-support/src/cluster_harness/node/lifecycle/spawn_full.rs b/nodedb-test-support/src/cluster_harness/node/lifecycle/spawn_full.rs index 36954469c..b88c03706 100644 --- a/nodedb-test-support/src/cluster_harness/node/lifecycle/spawn_full.rs +++ b/nodedb-test-support/src/cluster_harness/node/lifecycle/spawn_full.rs @@ -336,6 +336,12 @@ impl TestClusterNode { .unwrap_or_else(|p| p.into_inner()) .take(); + // The data-group recovery production boot runs, in the background: + // the harness starts nodes one at a time, and a group elects only + // once its peers run. It sets `data_groups_replayed`, which gates + // the forget of a departed vShard's Calvin state. + nodedb::bootstrap::data_group_recovery::spawn_data_group_recovery(&shared); + // CRDT constraint reconcile loop (leader-gated). The production server // spawns this from `spawn_background_loops`, which the harness does not // call; wire it directly so cluster tests exercise constraint delivery diff --git a/nodedb/src/bootstrap/data_group_recovery.rs b/nodedb/src/bootstrap/data_group_recovery.rs index 7c86cabae..6dbc89986 100644 --- a/nodedb/src/bootstrap/data_group_recovery.rs +++ b/nodedb/src/bootstrap/data_group_recovery.rs @@ -213,6 +213,7 @@ fn gate_view( /// Hold startup until every locally hosted data group has replayed its retained /// Raft log on this node. A group awaiting a snapshot install does not hold it. +/// On success it sets `SharedState::data_groups_replayed`. /// /// Refuses when `start_raft` installed no Raft status source. pub async fn await_data_group_recovery(shared: &Arc) -> anyhow::Result<()> { @@ -235,6 +236,9 @@ pub async fn await_data_group_recovery(shared: &Arc) -> anyhow::Res ); } info!("all local data raft groups replayed — opening client gateway"); + shared + .data_groups_replayed + .store(true, std::sync::atomic::Ordering::Release); return Ok(()); } @@ -258,6 +262,32 @@ pub async fn await_data_group_recovery(shared: &Arc) -> anyhow::Res } } +/// Run [`await_data_group_recovery`] in a loop on `shared`'s shutdown +/// watch until it succeeds, without holding the caller. A host that brings +/// nodes up one at a time uses it: a group here elects only once its peers +/// run. Each failed attempt is logged, and the next one starts at once. +pub fn spawn_data_group_recovery(shared: &Arc) { + let task_shared = Arc::clone(shared); + crate::control::shutdown::spawn_loop( + &shared.loop_registry, + &shared.shutdown, + "data_group_recovery", + crate::control::shutdown::ShutdownPhase::DrainingControlPlane, + move |mut shutdown| async move { + loop { + tokio::select! { + biased; + _ = shutdown.wait_cancelled() => break, + recovered = await_data_group_recovery(&task_shared) => match recovered { + Ok(()) => break, + Err(error) => warn!(%error, "data raft group recovery did not finish; retrying"), + }, + } + } + }, + ); +} + #[cfg(test)] mod tests { use super::*; diff --git a/nodedb/src/control/backup/cut.rs b/nodedb/src/control/backup/cut.rs index 5ea8eed68..8ae2bca06 100644 --- a/nodedb/src/control/backup/cut.rs +++ b/nodedb/src/control/backup/cut.rs @@ -72,7 +72,7 @@ pub(super) async fn consistent_cut(state: &SharedState) -> Result { /// Take the consistent cut at `watermark` on this node: wait until every /// write committed below it has a final outcome here. A remote source node /// runs it for the watermark the backup's coordinator picked. -pub(crate) async fn cut_at(state: &SharedState, watermark: u64) -> Result<(), Error> { +pub async fn cut_at(state: &SharedState, watermark: u64) -> Result<(), Error> { cut_at_point(state, watermark, 0).await } diff --git a/nodedb/src/control/backup/cut_order/ordered_cut.rs b/nodedb/src/control/backup/cut_order/ordered_cut.rs index 29b6fa860..ec8406aeb 100644 --- a/nodedb/src/control/backup/cut_order/ordered_cut.rs +++ b/nodedb/src/control/backup/cut_order/ordered_cut.rs @@ -25,6 +25,7 @@ use nodedb_physical::physical_plan::CutCaptureRequest; use crate::control::state::SharedState; use crate::control::wal_replication::ReplicatedWrite; +use crate::types::CoveredCutBarrier; /// The clock skew between two nodes a cut's window tolerates. const CLOCK_SKEW_MARGIN: Duration = Duration::from_secs(30); @@ -93,6 +94,25 @@ impl OrderedCut { capture: self.capture.clone(), } } + + /// This cut's barrier at `log_index`, as a data-group snapshot carries it. + pub fn covered_at(&self, log_index: u64) -> CoveredCutBarrier { + CoveredCutBarrier { + hlc: self.hlc, + restore_point: self.restore_point, + capture: self.capture.clone(), + log_index, + } + } + + /// The cut a data-group snapshot's `barrier` names. + pub fn from_covered(barrier: &CoveredCutBarrier) -> Self { + Self { + hlc: barrier.hlc, + restore_point: barrier.restore_point, + capture: barrier.capture.clone(), + } + } } /// The ordering window of one cut on this node. @@ -165,6 +185,22 @@ mod tests { assert_ne!(plain.key(), cut.key(), "the capture keeps two cuts apart"); } + #[test] + fn a_cut_round_trips_through_a_snapshot() { + let cut = OrderedCut { + hlc: 7, + restore_point: 3, + capture: Some(CutCaptureRequest { + request_id: 11, + database_id: 2, + tenants: vec![1], + }), + }; + let covered = cut.covered_at(40); + assert_eq!(covered.log_index, 40); + assert_eq!(OrderedCut::from_covered(&covered), cut); + } + #[test] fn the_hold_outlasts_the_proposal_window_by_the_skew_margin() { let window = CutWindow::of(1_000, Duration::from_secs(30)); diff --git a/nodedb/src/control/backup/cut_order/registry.rs b/nodedb/src/control/backup/cut_order/registry.rs index dcdb15e71..d63a0a98e 100644 --- a/nodedb/src/control/backup/cut_order/registry.rs +++ b/nodedb/src/control/backup/cut_order/registry.rs @@ -117,6 +117,20 @@ impl CutBarriers { self.group(key, group).and_then(|barrier| barrier.applied) } + /// Every tracked cut this node applied a barrier of in `group`, with the + /// barrier's log index, in cut key order. + pub(crate) fn applied_in_group(&self, group: u64) -> Vec<(OrderedCut, u64)> { + self.cuts + .lock() + .unwrap_or_else(|p| p.into_inner()) + .values() + .filter_map(|tracked| { + let applied = tracked.groups.get(&group)?.applied?; + Some((tracked.cut.clone(), applied)) + }) + .collect() + } + /// Note that this node proposed `key`'s barrier into `group` while it /// led the group. pub(crate) fn note_proposed(&self, key: CutKey, group: u64, proposed: Proposed) { @@ -214,6 +228,21 @@ mod tests { assert_eq!(barriers.cut(key), Some(cut(5))); } + #[test] + fn a_group_lists_the_barriers_it_applied_in_key_order() { + let barriers = CutBarriers::new(); + barriers.note_applied(&cut(9), 1, 60); + barriers.note_applied(&cut(5), 1, 40); + barriers.note_applied(&cut(7), 2, 50); + barriers.claim_driver(&cut(8)); + assert_eq!( + barriers.applied_in_group(1), + [(cut(5), 40), (cut(9), 60)], + "only applied barriers of the group, by cut key" + ); + assert_eq!(barriers.applied_in_group(3), []); + } + /// A lost proposal is forgotten only when it is the one recorded. #[test] fn a_lost_proposal_is_forgotten_once() { diff --git a/nodedb/src/control/cluster/calvin/scheduler/barrier_store.rs b/nodedb/src/control/cluster/calvin/scheduler/barrier_store.rs index ff200246d..6cc18e7cb 100644 --- a/nodedb/src/control/cluster/calvin/scheduler/barrier_store.rs +++ b/nodedb/src/control/cluster/calvin/scheduler/barrier_store.rs @@ -7,7 +7,11 @@ //! durable applied floor of a group asserts that every entry at or below it //! is durable, and boot resumes Raft delivery above it. So an entry below //! the floor is never delivered again, and its row is what a barrier of the -//! txn reads after a restart: +//! txn reads after a restart. +//! +//! A saved event is visible at once and durable before the floor that +//! covers it: [`save_event`] commits without persisting, and the apply +//! loop's settle pass runs [`persist_pending`] before it saves any floor. //! //! - [`restore_at_boot`] marks the txn of every stored row stored in its //! vShard's buffer, and removes the rows of finished txns; @@ -30,6 +34,9 @@ use crate::types::CutBarrierLog; /// Fold `event` into the stored row of `txn` on `vshard_id`. An event the /// row's log does not take, a copy it holds or one after its timeout, /// writes nothing. +/// +/// The row is visible at once. It is durable after the next +/// [`persist_pending`]. pub fn save_event( catalog: &SystemCatalog, vshard_id: u32, @@ -49,6 +56,12 @@ pub fn save_event( }) } +/// Persist every row [`save_event`] wrote since the last persist, in one +/// durable commit. On an error the rows stay pending for the next call. +pub fn persist_pending(catalog: &SystemCatalog) -> crate::Result<()> { + catalog.persist_calvin_barrier_logs() +} + /// The stored log of `txn` on `vshard_id`. A txn with no row has an empty /// log. pub fn load_log(catalog: &SystemCatalog, vshard_id: u32, txn: TxnId) -> crate::Result { @@ -98,7 +111,10 @@ fn is_applied(shared: &SharedState, vshard_id: u32, epoch: u64, position: u32) - /// remove the rows of finished txns. Boot runs this once the applied /// ledgers are filled, before the data-group apply loop and the schedulers /// start. -pub fn restore_at_boot(shared: &SharedState) -> crate::Result<()> { +/// +/// Returns the vShards that still hold rows. The first scheduler reconcile +/// purges those this node no longer hosts. +pub fn restore_at_boot(shared: &SharedState) -> crate::Result> { let catalog = shared.credentials.catalog(); let mut finished = Vec::new(); let mut waiting: BTreeMap> = BTreeMap::new(); @@ -113,6 +129,7 @@ pub fn restore_at_boot(shared: &SharedState) -> crate::Result<()> { } } catalog.remove_calvin_barrier_logs(&finished)?; + let holding: BTreeSet = waiting.keys().copied().collect(); for (vshard_id, txns) in waiting { shared .calvin @@ -120,7 +137,7 @@ pub fn restore_at_boot(shared: &SharedState) -> crate::Result<()> { .vshard(vshard_id) .mark_stored(txns); } - Ok(()) + Ok(holding) } /// The stored rows of the unfinished txns of `vshards`, for a snapshot of @@ -280,7 +297,11 @@ mod tests { .applied .install(VSHARD, NOT_YET_APPLIED_EPOCH, BTreeSet::from([(6, 0)])); - restore_at_boot(&state).expect("restore"); + assert_eq!( + restore_at_boot(&state).expect("restore"), + BTreeSet::from([VSHARD]), + "boot names the vShards that still hold rows" + ); let buffer = state.calvin.read_results.vshard(VSHARD); assert_eq!(buffer.waiting_txns(), 1); diff --git a/nodedb/src/control/cluster/calvin/scheduler/driver/core/cut_marker.rs b/nodedb/src/control/cluster/calvin/scheduler/driver/core/cut_marker.rs index d4b954665..22187a099 100644 --- a/nodedb/src/control/cluster/calvin/scheduler/driver/core/cut_marker.rs +++ b/nodedb/src/control/cluster/calvin/scheduler/driver/core/cut_marker.rs @@ -233,6 +233,40 @@ mod tests { assert!(!scheduler.release_cut_holds(), "nothing left to end"); } + /// A scheduler holding for a cut whose barrier a snapshot install carried + /// is woken by the install and releases its hold at once. + #[tokio::test] + async fn an_installed_barrier_ends_the_hold() { + use futures::FutureExt; + + let (mut scheduler, _dir) = build_test_scheduler(test_coll_vshard()); + let cut = fresh_cut(&scheduler); + scheduler.process_scheduler_input(marker(&cut)); + assert!(scheduler.held_for_cut(0)); + let group_id = scheduler + .data_group() + .expect("the fixture routes every vShard"); + assert!(!scheduler.release_cut_holds()); + + let shared = Arc::clone(&scheduler.shared); + let woken = shared.calvin.cut_barriers.notified(); + tokio::pin!(woken); + woken.as_mut().enable(); + crate::control::cluster::calvin_snapshot::cut_barriers::install_cut_barriers( + &shared, + group_id, + &[cut.covered_at(12)], + &[], + ) + .expect("the install records the barrier"); + assert!( + woken.now_or_never().is_some(), + "the install wakes the run loop" + ); + assert!(scheduler.release_cut_holds()); + assert!(!scheduler.held_for_cut(0)); + } + /// A second copy of the marker keeps the first copy's place, and a copy /// of a cut whose barrier applied holds nothing for long. #[tokio::test] diff --git a/nodedb/src/control/cluster/calvin/scheduler/driver/core/read_result.rs b/nodedb/src/control/cluster/calvin/scheduler/driver/core/read_result.rs index 53b0bc8bc..3a1917553 100644 --- a/nodedb/src/control/cluster/calvin/scheduler/driver/core/read_result.rs +++ b/nodedb/src/control/cluster/calvin/scheduler/driver/core/read_result.rs @@ -79,7 +79,7 @@ impl Scheduler { on the next pass" ); crate::diag::calvin_barrier_log_store_failed( - self.vshard_id, + Some(self.vshard_id), Some((txn_id.epoch, txn_id.position)), "load", &e, @@ -253,7 +253,7 @@ impl Scheduler { "calvin: the barrier rows of finished txns were not removed; retrying on \ the next stall tick" ); - crate::diag::calvin_barrier_log_store_failed(self.vshard_id, None, "remove", &e); + crate::diag::calvin_barrier_log_store_failed(Some(self.vshard_id), None, "remove", &e); } } @@ -281,7 +281,7 @@ impl Scheduler { "calvin: the barrier row of a finished txn was not removed" ); crate::diag::calvin_barrier_log_store_failed( - self.vshard_id, + Some(self.vshard_id), Some((txn_id.epoch, txn_id.position)), "remove", &e, diff --git a/nodedb/src/control/cluster/calvin_snapshot/capture.rs b/nodedb/src/control/cluster/calvin_snapshot/capture.rs index cc9fb8fea..064435443 100644 --- a/nodedb/src/control/cluster/calvin_snapshot/capture.rs +++ b/nodedb/src/control/cluster/calvin_snapshot/capture.rs @@ -18,6 +18,7 @@ use nodedb_cluster::calvin::SequencerEntry; use crate::Error; use crate::control::cluster::calvin::scheduler::barrier_store; +use crate::control::cluster::calvin_snapshot::cut_barriers::capture_cut_barriers; use crate::control::security::catalog::calvin_base::CalvinBase; use crate::control::state::SharedState; use crate::types::{GroupCalvinCut, VShardCalvinState}; @@ -103,18 +104,22 @@ pub async fn await_calvin_cut( } } -/// The group's Calvin cut through `through`: the applied positions of every -/// group vShard here, and the stored barrier logs of their unfinished txns. -/// The caller holds the group's apply gate exclusive, so no Calvin install -/// of these vShards runs and no barrier entry of the group applies. +/// The group's Calvin cut through `through`, for a snapshot through the +/// group's log index `cut_index`: the applied positions of every group +/// vShard here, the stored barrier logs of their unfinished txns, and the +/// group's backup cut barriers and floors. The caller holds the group's +/// apply gate exclusive, so no Calvin install of these vShards runs and no +/// entry of the group applies. /// -/// Fails when the barrier logs cannot be read whole (see +/// Fails when the barrier logs or cut floors cannot be read whole (see /// [`barrier_store::capture_group`]). The build retries on the next /// heartbeat. pub fn capture_calvin_cut( shared: &SharedState, + group_id: u64, group_vshards: &HashSet, through: u64, + cut_index: u64, ) -> Result { let ledgers = &shared.calvin.applied; let ids: BTreeSet = group_vshards.iter().copied().collect(); @@ -130,9 +135,12 @@ pub fn capture_calvin_cut( }) .collect(); let barrier_logs = barrier_store::capture_group(shared, &ids)?; + let (cut_barriers, cut_floors) = capture_cut_barriers(shared, group_id, cut_index)?; Ok(GroupCalvinCut { through, vshards, barrier_logs, + cut_barriers, + cut_floors, }) } diff --git a/nodedb/src/control/cluster/calvin_snapshot/cut_barriers.rs b/nodedb/src/control/cluster/calvin_snapshot/cut_barriers.rs new file mode 100644 index 000000000..b8aaf0d25 --- /dev/null +++ b/nodedb/src/control/cluster/calvin_snapshot/cut_barriers.rs @@ -0,0 +1,195 @@ +// SPDX-License-Identifier: BUSL-1.1 + +//! The backup cut barriers a data-group snapshot carries. +//! +//! A cut barrier applied in a group is replicated state, yet a replica learns +//! of it only from the entry. A receiver never applies the entries a snapshot +//! covers, so the snapshot carries two things for them: +//! +//! - every barrier of a cut whose window is open, with its log index. A cut +//! waiting on the receiver and a scheduler holding redo for the cut read it +//! as applied. +//! - the group's durable cut floors. Every later entry records a commit HLC +//! above them, on the receiver as on the builder. + +use crate::Error; +use crate::control::backup::cut_order::{CutWindow, OrderedCut}; +use crate::control::pitr::restore_point::{group_cut_floors, install_cut_floors}; +use crate::control::state::SharedState; +use crate::types::{CoveredCutBarrier, CutFloor}; + +/// The cut barriers of `group_id` a snapshot through `cut_index` carries. +/// The caller holds the group's apply gate exclusive, so no barrier of the +/// group applies meanwhile, and every applied one sits at or below the cut. +pub fn capture_cut_barriers( + shared: &SharedState, + group_id: u64, + cut_index: u64, +) -> Result<(Vec, Vec), Error> { + // no-determinism: the window bounds only which cuts still wait anywhere. + let now = shared.hlc_clock.now().wall_ns; + let barriers = shared + .calvin + .cut_barriers + .applied_in_group(group_id) + .into_iter() + .filter(|(cut, log_index)| { + *log_index <= cut_index && now <= CutWindow::on(shared, cut.hlc).hold_until + }) + .map(|(cut, log_index)| cut.covered_at(log_index)) + .collect(); + let floors = group_cut_floors(shared, group_id, cut_index)?; + Ok((barriers, floors)) +} + +/// Install the cut barriers a snapshot of `group_id` carried: the floors +/// first, durably, then each barrier as applied here, which wakes every +/// cut wait and every scheduler holding redo for it. +pub fn install_cut_barriers( + shared: &SharedState, + group_id: u64, + barriers: &[CoveredCutBarrier], + floors: &[CutFloor], +) -> crate::Result<()> { + install_cut_floors(shared, group_id, floors)?; + for barrier in barriers { + shared.calvin.cut_barriers.note_applied( + &OrderedCut::from_covered(barrier), + group_id, + barrier.log_index, + ); + } + Ok(()) +} + +#[cfg(test)] +mod tests { + use std::collections::HashSet; + use std::sync::{Arc, RwLock}; + use std::time::Duration; + + use nodedb_cluster::RoutingTable; + + use super::*; + use crate::bridge::dispatch::Dispatcher; + use crate::control::backup::cut_order::wait::await_group_barriers; + use crate::control::cluster::calvin_snapshot::{capture_calvin_cut, install_calvin_cut}; + use crate::wal::WalManager; + + const NODE: u64 = 1; + + /// A fresh node that hosts every group of its routing table, and one + /// of those groups. + fn node() -> (Arc, u64, tempfile::TempDir) { + let dir = tempfile::tempdir().unwrap(); + let wal = Arc::new(WalManager::open_for_testing(&dir.path().join("test.wal")).unwrap()); + let (dispatcher, _sides) = Dispatcher::new(1, 8); + let mut state = SharedState::new(dispatcher, wal).unwrap(); + let routing = RoutingTable::uniform(1, &[NODE], 1); + let group_id = routing.group_ids()[0]; + let shared = Arc::get_mut(&mut state).unwrap(); + shared.node_id = NODE; + shared.cluster_routing = Some(Arc::new(RwLock::new(routing))); + (state, group_id, dir) + } + + fn open_cut(state: &SharedState) -> OrderedCut { + OrderedCut { + hlc: state.hlc_clock.now().wall_ns, + restore_point: 0, + capture: None, + } + } + + #[test] + fn a_snapshot_carries_the_barriers_at_or_below_its_cut() { + let (leader, group_id, _dir) = node(); + let cut = open_cut(&leader); + let closed = OrderedCut { + hlc: 1, + ..cut.clone() + }; + let later = OrderedCut { + hlc: cut.hlc + 1, + ..cut.clone() + }; + leader.calvin.cut_barriers.note_applied(&cut, group_id, 40); + leader + .calvin + .cut_barriers + .note_applied(&closed, group_id, 30); + leader + .calvin + .cut_barriers + .note_applied(&later, group_id, 60); + crate::control::pitr::restore_point::persist_cut_floor(&leader, group_id, 40, cut.hlc) + .unwrap(); + crate::control::pitr::restore_point::persist_cut_floor(&leader, group_id, 60, later.hlc) + .unwrap(); + + let captured = capture_calvin_cut(&leader, group_id, &HashSet::new(), 7, 50).unwrap(); + assert_eq!( + captured.cut_barriers, + [cut.covered_at(40)], + "a closed window and a barrier above the cut stay behind" + ); + assert_eq!( + captured.cut_floors, + [CutFloor { + index: 40, + watermark: cut.hlc + }] + ); + } + + #[tokio::test] + async fn an_installed_barrier_ends_a_pending_cut_wait() { + let (leader, group_id, _leader_dir) = node(); + let cut = open_cut(&leader); + leader.calvin.cut_barriers.note_applied(&cut, group_id, 40); + crate::control::pitr::restore_point::persist_cut_floor(&leader, group_id, 40, cut.hlc) + .unwrap(); + let captured = capture_calvin_cut(&leader, group_id, &HashSet::new(), 7, 50).unwrap(); + + let (follower, follower_group, _follower_dir) = node(); + assert_eq!(follower_group, group_id); + let waiting = { + let follower = Arc::clone(&follower); + let cut = cut.clone(); + tokio::spawn(async move { + let deadline = tokio::time::Instant::now() + Duration::from_secs(10); + await_group_barriers(&follower, &cut, &[group_id], deadline).await + }) + }; + tokio::task::yield_now().await; + assert!(!waiting.is_finished(), "no barrier applied on the follower"); + + // The install raises the follower's applied index past the snapshot. + follower.group_watchers.bump(group_id, 50); + install_calvin_cut(&follower, group_id, &HashSet::new(), captured).unwrap(); + assert_eq!( + follower.calvin.cut_barriers.applied(cut.key(), group_id), + Some(40) + ); + assert_eq!( + follower.pitr.recorded_cuts().watermark_below(group_id, 51), + Some(cut.hlc), + "the apply loop raises later entries above the cut" + ); + assert_eq!( + follower + .credentials + .catalog() + .group_cut_floors(group_id) + .unwrap() + .len(), + 1, + "the floor outlives a restart" + ); + tokio::time::timeout(Duration::from_secs(5), waiting) + .await + .expect("the install wakes the wait") + .expect("the wait runs") + .expect("the cut wait succeeds"); + } +} diff --git a/nodedb/src/control/cluster/calvin_snapshot/install.rs b/nodedb/src/control/cluster/calvin_snapshot/install.rs index 582ec801d..a959aef7f 100644 --- a/nodedb/src/control/cluster/calvin_snapshot/install.rs +++ b/nodedb/src/control/cluster/calvin_snapshot/install.rs @@ -13,6 +13,8 @@ //! //! The stored dependent-read barrier logs of the group's vShards become the //! cut's too: the snapshot covers barrier entries this node never applies. +//! So do the group's backup cut barriers and cut floors (see +//! [`super::cut_barriers`]). //! //! A WAL `SnapshotInstalled` record of the group, written before this runs, //! makes boot recovery drop the vShards' applied markers from before it. @@ -20,6 +22,7 @@ use std::collections::{BTreeSet, HashSet}; use crate::control::cluster::calvin::scheduler::barrier_store; +use crate::control::cluster::calvin_snapshot::cut_barriers::install_cut_barriers; use crate::control::cluster::snapshot_install::SnapshotInstallError; use crate::control::security::catalog::calvin_applied::StoredCalvinApplied; use crate::control::state::SharedState; @@ -80,5 +83,12 @@ pub fn install_calvin_cut( .calvin .bases .record_snapshot(catalog, &ids, cut.through) - .map_err(settle_error) + .map_err(settle_error)?; + install_cut_barriers(shared, group_id, &cut.cut_barriers, &cut.cut_floors).map_err(|source| { + SnapshotInstallError::Settle { + group_id, + step: crate::control::cluster::snapshot_install::SettleStep::CutBarriers, + source, + } + }) } diff --git a/nodedb/src/control/cluster/calvin_snapshot/mod.rs b/nodedb/src/control/cluster/calvin_snapshot/mod.rs index 206f3654f..4473a2ede 100644 --- a/nodedb/src/control/cluster/calvin_snapshot/mod.rs +++ b/nodedb/src/control/cluster/calvin_snapshot/mod.rs @@ -1,6 +1,7 @@ // SPDX-License-Identifier: BUSL-1.1 pub mod capture; +pub mod cut_barriers; pub mod install; pub mod requirement; diff --git a/nodedb/src/control/cluster/calvin_snapshot/requirement.rs b/nodedb/src/control/cluster/calvin_snapshot/requirement.rs index 69f4767ca..144bfcb13 100644 --- a/nodedb/src/control/cluster/calvin_snapshot/requirement.rs +++ b/nodedb/src/control/cluster/calvin_snapshot/requirement.rs @@ -188,9 +188,11 @@ pub fn rebased(shared: &SharedState, served: &[u32]) -> Vec { /// Forget the Calvin state of `vshards`: this node left their groups. A /// later return catches up from nothing, so neither their bases nor their /// applied positions can survive. -pub fn forget_left(shared: &SharedState, vshards: &[u32]) { +/// +/// On an error the stored state stays, and the caller retries the forget. +pub fn forget_left(shared: &SharedState, vshards: &[u32]) -> crate::Result<()> { if vshards.is_empty() { - return; + return Ok(()); } let catalog = shared.credentials.catalog(); let resets: Vec = @@ -211,13 +213,15 @@ pub fn forget_left(shared: &SharedState, vshards: &[u32]) { let forgotten = catalog .replace_calvin_applied(&resets) .and_then(|()| shared.calvin.bases.forget(catalog, vshards)); - if let Err(error) = forgotten { + if let Err(error) = &forgotten { tracing::error!( ?vshards, %error, "calvin: the left vShards' Calvin state was not forgotten" ); + crate::diag::calvin_left_state_not_forgotten(vshards, error); } + forgotten } /// Keep the sequencer log range only for vShards of the data groups this diff --git a/nodedb/src/control/cluster/left_vshard_forget.rs b/nodedb/src/control/cluster/left_vshard_forget.rs new file mode 100644 index 000000000..265ce2f53 --- /dev/null +++ b/nodedb/src/control/cluster/left_vshard_forget.rs @@ -0,0 +1,413 @@ +// SPDX-License-Identifier: BUSL-1.1 + +//! Forget the stored Calvin state of the vShards this node left. +//! +//! A vShard that left keeps no applied positions, base or dependent-read +//! barrier rows here. A forget that fails leaves the vShard in +//! `CalvinLocalState::forget_pending`, and every scheduler reconcile pass +//! retries it. Boot adds every vShard that holds any stored Calvin state. +//! +//! A pending vShard whose group lists this node as a voter or learner +//! leaves the set with no forget: its state belongs to that membership. +//! +//! No pass forgets before boot recovery sets +//! `SharedState::data_groups_replayed`. Until every mounted data group +//! applied its committed log, a group can still re-apply a conf change, so +//! the routing table can omit a group this node replicates. + +use std::collections::BTreeSet; +use std::sync::RwLock; + +use nodedb_cluster::RoutingTable; + +use crate::control::cluster::calvin::scheduler::{NOT_YET_APPLIED_EPOCH, barrier_store}; +use crate::control::cluster::calvin_snapshot; +use crate::control::state::SharedState; + +/// Add to the pending set every vShard with stored Calvin state at boot: +/// `holding_rows`, the vShards with barrier rows, and every vShard with an +/// applied ledger or a base. Boot runs this once both are loaded. +pub fn seed_at_boot(shared: &SharedState, holding_rows: BTreeSet) { + let mut seeded = holding_rows; + seeded.extend( + shared + .calvin + .applied + .snapshot_all() + .into_iter() + // A reset ledger holds no applied position. + .filter(|state| { + state.fully_applied_epoch != NOT_YET_APPLIED_EPOCH || !state.tail.is_empty() + }) + .map(|state| state.vshard_id), + ); + seeded.extend(shared.calvin.bases.vshards()); + shared + .calvin + .forget_pending + .lock() + .unwrap_or_else(|p| p.into_inner()) + .extend(seeded); +} + +/// The vShards of every Raft group that lists `node_id` as a voter or a +/// learner, sorted. A learner applies the group's entries, so it holds the +/// group's Calvin state too. +pub fn replicated_vshards(routing: &RwLock, node_id: u64) -> Vec { + let routing = routing.read().unwrap_or_else(|p| p.into_inner()); + let mut vshards = Vec::new(); + for (group_id, info) in routing.group_members() { + if info.members.contains(&node_id) || info.learners.contains(&node_id) { + vshards.extend(routing.vshards_for_group(*group_id)); + } + } + vshards.sort_unstable(); + vshards.dedup(); + vshards +} + +/// Whether the routing table names every group this node replicates: boot +/// recovery saw every mounted data group apply its committed log. +fn membership_settled(shared: &SharedState) -> bool { + shared + .data_groups_replayed + .load(std::sync::atomic::Ordering::Acquire) +} + +/// Forget the stored Calvin state of `left`, the vShards this node no +/// longer hosts. Each vShard whose forget fails waits in the pending set. +pub fn forget_left_vshards(shared: &SharedState, left: &[u32]) { + if left.is_empty() { + return; + } + let state_forgotten = calvin_snapshot::forget_left(shared, left).is_ok(); + let mut failed = Vec::new(); + for &vshard_id in left { + let rows_forgotten = forget_rows(shared, vshard_id); + if !state_forgotten || !rows_forgotten { + failed.push(vshard_id); + } + } + if failed.is_empty() { + return; + } + shared + .calvin + .forget_pending + .lock() + .unwrap_or_else(|p| p.into_inner()) + .extend(failed); +} + +/// Retry the forget of every pending vShard. `replicated` is sorted: the +/// vShards of every group that lists this node as a voter or learner. A +/// replicated vShard leaves the set with no forget, and a forgotten one +/// leaves it. Before membership settles, the pass changes nothing. +pub fn retry_pending_forgets(shared: &SharedState, replicated: &[u32]) { + if !membership_settled(shared) { + return; + } + let pending: Vec = shared + .calvin + .forget_pending + .lock() + .unwrap_or_else(|p| p.into_inner()) + .iter() + .copied() + .collect(); + if pending.is_empty() { + return; + } + let mut settled = Vec::new(); + for vshard_id in pending { + if replicated.binary_search(&vshard_id).is_ok() { + settled.push(vshard_id); + continue; + } + // Both forgets are idempotent, so a retry runs both. + let state_forgotten = calvin_snapshot::forget_left(shared, &[vshard_id]).is_ok(); + let rows_forgotten = forget_rows(shared, vshard_id); + if state_forgotten && rows_forgotten { + tracing::info!( + vshard_id, + "calvin: the stored state of a vShard that left is forgotten" + ); + settled.push(vshard_id); + } + } + let mut set = shared + .calvin + .forget_pending + .lock() + .unwrap_or_else(|p| p.into_inner()); + for vshard_id in settled { + set.remove(&vshard_id); + } +} + +/// Remove every barrier row and the buffer of `vshard_id`. Returns whether +/// the rows went. +fn forget_rows(shared: &SharedState, vshard_id: u32) -> bool { + match barrier_store::forget_vshard(shared, vshard_id) { + Ok(()) => true, + Err(e) => { + tracing::warn!( + vshard_id, + error = %e, + "calvin: the barrier rows of a vShard that left were not removed; the next \ + reconcile pass retries" + ); + crate::diag::calvin_barrier_log_store_failed(Some(vshard_id), None, "forget", &e); + false + } + } +} + +#[cfg(test)] +mod tests { + use std::collections::BTreeSet; + use std::sync::Arc; + + use super::*; + use crate::bridge::dispatch::Dispatcher; + use crate::control::cluster::calvin::BarrierEvent; + use crate::control::cluster::calvin::scheduler::lock_manager::TxnId; + use crate::wal::WalManager; + + const VSHARD: u32 = 5; + const NODE: u64 = 1; + const OTHER: u64 = 2; + + /// Routing whose one data group holds every vShard, with `voters`. + fn routing(voters: &[u64]) -> RwLock { + RwLock::new(RoutingTable::uniform(1, voters, 1)) + } + + /// A state whose boot recovery has not run. + fn booting_state() -> (Arc, tempfile::TempDir) { + let dir = tempfile::tempdir().expect("tempdir"); + let wal = Arc::new(WalManager::open_for_testing(&dir.path().join("wal")).expect("wal")); + let (dispatcher, _sides) = Dispatcher::new(1, 64); + ( + SharedState::new(dispatcher, wal).expect("shared state"), + dir, + ) + } + + /// A state whose boot recovery finished. + fn state() -> (Arc, tempfile::TempDir) { + let (state, dir) = booting_state(); + state + .data_groups_replayed + .store(true, std::sync::atomic::Ordering::Release); + (state, dir) + } + + fn save_row(state: &SharedState, vshard_id: u32) { + barrier_store::save_event( + state.credentials.catalog(), + vshard_id, + TxnId::new(3, 0), + &BarrierEvent::Timeout, + ) + .expect("save the row"); + } + + fn row_vshards(state: &SharedState) -> BTreeSet { + state + .credentials + .catalog() + .load_calvin_barrier_logs(None) + .expect("load rows") + .into_iter() + .map(|row| row.vshard_id) + .collect() + } + + fn pending(state: &SharedState) -> BTreeSet { + state + .calvin + .forget_pending + .lock() + .expect("pending set") + .clone() + } + + /// A failed row removal keeps the vShard pending. The next pass that + /// finds it unhosted removes its rows and drops it. + #[test] + fn a_failed_forget_retries_on_the_next_pass() { + let (state, _dir) = state(); + save_row(&state, VSHARD); + save_row(&state, VSHARD + 1); + + state + .credentials + .catalog() + .fail_next_calvin_barrier_replace_for_test(); + forget_left_vshards(&state, &[VSHARD]); + assert_eq!(pending(&state), BTreeSet::from([VSHARD])); + assert_eq!(row_vshards(&state), BTreeSet::from([VSHARD, VSHARD + 1])); + + retry_pending_forgets(&state, &[VSHARD + 1]); + assert!(pending(&state).is_empty()); + assert_eq!( + row_vshards(&state), + BTreeSet::from([VSHARD + 1]), + "the hosted vShard keeps its rows" + ); + } + + /// A pending vShard hosted again leaves the set, and its rows stay. + #[test] + fn a_rehosted_vshard_leaves_the_set_without_a_purge() { + let (state, _dir) = state(); + save_row(&state, VSHARD); + state + .credentials + .catalog() + .fail_next_calvin_barrier_replace_for_test(); + forget_left_vshards(&state, &[VSHARD]); + assert_eq!(pending(&state), BTreeSet::from([VSHARD])); + + retry_pending_forgets(&state, &[VSHARD]); + assert!(pending(&state).is_empty()); + assert_eq!(row_vshards(&state), BTreeSet::from([VSHARD])); + } + + /// Boot names every vShard that holds rows. The first pass removes the + /// rows of the vShards this node does not host and keeps the rest. + #[test] + fn boot_rows_of_an_unhosted_vshard_go_on_the_first_pass() { + let (state, _dir) = state(); + save_row(&state, VSHARD); + save_row(&state, VSHARD + 1); + + let holding = barrier_store::restore_at_boot(&state).expect("restore"); + assert_eq!(holding, BTreeSet::from([VSHARD, VSHARD + 1])); + state + .calvin + .forget_pending + .lock() + .expect("pending set") + .extend(holding); + + retry_pending_forgets(&state, &[VSHARD + 1]); + assert!(pending(&state).is_empty()); + assert_eq!(row_vshards(&state), BTreeSet::from([VSHARD + 1])); + assert_eq!( + state.calvin.read_results.waiting_txns(VSHARD), + 0, + "the unhosted vShard's buffer goes with its rows" + ); + assert_eq!(state.calvin.read_results.waiting_txns(VSHARD + 1), 1); + } + + /// A pass that runs before boot recovery forgets nothing, even when its + /// routing omits a group this node replicates. Recovery waits for the + /// group's leader. The first pass after it keeps the state of every + /// replicated vShard. + #[tokio::test] + async fn an_early_pass_never_purges_a_replicated_vshard() { + let (state, _dir) = booting_state(); + let leader = Arc::new(std::sync::atomic::AtomicU64::new(0)); + let status_leader = Arc::clone(&leader); + let status_fn: Arc Vec + Send + Sync> = + Arc::new(move || { + vec![nodedb_cluster::GroupStatus { + group_id: 1, + role: "Leader".into(), + leader_id: status_leader.load(std::sync::atomic::Ordering::Acquire), + term: 1, + commit_index: 0, + last_applied: 0, + last_log_index: 0, + snapshot_index: 0, + replica_state: nodedb_raft::ReplicaState::Whole, + member_count: 1, + learner_count: 0, + vshard_count: 1, + }] + }); + assert!(state.raft_status_fn.set(status_fn).is_ok()); + + save_row(&state, VSHARD); + state.calvin.applied.install(VSHARD, 2, BTreeSet::new()); + let holding = barrier_store::restore_at_boot(&state).expect("restore"); + seed_at_boot(&state, holding); + assert_eq!(pending(&state), BTreeSet::from([VSHARD])); + + // The routing omits this node: membership has not settled. + retry_pending_forgets(&state, &replicated_vshards(&routing(&[OTHER]), NODE)); + assert_eq!(pending(&state), BTreeSet::from([VSHARD])); + assert_eq!(row_vshards(&state), BTreeSet::from([VSHARD])); + assert!(state.calvin.applied.get(VSHARD).is_some()); + + // The group elects, and boot recovery finishes. + leader.store(NODE, std::sync::atomic::Ordering::Release); + crate::bootstrap::data_group_recovery::await_data_group_recovery(&state) + .await + .expect("recovery"); + assert!( + state + .data_groups_replayed + .load(std::sync::atomic::Ordering::Acquire) + ); + retry_pending_forgets(&state, &replicated_vshards(&routing(&[NODE]), NODE)); + assert!(pending(&state).is_empty()); + assert_eq!(row_vshards(&state), BTreeSet::from([VSHARD])); + assert!(state.calvin.applied.get(VSHARD).is_some()); + } + + /// A learner replicates its group, so its vShards count as replicated. + #[test] + fn a_learner_replicates_its_groups_vshards() { + let mut table = RoutingTable::uniform(1, &[OTHER], 1); + table.set_group_learners(1, vec![NODE]); + assert!( + replicated_vshards(&RwLock::new(table), NODE) + .binary_search(&VSHARD) + .is_ok() + ); + assert!(replicated_vshards(&routing(&[OTHER]), NODE).is_empty()); + } + + /// An unreplicated vShard with only an applied ledger and a base is + /// forgotten on the first pass after the data groups replay. + #[test] + fn an_unreplicated_ledger_and_base_are_forgotten() { + let (state, _dir) = state(); + let catalog = state.credentials.catalog(); + state.calvin.applied.install(VSHARD, 4, BTreeSet::new()); + state + .calvin + .bases + .record_kept(catalog, &[(VSHARD, 1)]) + .expect("record the base"); + seed_at_boot(&state, BTreeSet::new()); + assert_eq!(pending(&state), BTreeSet::from([VSHARD])); + + retry_pending_forgets(&state, &replicated_vshards(&routing(&[OTHER]), NODE)); + assert!(pending(&state).is_empty()); + assert!(state.calvin.applied.get(VSHARD).is_none()); + assert!(state.calvin.bases.base(VSHARD).is_none()); + assert!( + catalog + .load_calvin_bases() + .expect("load bases") + .iter() + .all(|(vshard_id, _)| *vshard_id != VSHARD) + ); + assert!( + catalog + .load_all_calvin_applied() + .expect("load ledgers") + .iter() + .filter(|stored| stored.vshard_id == VSHARD) + .all(|stored| { + stored.fully_applied_epoch == NOT_YET_APPLIED_EPOCH && stored.tail.is_empty() + }), + "a stored ledger of the vShard is a reset one" + ); + } +} diff --git a/nodedb/src/control/cluster/mod.rs b/nodedb/src/control/cluster/mod.rs index 079e5a11c..5dcf47912 100644 --- a/nodedb/src/control/cluster/mod.rs +++ b/nodedb/src/control/cluster/mod.rs @@ -21,6 +21,7 @@ pub mod forwarded_budget; pub mod handle; pub mod init; pub mod leased_read; +pub mod left_vshard_forget; pub mod linearizable_read; pub mod metadata_applier; pub mod metadata_image; diff --git a/nodedb/src/control/cluster/snapshot_builder.rs b/nodedb/src/control/cluster/snapshot_builder.rs index 7beb0cc04..1f4771afd 100644 --- a/nodedb/src/control/cluster/snapshot_builder.rs +++ b/nodedb/src/control/cluster/snapshot_builder.rs @@ -360,8 +360,14 @@ impl nodedb_cluster::SnapshotBuilder for DataPlaneSnapshotBuilder { .map_err(|e| Box::new(e) as Box)?; // Read under the fence: the storage captured below holds exactly the // Calvin positions this names. - let calvin_cut = capture_calvin_cut(&self.shared, &group_vshards, calvin_through) - .map_err(|e| Box::new(e) as Box)?; + let calvin_cut = capture_calvin_cut( + &self.shared, + group_id, + &group_vshards, + calvin_through, + cut_index, + ) + .map_err(|e| Box::new(e) as Box)?; // Enumerate tenants and their databases from the system catalog — the // same source the backup orchestrator's catalog sections use. Every diff --git a/nodedb/src/control/cluster/snapshot_install/error.rs b/nodedb/src/control/cluster/snapshot_install/error.rs index a433c4b04..f24ffe12d 100644 --- a/nodedb/src/control/cluster/snapshot_install/error.rs +++ b/nodedb/src/control/cluster/snapshot_install/error.rs @@ -18,6 +18,8 @@ pub enum SettleStep { CalvinState, /// Replacing the group's open chunked redo streams. RedoStreams, + /// Recording the group's backup cut floors and barriers. + CutBarriers, } impl std::fmt::Display for SettleStep { @@ -29,6 +31,7 @@ impl std::fmt::Display for SettleStep { Self::EventLane => "event lane install", Self::CalvinState => "Calvin state install", Self::RedoStreams => "redo stream install", + Self::CutBarriers => "cut barrier install", }) } } diff --git a/nodedb/src/control/cluster/start_raft/hooks.rs b/nodedb/src/control/cluster/start_raft/hooks.rs index a68f13243..2709c23f2 100644 --- a/nodedb/src/control/cluster/start_raft/hooks.rs +++ b/nodedb/src/control/cluster/start_raft/hooks.rs @@ -119,7 +119,11 @@ pub(super) async fn build_hooks( // at or below a group's durable applied floor is never delivered again, // so its row is what a barrier reads from now on. A boot completion of a // snapshot install below replaces its group's rows. - crate::control::cluster::calvin::scheduler::barrier_store::restore_at_boot(shared)?; + let holding = + crate::control::cluster::calvin::scheduler::barrier_store::restore_at_boot(shared)?; + // The first reconcile pass after the data groups replay forgets the + // stored Calvin state of each vShard this node no longer replicates. + crate::control::cluster::left_vshard_forget::seed_at_boot(shared, holding); // Per-group snapshot builder for the SEND path: on the leader, build the // real serialized engine state for a lagging follower's group vshards diff --git a/nodedb/src/control/cluster/start_raft_helpers.rs b/nodedb/src/control/cluster/start_raft_helpers.rs index 04be91339..a4888f1e8 100644 --- a/nodedb/src/control/cluster/start_raft_helpers.rs +++ b/nodedb/src/control/cluster/start_raft_helpers.rs @@ -11,6 +11,7 @@ use crate::control::cluster::calvin::{ }; use crate::control::cluster::calvin_snapshot; use crate::control::cluster::handle::ClusterHandle; +use crate::control::cluster::left_vshard_forget; use crate::control::state::SharedState; /// The vShards whose scheduler this node started and has not stopped. @@ -41,8 +42,11 @@ fn hosted_vshards(routing: &RwLock, node_id: u64) /// from. Dropping the vShard's sequencer sender closes the scheduler's /// intake, and the scheduler exits. Its other channels and its lock table go /// too. It hands the barrier events of the txns it did not finish back to -/// the vShard's read-result buffer. The next reconcile that finds the vShard hosted spawns a new -/// scheduler from the vShard's base. +/// the vShard's read-result buffer. The next reconcile that finds the vShard +/// hosted spawns a new scheduler from the vShard's base. +/// +/// A vShard that left has its stored Calvin state forgotten (see +/// [`left_vshard_forget`]). fn retire_vshard_schedulers( hosted: &[u32], shared: &Arc, @@ -67,24 +71,10 @@ fn retire_vshard_schedulers( } (left, rebased) }; - calvin_snapshot::forget_left(shared, &left); - // A vShard that left this node keeps no barrier events here, in memory - // or stored. A rebased vShard keeps them for the scheduler that starts - // again. - for &vshard in &left { - if let Err(e) = - crate::control::cluster::calvin::scheduler::barrier_store::forget_vshard(shared, vshard) - { - // The rows stay. Boot marks them stored for a vShard with no - // scheduler, and a snapshot install of the vShard replaces them. - tracing::warn!( - vshard_id = vshard, - error = %e, - "calvin: the barrier rows of a vShard that left were not removed" - ); - crate::diag::calvin_barrier_log_store_failed(vshard, None, "remove", &e); - } - } + // A vShard that left this node keeps no Calvin state here, in memory or + // stored. A rebased vShard keeps its barrier events for the scheduler + // that starts again. + left_vshard_forget::forget_left_vshards(shared, &left); for &vshard in &rebased { tracing::info!( vshard_id = vshard, @@ -164,6 +154,11 @@ fn reconcile_vshard_schedulers(params: ReconcileSchedulersParams<'_>) -> crate:: } = params; let hosted = hosted_vshards(routing, node_id); + // Forgets that failed on an earlier pass, or wait since boot, go first. + left_vshard_forget::retry_pending_forgets( + shared, + &left_vshard_forget::replicated_vshards(routing, node_id), + ); retire_vshard_schedulers( &hosted, shared, diff --git a/nodedb/src/control/distributed_applier/apply_loop/calvin_read_result.rs b/nodedb/src/control/distributed_applier/apply_loop/calvin_read_result.rs index 2ad1f2fca..c541d6a37 100644 --- a/nodedb/src/control/distributed_applier/apply_loop/calvin_read_result.rs +++ b/nodedb/src/control/distributed_applier/apply_loop/calvin_read_result.rs @@ -13,9 +13,11 @@ //! //! The entry counts as durably applied once its row holds the event. Boot //! resumes Raft delivery above the group's durable applied floor, so the -//! row is what a barrier reads after a restart. A row write that fails -//! leaves the event in memory alone, and the entry breaks the applied -//! prefix: a restart delivers it again. +//! row is what a barrier reads after a restart. The row is visible at once +//! and durable before the floor that covers it: the settle pass persists +//! every saved row before it saves a floor. A row write that fails leaves +//! the event in memory alone, and the entry breaks the applied prefix: a +//! restart delivers it again. use crate::control::array_sync::raft_apply::AppliedPosition; use crate::control::cluster::calvin::BarrierEvent; @@ -253,7 +255,7 @@ fn drop_finished(ctx: ApplyContext<'_>, vshard_id: u32, txn: TxnId) { "calvin: the barrier row of a finished txn was not removed" ); crate::diag::calvin_barrier_log_store_failed( - vshard_id, + Some(vshard_id), Some((txn.epoch, txn.position)), "remove", &e, @@ -262,7 +264,8 @@ fn drop_finished(ctx: ApplyContext<'_>, vshard_id: u32, txn: TxnId) { } /// Save `event` in the row of `txn` on `vshard_id`, then hand it to the -/// vShard's buffer. Returns whether the row holds it. +/// vShard's buffer. Returns whether the row holds it. The settle pass +/// persists the row before a floor covers its entry. /// /// A txn with an event its row lacks skips the row: the row stays a prefix /// of the txn's events, so a restart that delivers the missing entry again @@ -282,7 +285,7 @@ fn save_and_buffer(ctx: ApplyContext<'_>, vshard_id: u32, txn: TxnId, event: Bar "calvin: the barrier event's row write failed; the event stays in memory" ); crate::diag::calvin_barrier_log_store_failed( - vshard_id, + Some(vshard_id), Some((txn.epoch, txn.position)), "save", &e, diff --git a/nodedb/src/control/distributed_applier/apply_loop/group_watch.rs b/nodedb/src/control/distributed_applier/apply_loop/group_watch.rs index 9d85000cc..1b661d4b3 100644 --- a/nodedb/src/control/distributed_applier/apply_loop/group_watch.rs +++ b/nodedb/src/control/distributed_applier/apply_loop/group_watch.rs @@ -5,8 +5,9 @@ //! floors, which raise the commit HLC of every entry after a cut barrier. use std::collections::{BTreeMap, HashMap}; +use std::sync::Arc; -use crate::control::pitr::restore_point::RecordedCut; +use crate::control::pitr::restore_point::RecordedCuts; /// Per-group apply state. #[derive(Debug, Default)] @@ -15,23 +16,29 @@ pub(super) struct GroupWatch { /// Per group, barrier log index to the lowest commit HLC an entry after /// that barrier records: one above the barrier's watermark. cut_floors: HashMap>, + /// The cut barriers this loop never applied: those of an earlier node + /// life, and those a snapshot install carried. + recorded: Arc, + /// The generation of `recorded` merged into `cut_floors`. + merged: u64, } impl GroupWatch { - /// A watch that knows the restore point cuts this node applied before a - /// restart: the entries it applies again after one of them record above - /// it, as they did the first time. - pub(super) fn with_cuts(cuts: &[RecordedCut]) -> Self { - let mut watch = Self::default(); - for cut in cuts { - watch.raise_cut(cut.group_id, cut.barrier_index, cut.hlc); - } + /// A watch that knows the cuts of `recorded`: the entries it applies + /// after one of them record above it, as on every other replica. + pub(super) fn with_cuts(recorded: Arc) -> Self { + let mut watch = Self { + recorded, + ..Self::default() + }; + watch.merge_recorded(); watch } /// Note that the apply loop applies `(group_id, log_index)`. Reports an /// index at or below one it already applied as a second apply. pub(super) fn note_apply(&mut self, group_id: u64, log_index: u64) { + self.merge_recorded(); let highest = self.highest_applied.entry(group_id).or_insert(0); if log_index <= *highest { crate::diag::raft_entry_reapplied(group_id, log_index, *highest); @@ -69,7 +76,28 @@ impl GroupWatch { .cut_floors .get(&group_id) .and_then(|floors| floors.range(..log_index).map(|(_, floor)| *floor).max()); - floor.map_or(write_hlc, |floor| write_hlc.max(floor)) + // A cut recorded since the last merge binds this entry too. + let unmerged = (self.recorded.generation() != self.merged) + .then(|| self.recorded.watermark_below(group_id, log_index)) + .flatten() + .map(|watermark| watermark.saturating_add(1)); + floor + .into_iter() + .chain(unmerged) + .max() + .map_or(write_hlc, |floor| write_hlc.max(floor)) + } + + /// Merge the cuts `recorded` holds once its generation moved. + fn merge_recorded(&mut self) { + let generation = self.recorded.generation(); + if generation == self.merged { + return; + } + for cut in self.recorded.all() { + self.raise_cut(cut.group_id, cut.barrier_index, cut.hlc); + } + self.merged = generation; } /// Fold every barrier below `log_index` into the highest of them: entries @@ -96,6 +124,7 @@ impl GroupWatch { #[cfg(test)] mod tests { use super::*; + use crate::control::pitr::restore_point::RecordedCut; #[test] fn an_entry_after_a_cut_records_above_the_cut() { @@ -117,6 +146,42 @@ mod tests { ); } + fn watch_of(cuts: &[RecordedCut]) -> GroupWatch { + let recorded = RecordedCuts::new(); + recorded.record(cuts); + GroupWatch::with_cuts(Arc::new(recorded)) + } + + /// A snapshot install carries a barrier the follower never applied. The + /// follower's next entries record above it, as the leader's do. + #[test] + fn an_installed_floor_binds_like_an_applied_one() { + let mut leader = GroupWatch::default(); + leader.note_apply(1, 10); + leader.raise_cut(1, 10, 100); + + let recorded = Arc::new(RecordedCuts::new()); + let mut follower = GroupWatch::with_cuts(Arc::clone(&recorded)); + assert_eq!(follower.commit_hlc(1, 11, 50), 50, "nothing installed yet"); + recorded.replace_group( + 1, + &[RecordedCut { + group_id: 1, + barrier_index: 10, + hlc: 100, + }], + ); + // Read before the entry's apply notes it, as the pipeline stamps it. + assert_eq!(follower.commit_hlc(1, 11, 50), leader.commit_hlc(1, 11, 50)); + follower.note_apply(1, 11); + leader.note_apply(1, 11); + assert_eq!(follower.commit_hlc(1, 11, 50), 101); + assert_eq!( + follower.commit_hlc(1, 12, 300), + leader.commit_hlc(1, 12, 300) + ); + } + #[test] fn an_entry_applied_again_after_a_restart_records_as_before() { let cuts = [ @@ -131,7 +196,7 @@ mod tests { hlc: 200, }, ]; - let mut watch = GroupWatch::with_cuts(&cuts); + let mut watch = watch_of(&cuts); // Delivery resumes between the two barriers. watch.note_apply(1, 15); assert_eq!(watch.commit_hlc(1, 15, 50), 101); @@ -205,7 +270,7 @@ mod tests { // The restart reads the floors back from the catalog. let catalog = SystemCatalog::open(&catalog_path).unwrap(); - let mut watch = GroupWatch::with_cuts(&load_recorded_cuts(&catalog).unwrap()); + let mut watch = watch_of(&load_recorded_cuts(&catalog).unwrap()); watch.note_apply(1, 11); assert_eq!(watch.commit_hlc(1, 11, 50), 101); } diff --git a/nodedb/src/control/distributed_applier/apply_loop/settle.rs b/nodedb/src/control/distributed_applier/apply_loop/settle.rs index b7c07fc58..d7b0d6aac 100644 --- a/nodedb/src/control/distributed_applier/apply_loop/settle.rs +++ b/nodedb/src/control/distributed_applier/apply_loop/settle.rs @@ -4,7 +4,9 @@ use std::collections::HashMap; +use crate::control::cluster::calvin::scheduler::barrier_store; use crate::control::server::dispatch_utils::publish_settled_changes; +use crate::control::state::SharedState; use super::bookkeeping::record_durable_apply; use super::context::ApplyContext; @@ -17,9 +19,14 @@ use super::lane::Lane; /// floor that covers them. An entry above the saved floor is delivered /// again after a restart and records its mark again, so every committed /// write keeps a durable mark. +/// +/// The dependent-read barrier rows saved before the pass persist before any +/// floor too: a floor covers a barrier entry only once its row is durable. pub(super) fn settle_lanes(ctx: ApplyContext<'_>, lanes: &mut HashMap) { let state = ctx.state; let tracker = ctx.tracker; + // Persisted once per pass, at the first floor that moved. + let mut barrier_rows_durable: Option = None; for (group_id, lane) in lanes { let first = lane.front_index(); let settled = lane.settle(tracker, &state.tenant_marks); @@ -34,6 +41,11 @@ pub(super) fn settle_lanes(ctx: ApplyContext<'_>, lanes: &mut HashMap if !lane.floor_pending() { continue; } + if !*barrier_rows_durable.get_or_insert_with(|| persist_barrier_rows(state)) { + // Every floor waits: one may cover a barrier entry whose row a + // crash loses. The next pass persists again. + continue; + } // One save per pass that moved the floor, never one per entry: // each save is an fsync, and the pass coalesces every apply that // finished before it. @@ -54,3 +66,97 @@ pub(super) fn settle_lanes(ctx: ApplyContext<'_>, lanes: &mut HashMap } } } + +/// Persist the barrier rows saved since the last persist. Returns whether +/// they are durable. +fn persist_barrier_rows(state: &SharedState) -> bool { + match barrier_store::persist_pending(state.credentials.catalog()) { + Ok(()) => true, + Err(error) => { + tracing::error!( + %error, + "apply loop: the dependent-read barrier rows did not persist; every applied \ + floor waits" + ); + crate::diag::calvin_barrier_log_store_failed(None, None, "persist", &error); + false + } + } +} + +#[cfg(test)] +mod tests { + use std::sync::{Arc, Mutex}; + + use nodedb_cluster::GroupAppliedWatchers; + + use super::*; + use crate::bridge::dispatch::Dispatcher; + use crate::control::cluster::calvin::BarrierEvent; + use crate::control::cluster::calvin::scheduler::lock_manager::TxnId; + use crate::control::distributed_applier::propose_tracker::ProposeTracker; + use crate::wal::WalManager; + + use super::super::lane::{Slot, SlotState}; + use super::super::proposal_gate::PrefixStep; + + const GROUP: u64 = 4; + + /// A concluded barrier entry at `log_index` whose row holds its event. + fn concluded(log_index: u64) -> Slot { + Slot { + log_index, + proposal_key: 0, + collection: None, + write_mark: None, + user_write: false, + state: SlotState::Concluded(PrefixStep::Record(true)), + } + } + + /// A barrier row persist fails: the pass saves no floor, so the floor + /// never passes the barrier entry. The next pass persists and saves it. + #[test] + fn the_floor_waits_for_the_barrier_rows_to_persist() { + let dir = tempfile::tempdir().expect("tempdir"); + let wal = Arc::new(WalManager::open_for_testing(&dir.path().join("wal")).expect("wal")); + let (dispatcher, _sides) = Dispatcher::new(1, 64); + let state = SharedState::new(dispatcher, wal).expect("shared state"); + let saved: Arc>> = Arc::new(Mutex::new(Vec::new())); + let sink_saved = Arc::clone(&saved); + let sink: Arc = + Arc::new(move |group_id, applied_index| { + sink_saved + .lock() + .expect("saved floors") + .push((group_id, applied_index)); + Ok::<(), crate::Error>(()) + }); + assert!(state.raft_applied_index_sink.set(sink).is_ok()); + let watchers = Arc::new(GroupAppliedWatchers::new()); + let tracker = Arc::new(ProposeTracker::new().with_group_watchers(watchers)); + let ctx = ApplyContext { + state: &state, + tracker: &tracker, + }; + + let catalog = state.credentials.catalog(); + barrier_store::save_event(catalog, 7, TxnId::new(3, 0), &BarrierEvent::Timeout) + .expect("save the row"); + let mut lane = Lane::new(GROUP); + lane.push(concluded(5)); + let mut lanes = HashMap::from([(GROUP, lane)]); + + catalog.fail_next_calvin_barrier_persist_for_test(); + settle_lanes(ctx, &mut lanes); + assert!( + saved.lock().expect("saved floors").is_empty(), + "no floor passes a row that is not durable" + ); + assert!(lanes[&GROUP].floor_pending()); + + settle_lanes(ctx, &mut lanes); + assert_eq!(*saved.lock().expect("saved floors"), vec![(GROUP, 5)]); + assert!(!lanes[&GROUP].floor_pending()); + } +} diff --git a/nodedb/src/control/pitr/restore_point/mod.rs b/nodedb/src/control/pitr/restore_point/mod.rs index f9c9ceb5c..f46c283ee 100644 --- a/nodedb/src/control/pitr/restore_point/mod.rs +++ b/nodedb/src/control/pitr/restore_point/mod.rs @@ -2,10 +2,15 @@ pub mod create; pub mod record; +pub mod recorded_cuts; pub mod seed; pub mod task; pub use create::{RestorePointError, create_restore_point, list_restore_points}; pub use record::{record_group_point, sequencer_hook, spawn_node_cut}; -pub use seed::{RecordedCut, load_recorded_cuts, persist_cut_floor, persist_until_durable}; +pub use recorded_cuts::RecordedCuts; +pub use seed::{ + RecordedCut, group_cut_floors, install_cut_floors, load_recorded_cuts, persist_cut_floor, + persist_until_durable, +}; pub use task::spawn_restore_point_task; diff --git a/nodedb/src/control/pitr/restore_point/recorded_cuts.rs b/nodedb/src/control/pitr/restore_point/recorded_cuts.rs new file mode 100644 index 000000000..4a0578217 --- /dev/null +++ b/nodedb/src/control/pitr/restore_point/recorded_cuts.rs @@ -0,0 +1,116 @@ +// SPDX-License-Identifier: BUSL-1.1 + +//! The cut barriers the apply loop raises commit HLCs above, beyond the ones +//! it applies itself: those the system catalog held at boot, and those a +//! data-group snapshot install carried. +//! +//! The apply loop merges every change into its own floors. A change bumps +//! the generation, so the loop reads the shared floors only after one. + +use std::collections::BTreeMap; +use std::sync::Mutex; +use std::sync::atomic::{AtomicU64, Ordering}; + +use super::seed::RecordedCut; + +/// Per group, barrier log index to watermark, and a generation that moves +/// on every change. +#[derive(Debug, Default)] +pub struct RecordedCuts { + floors: Mutex>>, + generation: AtomicU64, +} + +impl RecordedCuts { + pub fn new() -> Self { + Self::default() + } + + /// Add every cut of `cuts`. + pub fn record(&self, cuts: &[RecordedCut]) { + let mut floors = self.floors.lock().unwrap_or_else(|p| p.into_inner()); + for cut in cuts { + let watermark = floors + .entry(cut.group_id) + .or_default() + .entry(cut.barrier_index) + .or_insert(0); + *watermark = (*watermark).max(cut.hlc); + } + self.generation.fetch_add(1, Ordering::Release); + } + + /// Replace `group_id`'s cuts with `cuts`, the group's durable rows after + /// a snapshot install. + pub fn replace_group(&self, group_id: u64, cuts: &[RecordedCut]) { + let mut floors = self.floors.lock().unwrap_or_else(|p| p.into_inner()); + let group = floors.entry(group_id).or_default(); + group.clear(); + for cut in cuts.iter().filter(|cut| cut.group_id == group_id) { + let watermark = group.entry(cut.barrier_index).or_insert(0); + *watermark = (*watermark).max(cut.hlc); + } + self.generation.fetch_add(1, Ordering::Release); + } + + /// Moves on every change. Read it before [`Self::all`]: a change after + /// the read moves it again. + pub fn generation(&self) -> u64 { + self.generation.load(Ordering::Acquire) + } + + /// Every cut, in group and index order. + pub fn all(&self) -> Vec { + self.floors + .lock() + .unwrap_or_else(|p| p.into_inner()) + .iter() + .flat_map(|(group_id, group)| { + group.iter().map(|(barrier_index, hlc)| RecordedCut { + group_id: *group_id, + barrier_index: *barrier_index, + hlc: *hlc, + }) + }) + .collect() + } + + /// The highest watermark of a cut of `group_id` below `log_index`. + pub fn watermark_below(&self, group_id: u64, log_index: u64) -> Option { + self.floors + .lock() + .unwrap_or_else(|p| p.into_inner()) + .get(&group_id)? + .range(..log_index) + .map(|(_, hlc)| *hlc) + .max() + } +} + +#[cfg(test)] +mod tests { + use super::*; + + fn cut(group_id: u64, barrier_index: u64, hlc: u64) -> RecordedCut { + RecordedCut { + group_id, + barrier_index, + hlc, + } + } + + #[test] + fn an_install_replaces_only_its_group() { + let cuts = RecordedCuts::new(); + let start = cuts.generation(); + cuts.record(&[cut(1, 10, 100), cut(2, 5, 50)]); + assert_ne!(cuts.generation(), start); + assert_eq!(cuts.watermark_below(1, 11), Some(100)); + assert_eq!(cuts.watermark_below(1, 10), None, "the barrier's own place"); + + let before = cuts.generation(); + cuts.replace_group(1, &[cut(1, 30, 300), cut(2, 9, 90)]); + assert_ne!(cuts.generation(), before); + assert_eq!(cuts.all(), [cut(1, 30, 300), cut(2, 5, 50)]); + } +} diff --git a/nodedb/src/control/pitr/restore_point/seed.rs b/nodedb/src/control/pitr/restore_point/seed.rs index f10d22eb0..d6ebde921 100644 --- a/nodedb/src/control/pitr/restore_point/seed.rs +++ b/nodedb/src/control/pitr/restore_point/seed.rs @@ -19,6 +19,7 @@ use crate::control::cluster::metadata_applier::{MetadataApplyWedge, WedgeReport, use crate::control::security::catalog::SystemCatalog; use crate::control::security::catalog::cut_floors::StoredBarrier; use crate::control::state::SharedState; +use crate::types::CutFloor; /// One group's cut barrier. #[derive(Debug, Clone, Copy, PartialEq, Eq)] @@ -62,6 +63,59 @@ pub fn persist_cut_floor( .put_cut_floor(group_id, barrier, durable) } +/// The durable cut floors of `group_id` at or below `through`, in index +/// order, for a data-group snapshot to carry. +pub fn group_cut_floors( + state: &SharedState, + group_id: u64, + through: u64, +) -> crate::Result> { + Ok(state + .credentials + .catalog() + .group_cut_floors(group_id)? + .into_iter() + .filter(|barrier| barrier.index <= through) + .map(|barrier| CutFloor { + index: barrier.index, + watermark: barrier.watermark, + }) + .collect()) +} + +/// Durably record the cut floors `floors` a snapshot of `group_id` carried, +/// and hand the group's rows to the apply loop. The receiver never applies +/// the barriers the snapshot covers, yet every later entry records above them. +pub fn install_cut_floors( + state: &SharedState, + group_id: u64, + floors: &[CutFloor], +) -> crate::Result<()> { + if floors.is_empty() { + return Ok(()); + } + let catalog = state.credentials.catalog(); + let barriers: Vec = floors + .iter() + .map(|floor| StoredBarrier { + index: floor.index, + watermark: floor.watermark, + }) + .collect(); + catalog.put_cut_floors(group_id, &barriers, state.pitr.durable_applied(group_id))?; + let rows: Vec = catalog + .group_cut_floors(group_id)? + .into_iter() + .map(|barrier| RecordedCut { + group_id, + barrier_index: barrier.index, + hlc: barrier.watermark, + }) + .collect(); + state.pitr.recorded_cuts().replace_group(group_id, &rows); + Ok(()) +} + /// First wait between two attempts of a floor write. const FIRST_RETRY: Duration = Duration::from_millis(10); diff --git a/nodedb/src/control/pitr/state.rs b/nodedb/src/control/pitr/state.rs index 0a78a6d1b..f500032ef 100644 --- a/nodedb/src/control/pitr/state.rs +++ b/nodedb/src/control/pitr/state.rs @@ -10,7 +10,7 @@ use std::sync::{Arc, Mutex, OnceLock, RwLock}; use super::node_life::NodeLife; use super::pins::ColdPins; -use super::restore_point::RecordedCut; +use super::restore_point::{RecordedCut, RecordedCuts}; use crate::control::metrics::SystemMetrics; use crate::storage::snapshot::SnapshotCatalog; @@ -35,8 +35,9 @@ pub struct PitrState { /// Chunk ids the kept bases and the bases in progress list. chunk_pins: tokio::sync::RwLock, last_failure: Mutex>, - /// The cut barriers the system catalog held at boot. - recorded_cuts: OnceLock>, + /// The cut barriers the system catalog held at boot, and those a + /// snapshot install carried since. + recorded_cuts: Arc, /// Each group's durable applied index, as this life saved it. durable_applied: Mutex>, } @@ -60,15 +61,14 @@ impl PitrState { self.cluster_catalog.get() } - /// Install the cut barriers read from the system catalog at boot. A - /// second install keeps the first. + /// Add the cut barriers read from the system catalog at boot. pub fn install_recorded_cuts(&self, cuts: Vec) { - let _ = self.recorded_cuts.set(cuts); + self.recorded_cuts.record(&cuts); } - /// The cut barriers the system catalog held at boot. - pub fn recorded_cuts(&self) -> &[RecordedCut] { - self.recorded_cuts.get().map_or(&[], Vec::as_slice) + /// The cut barriers the apply loop reads beyond the ones it applies. + pub fn recorded_cuts(&self) -> Arc { + Arc::clone(&self.recorded_cuts) } /// Note that `group_id`'s durable applied index reached `index`. diff --git a/nodedb/src/control/security/catalog/calvin_barrier_logs.rs b/nodedb/src/control/security/catalog/calvin_barrier_logs.rs index 9be335917..59b343901 100644 --- a/nodedb/src/control/security/catalog/calvin_barrier_logs.rs +++ b/nodedb/src/control/security/catalog/calvin_barrier_logs.rs @@ -9,12 +9,18 @@ //! applied floor, so an entry at or below it is never delivered again. Its //! row is what a barrier of the txn reads after a restart. //! +//! A fold commits without persisting: the row is visible at once, and a +//! crash rolls it back. [`SystemCatalog::persist_calvin_barrier_logs`] +//! persists every such fold in one commit. The apply loop runs it before it +//! saves a floor, so the floor never covers an entry whose row a crash loses. +//! //! Each row holds the encoded barrier log of one txn on one vShard. This //! module stores the bytes and never decodes them. use std::collections::BTreeSet; +use std::sync::atomic::Ordering; -use redb::{ReadableDatabase, ReadableTable, TableDefinition}; +use redb::{Durability, ReadableDatabase, ReadableTable, TableDefinition}; use super::types::{SystemCatalog, catalog_err}; @@ -59,6 +65,9 @@ impl SystemCatalog { /// Replace the row of `(vshard_id, epoch, position)` with what `fold` /// returns for the row it holds, in one transaction. A fold that /// returns `None` keeps the row, and nothing commits. + /// + /// The commit is visible at once and persists with the next + /// [`Self::persist_calvin_barrier_logs`] or any durable commit. pub fn fold_calvin_barrier_log( &self, vshard_id: u32, @@ -66,10 +75,13 @@ impl SystemCatalog { position: u32, fold: impl FnOnce(Option<&[u8]>) -> crate::Result>>, ) -> crate::Result<()> { - let write_txn = self + let mut write_txn = self .db .begin_write() .map_err(|e| catalog_err("fold_calvin_barrier_log txn", e))?; + write_txn + .set_durability(Durability::None) + .map_err(|e| catalog_err("fold_calvin_barrier_log durability", e))?; { let mut table = write_txn .open_table(CALVIN_BARRIER_LOGS) @@ -90,7 +102,50 @@ impl SystemCatalog { } write_txn .commit() - .map_err(|e| catalog_err("commit calvin_barrier_logs", e)) + .map_err(|e| catalog_err("commit calvin_barrier_logs", e))?; + // Set after the commit: a persist that clears the flag first + // persists this commit too. + self.calvin_barrier_logs_pending + .store(true, Ordering::Release); + Ok(()) + } + + /// Persist every barrier log fold committed since the last persist, in + /// one durable commit. With no such fold, nothing commits. + /// + /// On an error the folds stay pending, so the next persist retries. + pub fn persist_calvin_barrier_logs(&self) -> crate::Result<()> { + #[cfg(test)] + if self + .fail_next_calvin_barrier_persist + .swap(false, Ordering::SeqCst) + { + return Err(crate::Error::Storage { + engine: "catalog".into(), + detail: "injected calvin_barrier_logs persist failure".into(), + }); + } + if !self + .calvin_barrier_logs_pending + .swap(false, Ordering::AcqRel) + { + return Ok(()); + } + let persisted = self + .db + .begin_write() + .map_err(|e| catalog_err("persist_calvin_barrier_logs txn", e)) + .and_then(|mut txn| { + txn.set_durability(Durability::Immediate) + .map_err(|e| catalog_err("persist_calvin_barrier_logs durability", e))?; + txn.commit() + .map_err(|e| catalog_err("persist_calvin_barrier_logs commit", e)) + }); + if persisted.is_err() { + self.calvin_barrier_logs_pending + .store(true, Ordering::Release); + } + persisted } /// The row of `(vshard_id, epoch, position)`, when one is stored. @@ -200,6 +255,16 @@ impl SystemCatalog { .map_err(|e| catalog_err("insert calvin_barrier_logs", e))?; } } + #[cfg(test)] + if self + .fail_next_calvin_barrier_replace + .swap(false, Ordering::SeqCst) + { + return Err(crate::Error::Storage { + engine: "catalog".into(), + detail: "injected calvin_barrier_logs replace failure".into(), + }); + } write_txn .commit() .map_err(|e| catalog_err("commit calvin_barrier_logs", e)) @@ -280,6 +345,72 @@ mod tests { assert_eq!(only, vec![row(4, 2, 0, b"y")]); } + /// The rows a crash at this instant leaves: the file as it stands, + /// opened from a copy while the live catalog keeps its own open. + fn rows_after_crash(path: &std::path::Path, copy: &std::path::Path) -> Vec<(u32, u64, u32)> { + std::fs::copy(path, copy).expect("copy the catalog file"); + let db = redb::Database::create(copy).expect("open the copy"); + let txn = db.begin_read().expect("read txn"); + let Ok(table) = txn.open_table(CALVIN_BARRIER_LOGS) else { + return Vec::new(); + }; + table + .iter() + .expect("iter") + .map(|row| row.expect("row").0.value()) + .collect() + } + + /// A fold is visible at once, and a crash rolls it back. The persist + /// after it makes every fold durable, and a second persist commits + /// nothing. + #[test] + fn a_fold_persists_only_with_the_persist() { + let dir = tempfile::tempdir().expect("tempdir"); + let path = dir.path().join("system.redb"); + let catalog = SystemCatalog::open(&path).expect("catalog"); + put(&catalog, &row(3, 7, 1, b"a")); + put(&catalog, &row(4, 2, 0, b"b")); + assert_eq!( + catalog.load_calvin_barrier_log(3, 7, 1).expect("load"), + Some(b"a".to_vec()), + "a fold is visible to reads" + ); + assert!( + rows_after_crash(&path, &dir.path().join("crash-1.redb")).is_empty(), + "a crash rolls the folds back" + ); + + catalog.persist_calvin_barrier_logs().expect("persist"); + assert_eq!( + rows_after_crash(&path, &dir.path().join("crash-2.redb")), + vec![(3, 7, 1), (4, 2, 0)] + ); + assert!( + !catalog.calvin_barrier_logs_pending.load(Ordering::Acquire), + "nothing waits to persist" + ); + catalog.persist_calvin_barrier_logs().expect("persist"); + } + + /// A failed persist keeps the folds pending, and the next persist + /// makes them durable. + #[test] + fn a_failed_persist_keeps_the_folds_pending() { + let dir = tempfile::tempdir().expect("tempdir"); + let path = dir.path().join("system.redb"); + let catalog = SystemCatalog::open(&path).expect("catalog"); + put(&catalog, &row(3, 7, 1, b"a")); + catalog.fail_next_calvin_barrier_persist_for_test(); + assert!(catalog.persist_calvin_barrier_logs().is_err()); + assert!(rows_after_crash(&path, &dir.path().join("crash-1.redb")).is_empty()); + catalog.persist_calvin_barrier_logs().expect("persist"); + assert_eq!( + rows_after_crash(&path, &dir.path().join("crash-2.redb")), + vec![(3, 7, 1)] + ); + } + /// A replace drops every row of the named vShards and keeps the rest. #[test] fn a_replace_swaps_the_named_vshards_rows() { diff --git a/nodedb/src/control/security/catalog/cut_floors.rs b/nodedb/src/control/security/catalog/cut_floors.rs index d9808c9c5..71190d620 100644 --- a/nodedb/src/control/security/catalog/cut_floors.rs +++ b/nodedb/src/control/security/catalog/cut_floors.rs @@ -49,6 +49,18 @@ impl SystemCatalog { group_id: u64, barrier: StoredBarrier, durable_applied: u64, + ) -> crate::Result<()> { + self.put_cut_floors(group_id, &[barrier], durable_applied) + } + + /// Record every barrier of `barriers` group `group_id` holds, in one + /// write. The barriers at or below `durable_applied` fold as in + /// [`Self::put_cut_floor`]. + pub fn put_cut_floors( + &self, + group_id: u64, + barriers: &[StoredBarrier], + durable_applied: u64, ) -> crate::Result<()> { crate::fail_point_err!("cut_floor::before_persist", |detail: String| { crate::Error::Internal { @@ -70,8 +82,10 @@ impl SystemCatalog { Some(bytes) => decode(bytes.value())?, None => StoredFloors::default(), }; - floors.barriers.retain(|held| held.index != barrier.index); - floors.barriers.push(barrier); + for barrier in barriers { + floors.barriers.retain(|held| held.index != barrier.index); + floors.barriers.push(*barrier); + } floors.barriers.sort_by_key(|held| held.index); let settled = floors .barriers @@ -105,6 +119,26 @@ impl SystemCatalog { txn.commit().map_err(|e| catalog_err("cut floor commit", e)) } + /// Every recorded barrier of `group_id`, in index order. + pub fn group_cut_floors(&self, group_id: u64) -> crate::Result> { + let txn = self + .db + .begin_read() + .map_err(|e| catalog_err("cut floor read txn", e))?; + let table = match txn.open_table(CUT_FLOORS) { + Ok(table) => table, + Err(TableError::TableDoesNotExist(_)) => return Ok(Vec::new()), + Err(e) => return Err(catalog_err("open cut floors", e)), + }; + match table + .get(group_id) + .map_err(|e| catalog_err("get cut floors", e))? + { + Some(bytes) => Ok(decode(bytes.value())?.barriers), + None => Ok(Vec::new()), + } + } + /// Every recorded barrier, as `(group_id, barrier)`. pub fn load_cut_floors(&self) -> crate::Result> { let txn = self @@ -170,4 +204,27 @@ mod tests { ] ); } + + #[test] + fn a_group_takes_several_barriers_in_one_write() { + let dir = tempfile::tempdir().unwrap(); + let catalog = SystemCatalog::open(&dir.path().join("system.redb")).unwrap(); + assert!(catalog.group_cut_floors(3).unwrap().is_empty()); + catalog.put_cut_floor(3, barrier(40, 400), 0).unwrap(); + catalog + .put_cut_floors(3, &[barrier(20, 200), barrier(10, 100)], 0) + .unwrap(); + catalog.put_cut_floor(4, barrier(7, 70), 0).unwrap(); + assert_eq!( + catalog.group_cut_floors(3).unwrap(), + [barrier(10, 100), barrier(20, 200), barrier(40, 400)] + ); + + // Durable through 25 folds the two barriers below it. + catalog.put_cut_floors(3, &[barrier(10, 100)], 25).unwrap(); + assert_eq!( + catalog.group_cut_floors(3).unwrap(), + [barrier(20, 200), barrier(40, 400)] + ); + } } diff --git a/nodedb/src/control/security/catalog/system_catalog.rs b/nodedb/src/control/security/catalog/system_catalog.rs index e91e91ddf..e410a1029 100644 --- a/nodedb/src/control/security/catalog/system_catalog.rs +++ b/nodedb/src/control/security/catalog/system_catalog.rs @@ -24,6 +24,13 @@ pub struct SystemCatalog { /// Committed DEFINE EVENT definitions, for readers that must not read /// redb. pub(super) event_defs: Arc, + /// Whether a barrier log row committed without persisting since the + /// last `persist_calvin_barrier_logs`. + pub(super) calvin_barrier_logs_pending: Arc, + #[cfg(test)] + pub(super) fail_next_calvin_barrier_persist: Arc, + #[cfg(test)] + pub(super) fail_next_calvin_barrier_replace: Arc, #[cfg(test)] pub(super) fail_next_user_counter_write: Arc, #[cfg(test)] @@ -61,6 +68,11 @@ impl SystemCatalog { db: Arc::new(db), crdt_signing_root: Arc::new(std::sync::RwLock::new(None)), event_defs: Arc::new(super::event_defs_index::EventDefsIndex::new()), + calvin_barrier_logs_pending: Arc::new(std::sync::atomic::AtomicBool::new(false)), + #[cfg(test)] + fail_next_calvin_barrier_persist: Arc::new(std::sync::atomic::AtomicBool::new(false)), + #[cfg(test)] + fail_next_calvin_barrier_replace: Arc::new(std::sync::atomic::AtomicBool::new(false)), #[cfg(test)] fail_next_user_counter_write: Arc::new(std::sync::atomic::AtomicBool::new(false)), #[cfg(test)] @@ -89,6 +101,11 @@ impl SystemCatalog { db: Arc::new(db), crdt_signing_root: Arc::new(std::sync::RwLock::new(None)), event_defs: Arc::new(super::event_defs_index::EventDefsIndex::new()), + calvin_barrier_logs_pending: Arc::new(std::sync::atomic::AtomicBool::new(false)), + #[cfg(test)] + fail_next_calvin_barrier_persist: Arc::new(std::sync::atomic::AtomicBool::new(false)), + #[cfg(test)] + fail_next_calvin_barrier_replace: Arc::new(std::sync::atomic::AtomicBool::new(false)), #[cfg(test)] fail_next_user_counter_write: Arc::new(std::sync::atomic::AtomicBool::new(false)), #[cfg(test)] @@ -140,6 +157,21 @@ impl SystemCatalog { .store(true, std::sync::atomic::Ordering::SeqCst); } + /// Make the next `persist_calvin_barrier_logs` raise. The pending rows + /// stay pending. + #[cfg(test)] + pub(crate) fn fail_next_calvin_barrier_persist_for_test(&self) { + self.fail_next_calvin_barrier_persist + .store(true, std::sync::atomic::Ordering::SeqCst); + } + + /// Make the next `replace_calvin_barrier_logs` raise before it commits. + #[cfg(test)] + pub(crate) fn fail_next_calvin_barrier_replace_for_test(&self) { + self.fail_next_calvin_barrier_replace + .store(true, std::sync::atomic::Ordering::SeqCst); + } + /// Bootstrap every `_system.*` table from the canonical registry — /// but only if at least one is actually missing. Probing read-only /// first keeps `open` byte-idempotent on an already-bootstrapped diff --git a/nodedb/src/control/state/calvin_bases.rs b/nodedb/src/control/state/calvin_bases.rs index 3c1fe7269..d03939e0b 100644 --- a/nodedb/src/control/state/calvin_bases.rs +++ b/nodedb/src/control/state/calvin_bases.rs @@ -133,6 +133,11 @@ impl CalvinBases { reaches } + /// Every vShard that holds a base here. + pub fn vshards(&self) -> BTreeSet { + self.state().bases.keys().copied().collect() + } + /// Stop waiting for every vShard not in `mounted`, the vShards of the /// data groups this node mounts: an unmounted group starts no scheduler. pub fn retain_waiting(&self, mounted: &BTreeSet) { diff --git a/nodedb/src/control/state/calvin_local.rs b/nodedb/src/control/state/calvin_local.rs index c612b64e3..0914cd96c 100644 --- a/nodedb/src/control/state/calvin_local.rs +++ b/nodedb/src/control/state/calvin_local.rs @@ -3,7 +3,7 @@ //! This node's in-process Calvin state, shared between the per-vShard //! schedulers and the Control-Plane paths that read or fence against them. -use std::collections::BTreeMap; +use std::collections::{BTreeMap, BTreeSet}; use std::sync::atomic::{AtomicU32, AtomicU64}; use std::sync::{Arc, Mutex, OnceLock}; @@ -102,6 +102,9 @@ pub struct CalvinLocalState { /// scheduler: read results and timeouts the data-group apply loop /// applied before the scheduler took them. pub read_results: CalvinReadResults, + /// vShards whose stored Calvin state waits for a forget. Each scheduler + /// reconcile forgets those this node does not host and drops the rest. + pub forget_pending: Mutex>, } impl CalvinLocalState { @@ -131,6 +134,7 @@ impl CalvinLocalState { caught_up: CaughtUpRegistry::default(), inboxes: CalvinInboxes::default(), read_results: CalvinReadResults::default(), + forget_pending: Mutex::new(BTreeSet::new()), } } } diff --git a/nodedb/src/control/state/fields.rs b/nodedb/src/control/state/fields.rs index 8e93257ec..5b4622932 100644 --- a/nodedb/src/control/state/fields.rs +++ b/nodedb/src/control/state/fields.rs @@ -168,6 +168,10 @@ pub struct SharedState { /// Never call it while holding a `cluster_routing` guard. pub raft_status_fn: std::sync::OnceLock Vec + Send + Sync>>, + /// Set once boot recovery saw every mounted data group apply its + /// committed log here. From then on the routing table names every group + /// this node replicates. + pub data_groups_replayed: std::sync::atomic::AtomicBool, /// Cluster observability handle. Set once by `start_raft`. pub cluster_observer: OnceLock>, /// Registry of standardized per-loop metrics. Populated by `start_raft`. diff --git a/nodedb/src/control/state/init.rs b/nodedb/src/control/state/init.rs index 6b20f3f09..a44590b75 100644 --- a/nodedb/src/control/state/init.rs +++ b/nodedb/src/control/state/init.rs @@ -157,6 +157,7 @@ impl SharedState { multi_raft: std::sync::OnceLock::new(), cluster_epoch: std::sync::OnceLock::new(), raft_status_fn: std::sync::OnceLock::new(), + data_groups_replayed: std::sync::atomic::AtomicBool::new(false), cluster_observer: std::sync::OnceLock::new(), loop_metrics_registry: nodedb_cluster::LoopMetricsRegistry::new(), per_vshard_metrics: crate::control::metrics::PerVShardMetricsRegistry::new(), diff --git a/nodedb/src/control/state/init_prod/open.rs b/nodedb/src/control/state/init_prod/open.rs index d418778b0..a4ab66d8f 100644 --- a/nodedb/src/control/state/init_prod/open.rs +++ b/nodedb/src/control/state/init_prod/open.rs @@ -228,6 +228,7 @@ impl SharedState { multi_raft: std::sync::OnceLock::new(), cluster_epoch: std::sync::OnceLock::new(), raft_status_fn: std::sync::OnceLock::new(), + data_groups_replayed: std::sync::atomic::AtomicBool::new(false), cluster_observer: std::sync::OnceLock::new(), loop_metrics_registry: nodedb_cluster::LoopMetricsRegistry::new(), per_vshard_metrics: crate::control::metrics::PerVShardMetricsRegistry::new(), diff --git a/nodedb/src/diag/context/mod.rs b/nodedb/src/diag/context/mod.rs index a1e6b2c64..db4048e4d 100644 --- a/nodedb/src/diag/context/mod.rs +++ b/nodedb/src/diag/context/mod.rs @@ -55,7 +55,9 @@ pub(in crate::diag) use quota::{ QuotaRowNotInstalled, QuotaRowWriteFailed, QuotaScopePurgeIncomplete, QuotaScopeReplayAborted, ScopeQuotaNotInstalled, }; -pub(in crate::diag) use raft_apply::{CalvinBarrierLogStoreFailed, RaftEntryReapplied}; +pub(in crate::diag) use raft_apply::{ + CalvinBarrierLogStoreFailed, CalvinLeftStateNotForgotten, RaftEntryReapplied, +}; pub(in crate::diag) use recovery::{ReplayRecordUnapplied, WalArchivalFailedTruncationHeld}; pub(in crate::diag) use redo_stream::{ RedoAbandonGivenUp, RedoSnapshotDebtNotRecorded, RedoStreamLostHere, diff --git a/nodedb/src/diag/context/raft_apply.rs b/nodedb/src/diag/context/raft_apply.rs index 13178ca52..25b8088b9 100644 --- a/nodedb/src/diag/context/raft_apply.rs +++ b/nodedb/src/diag/context/raft_apply.rs @@ -41,14 +41,15 @@ impl DomainContext for RaftEntryReapplied { } } -/// The catalog refused a read or write of a vShard's stored dependent-read -/// barrier log. +/// The catalog refused a read or write of the stored dependent-read +/// barrier logs. pub(in crate::diag) struct CalvinBarrierLogStoreFailed<'a> { - pub vshard_id: u32, - /// The txn's `(epoch, position)`. A removal of every row of the vShard + /// The vShard. A persist of every vShard's rows names none. + pub vshard_id: Option, + /// The txn's `(epoch, position)`. An operation on every row of a vShard /// names none. pub txn: Option<(u64, u32)>, - /// What the store did: `save`, `load` or `remove`. + /// What the store did: `save`, `load`, `remove`, `forget` or `persist`. pub op: &'static str, pub error_class: &'a str, } @@ -59,11 +60,12 @@ impl DomainContext for CalvinBarrierLogStoreFailed<'_> { } fn grouping_key(&self) -> String { - // One report per vShard and operation: the txn changes per event. - format!( - "calvin_barrier_log_store_failed:{}:{}", - self.op, self.vshard_id - ) + // One report per vShard and operation: the txn changes per event, + // and a retry loop repeats the same pair. + match self.vshard_id { + Some(vshard_id) => format!("calvin_barrier_log_store_failed:{}:{vshard_id}", self.op), + None => format!("calvin_barrier_log_store_failed:{}:all", self.op), + } } fn to_json(&self) -> Value { @@ -77,6 +79,15 @@ impl DomainContext for CalvinBarrierLogStoreFailed<'_> { "the scheduler keeps the txn's barrier open and reads the row \ again on its next pass" } + "forget" => { + "the rows of a vShard this node left stay stored. Every scheduler \ + reconcile pass removes them again until one succeeds" + } + "persist" => { + "the saved barrier rows are not durable yet. No applied floor moves \ + this pass, so a restart delivers their entries again. The next \ + settle pass persists again" + } _ => { "the row of a finished txn stays stored. The scheduler's next \ stall-tick sweep removes it" @@ -93,3 +104,34 @@ impl DomainContext for CalvinBarrierLogStoreFailed<'_> { }) } } + +/// The catalog refused the forget of the stored Calvin state of vShards +/// this node left: their applied positions and bases. +pub(in crate::diag) struct CalvinLeftStateNotForgotten<'a> { + pub vshards: &'a [u32], + pub error_class: &'a str, +} + +impl DomainContext for CalvinLeftStateNotForgotten<'_> { + fn domain_kind(&self) -> &'static str { + "nodedb.calvin_left_state_not_forgotten" + } + + fn grouping_key(&self) -> String { + // One root cause: the catalog refuses the write. The vShards are the + // occurrence, and a retry loop repeats them. + "calvin_left_state_not_forgotten".to_string() + } + + fn to_json(&self) -> Value { + json!({ + "vshards": self.vshards, + "error_class": self.error_class, + "effect": "the applied positions and base of a vShard this node left stay \ + stored. Every scheduler reconcile pass forgets them again until one \ + succeeds", + "operator_action": "check the system catalog's disk: free space, I/O errors, \ + and the redb file's permissions", + }) + } +} diff --git a/nodedb/src/diag/mod.rs b/nodedb/src/diag/mod.rs index cebbc6ede..7cb9e4d4c 100644 --- a/nodedb/src/diag/mod.rs +++ b/nodedb/src/diag/mod.rs @@ -12,8 +12,8 @@ pub use context::{DATABASE_SCOPE, IlpFlushOutcome, LostResponseWrite, TENANT_SCO pub use recording::{ IndexRebuildTarget, VectorBuildTarget, apply_panicked, batch_insert_without_surrogates, calvin_apply_halted, calvin_barrier_log_store_failed, calvin_completion_timeout, - catalog_apply_orphan_row, collection_purge_row_missing, columnar_segment_corrupt, - consumer_group_offsets_retained, continuous_aggregate_not_applied, + calvin_left_state_not_forgotten, catalog_apply_orphan_row, collection_purge_row_missing, + columnar_segment_corrupt, consumer_group_offsets_retained, continuous_aggregate_not_applied, crdt_dead_letter_not_enqueued, crdt_dead_letter_not_restored, crdt_dead_letter_not_stored, cut_barrier_not_placed, data_plane_core_fail_stopped, data_plane_response_lost, data_plane_responses_lost, descriptor_lease_not_renewed, entry_kind, fts_index_update_failed, diff --git a/nodedb/src/diag/recording/mod.rs b/nodedb/src/diag/recording/mod.rs index 8b8ba65c1..b3c64193b 100644 --- a/nodedb/src/diag/recording/mod.rs +++ b/nodedb/src/diag/recording/mod.rs @@ -55,7 +55,8 @@ pub use quota::{ quota_scope_replay_aborted, scope_quota_not_installed, }; pub use raft_apply::{ - calvin_barrier_log_store_failed, raft_entries_reapplied, raft_entry_reapplied, + calvin_barrier_log_store_failed, calvin_left_state_not_forgotten, raft_entries_reapplied, + raft_entry_reapplied, }; pub use recovery::{ batch_insert_without_surrogates, fts_index_update_failed, orphaned_index_entry_after_delete, diff --git a/nodedb/src/diag/recording/raft_apply.rs b/nodedb/src/diag/recording/raft_apply.rs index 8799ddacf..1b7b8510c 100644 --- a/nodedb/src/diag/recording/raft_apply.rs +++ b/nodedb/src/diag/recording/raft_apply.rs @@ -37,10 +37,11 @@ pub fn raft_entry_reapplied(group_id: u64, log_index: u64, highest_applied: u64) .emit(); } -/// Report a catalog error on a vShard's stored dependent-read barrier log. -/// Called from the sites that read and write the rows. `op` names the operation: `save`, `load` or `remove`. +/// Report a catalog error on the stored dependent-read barrier logs. +/// Called from the sites that read and write the rows. `op` names the +/// operation: `save`, `load`, `remove`, `forget` or `persist`. pub fn calvin_barrier_log_store_failed( - vshard_id: u32, + vshard_id: Option, txn: Option<(u64, u32)>, op: &'static str, err: &crate::Error, @@ -60,3 +61,20 @@ pub fn calvin_barrier_log_store_failed( .domain(&ctx) .emit(); } + +/// Report a catalog error that left the stored Calvin state of vShards this +/// node left. Called from `calvin_snapshot::forget_left`. +pub fn calvin_left_state_not_forgotten(vshards: &[u32], err: &crate::Error) { + let class = error_class(err); + let ctx = context::CalvinLeftStateNotForgotten { + vshards, + error_class: &class, + }; + let _ = Capture::new( + EventKind::Error, + "calvin state of a left vShard was not forgotten", + ) + .error_chain(error_chain_of(err)) + .domain(&ctx) + .emit(); +} diff --git a/nodedb/src/types/mod.rs b/nodedb/src/types/mod.rs index 22ba096bb..8bfadda76 100644 --- a/nodedb/src/types/mod.rs +++ b/nodedb/src/types/mod.rs @@ -22,4 +22,6 @@ pub use snapshot::{ ArrayCellsBlob, SurrogateBindEntry, TenantDataSnapshot, TsFlushedCollectionBlob, TsFlushedPartitionBlob, }; -pub use snapshot_calvin::{CutBarrierLog, GroupCalvinCut, VShardCalvinState}; +pub use snapshot_calvin::{ + CoveredCutBarrier, CutBarrierLog, CutFloor, GroupCalvinCut, VShardCalvinState, +}; diff --git a/nodedb/src/types/snapshot_calvin.rs b/nodedb/src/types/snapshot_calvin.rs index 585278355..0548753ec 100644 --- a/nodedb/src/types/snapshot_calvin.rs +++ b/nodedb/src/types/snapshot_calvin.rs @@ -37,6 +37,56 @@ pub struct GroupCalvinCut { /// group vShard. It holds every barrier entry of the group's log at or /// below the snapshot's index, which the receiver never applies. pub barrier_logs: Vec, + /// Every backup cut barrier the builder applied in the group at or below + /// the snapshot's index whose cut window is open, in cut key order. The + /// receiver never applies them, and a cut waiting there needs them. + pub cut_barriers: Vec, + /// The group's durable cut floors at or below the snapshot's index, in + /// index order. Every later entry records a commit HLC above them. + pub cut_floors: Vec, +} + +/// A backup cut's barrier a [`GroupCalvinCut`] covers. +#[derive( + Debug, + Clone, + PartialEq, + Eq, + serde::Serialize, + serde::Deserialize, + zerompk::ToMessagePack, + zerompk::FromMessagePack, +)] +#[msgpack(map)] +pub struct CoveredCutBarrier { + /// The cut's watermark: HLC wall time in nanoseconds. + pub hlc: u64, + /// The cluster restore point the cut takes, `0` for a backup's cut. + pub restore_point: u64, + /// A database backup's capture request. + pub capture: Option, + /// The log index of the cut's first barrier in the group. + pub log_index: u64, +} + +/// One durable cut floor of a group in a [`GroupCalvinCut`]. +#[derive( + Debug, + Clone, + Copy, + PartialEq, + Eq, + serde::Serialize, + serde::Deserialize, + zerompk::ToMessagePack, + zerompk::FromMessagePack, +)] +#[msgpack(map)] +pub struct CutFloor { + /// The barrier's log index. + pub index: u64, + /// The barrier's watermark: HLC wall time in nanoseconds. + pub watermark: u64, } /// One vShard's applied Calvin positions in a [`GroupCalvinCut`]. From 6efa1dab1bf45ff9f7cefdfc8efed407d413119d Mon Sep 17 00:00:00 2001 From: Farhan Syah Date: Sun, 11 Oct 2026 21:31:08 +0800 Subject: [PATCH 5/5] fix(cluster): tie peer liveness to the RPC timeout and carry one deadline through CRDT and gateway paths The QUIC idle timeout is now the RPC timeout, with keep-alives at a fifth of it, so a silent peer closes its connection and fails every pending read on it, while a live peer holds a slow handler open. The separate keep-alive and idle-timeout tunables are removed, and startup refuses an RPC timeout below 500 ms. The transport endpoint moves into its own module. A build with the failpoints feature can blackhole a node's datagrams to simulate a partition. CRDT admission becomes a directory module and, with the DDL, sync, Raft dispatch and gateway paths, takes a statement deadline instead of a fixed timeout. Gateway retries are reworked around that deadline, so the per-route attempts-left parameter goes away. The cluster test harness gains a convergence wait and the cluster cases are updated for the new liveness and deadline behaviour. --- .../cases/cluster_pitr_restore_point.rs | 47 +- .../cases/gateway_hung_node_budget.rs | 250 +++- .../cases/listeners_typed_not_leader.rs | 6 +- .../routing_redirect_hint_unhosted_group.rs | 31 +- .../cases/sync_constraint_version_fence.rs | 20 +- .../cases/timeseries_ring_drop_catch_up.rs | 57 +- .../common_suite/cases/ts_native_ingest.rs | 21 +- nodedb-cluster/Cargo.toml | 5 +- nodedb-cluster/src/routing.rs | 6 +- nodedb-cluster/src/rpc_codec/data_propose.rs | 111 -- nodedb-cluster/src/rpc_codec/mod.rs | 2 +- .../src/transport/client/attempt.rs | 176 ++- nodedb-cluster/src/transport/client/pool.rs | 126 +- nodedb-cluster/src/transport/client/send.rs | 21 +- .../src/transport/client/transport.rs | 73 +- nodedb-cluster/src/transport/config.rs | 37 +- nodedb-cluster/src/transport/endpoint.rs | 169 +++ nodedb-cluster/src/transport/liveness.rs | 104 ++ nodedb-cluster/src/transport/mod.rs | 2 + nodedb-cluster/src/transport/server.rs | 3 + .../cluster_harness/cluster/convergence.rs | 169 +++ .../src/cluster_harness/cluster/membership.rs | 133 +- .../src/cluster_harness/cluster/mod.rs | 5 +- .../src/cluster_harness/mod.rs | 2 +- .../src/native_harness/frames.rs | 34 +- nodedb-test-support/src/native_harness/mod.rs | 4 +- nodedb-types/src/config/tuning/network.rs | 15 +- nodedb/src/config/server/config.rs | 18 + nodedb/src/config/server/domain.rs | 16 + .../src/control/array_sync/inbound_propose.rs | 5 +- nodedb/src/control/backup/cut.rs | 16 +- nodedb/src/control/backup/restore/guard.rs | 4 +- .../cluster/array_cluster_exec/dispatch.rs | 23 +- .../cluster/start_raft/proposer_wiring.rs | 8 +- nodedb/src/control/crdt_admission.rs | 1333 ----------------- nodedb/src/control/crdt_admission/apply.rs | 680 +++++++++ nodedb/src/control/crdt_admission/classify.rs | 165 ++ nodedb/src/control/crdt_admission/fence.rs | 106 ++ nodedb/src/control/crdt_admission/legs.rs | 98 ++ nodedb/src/control/crdt_admission/mod.rs | 25 + nodedb/src/control/crdt_admission/policy.rs | 18 + nodedb/src/control/crdt_admission/request.rs | 73 + nodedb/src/control/crdt_admission/restore.rs | 201 +++ .../control/crdt_admission/test_support.rs | 149 ++ nodedb/src/control/crdt_admission/workflow.rs | 129 ++ nodedb/src/control/fail_gate.rs | 11 +- nodedb/src/control/gateway/cluster_error.rs | 152 +- nodedb/src/control/gateway/core.rs | 8 +- nodedb/src/control/gateway/dispatch_remote.rs | 48 +- .../control/gateway/dispatch_remote_stream.rs | 79 +- nodedb/src/control/gateway/dispatcher.rs | 9 - nodedb/src/control/gateway/retry.rs | 348 +++-- nodedb/src/control/gateway/stream.rs | 6 +- .../clone_materializer/dispatch.rs | 9 +- .../src/control/metadata_proposer/catalog.rs | 2 + .../metadata_proposer/catalog_batch.rs | 2 + .../planner/calvin/dependent_recon_finish.rs | 7 +- .../control/planner/calvin/submit/local.rs | 7 +- .../control/security/auth_lease/barrier.rs | 24 +- .../server/exchange/all_cores/read_cut.rs | 12 +- .../server/graph_dispatch/cluster_resolve.rs | 1 - .../src/control/server/graph_dispatch/hop.rs | 1 - .../server/graph_dispatch/keyed_read.rs | 1 - .../match_scatter/round_loop.rs | 1 - .../match_scatter/round_zero.rs | 1 - .../graph_dispatch/rag_fusion/stages.rs | 1 - nodedb/src/control/server/http/routes/crdt.rs | 4 +- .../server/native/dispatch/raw_dispatch.rs | 4 +- .../server/shared/clone_write/probes.rs | 9 +- .../control/server/shared/ddl/engine_apply.rs | 13 +- .../shared/ddl/neutral/convert/driver.rs | 3 +- .../server/shared/ddl/neutral/crdt_ops.rs | 7 +- .../shared/ddl/neutral/dsl/crdt_merge.rs | 7 +- .../ddl/neutral/graph_ops/rag_fusion.rs | 5 +- .../server/shared/ddl/neutral/last_value.rs | 5 +- .../shared/ddl/neutral/permission_tree.rs | 7 +- .../ddl/neutral/version_history/checkpoint.rs | 10 +- .../ddl/neutral/version_history/dispatch.rs | 7 +- .../ddl/neutral/version_history/restore.rs | 13 +- .../shared/ddl/sync_dispatch/dispatch.rs | 37 +- .../server/shared/ddl/sync_dispatch/mod.rs | 2 +- .../server/shared/ddl/user_dispatch.rs | 20 +- .../shared/session/array_fanout_stage.rs | 6 +- .../shared/session/commit/read_validation.rs | 1 - .../server/shared/session/commit/run.rs | 6 +- .../shared/session/ddl_authorization.rs | 1 + .../control/server/shared/session/deadline.rs | 5 + .../server/shared/session/leader_forward.rs | 12 +- .../server/shared/session/overlay_drop.rs | 10 +- .../server/shared/session/staging_gate.rs | 3 +- .../server/sync/async_dispatch/delta/apply.rs | 7 +- .../sync/async_dispatch/shape/snapshot.rs | 6 +- .../server/sync/raft_dispatch/propose.rs | 14 +- .../server/sync/raft_dispatch/response.rs | 10 +- .../server/sync/raft_dispatch/write.rs | 36 +- .../control/server/sync/timeseries_handler.rs | 1 + .../inproc/cases/system_task_call_sites.rs | 2 +- 97 files changed, 3460 insertions(+), 2239 deletions(-) create mode 100644 nodedb-cluster/src/transport/endpoint.rs create mode 100644 nodedb-cluster/src/transport/liveness.rs create mode 100644 nodedb-test-support/src/cluster_harness/cluster/convergence.rs delete mode 100644 nodedb/src/control/crdt_admission.rs create mode 100644 nodedb/src/control/crdt_admission/apply.rs create mode 100644 nodedb/src/control/crdt_admission/classify.rs create mode 100644 nodedb/src/control/crdt_admission/fence.rs create mode 100644 nodedb/src/control/crdt_admission/legs.rs create mode 100644 nodedb/src/control/crdt_admission/mod.rs create mode 100644 nodedb/src/control/crdt_admission/policy.rs create mode 100644 nodedb/src/control/crdt_admission/request.rs create mode 100644 nodedb/src/control/crdt_admission/restore.rs create mode 100644 nodedb/src/control/crdt_admission/test_support.rs create mode 100644 nodedb/src/control/crdt_admission/workflow.rs diff --git a/nodedb-cluster-tests/tests/common_suite/cases/cluster_pitr_restore_point.rs b/nodedb-cluster-tests/tests/common_suite/cases/cluster_pitr_restore_point.rs index 5a9f7fbe5..aebdb954f 100644 --- a/nodedb-cluster-tests/tests/common_suite/cases/cluster_pitr_restore_point.rs +++ b/nodedb-cluster-tests/tests/common_suite/cases/cluster_pitr_restore_point.rs @@ -44,14 +44,27 @@ async fn insert_rows(cluster: &TestCluster, prefix: &str) { for collection in COLLECTIONS { for i in 0..3 { let sql = format!("INSERT INTO {collection} (id, v) VALUES ('{prefix}-{i}', {i})"); - cluster.nodes[0] - .exec(&sql) - .await - .unwrap_or_else(|e| panic!("{sql}: {e}")); + write_on(cluster, 0, collection, &sql).await; } } } +/// Run the write `sql` into `collection` on node `index`. On error, panic +/// with the collection's group and every node's group positions, so the +/// failing hop shows. +async fn write_on(cluster: &TestCluster, index: usize, collection: &str, sql: &str) { + let node = &cluster.nodes[index]; + if let Err(error) = node.exec(sql).await { + panic!( + "{sql} on node {}: {error}\n{collection} is in group {:?}\n\ + groups (leader, commit, applied per node):\n{}", + node.node_id, + node.group_id_for_collection(collection), + cluster.group_positions_report() + ); + } +} + fn ids(prefixes: &[&str]) -> Vec { let mut ids: Vec = prefixes .iter() @@ -104,10 +117,12 @@ async fn create_collection(cluster: &TestCluster, collection: &str) { "CREATE COLLECTION {collection} (id STRING PRIMARY KEY, v INT) \ WITH (engine='document_strict')" ); - cluster - .exec_ddl_on_any_leader(&sql) - .await - .unwrap_or_else(|e| panic!("{sql}: {e}")); + if let Err(error) = cluster.exec_ddl_on_any_leader(&sql).await { + panic!( + "{sql}: {error}\ngroups (leader, commit, applied per node):\n{}", + cluster.group_positions_report() + ); + } } /// Wait until every node archived `point`, stop every node, restore each to @@ -185,10 +200,13 @@ async fn a_cluster_restores_every_group_to_a_restore_point() { ); } - cluster.nodes[1] - .exec("INSERT INTO pitr_a (id, v) VALUES ('after-restore', 9)") - .await - .unwrap_or_else(|e| panic!("a write after the restore: {e}")); + write_on( + &cluster, + 1, + "pitr_a", + "INSERT INTO pitr_a (id, v) VALUES ('after-restore', 9)", + ) + .await; cluster .wait_for_full_apply_convergence(Duration::from_secs(30)) .await; @@ -263,10 +281,7 @@ async fn a_ddl_after_the_watermark_is_absent_though_it_precedes_the_point() { create_collection(&cluster, "pitr_mid").await; for i in 0..3 { let sql = format!("INSERT INTO pitr_mid (id, v) VALUES ('mid-{i}', {i})"); - cluster.nodes[0] - .exec(&sql) - .await - .unwrap_or_else(|e| panic!("{sql}: {e}")); + write_on(&cluster, 0, "pitr_mid", &sql).await; } assert!( !creator.is_finished(), diff --git a/nodedb-cluster-tests/tests/common_suite/cases/gateway_hung_node_budget.rs b/nodedb-cluster-tests/tests/common_suite/cases/gateway_hung_node_budget.rs index 38e87b38b..23fbd6343 100644 --- a/nodedb-cluster-tests/tests/common_suite/cases/gateway_hung_node_budget.rs +++ b/nodedb-cluster-tests/tests/common_suite/cases/gateway_hung_node_budget.rs @@ -1,26 +1,28 @@ // SPDX-License-Identifier: BUSL-1.1 -//! A node that takes a forwarded request and never answers costs a statement -//! at most its one budget. +//! A live slow node gets a statement's whole budget. A dead node costs it +//! about one RPC timeout. //! -//! The execute gate `exec::before_execute` parks every request that the -//! leader of the collection's group receives. A follower forwards its -//! statements on the collection to that leader. +//! A follower forwards its statements on the collection to the leader of the +//! collection's group. The leader's execute gate `exec::before_execute` +//! slows or parks the requests it receives: //! -//! - A read waits only its first attempt's share of the budget, then -//! retries. The test releases the gate once the retry starts, and the -//! read returns its row inside the statement timeout. -//! - A write waits its whole budget, then reports an unknown outcome -//! (`57014`). It is never sent again. +//! - A slow leader keeps its connection alive, so a read waits for it and +//! returns its row with no retry. +//! - A parked leader never answers. A read and a write each wait their whole +//! budget and end `57014`, with no retry. +//! - A leader cut off by `transport::blackhole` goes silent. Its connection +//! hits the QUIC idle timeout, the read retries on the new leader, and it +//! returns its row within two RPC timeouts. //! //! Requires `--features failpoints`. #![cfg(feature = "failpoints")] -use std::path::Path; use std::time::{Duration, Instant}; use nodedb_cluster::rpc_codec::REPLY_MARGIN; +use nodedb_cluster::transport::endpoint::BLACKHOLE_FAIL_POINT; use nodedb_test_support::fail_point::{FailAction, FailGuard}; use nodedb_types::id::DatabaseId; @@ -29,37 +31,29 @@ use crate::common::cluster_harness::{GroupLeader, TestCluster, TestClusterNode, const COLLECTION: &str = "hung_exec"; const GATE: &str = "exec::before_execute"; -/// Every node's fixed RPC timeout. A read reserves this much plus the reply -/// margin for each attempt after its first. -const RPC_TIMEOUT_SECS: u64 = 3; -/// The read's statement timeout. Its first attempt gets -/// max(12 - 2 × (3 + 2), 3) = 3 s and waits 5 s with the reply margin, -/// which leaves the retry time to answer. -const READ_TIMEOUT: Duration = Duration::from_secs(12); -/// The write's statement timeout. A write waits all of it. -const WRITE_TIMEOUT: Duration = Duration::from_secs(4); -/// How often the test checks the follower's retry count. -const RETRY_POLL: Duration = Duration::from_millis(10); - -/// Release the gate at `release` once `node`'s gateway retries past -/// `retries_before`. Gives up after `give_up`, so a read that never retries -/// fails its own assertions instead of hanging the test. -async fn release_on_retry( - node: &TestClusterNode, - retries_before: u64, - release: &Path, - give_up: Duration, -) { - let started = Instant::now(); - while node.not_leader_retry_count() <= retries_before && started.elapsed() < give_up { - tokio::time::sleep(RETRY_POLL).await; - } - std::fs::write(release, b"release").expect("release the execute gate"); +/// Every node's RPC timeout, and so its QUIC idle timeout. +const RPC_TIMEOUT: Duration = Duration::from_secs(3); +/// How long a slow leader holds each request it receives. +const SLOW_EXECUTE: Duration = Duration::from_secs(8); +/// The statement timeout of a read sent to a slow leader. +const SLOW_READ_TIMEOUT: Duration = Duration::from_secs(12); +/// The statement timeout of a request sent to a parked leader. +const PARKED_TIMEOUT: Duration = Duration::from_secs(4); +/// The statement timeout of a read sent to a silent leader. +const SILENT_READ_TIMEOUT: Duration = Duration::from_secs(20); +/// Scheduling slack on the upper bounds. +const SLACK: Duration = Duration::from_secs(2); + +/// A 3-node cluster with one seeded row, the collection's group leader, and +/// a follower that forwards to it. +struct Setup { + cluster: TestCluster, + leader_id: u64, + follower: usize, } -#[tokio::test(flavor = "multi_thread", worker_threads = 6)] -async fn a_silent_leader_costs_a_read_one_retry_and_a_write_its_budget() { - let cluster = TestCluster::spawn_three_with_rpc_timeout(RPC_TIMEOUT_SECS) +async fn setup() -> Setup { + let cluster = TestCluster::spawn_three_with_rpc_timeout(RPC_TIMEOUT.as_secs()) .await .expect("3-node cluster"); cluster @@ -102,7 +96,6 @@ async fn a_silent_leader_costs_a_read_one_retry_and_a_write_its_budget() { .iter() .position(|n| n.node_id == leader_id) .expect("the group's leader is a cluster node"); - let follower = &cluster.nodes[(leader + 1) % 3]; cluster.nodes[leader] .client @@ -114,16 +107,85 @@ async fn a_silent_leader_costs_a_read_one_retry_and_a_write_its_budget() { cluster .wait_for_full_apply_convergence(Duration::from_secs(20)) .await; + Setup { + cluster, + leader_id, + follower: (leader + 1) % 3, + } +} - // A read: its first attempt times out, its retry answers. - follower - .client +async fn set_statement_timeout(node: &TestClusterNode, timeout: Duration) { + node.client .simple_query(&format!( - "SET statement_timeout = '{}s'", - READ_TIMEOUT.as_secs() + "SET statement_timeout = '{}ms'", + timeout.as_millis() )) .await - .expect("SET statement_timeout for the read"); + .expect("SET statement_timeout"); +} + +fn read_sql() -> String { + format!("SELECT id, val FROM {COLLECTION} WHERE id = 'k1'") +} + +fn row_count(messages: Vec) -> usize { + messages + .into_iter() + .filter(|m| matches!(m, tokio_postgres::SimpleQueryMessage::Row(_))) + .count() +} + +fn sqlstate(err: &tokio_postgres::Error) -> Option<&str> { + err.as_db_error().map(|db| db.code().code()) +} + +#[tokio::test(flavor = "multi_thread", worker_threads = 6)] +async fn a_slow_leader_gets_a_read_its_whole_budget() { + let Setup { + cluster, + leader_id, + follower, + } = setup().await; + let follower = &cluster.nodes[follower]; + set_statement_timeout(follower, SLOW_READ_TIMEOUT).await; + + let gate = FailGuard::for_node(leader_id, GATE, FailAction::Sleep(SLOW_EXECUTE)); + let retries_before = follower.not_leader_retry_count(); + let started = Instant::now(); + let read = follower.client.simple_query(&read_sql()).await; + let elapsed = started.elapsed(); + drop(gate); + + let rows = read.unwrap_or_else(|e| panic!("the slow leader answers: {}", db_detail(&e))); + assert_eq!(row_count(rows), 1, "the read returns the seeded row"); + assert_eq!( + follower.not_leader_retry_count(), + retries_before, + "a live slow leader is never retried" + ); + assert!( + elapsed >= SLOW_EXECUTE, + "the read returned in {elapsed:?}, before the leader ran it" + ); + assert!( + elapsed < SLOW_READ_TIMEOUT, + "the read finished in {elapsed:?}, past its {SLOW_READ_TIMEOUT:?} budget" + ); + + cluster.shutdown().await; +} + +#[tokio::test(flavor = "multi_thread", worker_threads = 6)] +async fn a_parked_leader_costs_a_read_its_budget() { + let Setup { + cluster, + leader_id, + follower, + } = setup().await; + let follower = &cluster.nodes[follower]; + set_statement_timeout(follower, PARKED_TIMEOUT).await; + + // A read: the live leader holds it, so it waits its whole budget. let read_dir = tempfile::tempdir().expect("read gate directory"); let read_release = read_dir.path().join("release"); let read_gate = FailGuard::for_node( @@ -132,43 +194,38 @@ async fn a_silent_leader_costs_a_read_one_retry_and_a_write_its_budget() { FailAction::WaitForFile(read_release.clone()), ); let retries_before = follower.not_leader_retry_count(); - let read_sql = format!("SELECT id, val FROM {COLLECTION} WHERE id = 'k1'"); let started = Instant::now(); - let (read, ()) = tokio::join!( - follower.client.simple_query(&read_sql), - release_on_retry(follower, retries_before, &read_release, READ_TIMEOUT * 2), - ); + let read = follower.client.simple_query(&read_sql()).await; let read_elapsed = started.elapsed(); + std::fs::write(&read_release, b"release").expect("release the execute gate"); drop(read_gate); - let rows = read - .unwrap_or_else(|e| panic!("the read retries past the silent leader: {}", db_detail(&e))) - .into_iter() - .filter(|m| matches!(m, tokio_postgres::SimpleQueryMessage::Row(_))) - .count(); - assert_eq!(rows, 1, "the retried read returns the seeded row"); + let err = read.expect_err("a read the leader never answers runs out of budget"); + assert_eq!( + sqlstate(&err), + Some("57014"), + "the read reports its deadline: {}", + db_detail(&err) + ); assert!( read_dir.path().join("release.parked").exists(), - "the leader took the read's first attempt and held it" + "the leader took the read and held it" + ); + assert_eq!( + follower.not_leader_retry_count(), + retries_before, + "a live leader that holds a read is never retried" ); assert!( - follower.not_leader_retry_count() > retries_before, - "the read retried after its first attempt went unanswered" + read_elapsed >= PARKED_TIMEOUT, + "the read waited {read_elapsed:?}, less than its {PARKED_TIMEOUT:?} budget" ); assert!( - read_elapsed < READ_TIMEOUT, - "the read finished in {read_elapsed:?}, past its {READ_TIMEOUT:?} statement budget" + read_elapsed < PARKED_TIMEOUT + REPLY_MARGIN + SLACK, + "the read waited {read_elapsed:?}, past its budget and reply margin" ); // A write: it waits its whole budget and is never sent again. - follower - .client - .simple_query(&format!( - "SET statement_timeout = '{}s'", - WRITE_TIMEOUT.as_secs() - )) - .await - .expect("SET statement_timeout for the write"); let write_dir = tempfile::tempdir().expect("write gate directory"); let write_release = write_dir.path().join("release"); let write_gate = FailGuard::for_node( @@ -190,7 +247,7 @@ async fn a_silent_leader_costs_a_read_one_retry_and_a_write_its_budget() { let err = write.expect_err("a write whose answer never arrives is an unknown outcome"); assert_eq!( - err.as_db_error().map(|db| db.code().code()), + sqlstate(&err), Some("57014"), "the write reports its deadline: {}", db_detail(&err) @@ -205,12 +262,53 @@ async fn a_silent_leader_costs_a_read_one_retry_and_a_write_its_budget() { "a write with a lost answer is never sent again" ); assert!( - write_elapsed >= WRITE_TIMEOUT, - "the write waited {write_elapsed:?}, less than its {WRITE_TIMEOUT:?} budget" + write_elapsed >= PARKED_TIMEOUT, + "the write waited {write_elapsed:?}, less than its {PARKED_TIMEOUT:?} budget" + ); + assert!( + write_elapsed < PARKED_TIMEOUT + REPLY_MARGIN + SLACK, + "the write waited {write_elapsed:?}, past its budget and reply margin" + ); + + cluster.shutdown().await; +} + +#[tokio::test(flavor = "multi_thread", worker_threads = 6)] +async fn a_silent_leader_costs_a_read_one_idle_timeout() { + let Setup { + cluster, + leader_id, + follower, + } = setup().await; + let follower = &cluster.nodes[follower]; + set_statement_timeout(follower, SILENT_READ_TIMEOUT).await; + + // The leader drops every datagram, both ways, while its process runs. + let blackhole = FailGuard::for_node( + leader_id, + BLACKHOLE_FAIL_POINT, + FailAction::Fail("cut off".into()), + ); + let retries_before = follower.not_leader_retry_count(); + let started = Instant::now(); + let read = follower.client.simple_query(&read_sql()).await; + let elapsed = started.elapsed(); + drop(blackhole); + + let rows = read + .unwrap_or_else(|e| panic!("the read retries past the silent leader: {}", db_detail(&e))); + assert_eq!( + row_count(rows), + 1, + "the retried read returns the seeded row" + ); + assert!( + follower.not_leader_retry_count() > retries_before, + "the read retried after the silent leader's connection timed out" ); assert!( - write_elapsed < (WRITE_TIMEOUT + REPLY_MARGIN) * 2, - "the write waited {write_elapsed:?}, more than one attempt's wait" + elapsed < RPC_TIMEOUT * 2, + "the read took {elapsed:?}, more than two {RPC_TIMEOUT:?} RPC timeouts" ); cluster.shutdown().await; diff --git a/nodedb-cluster-tests/tests/common_suite/cases/listeners_typed_not_leader.rs b/nodedb-cluster-tests/tests/common_suite/cases/listeners_typed_not_leader.rs index 545704b6f..b75c3878f 100644 --- a/nodedb-cluster-tests/tests/common_suite/cases/listeners_typed_not_leader.rs +++ b/nodedb-cluster-tests/tests/common_suite/cases/listeners_typed_not_leader.rs @@ -29,7 +29,7 @@ //! NotLeader") when the QUIC connection itself fails (e.g. sending to a node that //! doesn't exist). In that case the hinted leader in the error is the bad node_id //! itself, so the retry loop would update the routing table to the same bad node -//! and exhaust all 3 attempts — the client sees `NotLeader`, not success. +//! and retry until the statement deadline — the client sees `NotLeader`, not success. //! //! The retry-on-success path exists for a FUTURE scenario where Raft-aware //! execution on follower nodes explicitly returns `TypedClusterError::NotLeader` @@ -492,7 +492,7 @@ async fn native_not_leader_gateway_error_mapping() { #[tokio::test(flavor = "multi_thread", worker_threads = 4)] async fn not_leader_counter_increments_per_retry_attempt() { - use nodedb::control::gateway::retry::retry_not_leader; + use nodedb::control::gateway::retry::{RetryPacing, retry_not_leader}; use std::sync::atomic::{AtomicU64, AtomicUsize}; let counter = Arc::new(AtomicU64::new(0)); @@ -502,7 +502,7 @@ async fn not_leader_counter_increments_per_retry_attempt() { let call_count_inner = Arc::clone(&call_count); let deadline = tokio::time::Instant::now() + std::time::Duration::from_secs(30); - let result = retry_not_leader(None, deadline, move |attempt| { + let result = retry_not_leader(None, deadline, RetryPacing::default(), move |attempt| { let c = Arc::clone(&call_count_inner); let rc = Arc::clone(&counter_inner); async move { diff --git a/nodedb-cluster-tests/tests/common_suite/cases/routing_redirect_hint_unhosted_group.rs b/nodedb-cluster-tests/tests/common_suite/cases/routing_redirect_hint_unhosted_group.rs index 33919eb3f..5fc1e4c79 100644 --- a/nodedb-cluster-tests/tests/common_suite/cases/routing_redirect_hint_unhosted_group.rs +++ b/nodedb-cluster-tests/tests/common_suite/cases/routing_redirect_hint_unhosted_group.rs @@ -16,7 +16,7 @@ use common::cluster_harness::{TestCluster, TestClusterNode, wait_for}; use std::time::Duration; -use nodedb::control::gateway::retry::retry_not_leader; +use nodedb::control::gateway::retry::{RetryPacing, retry_not_leader}; use nodedb::types::VShardId; const CONVERGE: Duration = Duration::from_secs(30); @@ -198,18 +198,23 @@ async fn an_outsider_follows_a_redirect_to_the_new_leader_and_ignores_a_stale_on // A stale redirect naming the old leader at the old term, through the // gateway's retry path, leaves the hint on the new leader. let deadline = tokio::time::Instant::now() + std::time::Duration::from_secs(30); - let _ = retry_not_leader(Some(&**routing), deadline, |attempt| async move { - if attempt.index == 0 { - Err(nodedb::Error::NotLeader { - vshard_id: VShardId::new(vshard_id), - leader_node: old_leader, - leader_addr: String::new(), - leader_term: old_term, - }) - } else { - Ok::<(), nodedb::Error>(()) - } - }) + let _ = retry_not_leader( + Some(&**routing), + deadline, + RetryPacing::default(), + |attempt| async move { + if attempt.index == 0 { + Err(nodedb::Error::NotLeader { + vshard_id: VShardId::new(vshard_id), + leader_node: old_leader, + leader_addr: String::new(), + leader_term: old_term, + }) + } else { + Ok::<(), nodedb::Error>(()) + } + }, + ) .await; assert_eq!( hint(outsider, group_id), diff --git a/nodedb-cluster-tests/tests/common_suite/cases/sync_constraint_version_fence.rs b/nodedb-cluster-tests/tests/common_suite/cases/sync_constraint_version_fence.rs index b1d3c7df6..7ace48563 100644 --- a/nodedb-cluster-tests/tests/common_suite/cases/sync_constraint_version_fence.rs +++ b/nodedb-cluster-tests/tests/common_suite/cases/sync_constraint_version_fence.rs @@ -220,14 +220,11 @@ async fn peer_delta_fenced_until_constraint_installed() { "fenced delta A must not be imported, but crdt_state returned: {landed:?}" ); - // Install the constraint deterministically: drive one reconcile pass on - // every node (only the metadata leader actually proposes; the rest - // no-op), then poll until the UNIQUE(email) constraint is observed - // installed in node 0's validator. - for node in &cluster.nodes { - node.run_constraint_reconcile_once().await; - } + // Install the constraint on demand: a reconcile pass proposes only on the + // metadata leader, and leadership can move under load. Re-drive it on + // every node each round until node 0's validator holds UNIQUE(email). let deadline = Instant::now() + Duration::from_secs(15); + let mut accepted = 0usize; loop { let constraints = cluster.nodes[0] .crdt_constraints(TenantId::new(1), COLL) @@ -240,10 +237,15 @@ async fn peer_delta_fenced_until_constraint_installed() { } if Instant::now() >= deadline { panic!( - "UNIQUE(email) constraint on '{COLL}' did not install within 15s of driving \ - reconcile on demand; observed: {constraints:?}" + "UNIQUE(email) constraint on '{COLL}' did not install within 15s of re-driving \ + reconcile on every node ({accepted} proposals accepted); observed: \ + {constraints:?}; groups:\n{}", + cluster.group_positions_report() ); } + for node in &cluster.nodes { + accepted += node.run_constraint_reconcile_once().await; + } tokio::time::sleep(Duration::from_millis(200)).await; } diff --git a/nodedb-cluster-tests/tests/common_suite/cases/timeseries_ring_drop_catch_up.rs b/nodedb-cluster-tests/tests/common_suite/cases/timeseries_ring_drop_catch_up.rs index 0942bec47..c6f5a88f1 100644 --- a/nodedb-cluster-tests/tests/common_suite/cases/timeseries_ring_drop_catch_up.rs +++ b/nodedb-cluster-tests/tests/common_suite/cases/timeseries_ring_drop_catch_up.rs @@ -13,8 +13,8 @@ //! `TimeseriesIngest` opcode: a fresh column comes only from a raw ILP line //! (see `ts_native_ingest`). A parked apply holds every later entry of its //! group, so only A reaches the gates. B goes out once every replica's gate -//! holds A, and C once every replica committed B, so the log orders them as -//! sent: +//! holds A, and C once every replica's log holds B's ingest committed, so the +//! log orders them as sent: //! //! - A gives the new column `extra` a float. //! - B gives a new column `y` a float and names no `extra`. @@ -40,12 +40,13 @@ use common::cluster_harness::{TestCluster, TestClusterNode, wait_for}; use std::path::{Path, PathBuf}; use std::time::{Duration, Instant}; +use nodedb::control::wal_replication::{ReplicatedEntry, ReplicatedWrite}; use nodedb::event::cdc::consume::{ConsumeError, ConsumeParams, consume_local}; use nodedb_test_support::fail_point::{FailAction, FailGuard}; use nodedb_types::DatabaseId; use super::ts_native_ingest::{ - ingest_native, ingest_until_accepted, native_session, rejection_warnings, + ingest_native_within, ingest_until_accepted, native_session, rejection_warnings, }; const COLLECTION: &str = "ts_ring_drop"; @@ -118,6 +119,32 @@ fn group_commit_index(node: &TestClusterNode, group_id: u64) -> u64 { .map_or(0, |group| group.commit_index) } +/// Whether `node`'s log for `group_id` holds a committed ingest into the +/// collection above index `after`. +/// +/// One ingest is in flight at a time, and every earlier one sits at or below +/// `after`, so such an entry is the ingest just sent. +fn committed_ingest_above(node: &TestClusterNode, group_id: u64, after: u64) -> bool { + let Some(multi_raft) = node.shared.multi_raft.get() else { + return false; + }; + let mut multi_raft = multi_raft.lock().unwrap_or_else(|p| p.into_inner()); + let Some(raft) = multi_raft.groups_mut().get(&group_id) else { + return false; + }; + let collection = nodedb_types::QualifiedCollection::new(DatabaseId::DEFAULT, COLLECTION); + (after + 1..=raft.commit_index()) + .filter_map(|index| raft.log_entry_at(index)) + .filter_map(|entry| ReplicatedEntry::from_bytes(&entry.data)) + .any(|entry| { + matches!( + entry.write, + ReplicatedWrite::TimeseriesIngest { collection: ref written, .. } + if written == collection.as_str() + ) + }) +} + /// The new values of the change events `node` serves. fn event_values(node: &TestClusterNode) -> Vec { let params = ConsumeParams { @@ -199,8 +226,9 @@ async fn catch_up_after_an_apply_time_rejection_rebuilds_only_landed_rows() { format!("{COLLECTION} value=3.0,extra=\"text\" 4000000000"), ]; // A parked apply holds every later entry of its group, so only A reaches - // the gates. B and C each go out once every replica committed the line - // before it. + // the gates. B and C each go out once every replica's log holds the line + // before it committed. Each statement waits without a bound while parked: + // its reply comes only after the release. let group_id = cluster.nodes[0] .group_id_for_collection(COLLECTION) .expect("the collection maps to a data group"); @@ -219,7 +247,7 @@ async fn catch_up_after_an_apply_time_rejection_rebuilds_only_landed_rows() { .collect(); let mut session = native_session(node).await; sent.push(tokio::spawn(async move { - ingest_native(&mut session, 1, COLLECTION, &line).await + ingest_native_within(&mut session, 1, COLLECTION, &line, None).await })); if label == "A" { wait_for("every replica holds A's apply", CONVERGE, GATE_STEP, || { @@ -236,7 +264,7 @@ async fn catch_up_after_an_apply_time_rejection_rebuilds_only_landed_rows() { .nodes .iter() .zip(&committed_before) - .all(|(node, before)| group_commit_index(node, group_id) > *before) + .all(|(node, before)| committed_ingest_above(node, group_id, *before)) }, ) .await; @@ -248,9 +276,20 @@ async fn catch_up_after_an_apply_time_rejection_rebuilds_only_landed_rows() { for gate in &gates { gate.release(); } + // Every reply is due once the applies run, so the bound starts here. + let replied_by = tokio::time::Instant::now() + CONVERGE; let mut replies = Vec::new(); - for statement in sent { - replies.push(statement.await.expect("statement task")); + for (label, statement) in ["A", "B", "C"].into_iter().zip(sent) { + let reply = tokio::time::timeout_at(replied_by, statement) + .await + .unwrap_or_else(|_| { + panic!( + "{label} got no reply within {CONVERGE:?} of the release; groups:\n{}", + cluster.group_positions_report() + ) + }) + .expect("statement task"); + replies.push(reply); } let deadline = Instant::now() + CONVERGE; diff --git a/nodedb-cluster-tests/tests/common_suite/cases/ts_native_ingest.rs b/nodedb-cluster-tests/tests/common_suite/cases/ts_native_ingest.rs index 41ff198bb..5baf88ce6 100644 --- a/nodedb-cluster-tests/tests/common_suite/cases/ts_native_ingest.rs +++ b/nodedb-cluster-tests/tests/common_suite/cases/ts_native_ingest.rs @@ -12,7 +12,7 @@ use std::time::Duration; -use nodedb_test_support::native_harness::{do_handshake, send_request}; +use nodedb_test_support::native_harness::{RESPONSE_BOUND, do_handshake, send_request_within}; use nodedb_types::protocol::opcodes::ResponseStatus; use nodedb_types::protocol::text_fields::TextFields; use nodedb_types::protocol::{HelloFrame, NativeResponse, OpCode}; @@ -33,14 +33,28 @@ pub(super) async fn native_session(node: &TestClusterNode) -> TcpStream { } /// Ingest the raw ILP `lines` into `collection` through the native -/// `TimeseriesIngest` opcode. +/// `TimeseriesIngest` opcode, waiting at most [`RESPONSE_BOUND`] for the +/// response. pub(super) async fn ingest_native( stream: &mut TcpStream, seq: u64, collection: &str, lines: &str, ) -> NativeResponse { - send_request( + ingest_native_within(stream, seq, collection, lines, Some(RESPONSE_BOUND)).await +} + +/// [`ingest_native`] waiting at most `bound` for the response, or without +/// limit when `bound` is `None`. A test that parks the apply on purpose +/// bounds the wait itself. +pub(super) async fn ingest_native_within( + stream: &mut TcpStream, + seq: u64, + collection: &str, + lines: &str, + bound: Option, +) -> NativeResponse { + send_request_within( stream, seq, OpCode::TimeseriesIngest, @@ -50,6 +64,7 @@ pub(super) async fn ingest_native( format: Some("ilp".to_string()), ..Default::default() }, + bound, ) .await } diff --git a/nodedb-cluster/Cargo.toml b/nodedb-cluster/Cargo.toml index eb0b0403a..c8c04a79b 100644 --- a/nodedb-cluster/Cargo.toml +++ b/nodedb-cluster/Cargo.toml @@ -19,8 +19,9 @@ default = [] # `faultbox::init`; a no-op on wasm32 regardless. Off by default so the crate # stays dependency-light for embedders that report failures their own way. diagnostics = ["dep:faultbox"] -# Arms the Raft fail gates cluster tests use. Off by default. -failpoints = ["nodedb-raft/failpoints"] +# Arms the Raft fail gates and the transport blackhole cluster tests use. +# Off by default. +failpoints = ["nodedb-raft/failpoints", "nodedb-types/failpoints"] [dependencies] # Workspace crates diff --git a/nodedb-cluster/src/routing.rs b/nodedb-cluster/src/routing.rs index 87c97784d..2030d85bd 100644 --- a/nodedb-cluster/src/routing.rs +++ b/nodedb-cluster/src/routing.rs @@ -321,9 +321,11 @@ impl RoutingTable { self.group_members.len() } - /// All group IDs. + /// All group IDs, in ascending order. pub fn group_ids(&self) -> Vec { - self.group_members.keys().copied().collect() + let mut ids: Vec = self.group_members.keys().copied().collect(); + ids.sort_unstable(); + ids } /// Update the voting members of a Raft group (for membership changes). diff --git a/nodedb-cluster/src/rpc_codec/data_propose.rs b/nodedb-cluster/src/rpc_codec/data_propose.rs index b650d04ec..9d188e155 100644 --- a/nodedb-cluster/src/rpc_codec/data_propose.rs +++ b/nodedb-cluster/src/rpc_codec/data_propose.rs @@ -53,42 +53,6 @@ pub fn reply_wait(deadline_remaining_ms: u64) -> std::time::Duration { std::time::Duration::from_millis(deadline_remaining_ms).saturating_add(REPLY_MARGIN) } -/// The budget one send attempt gets out of the `remaining_ms` a statement has -/// left. `attempts_left` counts this attempt. -/// -/// A request that is not resend-safe gets all of `remaining_ms`: a lost -/// answer is an unknown outcome, so no later attempt follows. -/// -/// A resend-safe request reserves `rpc_timeout + REPLY_MARGIN` for each later -/// attempt: that attempt's floor plus its reply margin. It still gets at -/// least `rpc_timeout`, capped at `remaining_ms`. The last attempt gets all -/// that is left. -/// -/// Bound: each attempt waits [`reply_wait`] of its budget. A caller that -/// starts no attempt once the deadline passed waits at most the statement -/// budget plus one `REPLY_MARGIN` across every attempt. Each attempt's -/// budget is at most what is left, so only the attempt that reaches the -/// deadline runs past it, by its margin. -pub fn attempt_budget( - remaining_ms: u64, - resend_safe: bool, - attempts_left: usize, - rpc_timeout: std::time::Duration, -) -> u64 { - if !resend_safe || attempts_left <= 1 { - return remaining_ms; - } - let rpc_timeout_ms = u64::try_from(rpc_timeout.as_millis()).unwrap_or(u64::MAX); - let margin_ms = u64::try_from(REPLY_MARGIN.as_millis()).unwrap_or(u64::MAX); - let later_attempts = u64::try_from(attempts_left - 1).unwrap_or(u64::MAX); - let reserve = rpc_timeout_ms - .saturating_add(margin_ms) - .saturating_mul(later_attempts); - remaining_ms - .saturating_sub(reserve) - .max(remaining_ms.min(rpc_timeout_ms)) -} - /// The leader's deadline for a forward that carries `deadline_remaining_ms`. /// `None` when the budget is spent. /// @@ -328,81 +292,6 @@ mod tests { ); } - const RPC_TIMEOUT: std::time::Duration = std::time::Duration::from_secs(5); - - /// A write can have run on a node that never answered, so it never - /// retries and keeps the whole remainder on every attempt. - #[test] - fn a_write_attempt_gets_the_full_remainder() { - assert_eq!(attempt_budget(30_000, false, 3, RPC_TIMEOUT), 30_000); - assert_eq!(attempt_budget(30_000, false, 2, RPC_TIMEOUT), 30_000); - assert_eq!(attempt_budget(1_000, false, 3, RPC_TIMEOUT), 1_000); - } - - /// A read leaves one RPC timeout plus one reply margin for each later - /// attempt. - #[test] - fn a_read_attempt_reserves_an_rpc_timeout_and_margin_per_later_attempt() { - // 30 s - 2 × (5 s + 2 s) = 16 s; 30 s - (5 s + 2 s) = 23 s. - assert_eq!(attempt_budget(30_000, true, 3, RPC_TIMEOUT), 16_000); - assert_eq!(attempt_budget(30_000, true, 2, RPC_TIMEOUT), 23_000); - } - - /// A read with little left still gets one RPC timeout, never more than - /// what is left. - #[test] - fn a_read_attempt_keeps_at_least_one_rpc_timeout() { - assert_eq!(attempt_budget(15_000, true, 3, RPC_TIMEOUT), 5_000); - assert_eq!(attempt_budget(3_000, true, 3, RPC_TIMEOUT), 3_000); - assert_eq!(attempt_budget(0, true, 3, RPC_TIMEOUT), 0); - } - - /// Every attempt of a read sent to a node that never answers waits out - /// its reply. The waits together stay within the budget plus one margin, - /// for an RPC timeout above and below the margin. - #[test] - fn hung_read_attempts_fit_the_budget_plus_one_margin() { - let margin_ms = u64::try_from(REPLY_MARGIN.as_millis()).unwrap_or(u64::MAX); - for rpc_timeout_ms in [500, 1_000, 5_000] { - let rpc_timeout = std::time::Duration::from_millis(rpc_timeout_ms); - for budget_ms in [1_000, 4_000, 12_000, 30_000] { - let mut remaining = budget_ms; - let mut waited = 0; - for attempts_left in (1..=3).rev() { - if remaining == 0 { - break; - } - let attempt = attempt_budget(remaining, true, attempts_left, rpc_timeout); - assert!(attempt <= remaining); - let wait = attempt + margin_ms; - waited += wait; - remaining = remaining.saturating_sub(wait); - } - assert!( - waited <= budget_ms + margin_ms, - "rpc timeout {rpc_timeout_ms} ms, budget {budget_ms} ms: waited {waited} ms" - ); - } - } - } - - /// An RPC timeout below the reply margin still leaves each later - /// attempt its whole floor and margin. - #[test] - fn a_short_rpc_timeout_reserves_its_margin_too() { - let rpc_timeout = std::time::Duration::from_millis(500); - // 12 s - 2 × (0.5 s + 2 s) = 7 s. - assert_eq!(attempt_budget(12_000, true, 3, rpc_timeout), 7_000); - // 12 s - (0.5 s + 2 s) = 9.5 s. - assert_eq!(attempt_budget(12_000, true, 2, rpc_timeout), 9_500); - } - - /// The last attempt of a read gets all that is left. - #[test] - fn the_last_read_attempt_gets_all_that_is_left() { - assert_eq!(attempt_budget(30_000, true, 1, RPC_TIMEOUT), 30_000); - assert_eq!(attempt_budget(30_000, true, 0, RPC_TIMEOUT), 30_000); - } use crate::rpc_codec::{decode, encode}; fn roundtrip(target: ProposeTarget) -> DataProposeRequest { diff --git a/nodedb-cluster/src/rpc_codec/mod.rs b/nodedb-cluster/src/rpc_codec/mod.rs index 79d320230..e75b05d7c 100644 --- a/nodedb-cluster/src/rpc_codec/mod.rs +++ b/nodedb-cluster/src/rpc_codec/mod.rs @@ -55,7 +55,7 @@ pub use data_plane_error::{ }; pub use data_propose::{ DataProposeRequest, DataProposeResponse, ForwardedProposeRefusal, ProposeTarget, REPLY_MARGIN, - attempt_budget, forwarded_deadline, remaining_budget_ms, reply_wait, + forwarded_deadline, remaining_budget_ms, reply_wait, }; pub use execute::{ DescriptorVersionEntry, ExecuteRequest, ExecuteResponse, ExecuteStreamChunk, ExecuteStreamEnd, diff --git a/nodedb-cluster/src/transport/client/attempt.rs b/nodedb-cluster/src/transport/client/attempt.rs index 3ad90fd5c..e12c22649 100644 --- a/nodedb-cluster/src/transport/client/attempt.rs +++ b/nodedb-cluster/src/transport/client/attempt.rs @@ -24,6 +24,11 @@ //! A live peer acknowledges the request within its ACK delay. A read that //! times out on a connection that received datagrams meanwhile is a slow //! handler, and the connection stays pooled. +//! +//! A peer silent for the RPC timeout is dead or cut off: its connection hit +//! the QUIC idle timeout, or a dial got no answer. The call ends at once with +//! no resend to that peer, so the caller can try another node. A written +//! request then ends as `ClusterError::Unanswered`. use std::time::Duration; @@ -59,15 +64,18 @@ struct LinkFailure { /// Whether the whole request was written before the failure. The peer /// can have run a written request. written: bool, + /// The peer sent nothing for the RPC timeout. No resend goes to it. + silent: bool, } impl LinkFailure { - /// A failure before the request went out that leaves the pool as it is. - fn keep_connection(error: ClusterError) -> Self { + /// A dial the peer never answered. + fn from_dial(failed: super::pool::DialError) -> Self { Self { - error: Box::new(error), + error: Box::new(failed.error), condemned: None, written: false, + silent: failed.silent, } } @@ -85,10 +93,16 @@ impl LinkFailure { /// `Ok` holds the error kept while the next attempt goes out. `Err` ends the /// call. A written request that is not [`RaftRpc::resend_safe`] can have run /// on the peer. It is never resent and ends as `ClusterError::Unanswered`. +/// +/// A silent peer gets no resend: a written request ends as `Unanswered`, an +/// unsent one with its own error. fn resend_verdict(failure: LinkFailure, target: u64, resend_safe: bool) -> Result { - if failure.written && !resend_safe { + if failure.written && (failure.silent || !resend_safe) { return Err(failure.into_unanswered(target)); } + if failure.silent { + return Err(*failure.error); + } if RetryPolicy::is_retryable(&failure.error) { Ok(*failure.error) } else { @@ -234,15 +248,16 @@ impl NexarTransport { read_timeout: Duration, ) -> std::result::Result, LinkFailure> { let conn = self - .get_or_connect(target) + .connect_peer(target) .await - .map_err(LinkFailure::keep_connection)?; + .map_err(LinkFailure::from_dial)?; let stable_id = conn.stable_id(); if let Err(error) = self.verify_connection_target(&conn, target) { return Err(LinkFailure { error: Box::new(error), condemned: Some(stable_id), written: false, + silent: false, }); } let received_before = received_datagrams(&conn); @@ -255,11 +270,13 @@ impl NexarTransport { StreamFailure::Unanswered(error) => (error, false, true), }; let heard = received_datagrams(&conn) > received_before; - let broken = condemns_connection(timed_out, conn.close_reason().is_some(), heard); + let close_reason = conn.close_reason(); + let broken = condemns_connection(timed_out, close_reason.is_some(), heard); LinkFailure { error: Box::new(error), condemned: broken.then_some(stable_id), written, + silent: matches!(close_reason, Some(quinn::ConnectionError::TimedOut)), } }) } @@ -349,7 +366,13 @@ fn received_datagrams(conn: &quinn::Connection) -> u64 { mod tests { use nodedb_raft::message::RequestVoteRequest; - use super::super::transport::tests::{STALL, STALLED_GROUP, serve_echo}; + use std::net::SocketAddr; + use std::sync::Arc; + use std::sync::atomic::{AtomicBool, Ordering}; + + use super::super::transport::tests::{ + SLOW, SLOW_GROUP, STALL, STALLED_GROUP, serve_echo, serve_echo_with_rpc_timeout, + }; use super::*; use crate::rpc_codec::{FrameRefusal, PongResponse, RequestRefusal}; @@ -399,6 +422,106 @@ mod tests { assert_eq!(client.circuit_breaker().failure_count(1), 1); } + /// RPC timeout, and so QUIC idle timeout, of the liveness tests. + const LIVENESS_RPC_TIMEOUT: Duration = Duration::from_millis(300); + + fn vote(group_id: u64) -> RaftRpc { + RaftRpc::RequestVoteRequest(RequestVoteRequest { + term: 1, + candidate_id: 2, + last_log_index: 0, + last_log_term: 0, + group_id, + transfer: false, + }) + } + + /// A handler slower than the idle timeout on a live peer gets the whole + /// read timeout: keep-alives hold the connection open. + #[tokio::test] + async fn a_stalled_handler_on_a_live_peer_answers_past_the_idle_timeout() { + let (_server, client, _shutdown) = serve_echo_with_rpc_timeout(LIVENESS_RPC_TIMEOUT).await; + client.warm_peer(1).await.expect("warm"); + let pooled = client.peer_connection_stable_id(1).expect("pooled"); + assert!(SLOW > LIVENESS_RPC_TIMEOUT * 3); + + let reply = client + .send_rpc_with_read_timeout(1, vote(SLOW_GROUP), Duration::from_secs(2)) + .await + .expect("a live peer answers"); + + assert!( + matches!(reply, RaftRpc::RequestVoteResponse(_)), + "{reply:?}" + ); + assert_eq!(client.peer_connection_stable_id(1), Some(pooled)); + assert_eq!(client.circuit_breaker().failure_count(1), 0); + } + + /// A UDP relay between a client and a server that drops every datagram + /// once `cut` is set. + async fn relay(server: SocketAddr, cut: Arc) -> SocketAddr { + let socket = Arc::new( + tokio::net::UdpSocket::bind("127.0.0.1:0") + .await + .expect("bind relay"), + ); + let addr = socket.local_addr().expect("relay address"); + tokio::spawn(async move { + let mut client: Option = None; + let mut buf = vec![0u8; 65_536]; + while let Ok((len, from)) = socket.recv_from(&mut buf).await { + if cut.load(Ordering::SeqCst) { + continue; + } + let to = if from == server { + match client { + Some(client) => client, + None => continue, + } + } else { + client = Some(from); + server + }; + let _ = socket.send_to(&buf[..len], to).await; + } + }); + addr + } + + /// A peer cut off mid-read fails a long read at the idle timeout. The + /// read is not resent to it, and its connection leaves the pool. + #[tokio::test] + async fn a_silent_peer_fails_a_long_read_within_the_idle_timeout() { + let (server, client, _shutdown) = serve_echo_with_rpc_timeout(LIVENESS_RPC_TIMEOUT).await; + let cut = Arc::new(AtomicBool::new(false)); + client.register_peer(1, relay(server.local_addr(), Arc::clone(&cut)).await); + client.warm_peer(1).await.expect("warm through the relay"); + assert!(client.peer_connection_stable_id(1).is_some()); + + cut.store(true, Ordering::SeqCst); + let started = tokio::time::Instant::now(); + let err = client + .send_rpc_with_read_timeout(1, vote(STALLED_GROUP), Duration::from_secs(5)) + .await + .unwrap_err(); + let elapsed = started.elapsed(); + + assert!( + matches!(err, ClusterError::Unanswered { node_id: 1, .. }), + "{err:?}" + ); + let slack = Duration::from_millis(700); + assert!( + elapsed < LIVENESS_RPC_TIMEOUT + slack, + "failed after {elapsed:?}, idle timeout {LIVENESS_RPC_TIMEOUT:?}" + ); + assert!( + !client.peers.read().unwrap().contains_key(&1), + "the dead connection is evicted" + ); + } + fn lost(written: bool) -> LinkFailure { LinkFailure { error: Box::new(ClusterError::Transport { @@ -406,6 +529,7 @@ mod tests { }), condemned: None, written, + silent: false, } } @@ -439,6 +563,34 @@ mod tests { )); } + fn silent(written: bool) -> LinkFailure { + LinkFailure { + silent: true, + ..lost(written) + } + } + + /// A silent peer is dead or cut off. A written read is not resent to it + /// and ends unanswered, so the caller can try another node. + #[test] + fn a_written_request_to_a_silent_peer_ends_unanswered() { + for resend_safe in [false, true] { + match resend_verdict(silent(true), 2, resend_safe) { + Err(ClusterError::Unanswered { node_id: 2, .. }) => {} + other => panic!("expected Unanswered, got {other:?}"), + } + } + } + + /// A request a silent peer never got ends with its own error, unsent. + #[test] + fn an_unsent_request_to_a_silent_peer_is_not_resent() { + assert!(matches!( + resend_verdict(silent(false), 2, true), + Err(ClusterError::Transport { .. }) + )); + } + #[test] fn a_typed_refusal_is_a_final_answer() { let refusal = RaftRpc::RequestRefused(RequestRefusal::from(ClusterError::GroupNotFound { @@ -463,10 +615,10 @@ mod tests { #[test] fn only_the_outer_error_is_a_link_failure() { - let link = LinkFailure::keep_connection(ClusterError::Transport { - detail: "connection lost".into(), - }); - assert!(matches!(classify(1, Err(link)), Attempt::LinkFailed(_))); + assert!(matches!( + classify(1, Err(lost(false))), + Attempt::LinkFailed(_) + )); let unreadable = ClusterError::Codec { detail: "bad crc".into(), diff --git a/nodedb-cluster/src/transport/client/pool.rs b/nodedb-cluster/src/transport/client/pool.rs index bbb97c8c9..ee83c8e3e 100644 --- a/nodedb-cluster/src/transport/client/pool.rs +++ b/nodedb-cluster/src/transport/client/pool.rs @@ -71,15 +71,25 @@ impl NexarTransport { /// a dial that ended while it waited. Only a caller with neither dials, /// so a burst of sends to one peer opens one connection. pub(super) async fn get_or_connect(&self, target: u64) -> Result { + self.connect_peer(target) + .await + .map_err(|failed| failed.error) + } + + /// [`get_or_connect`](Self::get_or_connect), with whether the peer stayed + /// silent through the dial. + pub(super) async fn connect_peer( + &self, + target: u64, + ) -> std::result::Result { if let Some(conn) = self.pooled_connection(target) { return Ok(conn); } let addr = { let addrs = self.peer_addrs.read().unwrap_or_else(|p| p.into_inner()); - addrs - .get(&target) - .copied() - .ok_or(ClusterError::NodeUnreachable { node_id: target })? + addrs.get(&target).copied().ok_or(DialError::answered( + ClusterError::NodeUnreachable { node_id: target }, + ))? }; let waiting_since = Instant::now(); @@ -90,10 +100,14 @@ impl NexarTransport { return Ok(conn); } if let Some(failure) = last_dial.failed_since(waiting_since) { - return Err(ClusterError::Transport { - detail: format!( - "dial to node {target} at {addr} failed while this send waited: {failure}" - ), + return Err(DialError { + error: ClusterError::Transport { + detail: format!( + "dial to node {target} at {addr} failed while this send waited: {}", + failure.detail + ), + }, + silent: failure.silent, }); } self.drop_closed_connection(target); @@ -104,37 +118,51 @@ impl NexarTransport { peers.insert(target, conn.clone()); Ok(conn) } - Err(error) => { + Err(failed) => { last_dial.failure = Some(DialFailure { at: Instant::now(), - detail: error.to_string(), + detail: failed.error.to_string(), + silent: failed.silent, }); - Err(error) + Err(failed) } } } /// Connect to `target` at `addr` and negotiate the wire version. - async fn dial(&self, target: u64, addr: SocketAddr) -> Result { - // Connect — bounded by rpc_timeout so a hung QUIC handshake - // (peer not yet serving) doesn't block for the full 30s idle timeout. + /// + /// A peer that answers nothing within the RPC timeout fails the dial as + /// silent: it is dead or cut off, and a redial in the same call is waste. + async fn dial( + &self, + target: u64, + addr: SocketAddr, + ) -> std::result::Result { + let silent = || { + DialError::silent(ClusterError::Transport { + detail: format!( + "handshake timeout ({}ms) with node {target} at {addr}", + self.rpc_timeout.as_millis() + ), + }) + }; let connecting = self .listener .endpoint() .connect_with(self.client_config.clone(), addr, SNI_HOSTNAME) - .map_err(|e| ClusterError::Transport { - detail: format!("connect to node {target} at {addr}: {e}"), + .map_err(|e| { + DialError::answered(ClusterError::Transport { + detail: format!("connect to node {target} at {addr}: {e}"), + }) })?; let conn = tokio::time::timeout(self.rpc_timeout, connecting) .await - .map_err(|_| ClusterError::Transport { - detail: format!( - "handshake timeout ({}ms) with node {target} at {addr}", - self.rpc_timeout.as_millis() - ), - })? - .map_err(|e| ClusterError::Transport { - detail: format!("handshake with node {target} at {addr}: {e}"), + .map_err(|_| silent())? + .map_err(|e| match e { + quinn::ConnectionError::TimedOut => silent(), + other => DialError::answered(ClusterError::Transport { + detail: format!("handshake with node {target} at {addr}: {other}"), + }), })?; debug!(target, %addr, "connected to peer"); @@ -142,21 +170,18 @@ impl NexarTransport { // Open a dedicated bidi stream for the wire-version handshake. // This must complete before any RPC stream is opened on this connection. let agreed = { - let (mut hs_send, mut hs_recv) = - conn.open_bi().await.map_err(|e| ClusterError::Transport { + let (mut hs_send, mut hs_recv) = conn.open_bi().await.map_err(|e| { + DialError::answered(ClusterError::Transport { detail: format!("open handshake stream to node {target} at {addr}: {e}"), - })?; + }) + })?; let version = tokio::time::timeout( self.rpc_timeout, perform_version_handshake_client(&mut hs_send, &mut hs_recv), ) .await - .map_err(|_| ClusterError::Transport { - detail: format!( - "handshake timeout ({}ms) with node {target} at {addr}", - self.rpc_timeout.as_millis() - ), - })??; + .map_err(|_| silent())? + .map_err(DialError::answered)?; // Finish the handshake send stream — the server reads it until FIN. let _ = hs_send.finish(); version @@ -208,12 +233,40 @@ impl NexarTransport { } } +/// A failed dial, and whether the peer stayed silent through it. +#[derive(Debug)] +pub(super) struct DialError { + pub(super) error: ClusterError, + /// The peer sent nothing within the RPC timeout: it is dead or cut off. + pub(super) silent: bool, +} + +impl DialError { + /// A dial the peer refused or broke off: the peer is up. + fn answered(error: ClusterError) -> Self { + Self { + error, + silent: false, + } + } + + /// A dial the peer never answered within the RPC timeout. + fn silent(error: ClusterError) -> Self { + Self { + error, + silent: true, + } + } +} + /// The last failed dial to one peer. #[derive(Debug)] struct DialFailure { /// When the dial ended. at: Instant, detail: String, + /// The peer stayed silent through the dial. + silent: bool, } /// Outcome of the last dial to one peer, behind the peer's dial gate. @@ -224,11 +277,8 @@ struct LastDial { impl LastDial { /// The failure of a dial that ended at or after `since`. - fn failed_since(&self, since: Instant) -> Option<&str> { - self.failure - .as_ref() - .filter(|failure| failure.at >= since) - .map(|failure| failure.detail.as_str()) + fn failed_since(&self, since: Instant) -> Option<&DialFailure> { + self.failure.as_ref().filter(|failure| failure.at >= since) } } diff --git a/nodedb-cluster/src/transport/client/send.rs b/nodedb-cluster/src/transport/client/send.rs index fc6942836..d51ad9023 100644 --- a/nodedb-cluster/src/transport/client/send.rs +++ b/nodedb-cluster/src/transport/client/send.rs @@ -178,6 +178,9 @@ impl NexarTransport { /// There is NO retry wrapper on the stream body — retry only applies to the /// eager pre-first-frame phase, which the caller (coordinator) owns. Once /// the first frame has been observed, any error is terminal. + /// + /// The request is written before the stream returns, so a link failure + /// while reading it is `ClusterError::Unanswered`: the peer can have run it. pub async fn send_rpc_stream( &self, target: u64, @@ -211,7 +214,9 @@ impl NexarTransport { // server-side `accept_bi` stream is not reset early. let _send = send; loop { - let envelope = read_envelope(&mut recv).await?; + let envelope = read_envelope(&mut recv) + .await + .map_err(|e| written_stream_failure(target, e))?; let (fields, inner_frame) = auth_envelope::parse_envelope(&envelope, &auth.mac_key)?; if fields.from_node_id != target { @@ -361,6 +366,20 @@ impl NexarTransport { } } +/// The error of a failed read on a stream whose request is written. +/// +/// A link failure leaves the outcome unknown, so it is `Unanswered`. A +/// malformed frame keeps its own error. +fn written_stream_failure(target: u64, error: ClusterError) -> ClusterError { + if let ClusterError::Transport { detail } = error { + return ClusterError::Unanswered { + node_id: target, + detail, + }; + } + error +} + /// Map a terminal [`TypedClusterError`] carried by `ExecuteStreamEnd` into a /// [`ClusterError`] for the stream's `Err` item. /// diff --git a/nodedb-cluster/src/transport/client/transport.rs b/nodedb-cluster/src/transport/client/transport.rs index c635f8d89..0e4608cb0 100644 --- a/nodedb-cluster/src/transport/client/transport.rs +++ b/nodedb-cluster/src/transport/client/transport.rs @@ -15,6 +15,7 @@ use crate::error::{ClusterError, Result}; use crate::transport::auth_context::AuthContext; use crate::transport::config; use crate::transport::credentials::{self, TransportCredentials}; +use crate::transport::endpoint::RaftListener; use crate::transport::peer_identity_store::{NoopIdentityStore, PeerIdentityStore}; use crate::transport::topology_identity_store::TopologyIdentityStore; @@ -33,7 +34,7 @@ use crate::transport::topology_identity_store::TopologyIdentityStore; /// [`serve`]: Self::serve pub struct NexarTransport { pub(super) node_id: u64, - pub(super) listener: nexar::TransportListener, + pub(super) listener: RaftListener, pub(super) client_config: quinn::ClientConfig, /// Cached connections to peers. Stale connections are replaced on next use. pub(super) peers: RwLock>, @@ -114,8 +115,8 @@ impl NexarTransport { /// Create a new transport using values from `ClusterTransportTuning`. /// - /// All QUIC parameters (streams, windows, keep-alive, idle timeout) and - /// the RPC timeout are read from `tuning`. `creds` selects channel-level + /// All QUIC parameters (streams, windows) and the RPC timeout are read + /// from `tuning`. The idle timeout and keep-alive follow the RPC timeout. `creds` selects channel-level /// authentication. Use this in production so that operators can override /// defaults via the `[tuning.cluster_transport]` section of `config.toml`. /// @@ -185,14 +186,24 @@ impl NexarTransport { let (server_config, client_config) = match &creds { TransportCredentials::Mtls(tls) => ( - config::make_raft_server_config_mtls(tls, tuning, Arc::clone(&identity_store))?, - config::make_raft_client_config_mtls(tls, tuning, Arc::clone(&identity_store))?, + config::make_raft_server_config_mtls( + tls, + tuning, + rpc_timeout, + Arc::clone(&identity_store), + )?, + config::make_raft_client_config_mtls( + tls, + tuning, + rpc_timeout, + Arc::clone(&identity_store), + )?, ), TransportCredentials::Insecure => { credentials::announce_insecure_transport(node_id); ( - config::make_raft_server_config(tuning)?, - config::make_raft_client_config(tuning)?, + config::make_raft_server_config(tuning, rpc_timeout)?, + config::make_raft_client_config(tuning, rpc_timeout)?, ) } }; @@ -204,10 +215,7 @@ impl NexarTransport { let auth = Arc::new(AuthContext::from_credentials(node_id, &creds)); - let listener = nexar::TransportListener::bind_with_config(listen_addr, server_config) - .map_err(|e| ClusterError::Transport { - detail: format!("bind {listen_addr}: {e}"), - })?; + let listener = RaftListener::bind(node_id, listen_addr, server_config)?; info!( node_id, @@ -455,6 +463,12 @@ pub(super) mod tests { /// How long `EchoHandler` holds a vote request of [`STALLED_GROUP`]. pub(crate) const STALL: Duration = Duration::from_secs(2); + /// The group whose vote requests `EchoHandler` answers after [`SLOW`]. + pub(crate) const SLOW_GROUP: u64 = 504; + + /// How long `EchoHandler` holds a vote request of [`SLOW_GROUP`]. + pub(crate) const SLOW: Duration = Duration::from_secs(1); + /// AppendEntries requests `EchoHandler` refused for `UNHOSTED_GROUP`. static UNHOSTED_APPENDS: AtomicU32 = AtomicU32::new(0); @@ -480,6 +494,13 @@ pub(super) mod tests { replica: nodedb_raft::ReplicaState::Whole, })) } + RaftRpc::RequestVoteRequest(req) if req.group_id == SLOW_GROUP => { + tokio::time::sleep(SLOW).await; + Ok(RaftRpc::RequestVoteResponse(RequestVoteResponse { + term: req.term, + vote_granted: true, + })) + } RaftRpc::RequestVoteRequest(req) if req.group_id == STALLED_GROUP => { tokio::time::sleep(STALL).await; Ok(RaftRpc::RequestVoteResponse(RequestVoteResponse { @@ -975,6 +996,36 @@ pub(super) mod tests { (server, client, shutdown_tx) } + /// [`serve_echo`] with `rpc_timeout` on both nodes, so the QUIC idle + /// timeout is `rpc_timeout` too. + pub(crate) async fn serve_echo_with_rpc_timeout( + rpc_timeout: Duration, + ) -> ( + Arc, + Arc, + tokio::sync::watch::Sender, + ) { + let bind = |node_id| { + NexarTransport::with_timeout( + node_id, + "127.0.0.1:0".parse().unwrap(), + rpc_timeout, + TransportCredentials::Insecure, + ) + .unwrap() + }; + let server = Arc::new(bind(1)); + let client = Arc::new(bind(2)); + client.register_peer(1, server.local_addr()); + let (shutdown_tx, shutdown_rx) = tokio::sync::watch::channel(false); + let srv = server.clone(); + tokio::spawn(async move { + srv.serve(Arc::new(EchoHandler), shutdown_rx).await.unwrap(); + }); + tokio::time::sleep(Duration::from_millis(20)).await; + (server, client, shutdown_tx) + } + #[tokio::test] async fn a_handler_refusal_is_answered_and_not_resent() { let (_server, client, _shutdown) = serve_echo().await; diff --git a/nodedb-cluster/src/transport/config.rs b/nodedb-cluster/src/transport/config.rs index 36ac0042d..e23e9b7fd 100644 --- a/nodedb-cluster/src/transport/config.rs +++ b/nodedb-cluster/src/transport/config.rs @@ -9,6 +9,7 @@ use std::time::Duration; use nodedb_types::config::tuning::ClusterTransportTuning; use crate::error::{ClusterError, Result}; +use crate::transport::liveness::raft_transport_config; use crate::transport::peer_identity_store::PeerIdentityStore; use crate::transport::pinned_verifier::{PinnedClientVerifier, PinnedServerVerifier}; @@ -42,30 +43,6 @@ pub const SNI_HOSTNAME: &str = "nodedb"; /// constructing `NexarTransport::with_tuning()` with a custom `ClusterTransportTuning`. pub const DEFAULT_RPC_TIMEOUT: Duration = Duration::from_secs(5); -/// Transport config tuned for Raft RPCs in a datacenter. -/// -/// All values are read from `tuning`. Pass `&ClusterTransportTuning::default()` -/// to get the same behaviour as the previous hardcoded defaults. -pub fn raft_transport_config(tuning: &ClusterTransportTuning) -> quinn::TransportConfig { - let mut config = quinn::TransportConfig::default(); - // Raft RPCs use bidi streams: one per request-response pair. - config.max_concurrent_bidi_streams(quinn::VarInt::from_u32(tuning.quic_max_bi_streams)); - // Also allow uni streams for future migration/snapshot streaming. - config.max_concurrent_uni_streams(quinn::VarInt::from_u32(tuning.quic_max_uni_streams)); - // Datacenter tuning: generous windows, low RTT estimate. - config.receive_window(quinn::VarInt::from_u32(tuning.quic_receive_window)); - config.send_window(u64::from(tuning.quic_send_window)); - config.stream_receive_window(quinn::VarInt::from_u32(tuning.quic_stream_receive_window)); - config.initial_rtt(Duration::from_micros(100)); - config.keep_alive_interval(Some(Duration::from_secs(tuning.quic_keep_alive_secs))); - config.max_idle_timeout(Some( - Duration::from_secs(tuning.quic_idle_timeout_secs) - .try_into() - .expect("idle timeout fits IdleTimeout"), - )); - config -} - /// Build a QUIC server config with self-signed TLS (unauthenticated). /// /// **Crate-private** — reachable only via @@ -75,6 +52,7 @@ pub fn raft_transport_config(tuning: &ClusterTransportTuning) -> quinn::Transpor /// Production clusters use the mTLS variant. pub(crate) fn make_raft_server_config( tuning: &ClusterTransportTuning, + rpc_timeout: Duration, ) -> Result { ensure_rustls_crypto_provider(); let (cert, key) = nexar::transport::tls::generate_self_signed_cert().map_err(|e| { @@ -103,7 +81,7 @@ pub(crate) fn make_raft_server_config( })?; let mut server_config = quinn::ServerConfig::with_crypto(Arc::new(quic_crypto)); - server_config.transport_config(Arc::new(raft_transport_config(tuning))); + server_config.transport_config(Arc::new(raft_transport_config(tuning, rpc_timeout)?)); Ok(server_config) } @@ -116,6 +94,7 @@ pub(crate) fn make_raft_server_config( /// Production clusters use the mTLS variant. pub(crate) fn make_raft_client_config( tuning: &ClusterTransportTuning, + rpc_timeout: Duration, ) -> Result { ensure_rustls_crypto_provider(); let provider = rustls::crypto::ring::default_provider(); @@ -136,7 +115,7 @@ pub(crate) fn make_raft_client_config( })?; let mut client_config = quinn::ClientConfig::new(Arc::new(quic_crypto)); - client_config.transport_config(Arc::new(raft_transport_config(tuning))); + client_config.transport_config(Arc::new(raft_transport_config(tuning, rpc_timeout)?)); Ok(client_config) } @@ -188,6 +167,7 @@ pub struct TlsCredentials { pub fn make_raft_server_config_mtls( creds: &TlsCredentials, tuning: &ClusterTransportTuning, + rpc_timeout: Duration, identity_store: Arc, ) -> Result { ensure_rustls_crypto_provider(); @@ -247,7 +227,7 @@ pub fn make_raft_server_config_mtls( })?; let mut server_config = quinn::ServerConfig::with_crypto(Arc::new(quic_crypto)); - server_config.transport_config(Arc::new(raft_transport_config(tuning))); + server_config.transport_config(Arc::new(raft_transport_config(tuning, rpc_timeout)?)); Ok(server_config) } @@ -261,6 +241,7 @@ pub fn make_raft_server_config_mtls( pub fn make_raft_client_config_mtls( creds: &TlsCredentials, tuning: &ClusterTransportTuning, + rpc_timeout: Duration, identity_store: Arc, ) -> Result { ensure_rustls_crypto_provider(); @@ -320,7 +301,7 @@ pub fn make_raft_client_config_mtls( })?; let mut client_config = quinn::ClientConfig::new(Arc::new(quic_crypto)); - client_config.transport_config(Arc::new(raft_transport_config(tuning))); + client_config.transport_config(Arc::new(raft_transport_config(tuning, rpc_timeout)?)); Ok(client_config) } diff --git a/nodedb-cluster/src/transport/endpoint.rs b/nodedb-cluster/src/transport/endpoint.rs new file mode 100644 index 000000000..7de0b48f6 --- /dev/null +++ b/nodedb-cluster/src/transport/endpoint.rs @@ -0,0 +1,169 @@ +// SPDX-License-Identifier: BUSL-1.1 + +//! The QUIC endpoint a node serves and dials Raft RPCs on. +//! +//! A build with the `failpoints` feature binds the endpoint over a UDP socket +//! that drops every datagram while the node's `transport::blackhole` fail +//! point is armed. The node then looks partitioned to its peers while its +//! process keeps running. Any other build binds a plain UDP socket. + +use std::net::SocketAddr; + +use crate::error::{ClusterError, Result}; + +/// Node-scoped fail point that cuts a node off the network while armed. +pub const BLACKHOLE_FAIL_POINT: &str = "transport::blackhole"; + +/// A bound QUIC endpoint that accepts Raft RPC connections. +#[derive(Debug)] +pub struct RaftListener { + endpoint: quinn::Endpoint, + local_addr: SocketAddr, +} + +impl RaftListener { + /// Bind the endpoint of node `node_id` on `addr`. + pub fn bind(node_id: u64, addr: SocketAddr, config: quinn::ServerConfig) -> Result { + let endpoint = + bind_endpoint(node_id, addr, config).map_err(|e| ClusterError::Transport { + detail: format!("bind {addr}: {e}"), + })?; + let local_addr = endpoint.local_addr().map_err(|e| ClusterError::Transport { + detail: format!("local address of {addr}: {e}"), + })?; + Ok(Self { + endpoint, + local_addr, + }) + } + + /// Accept the next incoming QUIC connection. + pub async fn accept(&self) -> Result { + let incoming = self + .endpoint + .accept() + .await + .ok_or_else(|| ClusterError::Transport { + detail: "endpoint closed".into(), + })?; + incoming.await.map_err(|e| ClusterError::Transport { + detail: format!("accept: {e}"), + }) + } + + /// The local address the endpoint is bound to. + pub fn local_addr(&self) -> SocketAddr { + self.local_addr + } + + /// The endpoint, which also dials outbound connections. + pub fn endpoint(&self) -> &quinn::Endpoint { + &self.endpoint + } +} + +#[cfg(not(feature = "failpoints"))] +fn bind_endpoint( + _node_id: u64, + addr: SocketAddr, + config: quinn::ServerConfig, +) -> std::io::Result { + quinn::Endpoint::server(config, addr) +} + +#[cfg(feature = "failpoints")] +fn bind_endpoint( + node_id: u64, + addr: SocketAddr, + config: quinn::ServerConfig, +) -> std::io::Result { + let socket = std::net::UdpSocket::bind(addr)?; + let runtime = + quinn::default_runtime().ok_or_else(|| std::io::Error::other("no async runtime found"))?; + let inner = runtime.wrap_udp_socket(socket)?; + let socket = std::sync::Arc::new(blackhole::BlackholeSocket::new(node_id, inner)); + quinn::Endpoint::new_with_abstract_socket( + quinn::EndpointConfig::default(), + Some(config), + socket, + runtime, + ) +} + +#[cfg(feature = "failpoints")] +mod blackhole { + use std::io::{self, IoSliceMut}; + use std::net::SocketAddr; + use std::pin::Pin; + use std::sync::Arc; + use std::task::{Context, Poll}; + + use nodedb_types::fail_point::{FailScope, lookup}; + use quinn::udp::{RecvMeta, Transmit}; + use quinn::{AsyncUdpSocket, UdpPoller}; + + use super::BLACKHOLE_FAIL_POINT; + + /// A UDP socket that drops every datagram, both ways, while its node's + /// blackhole fail point is armed. + #[derive(Debug)] + pub(super) struct BlackholeSocket { + node_id: u64, + inner: Arc, + } + + impl BlackholeSocket { + pub(super) fn new(node_id: u64, inner: Arc) -> Self { + Self { node_id, inner } + } + + fn cut_off(&self) -> bool { + lookup(FailScope::Node(self.node_id), BLACKHOLE_FAIL_POINT).is_some() + } + } + + impl AsyncUdpSocket for BlackholeSocket { + fn create_io_poller(self: Arc) -> Pin> { + Arc::clone(&self.inner).create_io_poller() + } + + fn try_send(&self, transmit: &Transmit) -> io::Result<()> { + if self.cut_off() { + return Ok(()); + } + self.inner.try_send(transmit) + } + + fn poll_recv( + &self, + cx: &mut Context, + bufs: &mut [IoSliceMut<'_>], + meta: &mut [RecvMeta], + ) -> Poll> { + // Received datagrams are drained and dropped until the inner + // socket has none left, so it registers the waker. + loop { + match self.inner.poll_recv(cx, bufs, meta) { + Poll::Ready(Ok(_)) if self.cut_off() => {} + other => return other, + } + } + } + + fn local_addr(&self) -> io::Result { + self.inner.local_addr() + } + + fn max_transmit_segments(&self) -> usize { + self.inner.max_transmit_segments() + } + + fn max_receive_segments(&self) -> usize { + self.inner.max_receive_segments() + } + + fn may_fragment(&self) -> bool { + self.inner.may_fragment() + } + } +} diff --git a/nodedb-cluster/src/transport/liveness.rs b/nodedb-cluster/src/transport/liveness.rs new file mode 100644 index 000000000..43557c6af --- /dev/null +++ b/nodedb-cluster/src/transport/liveness.rs @@ -0,0 +1,104 @@ +// SPDX-License-Identifier: BUSL-1.1 + +//! QUIC transport settings for Raft RPCs, and peer liveness. +//! +//! The QUIC idle timeout is the RPC timeout. A peer that sends nothing for +//! that long, keep-alives included, is dead: its connection closes and every +//! pending read on it fails. A live peer's keep-alives hold the connection +//! open while a slow handler runs, so a long read gets its whole budget. + +use std::time::Duration; + +use nodedb_types::config::tuning::ClusterTransportTuning; + +use crate::error::{ClusterError, Result}; + +/// Smallest RPC timeout a node starts with. +/// +/// The RPC timeout is also the QUIC idle timeout. A shorter one drops a live +/// peer whose keep-alives meet a short scheduling stall. +pub const MIN_RPC_TIMEOUT: Duration = Duration::from_millis(500); + +/// Shortest keep-alive interval, so a short RPC timeout cannot flood a peer. +const MIN_KEEP_ALIVE: Duration = Duration::from_millis(100); + +/// Keep-alives sent per idle timeout. Several must be lost in a row before +/// a live peer reads as silent. +const KEEP_ALIVES_PER_IDLE_TIMEOUT: u32 = 5; + +/// The keep-alive interval for `rpc_timeout`: a fifth of it, at least 100 ms. +pub fn keep_alive_interval(rpc_timeout: Duration) -> Duration { + (rpc_timeout / KEEP_ALIVES_PER_IDLE_TIMEOUT).max(MIN_KEEP_ALIVE) +} + +/// Transport config tuned for Raft RPCs in a datacenter. +/// +/// Peer liveness is the QUIC idle timeout, set to `rpc_timeout`. Keep-alives +/// hold a live connection open while a handler runs past it. A peer silent +/// for `rpc_timeout` closes the connection, and every pending read on it fails. +pub fn raft_transport_config( + tuning: &ClusterTransportTuning, + rpc_timeout: Duration, +) -> Result { + let idle_timeout = + quinn::IdleTimeout::try_from(rpc_timeout).map_err(|e| ClusterError::Config { + detail: format!( + "rpc timeout {}ms does not fit a QUIC idle timeout: {e}", + rpc_timeout.as_millis() + ), + })?; + let mut config = quinn::TransportConfig::default(); + // Raft RPCs use bidi streams: one per request-response pair. + config.max_concurrent_bidi_streams(quinn::VarInt::from_u32(tuning.quic_max_bi_streams)); + // Also allow uni streams for future migration/snapshot streaming. + config.max_concurrent_uni_streams(quinn::VarInt::from_u32(tuning.quic_max_uni_streams)); + // Datacenter tuning: generous windows, low RTT estimate. + config.receive_window(quinn::VarInt::from_u32(tuning.quic_receive_window)); + config.send_window(u64::from(tuning.quic_send_window)); + config.stream_receive_window(quinn::VarInt::from_u32(tuning.quic_stream_receive_window)); + config.initial_rtt(Duration::from_micros(100)); + config.keep_alive_interval(Some(keep_alive_interval(rpc_timeout))); + config.max_idle_timeout(Some(idle_timeout)); + Ok(config) +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn the_keep_alive_is_a_fifth_of_the_rpc_timeout() { + assert_eq!( + keep_alive_interval(Duration::from_secs(5)), + Duration::from_secs(1) + ); + assert_eq!( + keep_alive_interval(Duration::from_millis(500)), + Duration::from_millis(100) + ); + } + + #[test] + fn the_keep_alive_never_drops_below_its_floor() { + assert_eq!( + keep_alive_interval(Duration::from_millis(300)), + MIN_KEEP_ALIVE + ); + } + + #[test] + fn the_idle_timeout_follows_the_rpc_timeout() { + let tuning = ClusterTransportTuning::default(); + assert!(raft_transport_config(&tuning, Duration::from_millis(300)).is_ok()); + assert!(raft_transport_config(&tuning, Duration::from_secs(5)).is_ok()); + } + + #[test] + fn an_idle_timeout_past_the_quic_range_is_refused() { + let tuning = ClusterTransportTuning::default(); + assert!(matches!( + raft_transport_config(&tuning, Duration::MAX), + Err(ClusterError::Config { .. }) + )); + } +} diff --git a/nodedb-cluster/src/transport/mod.rs b/nodedb-cluster/src/transport/mod.rs index 5e0cecad5..ab1325967 100644 --- a/nodedb-cluster/src/transport/mod.rs +++ b/nodedb-cluster/src/transport/mod.rs @@ -4,8 +4,10 @@ pub mod auth_context; pub mod client; pub mod config; pub mod credentials; +pub mod endpoint; mod frame_io; mod identity_admission; +pub mod liveness; pub mod peer_identity_store; pub mod peer_identity_verifier; pub mod pinned_verifier; diff --git a/nodedb-cluster/src/transport/server.rs b/nodedb-cluster/src/transport/server.rs index a52417851..95f1e637c 100644 --- a/nodedb-cluster/src/transport/server.rs +++ b/nodedb-cluster/src/transport/server.rs @@ -343,9 +343,12 @@ async fn handle_stream( reply_and_finish(&mut send, &auth, &response, "response").await }; + // A dead connection can carry no answer, so its request stops using the + // executor. Handlers already tolerate the drop: shutdown drops `work` too. let outcome = tokio::select! { biased; _ = shutdown.changed() => Ok(false), + _ = conn.closed() => Ok(false), result = work => result.map(|()| true), }; if let Ok(true) = outcome { diff --git a/nodedb-test-support/src/cluster_harness/cluster/convergence.rs b/nodedb-test-support/src/cluster_harness/cluster/convergence.rs new file mode 100644 index 000000000..e6b7a82b3 --- /dev/null +++ b/nodedb-test-support/src/cluster_harness/cluster/convergence.rs @@ -0,0 +1,169 @@ +// SPDX-License-Identifier: BUSL-1.1 + +//! Apply-convergence barriers. +//! +//! A barrier waits until every node applied every entry its hosted groups +//! committed when the barrier looked. On timeout it panics with each lagging +//! group's commit target and every node's commit and applied index. +//! +//! The target of a group is the highest commit index any node reports for it. +//! A node's own applied index alone is no target: after a restart, every node +//! can hold the same applied index while committed entries above it are +//! unapplied everywhere. + +use std::collections::BTreeMap; +use std::fmt::Write as _; +use std::time::{Duration, Instant}; + +use super::TestCluster; + +/// One hosted group's positions on one node. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub struct GroupPosition { + pub node_id: u64, + pub group_id: u64, + /// The leader this node knows for the group, `0` when it knows none. + pub leader_id: u64, + pub commit: u64, + /// This node's apply watermark for the group. + pub applied: u64, +} + +impl TestCluster { + /// Every hosted group's positions on every node, keyed by group. A node + /// that runs no Raft yet reports nothing. + pub fn group_positions(&self) -> BTreeMap> { + let mut groups: BTreeMap> = BTreeMap::new(); + for node in &self.nodes { + let Some(status_fn) = node.shared.raft_status_fn.get() else { + continue; + }; + for status in status_fn() { + groups + .entry(status.group_id) + .or_default() + .push(GroupPosition { + node_id: node.node_id, + group_id: status.group_id, + leader_id: status.leader_id, + commit: status.commit_index, + applied: node.shared.applied_index_watcher(status.group_id).current(), + }); + } + } + groups + } + + /// [`Self::group_positions`] as text, one group per line, for a panic + /// message. + pub fn group_positions_report(&self) -> String { + render(&self.group_positions(), &BTreeMap::new()) + } + + /// Block until every node applied every entry its hosted groups + /// committed. Groups and commits are re-read on every poll, so a group + /// that appears late is covered. + /// + /// # Panics + /// + /// At `timeout`, naming each lagging group with its commit target and + /// every node's commit and applied index. + pub async fn wait_for_full_apply_convergence(&self, timeout: Duration) { + self.wait_for_groups_applied(timeout, None).await; + } + + /// [`Self::wait_for_full_apply_convergence`] for the metadata group only. + pub(crate) async fn wait_for_metadata_apply_convergence(&self, timeout: Duration) { + self.wait_for_groups_applied(timeout, Some(nodedb_cluster::METADATA_GROUP_ID)) + .await; + } + + async fn wait_for_groups_applied(&self, timeout: Duration, only: Option) { + let deadline = Instant::now() + timeout; + loop { + // Commit targets first, apply positions after: an applied index + // only grows, so a node at or past the target applied every + // entry committed when the targets were read. + let targets = commit_targets(&self.group_positions(), only); + let mut positions = self.group_positions(); + positions.retain(|group_id, _| targets.contains_key(group_id)); + let lagging = lagging_groups(&positions, &targets); + let silent = self.nodes_without_status(); + if lagging.is_empty() && silent.is_empty() { + return; + } + if Instant::now() >= deadline { + let lagging_positions: BTreeMap> = positions + .into_iter() + .filter(|(group_id, _)| lagging.contains(group_id)) + .collect(); + panic!( + "the cluster did not apply every committed entry within {timeout:?}.\n\ + nodes reporting no group status: {silent:?}\n\ + lagging groups:\n{}", + render(&lagging_positions, &targets) + ); + } + tokio::time::sleep(Duration::from_millis(5)).await; + } + } + + /// Nodes whose Raft status is not installed yet. + fn nodes_without_status(&self) -> Vec { + self.nodes + .iter() + .filter(|node| node.shared.raft_status_fn.get().is_none()) + .map(|node| node.node_id) + .collect() + } +} + +/// The highest commit index any node reports, per group. +fn commit_targets( + groups: &BTreeMap>, + only: Option, +) -> BTreeMap { + groups + .iter() + .filter(|(group_id, _)| only.is_none_or(|only| only == **group_id)) + .map(|(group_id, positions)| { + let target = positions.iter().map(|p| p.commit).max().unwrap_or(0); + (*group_id, target) + }) + .collect() +} + +/// Groups where some hosting node applied less than the group's target. +fn lagging_groups( + groups: &BTreeMap>, + targets: &BTreeMap, +) -> Vec { + groups + .iter() + .filter(|(group_id, positions)| { + let target = targets.get(group_id).copied().unwrap_or(0); + positions.iter().any(|p| p.applied < target) + }) + .map(|(group_id, _)| *group_id) + .collect() +} + +fn render(groups: &BTreeMap>, targets: &BTreeMap) -> String { + let mut out = String::new(); + for (group_id, positions) in groups { + let _ = write!(out, " group {group_id}"); + if let Some(target) = targets.get(group_id) { + let _ = write!(out, " (commit target {target})"); + } + let _ = write!(out, ":"); + for p in positions { + let _ = write!( + out, + " [node {} leader {} commit {} applied {}]", + p.node_id, p.leader_id, p.commit, p.applied + ); + } + out.push('\n'); + } + out +} diff --git a/nodedb-test-support/src/cluster_harness/cluster/membership.rs b/nodedb-test-support/src/cluster_harness/cluster/membership.rs index 7c5fb72fe..e51f786ca 100644 --- a/nodedb-test-support/src/cluster_harness/cluster/membership.rs +++ b/nodedb-test-support/src/cluster_harness/cluster/membership.rs @@ -96,17 +96,10 @@ impl TestCluster { /// the retry loop tries the next node on failure so the test /// doesn't have to discover the leader explicitly. /// - /// After the DDL is accepted, **blocks until every node's - /// metadata applier has caught up to the proposer's applied - /// index**. `propose_catalog_entry` already waits for the entry - /// to be applied on the proposing node before returning, but - /// followers apply asynchronously — without this barrier a - /// subsequent `wait_for("x visible on every node")` would race - /// the follower appliers and trip its timeout on the cold-start - /// attempt. Polling the watermark directly is O(applied_index) - /// and converges as soon as the followers drain their commit - /// queues, so it's both strictly more correct and strictly - /// faster than waiting on the visibility check itself. + /// After the DDL is accepted, blocks until every node applied the + /// metadata group through its commit index. Followers apply + /// asynchronously, so a later visibility check otherwise races their + /// appliers. Panics with every node's positions after 10 s. pub async fn exec_ddl_on_any_leader(&self, sql: &str) -> Result { let deadline = std::time::Instant::now() + Duration::from_secs(10); let mut last_err = String::new(); @@ -114,7 +107,8 @@ impl TestCluster { for (idx, node) in self.nodes.iter().enumerate() { match node.exec(sql).await { Ok(()) => { - self.wait_for_applied_index_convergence(idx).await; + self.wait_for_metadata_apply_convergence(Duration::from_secs(10)) + .await; return Ok(idx); } Err(e) => last_err = e, @@ -127,121 +121,6 @@ impl TestCluster { )) } - /// Block until every node's metadata applier has caught up to the - /// proposer's current applied index. Called after every successful - /// DDL by `exec_ddl_on_any_leader`. - async fn wait_for_applied_index_convergence(&self, proposer_idx: usize) { - let group_id = nodedb_cluster::METADATA_GROUP_ID; - let target = self.nodes[proposer_idx] - .shared - .applied_index_watcher(group_id) - .current(); - if target == 0 { - return; - } - let deadline = std::time::Instant::now() + Duration::from_secs(10); - loop { - let all_caught_up = self - .nodes - .iter() - .all(|n| n.shared.applied_index_watcher(group_id).current() >= target); - if all_caught_up { - return; - } - if std::time::Instant::now() >= deadline { - // Don't panic — the caller's own `wait_for` assertion - // will report the specific visibility failure with a - // better error than "convergence timed out". - return; - } - tokio::time::sleep(Duration::from_millis(5)).await; - } - } - - /// Block until every node's per-group applied watermark has - /// caught up to the maximum observed across the cluster for that - /// group. This is the deterministic barrier for "every Raft - /// group has fully propagated" — replaces the SQL-polling - /// pattern (`wait_for_async("rows visible from node N", ...)`) - /// that races the follower applier under load. - /// - /// For every group registered on *any* node, the target is - /// `max(applied_index across all nodes)`. Each node then waits - /// for that group's local watcher to reach the target. New - /// groups that show up partway through (e.g. a vshard the test - /// has not written to yet) are handled by re-snapshotting on - /// every iteration of the outer poll, so the helper is - /// idempotent against late-bound group registration. - /// - /// Returns once every (node, group) pair has converged or the - /// deadline expires. On expiry, falls through silently — the - /// caller's own assertion will surface the specific row-level - /// failure with a more useful error than "convergence timed - /// out". - pub async fn wait_for_full_apply_convergence(&self, timeout: Duration) { - let deadline = std::time::Instant::now() + timeout; - loop { - // Targets: for each group_id seen on any node, take the - // max applied_index. Asymmetric group membership is - // expected — replication factor may be < node count, so - // not every group is hosted on every node. - let mut targets: std::collections::HashMap = std::collections::HashMap::new(); - for node in &self.nodes { - for (gid, applied) in node.shared.group_watchers().snapshot() { - let entry = targets.entry(gid).or_insert(0); - if applied > *entry { - *entry = applied; - } - } - } - - // Group membership is read from the routing table (the - // authoritative source) rather than inferred from the - // watcher registry. Inferring from the registry has a - // cold-start race: a follower that hosts group X but - // hasn't yet applied its first entry has no registry - // entry, would be treated as "not hosted", and the - // helper would return prematurely. Routing knows the - // members + learners list as soon as the conf-change - // commits. - let all_caught_up = { - let routing = self.nodes[0] - .shared - .cluster_routing - .as_ref() - .expect("cluster_routing") - .read() - .unwrap_or_else(|p| p.into_inner()); - - self.nodes.iter().all(|node| { - let nid = node.shared.node_id; - let watcher = node.shared.group_watchers(); - targets.iter().all(|(&gid, &target)| { - let hosts = routing - .group_info(gid) - .map(|info| info.members.contains(&nid) || info.learners.contains(&nid)) - .unwrap_or(false); - if !hosts { - return true; - } - watcher.get_or_create(gid).current() >= target - }) - }) - }; - if all_caught_up { - return; - } - if std::time::Instant::now() >= deadline { - // Falls through silently — the caller's own - // assertion will surface the specific row-level - // failure with a more useful error than - // "convergence timed out". - return; - } - tokio::time::sleep(Duration::from_millis(5)).await; - } - } - /// Cooperatively shut down every node. Reverse order so peers /// observe their neighbours' drop without rejecting inbound /// traffic on an already-closed transport. diff --git a/nodedb-test-support/src/cluster_harness/cluster/mod.rs b/nodedb-test-support/src/cluster_harness/cluster/mod.rs index d0ed6e4ba..6ee9896c8 100644 --- a/nodedb-test-support/src/cluster_harness/cluster/mod.rs +++ b/nodedb-test-support/src/cluster_harness/cluster/mod.rs @@ -4,12 +4,14 @@ //! //! [`TestCluster`] + [`ClusterSpawnConfig`] type definitions live in //! [`types`]; spawn-variant convenience wrappers in [`spawn_variants`]; -//! the bringup body in [`bringup`]; the convergence barriers in [`ready`]; +//! the bringup body in [`bringup`]; the readiness barriers in [`ready`]; +//! the apply-convergence barriers in [`convergence`]; //! the in-place restart in [`restart`]; post-spawn membership + DDL helpers //! in [`membership`]; data-group leadership moves and waits in //! [`leadership`]. mod bringup; +mod convergence; mod leadership; mod membership; mod ready; @@ -17,6 +19,7 @@ mod restart; mod spawn_variants; mod types; +pub use convergence::GroupPosition; pub use leadership::GroupLeader; pub use restart::{StoppedCluster, StoppedMember, StoppedNodeInfo}; pub use types::TestCluster; diff --git a/nodedb-test-support/src/cluster_harness/mod.rs b/nodedb-test-support/src/cluster_harness/mod.rs index eb4f9f0e3..4aa023190 100644 --- a/nodedb-test-support/src/cluster_harness/mod.rs +++ b/nodedb-test-support/src/cluster_harness/mod.rs @@ -22,7 +22,7 @@ pub mod retriable; pub mod shared_steps; pub mod wait; -pub use cluster::{GroupLeader, StoppedCluster, StoppedNodeInfo, TestCluster}; +pub use cluster::{GroupLeader, GroupPosition, StoppedCluster, StoppedNodeInfo, TestCluster}; pub use node::TestClusterNode; pub use pitr::PitrStorage; pub use retriable::{is_no_serving_leader, read_once_a_leader_exists}; diff --git a/nodedb-test-support/src/native_harness/frames.rs b/nodedb-test-support/src/native_harness/frames.rs index f99c9f2b7..2e549ad01 100644 --- a/nodedb-test-support/src/native_harness/frames.rs +++ b/nodedb-test-support/src/native_harness/frames.rs @@ -132,11 +132,29 @@ pub async fn read_frame(stream: &mut TcpStream) -> Option> { /// must send one JSON frame before calling this. Shared by [`send_sql`] and /// any test driving a direct-op opcode (`PointGet`, `RangeScan`, /// `VectorSearch`, `KvBatchPut`, ...) directly rather than through SQL text. +/// +/// Waits at most [`RESPONSE_BOUND`] for the response. pub async fn send_request( stream: &mut TcpStream, seq: u64, op: OpCode, fields: TextFields, +) -> NativeResponse { + send_request_within(stream, seq, op, fields, Some(RESPONSE_BOUND)).await +} + +/// How long [`send_request`] waits for a response. +pub const RESPONSE_BOUND: Duration = Duration::from_secs(5); + +/// [`send_request`] waiting at most `bound` for the response, or without +/// limit when `bound` is `None`. A caller that parks the server on purpose +/// bounds the wait itself. +pub async fn send_request_within( + stream: &mut TcpStream, + seq: u64, + op: OpCode, + fields: TextFields, + bound: Option, ) -> NativeResponse { let req = NativeRequest { op, @@ -146,14 +164,18 @@ pub async fn send_request( let json_bytes = sonic_rs::to_vec(&req).expect("json encode"); write_frame(stream, &json_bytes).await; - let deadline = tokio::time::Instant::now() + Duration::from_secs(5); + let deadline = bound.map(|bound| tokio::time::Instant::now() + bound); let mut accumulated: Option = None; loop { - let remaining = deadline.saturating_duration_since(tokio::time::Instant::now()); - let response_payload = tokio::time::timeout(remaining, read_frame(stream)) - .await - .expect("timeout waiting for response") - .expect("response frame"); + let response_payload = match deadline { + Some(deadline) => tokio::time::timeout_at(deadline, read_frame(stream)) + .await + .unwrap_or_else(|_| { + panic!("no response to request seq {seq} ({op:?}) within {bound:?}") + }), + None => read_frame(stream).await, + } + .expect("response frame"); let response: NativeResponse = sonic_rs::from_slice(&response_payload).expect("json decode NativeResponse"); if response.seq == seq { diff --git a/nodedb-test-support/src/native_harness/mod.rs b/nodedb-test-support/src/native_harness/mod.rs index cb0df1c10..9bdf9d15f 100644 --- a/nodedb-test-support/src/native_harness/mod.rs +++ b/nodedb-test-support/src/native_harness/mod.rs @@ -10,7 +10,7 @@ mod frames; mod server; pub use frames::{ - do_handshake, do_handshake_from, open_trust_session, read_frame, send_api_key_auth, - send_request, send_sql, write_frame, + RESPONSE_BOUND, do_handshake, do_handshake_from, open_trust_session, read_frame, + send_api_key_auth, send_request, send_request_within, send_sql, write_frame, }; pub use server::NativeTestServer; diff --git a/nodedb-types/src/config/tuning/network.rs b/nodedb-types/src/config/tuning/network.rs index 6f2ef37c6..f6b72d335 100644 --- a/nodedb-types/src/config/tuning/network.rs +++ b/nodedb-types/src/config/tuning/network.rs @@ -170,12 +170,11 @@ pub struct ClusterTransportTuning { /// [`Self::election_timeout_min_ms`]. #[serde(default)] pub election_timeout_max_ms: u64, + /// Timeout of one RPC round trip, and the QUIC idle timeout: a peer + /// silent this long is dead. Keep-alives go out at a fifth of it. + /// Startup refuses a value below 500 ms. #[serde(default = "default_rpc_timeout_secs")] pub rpc_timeout_secs: u64, - #[serde(default = "default_quic_keep_alive_secs")] - pub quic_keep_alive_secs: u64, - #[serde(default = "default_quic_idle_timeout_secs")] - pub quic_idle_timeout_secs: u64, #[serde(default = "default_quic_max_bi_streams")] pub quic_max_bi_streams: u32, #[serde(default = "default_quic_max_uni_streams")] @@ -262,8 +261,6 @@ impl Default for ClusterTransportTuning { election_timeout_min_ms: 0, election_timeout_max_ms: 0, rpc_timeout_secs: default_rpc_timeout_secs(), - quic_keep_alive_secs: default_quic_keep_alive_secs(), - quic_idle_timeout_secs: default_quic_idle_timeout_secs(), quic_max_bi_streams: default_quic_max_bi_streams(), quic_max_uni_streams: default_quic_max_uni_streams(), quic_receive_window: default_quic_receive_window(), @@ -309,12 +306,6 @@ fn default_election_timeout_max_secs() -> u64 { fn default_rpc_timeout_secs() -> u64 { 5 } -fn default_quic_keep_alive_secs() -> u64 { - 5 -} -fn default_quic_idle_timeout_secs() -> u64 { - 30 -} fn default_quic_max_bi_streams() -> u32 { 256 } diff --git a/nodedb/src/config/server/config.rs b/nodedb/src/config/server/config.rs index b83609e69..4c09766e3 100644 --- a/nodedb/src/config/server/config.rs +++ b/nodedb/src/config/server/config.rs @@ -422,6 +422,24 @@ mod tests { assert!(msg.contains("at least 10 seconds"), "{msg}"); } + /// The RPC timeout is the QUIC idle timeout, so one under its floor is + /// refused at startup. + #[test] + fn from_file_rejects_an_rpc_timeout_below_the_floor() { + let path = write_temp_config( + "nodedb-domain-rpc-timeout.toml", + "[tuning.cluster_transport]\nrpc_timeout_secs = 0\n", + ); + let err = ServerConfig::from_file(&path).unwrap_err(); + std::fs::remove_file(&path).ok(); + let msg = err.to_string(); + assert!( + msg.contains("tuning.cluster_transport.rpc_timeout_secs"), + "{msg}" + ); + assert!(msg.contains("at least 500 ms"), "{msg}"); + } + #[test] fn from_file_rejects_a_wal_write_buffer_under_the_floor() { let path = write_temp_config( diff --git a/nodedb/src/config/server/domain.rs b/nodedb/src/config/server/domain.rs index b152a1300..375d79b4c 100644 --- a/nodedb/src/config/server/domain.rs +++ b/nodedb/src/config/server/domain.rs @@ -120,6 +120,7 @@ pub(super) fn validate_domain(config: &ServerConfig) -> crate::Result<()> { } validate_calvin(config)?; + validate_rpc_timeout(config)?; if let Some(cluster) = config.cluster.as_ref() { positive_u64( @@ -148,6 +149,21 @@ pub(super) fn validate_domain(config: &ServerConfig) -> crate::Result<()> { Ok(()) } +/// Checks the RPC timeout against its floor. It is also the QUIC idle +/// timeout, so a shorter one drops live peers. +fn validate_rpc_timeout(config: &ServerConfig) -> crate::Result<()> { + let floor = nodedb_cluster::transport::liveness::MIN_RPC_TIMEOUT; + let secs = config.tuning.cluster_transport.rpc_timeout_secs; + if std::time::Duration::from_secs(secs) < floor { + return Err(reject( + "tuning.cluster_transport.rpc_timeout_secs", + secs, + &format!("a timeout of at least {} ms", floor.as_millis()), + )); + } + Ok(()) +} + /// Checks `[tuning.calvin]`. A zero channel capacity panics the channel /// constructor, and a zero bound stalls every scheduler. fn validate_calvin(config: &ServerConfig) -> crate::Result<()> { diff --git a/nodedb/src/control/array_sync/inbound_propose.rs b/nodedb/src/control/array_sync/inbound_propose.rs index a11379cdc..c225da50c 100644 --- a/nodedb/src/control/array_sync/inbound_propose.rs +++ b/nodedb/src/control/array_sync/inbound_propose.rs @@ -96,8 +96,9 @@ impl OriginArrayInbound { )) })? .as_ref(); - let deadline = tokio::time::Instant::now() - + std::time::Duration::from_secs(self.shared().tuning.network.default_deadline_secs); + let deadline = + crate::control::server::shared::session::StatementDeadline::of_statement(self.shared()) + .tokio_instant(); match async_proposer(vshard_id, idempotency_key, data, deadline).await { Ok((_payload, _committed_version)) => Ok(()), Err(e) => { diff --git a/nodedb/src/control/backup/cut.rs b/nodedb/src/control/backup/cut.rs index 8ae2bca06..6d76047e3 100644 --- a/nodedb/src/control/backup/cut.rs +++ b/nodedb/src/control/backup/cut.rs @@ -48,12 +48,11 @@ //! //! [`ReplicatedWrite::CutBarrier`]: crate::control::wal_replication::ReplicatedWrite::CutBarrier -use std::time::Duration; - use nodedb_physical::physical_plan::CutCaptureRequest; use nodedb_types::Hlc; use crate::Error; +use crate::control::server::shared::session::StatementDeadline; use crate::control::state::SharedState; use super::cut_order::OrderedCut; @@ -118,8 +117,9 @@ async fn cut_barriers(state: &SharedState, cut: OrderedCut) -> Result<(), Error> state .hlc_clock .update(Hlc::new(cut.hlc.saturating_add(1), 0)); - let timeout = Duration::from_secs(state.tuning.network.default_deadline_secs); - let deadline = tokio::time::Instant::now() + timeout; + let statement = StatementDeadline::of_statement(state); + let budget_ms = statement.remaining().as_millis(); + let deadline = statement.tokio_instant(); // Read after the watermark: a record minted after this read stamps its // write above the watermark. let target = state.outcome_floor.max_noted(); @@ -145,11 +145,11 @@ async fn cut_barriers(state: &SharedState, cut: OrderedCut) -> Result<(), Error> return Err(Error::Internal { detail: format!( "backup: writes minted at or below WAL LSN {} had no final outcome within \ - {}s, so the backup cannot take a consistent cut (outcome floor at {}). \ - Retry the backup. A write window held until restart keeps the floor \ - below it: restart the node if the floor does not move", + the statement deadline ({} ms), so the backup cannot take a consistent cut \ + (outcome floor at {}). Retry the backup. A write window held until restart \ + keeps the floor below it: restart the node if the floor does not move", target.as_u64(), - timeout.as_secs(), + budget_ms, state.outcome_floor.floor().as_u64(), ), }); diff --git a/nodedb/src/control/backup/restore/guard.rs b/nodedb/src/control/backup/restore/guard.rs index 8e4c64467..2e8570039 100644 --- a/nodedb/src/control/backup/restore/guard.rs +++ b/nodedb/src/control/backup/restore/guard.rs @@ -37,6 +37,7 @@ use crate::Error; use crate::control::security::auth_fence::cluster::{ confirmed_read_index, hosts_group, routed_groups, wait_applied, }; +use crate::control::server::shared::session::StatementDeadline; use crate::control::state::SharedState; use crate::control::state::tenant_marks::GroupMark; use crate::types::{DatabaseId, TraceId}; @@ -92,8 +93,7 @@ pub(super) async fn newest_committed_write( } }; // The statement deadline, shared by every group and every attempt. - let deadline = tokio::time::Instant::now() - + Duration::from_secs(state.tuning.network.default_deadline_secs); + let deadline = StatementDeadline::of_statement(state).tokio_instant(); let groups: Vec = routed_groups(state) .into_iter() .filter(|group| *group != METADATA_GROUP_ID && *group != SEQUENCER_GROUP_ID) diff --git a/nodedb/src/control/cluster/array_cluster_exec/dispatch.rs b/nodedb/src/control/cluster/array_cluster_exec/dispatch.rs index 0ff2e34e1..4a14acece 100644 --- a/nodedb/src/control/cluster/array_cluster_exec/dispatch.rs +++ b/nodedb/src/control/cluster/array_cluster_exec/dispatch.rs @@ -16,7 +16,7 @@ use nodedb_cluster::distributed_array::ArrayLocalExecutor; use nodedb_cluster::distributed_array::handler::handle_array_shard_rpc; use nodedb_cluster::distributed_array::rpc::ShardRpcDispatch; use nodedb_cluster::error::{ClusterError, Result as ClusterResult}; -use nodedb_cluster::rpc_codec::{RaftRpc, attempt_budget, remaining_budget_ms}; +use nodedb_cluster::rpc_codec::{RaftRpc, remaining_budget_ms}; use nodedb_cluster::wire::{VShardEnvelope, VShardMessageType}; use nodedb_cluster::{NexarTransport, RoutingTable}; @@ -68,20 +68,15 @@ const RESEND_PAUSE: std::time::Duration = std::time::Duration::from_millis(50); #[async_trait] impl ShardRpcDispatch for NexarArrayDispatch { - /// Every send shares the `timeout_ms` budget. A read leaves an RPC - /// timeout for its resend, so a silent shard cannot spend it all. + /// Every send gets what is left of the `timeout_ms` budget. A live shard + /// can use all of it. A dead shard fails the send at the QUIC idle + /// timeout, and the resend reaches the shard's current leader. async fn call(&self, req: VShardEnvelope, timeout_ms: u64) -> ClusterResult { let deadline = tokio::time::Instant::now() + std::time::Duration::from_millis(timeout_ms); let resend_safe = is_resend_safe(req.msg_type); let mut attempts_left = CALL_ATTEMPTS; loop { - let attempt_ms = attempt_budget( - remaining_budget_ms(deadline), - resend_safe, - attempts_left, - self.transport.rpc_timeout(), - ); - match self.call_once(&req, attempt_ms).await { + match self.call_once(&req, remaining_budget_ms(deadline)).await { Ok(resp) => return Ok(resp), // A resend starts only while the budget has time left. Err(e) @@ -169,8 +164,8 @@ impl NexarArrayDispatch { // Wrap in RaftRpc and send. The remote node's handler decodes the // envelope bytes and dispatches to the appropriate shard handler. let rpc = RaftRpc::VShardEnvelope(envelope_bytes); - // This attempt's budget bounds the reply wait, never the transport's - // fixed RPC timeout. + // The call's remaining budget bounds the reply wait, never the + // transport's fixed RPC timeout. let reply_wait = std::time::Duration::from_millis(timeout_ms); let resp_rpc = tokio::time::timeout( reply_wait, @@ -178,8 +173,8 @@ impl NexarArrayDispatch { .send_rpc_with_read_timeout(node_id, rpc, reply_wait), ) .await - // The request can have reached the shard: `call` resends a timeout - // only for a resend-safe request. + // The request can have reached the shard. The budget is spent, so + // `call` starts no resend. .map_err(|_| nodedb_cluster::error::ClusterError::ShardTimeout { vshard_id: req.vshard_id, elapsed_ms: timeout_ms, diff --git a/nodedb/src/control/cluster/start_raft/proposer_wiring.rs b/nodedb/src/control/cluster/start_raft/proposer_wiring.rs index 264ae0d08..2b79defe7 100644 --- a/nodedb/src/control/cluster/start_raft/proposer_wiring.rs +++ b/nodedb/src/control/cluster/start_raft/proposer_wiring.rs @@ -225,8 +225,8 @@ fn install_async_proposer( // Held weakly for the same cycle-breaking reason as `raft_proposer` above: // the proposer lives on `SharedState`. let state_for_proposer = Arc::downgrade(shared); - let async_submit: Arc = - Arc::new(move |vshard_id, idempotency_key, data, deadline| { + let async_submit: Arc = Arc::new( + move |vshard_id, idempotency_key, data, deadline| { let rl_weak = raft_loop_async.clone(); let tk = tracker_for_proposer.clone(); let state_weak = state_for_proposer.clone(); @@ -307,6 +307,7 @@ fn install_async_proposer( group_id, through: log_index, }], + crate::control::server::shared::session::StatementDeadline::of_statement(&state).instant(), ) .await?; } @@ -320,7 +321,8 @@ fn install_async_proposer( applied, }) }) - }); + }, + ); crate::control::vshard_admission::install_async_raft_proposer(shared, async_submit) } diff --git a/nodedb/src/control/crdt_admission.rs b/nodedb/src/control/crdt_admission.rs deleted file mode 100644 index 560ff47ed..000000000 --- a/nodedb/src/control/crdt_admission.rs +++ /dev/null @@ -1,1333 +0,0 @@ -// SPDX-License-Identifier: BUSL-1.1 - -//! Serialized preview, policy, and fenced CRDT apply admission. - -use std::time::Duration; - -use nodedb_physical::physical_plan::CrdtOp; -use nodedb_types::CrdtPreviewResult; - -use crate::bridge::envelope::{ErrorCode, PhysicalPlan, Status}; -use crate::control::server::shared::authorization::AuthorizedTask; -use crate::control::state::SharedState; -use crate::control::wal_replication::{ReplicableWrite, to_replicated_entry}; -use crate::event::EventSource; -use crate::types::{DatabaseId, TenantId, VShardId}; - -pub trait CrdtPostImagePolicy: Send + Sync { - fn evaluate(&self, preview: &CrdtPreviewResult) -> crate::Result<()>; -} - -/// Explicit policy for trusted internal callers. -pub struct TrustedInternalCrdtPolicy; - -impl CrdtPostImagePolicy for TrustedInternalCrdtPolicy { - fn evaluate(&self, _preview: &CrdtPreviewResult) -> crate::Result<()> { - Ok(()) - } -} - -const FRONTIER_RETRY_LIMIT: usize = 8; - -pub struct AuthorizedCrdtApplyAdmissionRequest<'a> { - pub authorized: AuthorizedTask, - /// The db-qualified collection (`QualifiedCollection::as_str`), the same - /// string the plan's `CrdtOp::Apply` carries. The preview and the apply - /// address the Data Plane by it, and the catalog lookup de-qualifies it. - pub collection: &'a str, - pub timeout: Duration, - pub event_source: EventSource, - pub policy: &'a dyn CrdtPostImagePolicy, -} - -pub struct CrdtApplyAdmissionRequest<'a> { - pub tenant_id: TenantId, - pub database_id: DatabaseId, - /// The db-qualified collection, equal to the plan's `CrdtOp::Apply` - /// collection. - pub collection: &'a str, - pub plan: PhysicalPlan, - pub timeout: Duration, - pub event_source: EventSource, - pub policy: &'a dyn CrdtPostImagePolicy, -} - -pub struct CrdtAdmissionOutcome { - pub payload: Vec, - /// The versions the apply stamped: the entry's log position on the - /// written vShard. - pub write_versions: crate::types::ReadVersions, - /// Operations the admitted delta encoded that the target document already - /// knew, measured by the preview that fenced this apply. Distinguishes a - /// client whose writes land from one whose writes are being absorbed — - /// both otherwise produce the same successful payload. - pub trimmed_ops: u64, -} - -impl CrdtAdmissionOutcome { - /// Attach the trim count measured by the preview that fenced this apply. - fn with_trimmed_ops(self, trimmed_ops: u64) -> Self { - Self { - trimmed_ops, - ..self - } - } -} - -pub struct CrdtRestoreAdmissionRequest<'a> { - pub tenant_id: TenantId, - pub database_id: DatabaseId, - /// The db-qualified collection the generated restore ops address. - pub collection: &'a str, - pub document_id: &'a str, - pub target_version_json: &'a str, - pub surrogate: nodedb_types::Surrogate, - pub peer_id: u64, - pub timeout: Duration, - pub event_source: EventSource, - pub policy: &'a dyn CrdtPostImagePolicy, -} - -struct CrdtAdmissionWorkflow<'a> { - state: &'a SharedState, - tenant_id: TenantId, - database_id: DatabaseId, - vshard_id: VShardId, - collection: &'a str, - timeout: Duration, - event_source: EventSource, - policy: &'a dyn CrdtPostImagePolicy, -} - -/// Whether an operation changes the Loro frontier and must serialize with an -/// admission preview when it is dispatched to this node's cores without a -/// proposal. -pub fn changes_crdt_frontier(op: &CrdtOp) -> bool { - match op { - CrdtOp::Apply { .. } - | CrdtOp::ApplyAuthenticated { .. } - | CrdtOp::ImportSnapshot { .. } - | CrdtOp::ListInsert { .. } - | CrdtOp::ListDelete { .. } - | CrdtOp::ListMove { .. } - | CrdtOp::DocUpsert { .. } - | CrdtOp::DocDelete { .. } - | CrdtOp::RestoreToVersion { .. } => true, - CrdtOp::Read { .. } - | CrdtOp::PreviewApply { .. } - | CrdtOp::GetVersionVector { .. } - | CrdtOp::ExportDelta { .. } - | CrdtOp::CompactAtVersion { .. } - | CrdtOp::SetConstraints { .. } - | CrdtOp::DropConstraints { .. } - | CrdtOp::ReadConstraints { .. } - | CrdtOp::SetPolicy { .. } - | CrdtOp::GetPolicy { .. } - | CrdtOp::ReadAtVersion { .. } => false, - } -} - -/// Preview, authorize, fence, and durably apply one externally authorized delta. -pub async fn dispatch_authorized_crdt_apply_admitted_outcome( - state: &SharedState, - request: AuthorizedCrdtApplyAdmissionRequest<'_>, -) -> crate::Result { - let AuthorizedCrdtApplyAdmissionRequest { - authorized, - collection, - timeout, - event_source, - policy, - } = request; - enforce_external_signing_policy(state, &authorized, collection)?; - let task = authorized.into_physical_task(); - dispatch_crdt_apply_admitted_outcome( - state, - CrdtApplyAdmissionRequest { - tenant_id: task.tenant_id, - database_id: task.database_id, - collection, - plan: task.plan, - timeout, - event_source, - policy, - }, - ) - .await -} - -/// `collection` is db-qualified. The catalog keys collections by the bare -/// name, so the lookup de-qualifies it first. -fn enforce_external_signing_policy( - state: &SharedState, - authorized: &AuthorizedTask, - collection: &str, -) -> crate::Result<()> { - let bare = crate::control::target_identity::naming::bare_collection_name( - authorized.database_id(), - collection, - ); - let stored = state - .credentials - .catalog() - .get_collection( - authorized.database_id(), - authorized.tenant_id().as_u64(), - &bare, - )? - .ok_or_else(|| crate::Error::CollectionNotFound { - tenant_id: authorized.tenant_id(), - collection: bare.clone(), - })?; - if stored.crdt_signing_required - && matches!(authorized.plan(), PhysicalPlan::Crdt(CrdtOp::Apply { .. })) - { - return Err(crate::Error::RejectedAuthz { - tenant_id: authorized.tenant_id(), - resource: format!("collection:{bare}:unsigned_crdt_delta_requires_authenticated_sync"), - }); - } - Ok(()) -} - -/// Preview, authorize, fence, and durably apply one trusted-internal CRDT delta. -pub(crate) async fn dispatch_crdt_apply_admitted_outcome( - state: &SharedState, - request: CrdtApplyAdmissionRequest<'_>, -) -> crate::Result { - let CrdtApplyAdmissionRequest { - tenant_id, - database_id, - collection, - plan, - timeout, - event_source, - policy, - } = request; - // Every CRDT apply converges here; the lease gates it on a drained - // collection and lives until the apply's outcome. - let _lease = crate::control::server::shared::clone_write::write_lease( - state, - tenant_id, - database_id, - &plan, - ) - .await?; - let (document_id, delta) = match &plan { - PhysicalPlan::Crdt( - CrdtOp::Apply { - collection: plan_collection, - document_id, - delta, - expected_frontier_digest: None, - .. - } - | CrdtOp::ApplyAuthenticated { - collection: plan_collection, - document_id, - delta, - expected_frontier_digest: None, - .. - }, - ) if plan_collection.as_str() == collection => (document_id.clone(), delta.clone()), - PhysicalPlan::Crdt( - CrdtOp::Apply { - expected_frontier_digest: Some(_), - .. - } - | CrdtOp::ApplyAuthenticated { - expected_frontier_digest: Some(_), - .. - }, - ) => return Err(crate::Error::CrdtAdmissionCallerFence), - _ => { - return Err(crate::Error::CrdtAdmissionInvalidPlan { - reason: "expected an unfenced CRDT Apply for the supplied collection", - }); - } - }; - // `collection` is the plan's database-qualified name. - let vshard_id = - nodedb_types::CollectionKey::from_qualified_str(database_id, collection)?.vshard(); - let workflow = CrdtAdmissionWorkflow { - state, - tenant_id, - database_id, - vshard_id, - collection, - timeout, - event_source, - policy, - }; - tokio::time::timeout( - timeout, - state.vshard_admission_sequencer.run_after_proposed( - vshard_id, - |last_proposed| async move { - caught_up(&workflow, last_proposed).await?; - admit_apply_locked(&workflow, plan, &document_id, &delta).await - }, - ), - ) - .await - .map_err(|_| crate::Error::CrdtAdmissionTimeout { - vshard_id, - timeout_ms: timeout_ms(timeout), - })? -} - -/// Wait until this node applied every write admitted to the workflow's -/// vShard before this admission took the slot, so its preview reads them. -async fn caught_up( - workflow: &CrdtAdmissionWorkflow<'_>, - last_proposed: Option, -) -> crate::Result<()> { - crate::control::vshard_admission::await_admitted_applies( - workflow.state, - workflow.vshard_id, - last_proposed, - std::time::Instant::now() + workflow.timeout, - ) - .await -} - -async fn admit_apply_locked( - workflow: &CrdtAdmissionWorkflow<'_>, - plan: PhysicalPlan, - document_id: &str, - delta: &[u8], -) -> crate::Result { - for _attempt in 0..FRONTIER_RETRY_LIMIT { - let preview = preview(workflow, document_id, delta).await?; - workflow.policy.evaluate(&preview)?; - let fenced = stamp_fence(plan.clone(), preview.frontier_digest)?; - match apply_fenced(workflow, fenced).await { - Err(crate::Error::DataPlane(ErrorCode::CrdtFrontierMismatch { .. })) => {} - // The trim count belongs to the preview that fenced *this* attempt. - // A retry re-previews against the advanced frontier and produces its - // own count, so the two are never mixed. - result => { - return result.map(|outcome| outcome.with_trimmed_ops(preview.trimmed_ops)); - } - } - } - Err(crate::Error::CrdtAdmissionRetriesExhausted { - vshard_id: workflow.vshard_id, - attempts: FRONTIER_RETRY_LIMIT, - }) -} - -async fn preview( - workflow: &CrdtAdmissionWorkflow<'_>, - document_id: &str, - delta: &[u8], -) -> crate::Result { - let response = tokio::time::timeout( - workflow.timeout, - crate::control::server::shared::ddl::sync_dispatch::dispatch_system_response_with_source( - workflow.state, - crate::control::server::shared::ddl::sync_dispatch::SystemTask::new( - crate::control::server::shared::ddl::sync_dispatch::SystemReason::AdmittedContinuation, - workflow.tenant_id, - nodedb_types::CollectionKey::from_qualified_str(workflow.database_id, workflow.collection)?, - PhysicalPlan::Crdt(CrdtOp::PreviewApply { - collection: nodedb_types::QualifiedCollection::from_stored( - workflow.collection.to_owned(), - ), - document_id: document_id.to_owned(), - delta: delta.to_vec(), - }), - ), - workflow.timeout, - workflow.event_source, - ), - ) - .await - .map_err(|_| crate::Error::CrdtAdmissionTimeout { - vshard_id: workflow.vshard_id, - timeout_ms: timeout_ms(workflow.timeout), - })??; - if response.status != Status::Ok { - return Err(response_error(&response)); - } - zerompk::from_msgpack(response.payload.as_bytes()).map_err(|error| crate::Error::Internal { - detail: format!("CRDT preview payload decode: {error}"), - }) -} - -fn stamp_fence(plan: PhysicalPlan, digest: [u8; 32]) -> crate::Result { - match plan { - PhysicalPlan::Crdt(CrdtOp::Apply { - collection, - document_id, - delta, - peer_id, - mutation_id, - surrogate, - provenance, - constraint_version_required, - expected_frontier_digest: None, - }) => Ok(PhysicalPlan::Crdt(CrdtOp::Apply { - collection, - document_id, - delta, - peer_id, - mutation_id, - surrogate, - provenance, - constraint_version_required, - expected_frontier_digest: Some(digest), - })), - PhysicalPlan::Crdt(CrdtOp::ApplyAuthenticated { - collection, - document_id, - delta, - peer_id, - mutation_id, - surrogate, - provenance, - constraint_version_required, - expected_frontier_digest: None, - auth_user_id, - auth_device_id, - auth_seq_no, - delta_signature, - signing_required, - }) => Ok(PhysicalPlan::Crdt(CrdtOp::ApplyAuthenticated { - collection, - document_id, - delta, - peer_id, - mutation_id, - surrogate, - provenance, - constraint_version_required, - expected_frontier_digest: Some(digest), - auth_user_id, - auth_device_id, - auth_seq_no, - delta_signature, - signing_required, - })), - _ => Err(crate::Error::CrdtAdmissionInvalidPlan { - reason: "validated CRDT Apply plan was changed before fence stamping", - }), - } -} - -/// Payload-only wrapper for an externally authorized caller. -pub async fn dispatch_authorized_crdt_apply_admitted( - state: &SharedState, - request: AuthorizedCrdtApplyAdmissionRequest<'_>, -) -> crate::Result> { - Ok( - dispatch_authorized_crdt_apply_admitted_outcome(state, request) - .await? - .payload, - ) -} - -/// Payload-only compatibility wrapper for trusted internal test callers. -#[cfg(test)] -pub(crate) async fn dispatch_crdt_apply_admitted( - state: &SharedState, - request: CrdtApplyAdmissionRequest<'_>, -) -> crate::Result> { - Ok(dispatch_crdt_apply_admitted_outcome(state, request) - .await? - .payload) -} - -/// Generate a restore delta and admit it without releasing the vShard slot. -/// A stale apply fence regenerates the delta from authoritative state before -/// retrying, so neither historical projection nor policy output is reused. -pub(crate) async fn dispatch_crdt_restore_admitted( - state: &SharedState, - request: CrdtRestoreAdmissionRequest<'_>, -) -> crate::Result> { - let CrdtRestoreAdmissionRequest { - tenant_id, - database_id, - collection, - document_id, - target_version_json, - surrogate, - peer_id, - timeout, - event_source, - policy, - } = request; - // The restore writes a delta like any apply; the lease gates it on a - // drained collection and lives until its outcome. - let _lease = crate::control::server::shared::clone_write::collections_write_lease( - state, - tenant_id, - database_id, - [collection.to_owned()], - ) - .await?; - // `collection` is the plan's database-qualified name. - let vshard_id = - nodedb_types::CollectionKey::from_qualified_str(database_id, collection)?.vshard(); - let workflow = CrdtAdmissionWorkflow { - state, - tenant_id, - database_id, - vshard_id, - collection, - timeout, - event_source, - policy, - }; - tokio::time::timeout( - timeout, - state.vshard_admission_sequencer.run_after_proposed( - vshard_id, - |last_proposed| async move { - caught_up(&workflow, last_proposed).await?; - for _attempt in 0..FRONTIER_RETRY_LIMIT { - let delta = generate_restore_delta( - &workflow, - document_id, - target_version_json, - surrogate, - ) - .await?; - if delta.is_empty() { - return Ok(None); - } - let plan = PhysicalPlan::Crdt(CrdtOp::Apply { - collection: nodedb_types::QualifiedCollection::from_stored( - collection.to_owned(), - ), - document_id: document_id.to_owned(), - delta: delta.clone(), - peer_id, - mutation_id: 0, - surrogate, - provenance: None, - constraint_version_required: 0, - expected_frontier_digest: None, - }); - let preview = preview(&workflow, document_id, &delta).await?; - workflow.policy.evaluate(&preview)?; - match apply_fenced(&workflow, stamp_fence(plan, preview.frontier_digest)?).await - { - Err(crate::Error::DataPlane(ErrorCode::CrdtFrontierMismatch { - .. - })) => {} - result => return result.map(Some), - } - } - Err(crate::Error::CrdtAdmissionRetriesExhausted { - vshard_id, - attempts: FRONTIER_RETRY_LIMIT, - }) - }, - ), - ) - .await - .map_err(|_| crate::Error::CrdtAdmissionTimeout { - vshard_id, - timeout_ms: timeout_ms(timeout), - })? -} - -async fn generate_restore_delta( - workflow: &CrdtAdmissionWorkflow<'_>, - document_id: &str, - target_version_json: &str, - surrogate: nodedb_types::Surrogate, -) -> crate::Result> { - let response = - crate::control::server::shared::ddl::sync_dispatch::dispatch_system_response_with_source( - workflow.state, - crate::control::server::shared::ddl::sync_dispatch::SystemTask::new( - crate::control::server::shared::ddl::sync_dispatch::SystemReason::AdmittedContinuation, - workflow.tenant_id, - nodedb_types::CollectionKey::from_qualified_str(workflow.database_id, workflow.collection)?, - PhysicalPlan::Crdt(CrdtOp::RestoreToVersion { - collection: nodedb_types::QualifiedCollection::from_stored( - workflow.collection.to_owned(), - ), - document_id: document_id.to_owned(), - target_version_json: target_version_json.to_owned(), - surrogate, - }), - ), - workflow.timeout, - workflow.event_source, - ) - .await?; - if response.status != Status::Ok { - return Err(response_error(&response)); - } - Ok(response.payload.to_vec()) -} - -async fn apply_fenced( - workflow: &CrdtAdmissionWorkflow<'_>, - plan: PhysicalPlan, -) -> crate::Result { - let raw = workflow.state.raw_async_raft_proposer()?; - let entry = to_replicated_entry( - workflow.tenant_id, - workflow.database_id, - workflow.vshard_id, - &ReplicableWrite::decide_for_replication(&plan)?, - )? - .ok_or(crate::Error::CrdtAdmissionInvalidPlan { - reason: "admitted CRDT Apply has no replicated form", - })? - // Every replica gives the write the source this node dispatches it - // with. - .with_event_source(workflow.event_source); - let outcome = tokio::time::timeout( - workflow.timeout, - crate::control::wal_replication::propose_replicated_entry( - workflow.state, - raw, - entry, - tokio::time::Instant::now() + workflow.timeout, - ), - ) - .await - .map_err(|_| crate::Error::CrdtAdmissionTimeout { - vshard_id: workflow.vshard_id, - timeout_ms: timeout_ms(workflow.timeout), - })??; - // This node's apply of the entry recorded its commit HLC. - Ok(CrdtAdmissionOutcome { - payload: outcome.0, - write_versions: outcome.1, - trimmed_ops: 0, - }) -} - -fn timeout_ms(timeout: Duration) -> u64 { - u64::try_from(timeout.as_millis()).unwrap_or(u64::MAX) -} - -fn response_error(response: &crate::bridge::envelope::Response) -> crate::Error { - match response.error_code.as_deref() { - Some(code) => crate::Error::DataPlane(code.clone()), - None => crate::Error::Internal { - detail: String::from_utf8_lossy(&response.payload).into_owned(), - }, - } -} - -#[cfg(test)] -mod tests { - use std::sync::atomic::{AtomicUsize, Ordering}; - use std::sync::{Arc, Mutex}; - use std::time::Instant; - - use nodedb_types::{CrdtPreviewResult, Surrogate}; - use tokio::time::Duration; - - use super::*; - use crate::bridge::dispatch::{BridgeResponse, CoreChannelDataSide, Dispatcher}; - use crate::bridge::envelope::{Response, Status}; - use crate::control::state::SharedState; - use crate::types::{Lsn, RequestId}; - use crate::wal::WalManager; - use nodedb_types::QualifiedCollection; - - fn collection() -> String { - "docs".to_owned() - } - - fn admission_request<'a>(policy: &'a dyn CrdtPostImagePolicy) -> CrdtApplyAdmissionRequest<'a> { - CrdtApplyAdmissionRequest { - tenant_id: TenantId::new(1), - database_id: DatabaseId::DEFAULT, - collection: "docs", - plan: apply_plan(), - timeout: Duration::from_secs(1), - event_source: EventSource::User, - policy, - } - } - - fn apply_plan() -> PhysicalPlan { - PhysicalPlan::Crdt(CrdtOp::Apply { - collection: QualifiedCollection::new(DatabaseId::DEFAULT, &collection()), - document_id: "doc-1".into(), - delta: vec![0x91, 0x01], - peer_id: 7, - mutation_id: 9, - surrogate: Surrogate::new(1), - provenance: None, - constraint_version_required: 0, - expected_frontier_digest: None, - }) - } - - fn fixture() -> (Arc, CoreChannelDataSide, tempfile::TempDir) { - let directory = tempfile::tempdir().expect("temporary WAL directory"); - let wal = Arc::new( - WalManager::open_for_testing(&directory.path().join("admission.wal")) - .expect("test WAL"), - ); - let (dispatcher, mut sides) = Dispatcher::new(1, 64); - let side = sides.pop().expect("one data side"); - let state = SharedState::new(dispatcher, wal).expect("shared state"); - (state, side, directory) - } - - fn response(request_id: RequestId, payload: Vec) -> Response { - Response { - request_id, - status: Status::Ok, - attempt: 1, - partial: false, - payload: payload.into(), - watermark_lsn: Lsn::ZERO, - error_code: None, - stage_vote: None, - read_versions: crate::types::ReadVersions::new(), - write_set: Vec::new(), - } - } - - async fn respond_n( - state: Arc, - mut side: CoreChannelDataSide, - n: usize, - mut f: F, - ) where - F: FnMut(crate::bridge::envelope::Request) -> Response + Send + 'static, - { - let deadline = Instant::now() + Duration::from_secs(2); - let mut handled = 0; - while handled < n && Instant::now() < deadline { - if let Ok(request) = side.request_rx.try_pop() { - let response = f(request.inner); - side.response_tx - .try_push(BridgeResponse { inner: response }) - .expect("fake data-plane response queue has capacity"); - handled += 1; - } - state.poll_and_route_responses(); - tokio::task::yield_now().await; - } - assert_eq!(handled, n, "fake data-plane received expected requests"); - state.poll_and_route_responses(); - } - - struct RecordingPolicy { - seen: Arc>>>, - reject: bool, - } - - impl CrdtPostImagePolicy for RecordingPolicy { - fn evaluate(&self, preview: &CrdtPreviewResult) -> crate::Result<()> { - self.seen - .lock() - .expect("test policy lock") - .push(preview.post_image_msgpack.clone()); - if self.reject { - Err(crate::Error::CrdtAdmissionCallerFence) - } else { - Ok(()) - } - } - } - - #[tokio::test] - async fn admitted_apply_previews_exact_post_image_before_fenced_apply() { - let (state, side, _directory) = fixture(); - let digest = [0x4d; 32]; - let post_image = vec![0x81, 0xa2, b'o', b'k', 0xc3]; - let preview_payload = zerompk::to_msgpack_vec(&CrdtPreviewResult { - post_image_msgpack: post_image.clone(), - imported_ops: 1, - trimmed_ops: 0, - frontier_digest: digest, - }) - .expect("preview payload"); - let preview_fields = Arc::new(Mutex::new(Vec::new())); - let fields = Arc::clone(&preview_fields); - let responder = - tokio::spawn(respond_n( - Arc::clone(&state), - side, - 1, - move |request| match request.plan { - PhysicalPlan::Crdt(CrdtOp::PreviewApply { - collection, - document_id, - delta, - }) => { - fields - .lock() - .expect("fields lock") - .push((collection, document_id, delta)); - response(request.request_id, preview_payload.clone()) - } - other => panic!("unexpected request: {other:?}"), - }, - )); - let apply_fences = Arc::new(Mutex::new(Vec::new())); - let fences = Arc::clone(&apply_fences); - let raw: Arc = - Arc::new(move |_shard, _key, bytes, _deadline| { - let fences = Arc::clone(&fences); - Box::pin(async move { - let entry = - crate::control::wal_replication::ReplicatedEntry::from_bytes(&bytes) - .expect("replicated entry"); - if let crate::control::wal_replication::ReplicatedWrite::CrdtApplyFenced { - expected_frontier_digest, - .. - } = entry.write - { - fences - .lock() - .expect("fences lock") - .push(expected_frontier_digest); - } - Ok((Vec::new(), crate::types::ReadVersions::new())) - }) - }); - crate::control::vshard_admission::install_async_raft_proposer( - &state, - crate::control::vshard_admission::applying_submit(raw), - ) - .expect("install raw/sequenced proposer pair"); - let seen = Arc::new(Mutex::new(Vec::new())); - let policy = RecordingPolicy { - seen: Arc::clone(&seen), - reject: false, - }; - dispatch_crdt_apply_admitted(&state, admission_request(&policy)) - .await - .expect("admitted apply"); - responder.await.expect("responder completes"); - assert_eq!(*seen.lock().expect("seen lock"), vec![post_image]); - assert_eq!(preview_fields.lock().expect("fields lock").len(), 1); - assert_eq!(*apply_fences.lock().expect("fences lock"), vec![digest]); - } - - #[tokio::test] - async fn denied_policy_dispatches_only_preview_and_does_not_advance_hlc() { - let (state, side, _directory) = fixture(); - let preview_payload = zerompk::to_msgpack_vec(&CrdtPreviewResult { - post_image_msgpack: vec![0xc0], - imported_ops: 0, - trimmed_ops: 0, - frontier_digest: [7; 32], - }) - .expect("preview payload"); - let responder = tokio::spawn(respond_n(Arc::clone(&state), side, 1, move |request| { - assert!(matches!( - request.plan, - PhysicalPlan::Crdt(CrdtOp::PreviewApply { .. }) - )); - response(request.request_id, preview_payload.clone()) - })); - let seen = Arc::new(Mutex::new(Vec::new())); - let policy = RecordingPolicy { seen, reject: true }; - let before_hlc = state.tenant_write_mark(1); - let result = dispatch_crdt_apply_admitted(&state, admission_request(&policy)).await; - responder.await.expect("responder completes"); - assert!(matches!( - result, - Err(crate::Error::CrdtAdmissionCallerFence) - )); - assert_eq!( - state.tenant_write_mark(1), - before_hlc, - "policy rejection must not advance the tenant write HLC" - ); - state.wal.sync().expect("sync wal"); - assert!( - state.wal.replay().expect("replay wal").is_empty(), - "policy rejection must not append a durable CRDT delta" - ); - } - - #[tokio::test] - async fn admitted_cluster_apply_uses_raw_proposer_with_fenced_entry() { - let (state, side, _directory) = fixture(); - let digest = [0xa5; 32]; - let preview_payload = zerompk::to_msgpack_vec(&CrdtPreviewResult { - post_image_msgpack: vec![0xc0], - imported_ops: 1, - trimmed_ops: 0, - frontier_digest: digest, - }) - .expect("preview payload"); - let responder = tokio::spawn(respond_n(Arc::clone(&state), side, 1, move |request| { - assert!(matches!( - request.plan, - PhysicalPlan::Crdt(CrdtOp::PreviewApply { .. }) - )); - response(request.request_id, preview_payload.clone()) - })); - let fenced = Arc::new(Mutex::new(Vec::new())); - let observed = Arc::clone(&fenced); - let raw: Arc = - Arc::new(move |_shard, _key, bytes, _deadline| { - let observed = Arc::clone(&observed); - Box::pin(async move { - let entry = - crate::control::wal_replication::ReplicatedEntry::from_bytes(&bytes) - .expect("replicated entry"); - match entry.write { - crate::control::wal_replication::ReplicatedWrite::CrdtApplyFenced { - expected_frontier_digest, - .. - } => { - observed - .lock() - .expect("fence lock") - .push(expected_frontier_digest); - } - other => panic!("unexpected replicated write: {other:?}"), - } - Ok((Vec::new(), crate::types::ReadVersions::new())) - }) - }); - crate::control::vshard_admission::install_async_raft_proposer( - &state, - crate::control::vshard_admission::applying_submit(raw), - ) - .expect("install raw/sequenced proposer pair"); - let seen = Arc::new(Mutex::new(Vec::new())); - let policy = RecordingPolicy { - seen, - reject: false, - }; - dispatch_crdt_apply_admitted(&state, admission_request(&policy)) - .await - .expect("cluster admitted apply"); - responder.await.expect("responder completes"); - assert_eq!(*fenced.lock().expect("fence lock"), vec![digest]); - } - - #[tokio::test] - async fn stale_frontier_retries_preview_and_policy_before_success() { - let (state, side, _directory) = fixture(); - let digest = [0x82; 32]; - let preview_payload = zerompk::to_msgpack_vec(&CrdtPreviewResult { - post_image_msgpack: vec![0xc0], - imported_ops: 0, - trimmed_ops: 0, - frontier_digest: digest, - }) - .expect("preview payload"); - let responder = tokio::spawn(respond_n(Arc::clone(&state), side, 2, move |request| { - assert!(matches!( - request.plan, - PhysicalPlan::Crdt(CrdtOp::PreviewApply { .. }) - )); - response(request.request_id, preview_payload.clone()) - })); - let fences = Arc::new(AtomicUsize::new(0)); - let count = Arc::clone(&fences); - let raw: Arc = - Arc::new(move |_shard, _key, _bytes, _deadline| { - let count = Arc::clone(&count); - Box::pin(async move { - if count.fetch_add(1, Ordering::SeqCst) == 0 { - Err(crate::Error::DataPlane(ErrorCode::CrdtFrontierMismatch { - expected: digest, - actual: [0; 32], - })) - } else { - Ok((Vec::new(), crate::types::ReadVersions::new())) - } - }) - }); - crate::control::vshard_admission::install_async_raft_proposer( - &state, - crate::control::vshard_admission::applying_submit(raw), - ) - .expect("install raw/sequenced proposer pair"); - let seen = Arc::new(Mutex::new(Vec::new())); - let policy = RecordingPolicy { - seen: Arc::clone(&seen), - reject: false, - }; - dispatch_crdt_apply_admitted(&state, admission_request(&policy)) - .await - .expect("second fence succeeds"); - responder.await.expect("responder completes"); - assert_eq!(fences.load(Ordering::SeqCst), 2); - assert_eq!(seen.lock().expect("policy lock").len(), 2); - } - - #[tokio::test] - async fn stale_restore_regenerates_before_retrying_admission() { - let (state, side, _directory) = fixture(); - let delta = vec![0x91, 0x02]; - let preview_payload = zerompk::to_msgpack_vec(&CrdtPreviewResult { - post_image_msgpack: vec![0xc0], - imported_ops: 1, - trimmed_ops: 0, - frontier_digest: [0x84; 32], - }) - .expect("preview payload"); - let request_index = Arc::new(AtomicUsize::new(0)); - let index = Arc::clone(&request_index); - let responder = tokio::spawn(respond_n(Arc::clone(&state), side, 4, move |request| { - let is_restore = matches!( - &request.plan, - PhysicalPlan::Crdt(CrdtOp::RestoreToVersion { .. }) - ); - match index.fetch_add(1, Ordering::SeqCst) { - 0 | 2 => assert!(is_restore), - 1 | 3 => assert!(matches!( - &request.plan, - PhysicalPlan::Crdt(CrdtOp::PreviewApply { .. }) - )), - _ => unreachable!("responder only handles four requests"), - } - let payload = if is_restore { - delta.clone() - } else { - preview_payload.clone() - }; - response(request.request_id, payload) - })); - let attempts = Arc::new(AtomicUsize::new(0)); - let raw: Arc = { - let attempts = Arc::clone(&attempts); - Arc::new(move |_shard, _key, _bytes, _deadline| { - let attempts = Arc::clone(&attempts); - Box::pin(async move { - if attempts.fetch_add(1, Ordering::SeqCst) == 0 { - Err(crate::Error::DataPlane(ErrorCode::CrdtFrontierMismatch { - expected: [0x84; 32], - actual: [0; 32], - })) - } else { - Ok((Vec::new(), crate::types::ReadVersions::new())) - } - }) - }) - }; - crate::control::vshard_admission::install_async_raft_proposer( - &state, - crate::control::vshard_admission::applying_submit(raw), - ) - .expect("install proposer"); - let policy = RecordingPolicy { - seen: Arc::new(Mutex::new(Vec::new())), - reject: false, - }; - dispatch_crdt_restore_admitted( - &state, - CrdtRestoreAdmissionRequest { - tenant_id: TenantId::new(1), - database_id: DatabaseId::DEFAULT, - collection: "docs", - document_id: "doc-1", - target_version_json: "{}", - surrogate: Surrogate::new(1), - peer_id: 1, - timeout: Duration::from_secs(1), - event_source: EventSource::User, - policy: &policy, - }, - ) - .await - .expect("second restore admission succeeds"); - responder.await.expect("responder completes"); - assert_eq!(request_index.load(Ordering::SeqCst), 4); - assert_eq!(attempts.load(Ordering::SeqCst), 2); - } - - #[tokio::test] - async fn stale_frontier_retries_preview_and_policy_then_exhausts_typed_error() { - let (state, side, _directory) = fixture(); - let digest = [0x81; 32]; - let preview_payload = zerompk::to_msgpack_vec(&CrdtPreviewResult { - post_image_msgpack: vec![0xc0], - imported_ops: 0, - trimmed_ops: 0, - frontier_digest: digest, - }) - .expect("preview payload"); - let responder = tokio::spawn(respond_n( - Arc::clone(&state), - side, - FRONTIER_RETRY_LIMIT, - move |request| { - assert!(matches!( - request.plan, - PhysicalPlan::Crdt(CrdtOp::PreviewApply { .. }) - )); - response(request.request_id, preview_payload.clone()) - }, - )); - let fenced_count = Arc::new(AtomicUsize::new(0)); - let count = Arc::clone(&fenced_count); - let raw: Arc = - Arc::new(move |_shard, _key, _bytes, _deadline| { - let count = Arc::clone(&count); - Box::pin(async move { - count.fetch_add(1, Ordering::SeqCst); - Err(crate::Error::DataPlane(ErrorCode::CrdtFrontierMismatch { - expected: digest, - actual: [0; 32], - })) - }) - }); - crate::control::vshard_admission::install_async_raft_proposer( - &state, - crate::control::vshard_admission::applying_submit(raw), - ) - .expect("install raw/sequenced proposer pair"); - let seen = Arc::new(Mutex::new(Vec::new())); - let policy = RecordingPolicy { - seen: Arc::clone(&seen), - reject: false, - }; - let result = dispatch_crdt_apply_admitted(&state, admission_request(&policy)).await; - responder.await.expect("responder completes"); - assert!(matches!( - result, - Err(crate::Error::CrdtAdmissionRetriesExhausted { - attempts: FRONTIER_RETRY_LIMIT, - .. - }) - )); - assert_eq!(fenced_count.load(Ordering::SeqCst), FRONTIER_RETRY_LIMIT); - assert_eq!( - seen.lock().expect("policy lock").len(), - FRONTIER_RETRY_LIMIT - ); - } - - #[tokio::test] - async fn signed_delta_collection_rejects_plain_external_apply_before_preview() { - let (state, _side, _directory) = fixture(); - let tenant_id = TenantId::new(1); - let mut collection = crate::control::security::catalog::StoredCollection::stamped_for_test( - 1, "docs", "owner", - ); - collection.crdt = true; - collection.crdt_signing_required = true; - state - .credentials - .catalog() - .put_collection(DatabaseId::DEFAULT, &collection) - .expect("store signed-delta collection"); - let task = nodedb_physical::physical_task::PhysicalTask { - tenant_id, - database_id: DatabaseId::DEFAULT, - vshard_id: nodedb_types::CollectionKey::from_bare(DatabaseId::DEFAULT, "docs").vshard(), - plan: apply_plan(), - post_set_op: nodedb_physical::physical_task::PostSetOp::None, - txn_id: None, - }; - let identity = - crate::control::security::identity::AuthenticatedIdentity::new_internal_service( - 1, - "crdt-test", - tenant_id, - Vec::new(), - true, - None, - crate::control::security::identity::AuthenticatedIdentity::default_database_set( - true, - ), - ); - let authorized = crate::control::server::shared::authorization::authorize_task_set( - &identity, - std::slice::from_ref(&task), - &state.permissions, - &state.roles, - &crate::control::security::audit::NoopAuditEmitter, - ) - .expect("authorize test task") - .into_tasks() - .into_iter() - .next() - .expect("one authorized task"); - let result = dispatch_authorized_crdt_apply_admitted( - &state, - AuthorizedCrdtApplyAdmissionRequest { - authorized, - collection: "docs", - timeout: Duration::from_millis(10), - event_source: EventSource::User, - policy: &TrustedInternalCrdtPolicy, - }, - ) - .await; - assert!(matches!(result, Err(crate::Error::RejectedAuthz { .. }))); - } - - #[tokio::test] - async fn generic_replicated_dispatch_rejects_unadmitted_apply() { - let (state, _side, _directory) = fixture(); - let tenant_id = TenantId::new(1); - let task = nodedb_physical::physical_task::PhysicalTask { - tenant_id, - database_id: DatabaseId::DEFAULT, - vshard_id: nodedb_types::CollectionKey::from_bare(DatabaseId::DEFAULT, "docs").vshard(), - plan: apply_plan(), - post_set_op: nodedb_physical::physical_task::PostSetOp::None, - txn_id: None, - }; - let identity = - crate::control::security::identity::AuthenticatedIdentity::new_internal_service( - 1, - "crdt-test", - tenant_id, - Vec::new(), - true, - None, - crate::control::security::identity::AuthenticatedIdentity::default_database_set( - true, - ), - ); - let authorized = crate::control::server::shared::authorization::authorize_task_set( - &identity, - std::slice::from_ref(&task), - &state.permissions, - &state.roles, - &crate::control::security::audit::NoopAuditEmitter, - ) - .expect("authorize test task") - .into_tasks() - .into_iter() - .next() - .expect("one authorized task"); - let result = crate::control::server::sync::raft_dispatch::dispatch_write_replicated( - &state, - "docs", - authorized, - EventSource::User, - ) - .await; - assert!(matches!( - result, - Err(crate::Error::CrdtApplyRequiresAdmission) - )); - } - - fn assert_frontier_mutation(op: CrdtOp) { - assert!(changes_crdt_frontier(&op), "{op:?} must serialize"); - } - - fn assert_frontier_read(op: CrdtOp) { - assert!(!changes_crdt_frontier(&op), "{op:?} must not serialize"); - } - - #[test] - fn frontier_classifier_covers_every_crdt_operation_category() { - let surrogate = Surrogate::new(1); - assert_frontier_mutation(CrdtOp::Apply { - collection: QualifiedCollection::new(DatabaseId::DEFAULT, &collection()), - document_id: "id".into(), - delta: Vec::new(), - peer_id: 1, - mutation_id: 1, - surrogate, - provenance: None, - constraint_version_required: 0, - expected_frontier_digest: None, - }); - assert_frontier_mutation(CrdtOp::ImportSnapshot { - tenant_id: 1, - collection: QualifiedCollection::new(DatabaseId::DEFAULT, &collection()), - bytes: Vec::new(), - }); - assert_frontier_mutation(CrdtOp::RestoreToVersion { - collection: QualifiedCollection::new(DatabaseId::DEFAULT, &collection()), - document_id: "id".into(), - target_version_json: "{}".into(), - surrogate, - }); - assert_frontier_mutation(CrdtOp::ListInsert { - collection: QualifiedCollection::new(DatabaseId::DEFAULT, &collection()), - document_id: "id".into(), - list_path: "blocks".into(), - index: 0, - fields_json: "{}".into(), - surrogate, - }); - assert_frontier_mutation(CrdtOp::ListDelete { - collection: QualifiedCollection::new(DatabaseId::DEFAULT, &collection()), - document_id: "id".into(), - list_path: "blocks".into(), - index: 0, - surrogate, - }); - assert_frontier_mutation(CrdtOp::ListMove { - collection: QualifiedCollection::new(DatabaseId::DEFAULT, &collection()), - document_id: "id".into(), - list_path: "blocks".into(), - from_index: 0, - to_index: 1, - surrogate, - }); - assert_frontier_mutation(CrdtOp::DocUpsert { - collection: QualifiedCollection::new(DatabaseId::DEFAULT, &collection()), - document_id: "id".into(), - fields_json: "{}".into(), - surrogate, - partial: false, - verb: nodedb_physical::physical_plan::CrdtWriteVerb::Insert, - returning: None, - rls_filters: Vec::new(), - }); - assert_frontier_mutation(CrdtOp::DocDelete { - collection: QualifiedCollection::new(DatabaseId::DEFAULT, &collection()), - document_id: "id".into(), - surrogate: Some(surrogate), - returning: None, - rls_filters: Vec::new(), - }); - - assert_frontier_read(CrdtOp::Read { - collection: QualifiedCollection::new(DatabaseId::DEFAULT, &collection()), - document_id: "id".into(), - }); - assert_frontier_read(CrdtOp::PreviewApply { - collection: QualifiedCollection::new(DatabaseId::DEFAULT, &collection()), - document_id: "id".into(), - delta: Vec::new(), - }); - assert_frontier_read(CrdtOp::GetVersionVector { - collection: QualifiedCollection::new(DatabaseId::DEFAULT, &collection()), - }); - assert_frontier_read(CrdtOp::ExportDelta { - collection: QualifiedCollection::new(DatabaseId::DEFAULT, &collection()), - from_version_json: "{}".into(), - }); - assert_frontier_read(CrdtOp::CompactAtVersion { - collection: QualifiedCollection::new(DatabaseId::DEFAULT, &collection()), - target_version_json: "{}".into(), - }); - assert_frontier_read(CrdtOp::SetConstraints { - collection: QualifiedCollection::new(DatabaseId::DEFAULT, &collection()), - constraint_version: 1, - constraints: Vec::new(), - }); - assert_frontier_read(CrdtOp::DropConstraints { - collection: QualifiedCollection::new(DatabaseId::DEFAULT, &collection()), - constraint_version: 1, - }); - assert_frontier_read(CrdtOp::ReadConstraints { - collection: QualifiedCollection::new(DatabaseId::DEFAULT, &collection()), - }); - assert_frontier_read(CrdtOp::SetPolicy { - collection: QualifiedCollection::new(DatabaseId::DEFAULT, &collection()), - policy_json: "{}".into(), - }); - assert_frontier_read(CrdtOp::GetPolicy { - collection: QualifiedCollection::new(DatabaseId::DEFAULT, &collection()), - }); - assert_frontier_read(CrdtOp::ReadAtVersion { - collection: QualifiedCollection::new(DatabaseId::DEFAULT, &collection()), - document_id: "id".into(), - version_vector_json: "{}".into(), - }); - } -} diff --git a/nodedb/src/control/crdt_admission/apply.rs b/nodedb/src/control/crdt_admission/apply.rs new file mode 100644 index 000000000..dc4921d74 --- /dev/null +++ b/nodedb/src/control/crdt_admission/apply.rs @@ -0,0 +1,680 @@ +// SPDX-License-Identifier: BUSL-1.1 + +//! Preview, authorize, fence, and durably apply one CRDT delta. + +use nodedb_physical::physical_plan::CrdtOp; + +use crate::bridge::envelope::{ErrorCode, PhysicalPlan}; +use crate::control::server::shared::authorization::AuthorizedTask; +use crate::control::state::SharedState; + +use super::fence::{apply_fenced, stamp_fence}; +use super::legs::preview; +use super::request::{ + AuthorizedCrdtApplyAdmissionRequest, CrdtAdmissionOutcome, CrdtApplyAdmissionRequest, +}; +use super::workflow::{CrdtAdmissionWorkflow, FRONTIER_RETRY_LIMIT}; + +/// Preview, authorize, fence, and durably apply one externally authorized delta. +pub async fn dispatch_authorized_crdt_apply_admitted_outcome( + state: &SharedState, + request: AuthorizedCrdtApplyAdmissionRequest<'_>, +) -> crate::Result { + let AuthorizedCrdtApplyAdmissionRequest { + authorized, + collection, + deadline, + event_source, + policy, + } = request; + enforce_external_signing_policy(state, &authorized, collection)?; + let task = authorized.into_physical_task(); + dispatch_crdt_apply_admitted_outcome( + state, + CrdtApplyAdmissionRequest { + tenant_id: task.tenant_id, + database_id: task.database_id, + collection, + plan: task.plan, + deadline, + event_source, + policy, + }, + ) + .await +} + +/// `collection` is db-qualified. The catalog keys collections by the bare +/// name, so the lookup de-qualifies it first. +fn enforce_external_signing_policy( + state: &SharedState, + authorized: &AuthorizedTask, + collection: &str, +) -> crate::Result<()> { + let bare = crate::control::target_identity::naming::bare_collection_name( + authorized.database_id(), + collection, + ); + let stored = state + .credentials + .catalog() + .get_collection( + authorized.database_id(), + authorized.tenant_id().as_u64(), + &bare, + )? + .ok_or_else(|| crate::Error::CollectionNotFound { + tenant_id: authorized.tenant_id(), + collection: bare.clone(), + })?; + if stored.crdt_signing_required + && matches!(authorized.plan(), PhysicalPlan::Crdt(CrdtOp::Apply { .. })) + { + return Err(crate::Error::RejectedAuthz { + tenant_id: authorized.tenant_id(), + resource: format!("collection:{bare}:unsigned_crdt_delta_requires_authenticated_sync"), + }); + } + Ok(()) +} + +/// Preview, authorize, fence, and durably apply one trusted-internal CRDT delta. +pub(crate) async fn dispatch_crdt_apply_admitted_outcome( + state: &SharedState, + request: CrdtApplyAdmissionRequest<'_>, +) -> crate::Result { + let CrdtApplyAdmissionRequest { + tenant_id, + database_id, + collection, + plan, + deadline, + event_source, + policy, + } = request; + // Every CRDT apply converges here; the lease gates it on a drained + // collection and lives until the apply's outcome. + let _lease = crate::control::server::shared::clone_write::write_lease( + state, + tenant_id, + database_id, + &plan, + ) + .await?; + let (document_id, delta) = match &plan { + PhysicalPlan::Crdt( + CrdtOp::Apply { + collection: plan_collection, + document_id, + delta, + expected_frontier_digest: None, + .. + } + | CrdtOp::ApplyAuthenticated { + collection: plan_collection, + document_id, + delta, + expected_frontier_digest: None, + .. + }, + ) if plan_collection.as_str() == collection => (document_id.clone(), delta.clone()), + PhysicalPlan::Crdt( + CrdtOp::Apply { + expected_frontier_digest: Some(_), + .. + } + | CrdtOp::ApplyAuthenticated { + expected_frontier_digest: Some(_), + .. + }, + ) => return Err(crate::Error::CrdtAdmissionCallerFence), + _ => { + return Err(crate::Error::CrdtAdmissionInvalidPlan { + reason: "expected an unfenced CRDT Apply for the supplied collection", + }); + } + }; + let workflow = CrdtAdmissionWorkflow::new( + state, + tenant_id, + database_id, + collection, + deadline, + event_source, + policy, + )?; + workflow + .run(|| admit_apply_locked(&workflow, plan, &document_id, &delta)) + .await +} + +async fn admit_apply_locked( + workflow: &CrdtAdmissionWorkflow<'_>, + plan: PhysicalPlan, + document_id: &str, + delta: &[u8], +) -> crate::Result { + for _attempt in 0..FRONTIER_RETRY_LIMIT { + let preview = preview(workflow, document_id, delta).await?; + workflow.policy.evaluate(&preview)?; + let fenced = stamp_fence(plan.clone(), preview.frontier_digest)?; + match apply_fenced(workflow, fenced).await { + Err(crate::Error::DataPlane(ErrorCode::CrdtFrontierMismatch { .. })) => {} + // The trim count belongs to the preview that fenced *this* attempt. + // A retry re-previews against the advanced frontier and produces its + // own count, so the two are never mixed. + result => { + return result.map(|outcome| outcome.with_trimmed_ops(preview.trimmed_ops)); + } + } + } + Err(crate::Error::CrdtAdmissionRetriesExhausted { + vshard_id: workflow.vshard_id, + attempts: FRONTIER_RETRY_LIMIT, + }) +} + +/// Payload-only wrapper for an externally authorized caller. +pub async fn dispatch_authorized_crdt_apply_admitted( + state: &SharedState, + request: AuthorizedCrdtApplyAdmissionRequest<'_>, +) -> crate::Result> { + Ok( + dispatch_authorized_crdt_apply_admitted_outcome(state, request) + .await? + .payload, + ) +} + +#[cfg(test)] +mod tests { + use std::sync::atomic::{AtomicUsize, Ordering}; + use std::sync::{Arc, Mutex}; + use std::time::Duration; + + use nodedb_types::CrdtPreviewResult; + + use super::*; + use crate::control::crdt_admission::policy::TrustedInternalCrdtPolicy; + use crate::control::crdt_admission::test_support::{ + RecordingPolicy, admission_request, admission_request_until, apply_plan, + dispatch_crdt_apply_admitted, fixture, preview_payload, respond_n, response, + }; + use crate::control::server::shared::session::StatementDeadline; + use crate::event::EventSource; + use crate::types::{DatabaseId, TenantId}; + + #[tokio::test] + async fn admitted_apply_previews_exact_post_image_before_fenced_apply() { + let (state, side, _directory) = fixture(); + let digest = [0x4d; 32]; + let post_image = vec![0x81, 0xa2, b'o', b'k', 0xc3]; + let preview_payload = zerompk::to_msgpack_vec(&CrdtPreviewResult { + post_image_msgpack: post_image.clone(), + imported_ops: 1, + trimmed_ops: 0, + frontier_digest: digest, + }) + .expect("preview payload"); + let preview_fields = Arc::new(Mutex::new(Vec::new())); + let fields = Arc::clone(&preview_fields); + let responder = + tokio::spawn(respond_n( + Arc::clone(&state), + side, + 1, + move |request| match request.plan { + PhysicalPlan::Crdt(CrdtOp::PreviewApply { + collection, + document_id, + delta, + }) => { + fields + .lock() + .expect("fields lock") + .push((collection, document_id, delta)); + response(request.request_id, preview_payload.clone()) + } + other => panic!("unexpected request: {other:?}"), + }, + )); + let apply_fences = Arc::new(Mutex::new(Vec::new())); + let fences = Arc::clone(&apply_fences); + let raw: Arc = + Arc::new(move |_shard, _key, bytes, _deadline| { + let fences = Arc::clone(&fences); + Box::pin(async move { + let entry = + crate::control::wal_replication::ReplicatedEntry::from_bytes(&bytes) + .expect("replicated entry"); + if let crate::control::wal_replication::ReplicatedWrite::CrdtApplyFenced { + expected_frontier_digest, + .. + } = entry.write + { + fences + .lock() + .expect("fences lock") + .push(expected_frontier_digest); + } + Ok((Vec::new(), crate::types::ReadVersions::new())) + }) + }); + crate::control::vshard_admission::install_async_raft_proposer( + &state, + crate::control::vshard_admission::applying_submit(raw), + ) + .expect("install raw/sequenced proposer pair"); + let seen = Arc::new(Mutex::new(Vec::new())); + let policy = RecordingPolicy { + seen: Arc::clone(&seen), + reject: false, + }; + dispatch_crdt_apply_admitted(&state, admission_request(&policy)) + .await + .expect("admitted apply"); + responder.await.expect("responder completes"); + assert_eq!(*seen.lock().expect("seen lock"), vec![post_image]); + assert_eq!(preview_fields.lock().expect("fields lock").len(), 1); + assert_eq!(*apply_fences.lock().expect("fences lock"), vec![digest]); + } + + #[tokio::test] + async fn denied_policy_dispatches_only_preview_and_does_not_advance_hlc() { + let (state, side, _directory) = fixture(); + let preview_payload = zerompk::to_msgpack_vec(&CrdtPreviewResult { + post_image_msgpack: vec![0xc0], + imported_ops: 0, + trimmed_ops: 0, + frontier_digest: [7; 32], + }) + .expect("preview payload"); + let responder = tokio::spawn(respond_n(Arc::clone(&state), side, 1, move |request| { + assert!(matches!( + request.plan, + PhysicalPlan::Crdt(CrdtOp::PreviewApply { .. }) + )); + response(request.request_id, preview_payload.clone()) + })); + let seen = Arc::new(Mutex::new(Vec::new())); + let policy = RecordingPolicy { seen, reject: true }; + let before_hlc = state.tenant_write_mark(1); + let result = dispatch_crdt_apply_admitted(&state, admission_request(&policy)).await; + responder.await.expect("responder completes"); + assert!(matches!( + result, + Err(crate::Error::CrdtAdmissionCallerFence) + )); + assert_eq!( + state.tenant_write_mark(1), + before_hlc, + "policy rejection must not advance the tenant write HLC" + ); + state.wal.sync().expect("sync wal"); + assert!( + state.wal.replay().expect("replay wal").is_empty(), + "policy rejection must not append a durable CRDT delta" + ); + } + + #[tokio::test] + async fn admitted_cluster_apply_uses_raw_proposer_with_fenced_entry() { + let (state, side, _directory) = fixture(); + let digest = [0xa5; 32]; + let preview_payload = zerompk::to_msgpack_vec(&CrdtPreviewResult { + post_image_msgpack: vec![0xc0], + imported_ops: 1, + trimmed_ops: 0, + frontier_digest: digest, + }) + .expect("preview payload"); + let responder = tokio::spawn(respond_n(Arc::clone(&state), side, 1, move |request| { + assert!(matches!( + request.plan, + PhysicalPlan::Crdt(CrdtOp::PreviewApply { .. }) + )); + response(request.request_id, preview_payload.clone()) + })); + let fenced = Arc::new(Mutex::new(Vec::new())); + let observed = Arc::clone(&fenced); + let raw: Arc = + Arc::new(move |_shard, _key, bytes, _deadline| { + let observed = Arc::clone(&observed); + Box::pin(async move { + let entry = + crate::control::wal_replication::ReplicatedEntry::from_bytes(&bytes) + .expect("replicated entry"); + match entry.write { + crate::control::wal_replication::ReplicatedWrite::CrdtApplyFenced { + expected_frontier_digest, + .. + } => { + observed + .lock() + .expect("fence lock") + .push(expected_frontier_digest); + } + other => panic!("unexpected replicated write: {other:?}"), + } + Ok((Vec::new(), crate::types::ReadVersions::new())) + }) + }); + crate::control::vshard_admission::install_async_raft_proposer( + &state, + crate::control::vshard_admission::applying_submit(raw), + ) + .expect("install raw/sequenced proposer pair"); + let seen = Arc::new(Mutex::new(Vec::new())); + let policy = RecordingPolicy { + seen, + reject: false, + }; + dispatch_crdt_apply_admitted(&state, admission_request(&policy)) + .await + .expect("cluster admitted apply"); + responder.await.expect("responder completes"); + assert_eq!(*fenced.lock().expect("fence lock"), vec![digest]); + } + + #[tokio::test] + async fn stale_frontier_retries_preview_and_policy_before_success() { + let (state, side, _directory) = fixture(); + let digest = [0x82; 32]; + let preview_payload = zerompk::to_msgpack_vec(&CrdtPreviewResult { + post_image_msgpack: vec![0xc0], + imported_ops: 0, + trimmed_ops: 0, + frontier_digest: digest, + }) + .expect("preview payload"); + let responder = tokio::spawn(respond_n(Arc::clone(&state), side, 2, move |request| { + assert!(matches!( + request.plan, + PhysicalPlan::Crdt(CrdtOp::PreviewApply { .. }) + )); + response(request.request_id, preview_payload.clone()) + })); + let fences = Arc::new(AtomicUsize::new(0)); + let count = Arc::clone(&fences); + let raw: Arc = + Arc::new(move |_shard, _key, _bytes, _deadline| { + let count = Arc::clone(&count); + Box::pin(async move { + if count.fetch_add(1, Ordering::SeqCst) == 0 { + Err(crate::Error::DataPlane(ErrorCode::CrdtFrontierMismatch { + expected: digest, + actual: [0; 32], + })) + } else { + Ok((Vec::new(), crate::types::ReadVersions::new())) + } + }) + }); + crate::control::vshard_admission::install_async_raft_proposer( + &state, + crate::control::vshard_admission::applying_submit(raw), + ) + .expect("install raw/sequenced proposer pair"); + let seen = Arc::new(Mutex::new(Vec::new())); + let policy = RecordingPolicy { + seen: Arc::clone(&seen), + reject: false, + }; + dispatch_crdt_apply_admitted(&state, admission_request(&policy)) + .await + .expect("second fence succeeds"); + responder.await.expect("responder completes"); + assert_eq!(fences.load(Ordering::SeqCst), 2); + assert_eq!(seen.lock().expect("policy lock").len(), 2); + } + + #[tokio::test] + async fn stale_frontier_retries_preview_and_policy_then_exhausts_typed_error() { + let (state, side, _directory) = fixture(); + let digest = [0x81; 32]; + let preview_payload = zerompk::to_msgpack_vec(&CrdtPreviewResult { + post_image_msgpack: vec![0xc0], + imported_ops: 0, + trimmed_ops: 0, + frontier_digest: digest, + }) + .expect("preview payload"); + let responder = tokio::spawn(respond_n( + Arc::clone(&state), + side, + FRONTIER_RETRY_LIMIT, + move |request| { + assert!(matches!( + request.plan, + PhysicalPlan::Crdt(CrdtOp::PreviewApply { .. }) + )); + response(request.request_id, preview_payload.clone()) + }, + )); + let fenced_count = Arc::new(AtomicUsize::new(0)); + let count = Arc::clone(&fenced_count); + let raw: Arc = + Arc::new(move |_shard, _key, _bytes, _deadline| { + let count = Arc::clone(&count); + Box::pin(async move { + count.fetch_add(1, Ordering::SeqCst); + Err(crate::Error::DataPlane(ErrorCode::CrdtFrontierMismatch { + expected: digest, + actual: [0; 32], + })) + }) + }); + crate::control::vshard_admission::install_async_raft_proposer( + &state, + crate::control::vshard_admission::applying_submit(raw), + ) + .expect("install raw/sequenced proposer pair"); + let seen = Arc::new(Mutex::new(Vec::new())); + let policy = RecordingPolicy { + seen: Arc::clone(&seen), + reject: false, + }; + let result = dispatch_crdt_apply_admitted(&state, admission_request(&policy)).await; + responder.await.expect("responder completes"); + assert!(matches!( + result, + Err(crate::Error::CrdtAdmissionRetriesExhausted { + attempts: FRONTIER_RETRY_LIMIT, + .. + }) + )); + assert_eq!(fenced_count.load(Ordering::SeqCst), FRONTIER_RETRY_LIMIT); + assert_eq!( + seen.lock().expect("policy lock").len(), + FRONTIER_RETRY_LIMIT + ); + } + + #[tokio::test] + async fn signed_delta_collection_rejects_plain_external_apply_before_preview() { + let (state, _side, _directory) = fixture(); + let tenant_id = TenantId::new(1); + let mut collection = crate::control::security::catalog::StoredCollection::stamped_for_test( + 1, "docs", "owner", + ); + collection.crdt = true; + collection.crdt_signing_required = true; + state + .credentials + .catalog() + .put_collection(DatabaseId::DEFAULT, &collection) + .expect("store signed-delta collection"); + let task = nodedb_physical::physical_task::PhysicalTask { + tenant_id, + database_id: DatabaseId::DEFAULT, + vshard_id: nodedb_types::CollectionKey::from_bare(DatabaseId::DEFAULT, "docs").vshard(), + plan: apply_plan(), + post_set_op: nodedb_physical::physical_task::PostSetOp::None, + txn_id: None, + }; + let identity = + crate::control::security::identity::AuthenticatedIdentity::new_internal_service( + 1, + "crdt-test", + tenant_id, + Vec::new(), + true, + None, + crate::control::security::identity::AuthenticatedIdentity::default_database_set( + true, + ), + ); + let authorized = crate::control::server::shared::authorization::authorize_task_set( + &identity, + std::slice::from_ref(&task), + &state.permissions, + &state.roles, + &crate::control::security::audit::NoopAuditEmitter, + ) + .expect("authorize test task") + .into_tasks() + .into_iter() + .next() + .expect("one authorized task"); + let result = dispatch_authorized_crdt_apply_admitted( + &state, + AuthorizedCrdtApplyAdmissionRequest { + authorized, + collection: "docs", + deadline: StatementDeadline::after(Duration::from_millis(10)), + event_source: EventSource::User, + policy: &TrustedInternalCrdtPolicy, + }, + ) + .await; + assert!(matches!(result, Err(crate::Error::RejectedAuthz { .. }))); + } + + #[tokio::test] + async fn generic_replicated_dispatch_rejects_unadmitted_apply() { + let (state, _side, _directory) = fixture(); + let tenant_id = TenantId::new(1); + let task = nodedb_physical::physical_task::PhysicalTask { + tenant_id, + database_id: DatabaseId::DEFAULT, + vshard_id: nodedb_types::CollectionKey::from_bare(DatabaseId::DEFAULT, "docs").vshard(), + plan: apply_plan(), + post_set_op: nodedb_physical::physical_task::PostSetOp::None, + txn_id: None, + }; + let identity = + crate::control::security::identity::AuthenticatedIdentity::new_internal_service( + 1, + "crdt-test", + tenant_id, + Vec::new(), + true, + None, + crate::control::security::identity::AuthenticatedIdentity::default_database_set( + true, + ), + ); + let authorized = crate::control::server::shared::authorization::authorize_task_set( + &identity, + std::slice::from_ref(&task), + &state.permissions, + &state.roles, + &crate::control::security::audit::NoopAuditEmitter, + ) + .expect("authorize test task") + .into_tasks() + .into_iter() + .next() + .expect("one authorized task"); + let result = crate::control::server::sync::raft_dispatch::dispatch_write_replicated( + &state, + "docs", + authorized, + StatementDeadline::of_statement(&state), + EventSource::User, + ) + .await; + assert!(matches!( + result, + Err(crate::Error::CrdtApplyRequiresAdmission) + )); + } + + /// A deadline that passes while the preview is unanswered refuses the + /// delta, and nothing is proposed. + #[tokio::test] + async fn a_deadline_before_the_proposal_is_a_definite_refusal() { + let (state, _side, _directory) = fixture(); + let proposals = Arc::new(AtomicUsize::new(0)); + let count = Arc::clone(&proposals); + let raw: Arc = + Arc::new(move |_shard, _key, _bytes, _deadline| { + count.fetch_add(1, Ordering::SeqCst); + Box::pin(async { Ok((Vec::new(), crate::types::ReadVersions::new())) }) + }); + crate::control::vshard_admission::install_async_raft_proposer( + &state, + crate::control::vshard_admission::applying_submit(raw), + ) + .expect("install proposer"); + let policy = TrustedInternalCrdtPolicy; + + let result = dispatch_crdt_apply_admitted( + &state, + admission_request_until(&policy, StatementDeadline::after(Duration::from_millis(50))), + ) + .await; + + assert!( + matches!(result, Err(crate::Error::CrdtAdmissionTimeout { .. })), + "got {result:?}" + ); + assert_eq!(proposals.load(Ordering::SeqCst), 0, "nothing was proposed"); + } + + /// A deadline that passes while the proposal is in flight leaves the + /// outcome unknown. + #[tokio::test] + async fn a_deadline_during_the_proposal_is_an_unknown_outcome() { + let (state, side, _directory) = fixture(); + let payload = preview_payload([0x5a; 32]); + let responder = tokio::spawn(respond_n(Arc::clone(&state), side, 1, move |request| { + assert!(matches!( + request.plan, + PhysicalPlan::Crdt(CrdtOp::PreviewApply { .. }) + )); + response(request.request_id, payload.clone()) + })); + let proposals = Arc::new(AtomicUsize::new(0)); + let count = Arc::clone(&proposals); + let raw: Arc = + Arc::new(move |_shard, _key, _bytes, _deadline| { + count.fetch_add(1, Ordering::SeqCst); + Box::pin(std::future::pending::< + crate::Result<(Vec, crate::types::ReadVersions)>, + >()) + }); + crate::control::vshard_admission::install_async_raft_proposer( + &state, + crate::control::vshard_admission::applying_submit(raw), + ) + .expect("install proposer"); + let policy = TrustedInternalCrdtPolicy; + + let result = dispatch_crdt_apply_admitted( + &state, + admission_request_until( + &policy, + StatementDeadline::after(Duration::from_millis(300)), + ), + ) + .await; + responder.await.expect("responder completes"); + + assert!( + matches!(result, Err(crate::Error::DeadlineExceeded { .. })), + "got {result:?}" + ); + assert_eq!(proposals.load(Ordering::SeqCst), 1, "the proposal was sent"); + } +} diff --git a/nodedb/src/control/crdt_admission/classify.rs b/nodedb/src/control/crdt_admission/classify.rs new file mode 100644 index 000000000..000905515 --- /dev/null +++ b/nodedb/src/control/crdt_admission/classify.rs @@ -0,0 +1,165 @@ +// SPDX-License-Identifier: BUSL-1.1 + +//! Which CRDT operations move a document's Loro frontier. + +use nodedb_physical::physical_plan::CrdtOp; + +/// Whether an operation changes the Loro frontier and must serialize with an +/// admission preview when it is dispatched to this node's cores without a +/// proposal. +pub fn changes_crdt_frontier(op: &CrdtOp) -> bool { + match op { + CrdtOp::Apply { .. } + | CrdtOp::ApplyAuthenticated { .. } + | CrdtOp::ImportSnapshot { .. } + | CrdtOp::ListInsert { .. } + | CrdtOp::ListDelete { .. } + | CrdtOp::ListMove { .. } + | CrdtOp::DocUpsert { .. } + | CrdtOp::DocDelete { .. } + | CrdtOp::RestoreToVersion { .. } => true, + CrdtOp::Read { .. } + | CrdtOp::PreviewApply { .. } + | CrdtOp::GetVersionVector { .. } + | CrdtOp::ExportDelta { .. } + | CrdtOp::CompactAtVersion { .. } + | CrdtOp::SetConstraints { .. } + | CrdtOp::DropConstraints { .. } + | CrdtOp::ReadConstraints { .. } + | CrdtOp::SetPolicy { .. } + | CrdtOp::GetPolicy { .. } + | CrdtOp::ReadAtVersion { .. } => false, + } +} + +#[cfg(test)] +mod tests { + use nodedb_types::{QualifiedCollection, Surrogate}; + + use super::*; + use crate::types::DatabaseId; + + fn collection() -> String { + "docs".to_owned() + } + + fn assert_frontier_mutation(op: CrdtOp) { + assert!(changes_crdt_frontier(&op), "{op:?} must serialize"); + } + + fn assert_frontier_read(op: CrdtOp) { + assert!(!changes_crdt_frontier(&op), "{op:?} must not serialize"); + } + + #[test] + fn frontier_classifier_covers_every_crdt_operation_category() { + let surrogate = Surrogate::new(1); + assert_frontier_mutation(CrdtOp::Apply { + collection: QualifiedCollection::new(DatabaseId::DEFAULT, &collection()), + document_id: "id".into(), + delta: Vec::new(), + peer_id: 1, + mutation_id: 1, + surrogate, + provenance: None, + constraint_version_required: 0, + expected_frontier_digest: None, + }); + assert_frontier_mutation(CrdtOp::ImportSnapshot { + tenant_id: 1, + collection: QualifiedCollection::new(DatabaseId::DEFAULT, &collection()), + bytes: Vec::new(), + }); + assert_frontier_mutation(CrdtOp::RestoreToVersion { + collection: QualifiedCollection::new(DatabaseId::DEFAULT, &collection()), + document_id: "id".into(), + target_version_json: "{}".into(), + surrogate, + }); + assert_frontier_mutation(CrdtOp::ListInsert { + collection: QualifiedCollection::new(DatabaseId::DEFAULT, &collection()), + document_id: "id".into(), + list_path: "blocks".into(), + index: 0, + fields_json: "{}".into(), + surrogate, + }); + assert_frontier_mutation(CrdtOp::ListDelete { + collection: QualifiedCollection::new(DatabaseId::DEFAULT, &collection()), + document_id: "id".into(), + list_path: "blocks".into(), + index: 0, + surrogate, + }); + assert_frontier_mutation(CrdtOp::ListMove { + collection: QualifiedCollection::new(DatabaseId::DEFAULT, &collection()), + document_id: "id".into(), + list_path: "blocks".into(), + from_index: 0, + to_index: 1, + surrogate, + }); + assert_frontier_mutation(CrdtOp::DocUpsert { + collection: QualifiedCollection::new(DatabaseId::DEFAULT, &collection()), + document_id: "id".into(), + fields_json: "{}".into(), + surrogate, + partial: false, + verb: nodedb_physical::physical_plan::CrdtWriteVerb::Insert, + returning: None, + rls_filters: Vec::new(), + }); + assert_frontier_mutation(CrdtOp::DocDelete { + collection: QualifiedCollection::new(DatabaseId::DEFAULT, &collection()), + document_id: "id".into(), + surrogate: Some(surrogate), + returning: None, + rls_filters: Vec::new(), + }); + + assert_frontier_read(CrdtOp::Read { + collection: QualifiedCollection::new(DatabaseId::DEFAULT, &collection()), + document_id: "id".into(), + }); + assert_frontier_read(CrdtOp::PreviewApply { + collection: QualifiedCollection::new(DatabaseId::DEFAULT, &collection()), + document_id: "id".into(), + delta: Vec::new(), + }); + assert_frontier_read(CrdtOp::GetVersionVector { + collection: QualifiedCollection::new(DatabaseId::DEFAULT, &collection()), + }); + assert_frontier_read(CrdtOp::ExportDelta { + collection: QualifiedCollection::new(DatabaseId::DEFAULT, &collection()), + from_version_json: "{}".into(), + }); + assert_frontier_read(CrdtOp::CompactAtVersion { + collection: QualifiedCollection::new(DatabaseId::DEFAULT, &collection()), + target_version_json: "{}".into(), + }); + assert_frontier_read(CrdtOp::SetConstraints { + collection: QualifiedCollection::new(DatabaseId::DEFAULT, &collection()), + constraint_version: 1, + constraints: Vec::new(), + }); + assert_frontier_read(CrdtOp::DropConstraints { + collection: QualifiedCollection::new(DatabaseId::DEFAULT, &collection()), + constraint_version: 1, + }); + assert_frontier_read(CrdtOp::ReadConstraints { + collection: QualifiedCollection::new(DatabaseId::DEFAULT, &collection()), + }); + assert_frontier_read(CrdtOp::SetPolicy { + collection: QualifiedCollection::new(DatabaseId::DEFAULT, &collection()), + policy_json: "{}".into(), + }); + assert_frontier_read(CrdtOp::GetPolicy { + collection: QualifiedCollection::new(DatabaseId::DEFAULT, &collection()), + }); + assert_frontier_read(CrdtOp::ReadAtVersion { + collection: QualifiedCollection::new(DatabaseId::DEFAULT, &collection()), + document_id: "id".into(), + version_vector_json: "{}".into(), + }); + } +} diff --git a/nodedb/src/control/crdt_admission/fence.rs b/nodedb/src/control/crdt_admission/fence.rs new file mode 100644 index 000000000..6c70aaa90 --- /dev/null +++ b/nodedb/src/control/crdt_admission/fence.rs @@ -0,0 +1,106 @@ +// SPDX-License-Identifier: BUSL-1.1 + +//! Stamp the previewed frontier on a delta and propose it fenced. + +use nodedb_physical::physical_plan::CrdtOp; + +use crate::bridge::envelope::PhysicalPlan; +use crate::control::wal_replication::{ReplicableWrite, to_replicated_entry}; + +use super::request::CrdtAdmissionOutcome; +use super::workflow::CrdtAdmissionWorkflow; + +/// `plan` with `digest` as the frontier its apply expects. +pub(super) fn stamp_fence(plan: PhysicalPlan, digest: [u8; 32]) -> crate::Result { + match plan { + PhysicalPlan::Crdt(CrdtOp::Apply { + collection, + document_id, + delta, + peer_id, + mutation_id, + surrogate, + provenance, + constraint_version_required, + expected_frontier_digest: None, + }) => Ok(PhysicalPlan::Crdt(CrdtOp::Apply { + collection, + document_id, + delta, + peer_id, + mutation_id, + surrogate, + provenance, + constraint_version_required, + expected_frontier_digest: Some(digest), + })), + PhysicalPlan::Crdt(CrdtOp::ApplyAuthenticated { + collection, + document_id, + delta, + peer_id, + mutation_id, + surrogate, + provenance, + constraint_version_required, + expected_frontier_digest: None, + auth_user_id, + auth_device_id, + auth_seq_no, + delta_signature, + signing_required, + }) => Ok(PhysicalPlan::Crdt(CrdtOp::ApplyAuthenticated { + collection, + document_id, + delta, + peer_id, + mutation_id, + surrogate, + provenance, + constraint_version_required, + expected_frontier_digest: Some(digest), + auth_user_id, + auth_device_id, + auth_seq_no, + delta_signature, + signing_required, + })), + _ => Err(crate::Error::CrdtAdmissionInvalidPlan { + reason: "validated CRDT Apply plan was changed before fence stamping", + }), + } +} + +/// Propose the fenced apply and wait for this node's apply of it. +pub(super) async fn apply_fenced( + workflow: &CrdtAdmissionWorkflow<'_>, + plan: PhysicalPlan, +) -> crate::Result { + let raw = workflow.state.raw_async_raft_proposer()?; + let entry = to_replicated_entry( + workflow.tenant_id, + workflow.database_id, + workflow.vshard_id, + &ReplicableWrite::decide_for_replication(&plan)?, + )? + .ok_or(crate::Error::CrdtAdmissionInvalidPlan { + reason: "admitted CRDT Apply has no replicated form", + })? + // Every replica gives the write the source this node dispatches it + // with. + .with_event_source(workflow.event_source); + let (payload, write_versions) = workflow + .propose(crate::control::wal_replication::propose_replicated_entry( + workflow.state, + raw, + entry, + workflow.deadline.tokio_instant(), + )) + .await?; + // This node's apply of the entry recorded its commit HLC. + Ok(CrdtAdmissionOutcome { + payload, + write_versions, + trimmed_ops: 0, + }) +} diff --git a/nodedb/src/control/crdt_admission/legs.rs b/nodedb/src/control/crdt_admission/legs.rs new file mode 100644 index 000000000..5a4614fb0 --- /dev/null +++ b/nodedb/src/control/crdt_admission/legs.rs @@ -0,0 +1,98 @@ +// SPDX-License-Identifier: BUSL-1.1 + +//! The Data-Plane reads an admission runs before it proposes: the preview of +//! a delta, and the generation of a restore delta. +//! +//! Both are legs of a request whose capability was consumed at its entry +//! point, so they go through the system door. + +use nodedb_physical::physical_plan::CrdtOp; +use nodedb_types::CrdtPreviewResult; + +use crate::bridge::envelope::{PhysicalPlan, Status}; +use crate::control::server::shared::ddl::sync_dispatch::{ + SystemReason, SystemTask, dispatch_system_response_with_source, +}; + +use super::workflow::CrdtAdmissionWorkflow; + +/// The post-image and frontier digest `delta` will produce on the document. +pub(super) async fn preview( + workflow: &CrdtAdmissionWorkflow<'_>, + document_id: &str, + delta: &[u8], +) -> crate::Result { + let response = dispatch_system_response_with_source( + workflow.state, + SystemTask::new( + SystemReason::AdmittedContinuation, + workflow.tenant_id, + nodedb_types::CollectionKey::from_qualified_str( + workflow.database_id, + workflow.collection, + )?, + PhysicalPlan::Crdt(CrdtOp::PreviewApply { + collection: nodedb_types::QualifiedCollection::from_stored( + workflow.collection.to_owned(), + ), + document_id: document_id.to_owned(), + delta: delta.to_vec(), + }), + ), + workflow.deadline, + workflow.event_source, + ) + .await + .map_err(|error| workflow.before_propose(error))?; + if response.status != Status::Ok { + return Err(response_error(&response)); + } + zerompk::from_msgpack(response.payload.as_bytes()).map_err(|error| crate::Error::Internal { + detail: format!("CRDT preview payload decode: {error}"), + }) +} + +/// The delta that takes the document back to `target_version_json`. +pub(super) async fn generate_restore_delta( + workflow: &CrdtAdmissionWorkflow<'_>, + document_id: &str, + target_version_json: &str, + surrogate: nodedb_types::Surrogate, +) -> crate::Result> { + let response = dispatch_system_response_with_source( + workflow.state, + SystemTask::new( + SystemReason::AdmittedContinuation, + workflow.tenant_id, + nodedb_types::CollectionKey::from_qualified_str( + workflow.database_id, + workflow.collection, + )?, + PhysicalPlan::Crdt(CrdtOp::RestoreToVersion { + collection: nodedb_types::QualifiedCollection::from_stored( + workflow.collection.to_owned(), + ), + document_id: document_id.to_owned(), + target_version_json: target_version_json.to_owned(), + surrogate, + }), + ), + workflow.deadline, + workflow.event_source, + ) + .await + .map_err(|error| workflow.before_propose(error))?; + if response.status != Status::Ok { + return Err(response_error(&response)); + } + Ok(response.payload.to_vec()) +} + +fn response_error(response: &crate::bridge::envelope::Response) -> crate::Error { + match response.error_code.as_deref() { + Some(code) => crate::Error::DataPlane(code.clone()), + None => crate::Error::Internal { + detail: String::from_utf8_lossy(&response.payload).into_owned(), + }, + } +} diff --git a/nodedb/src/control/crdt_admission/mod.rs b/nodedb/src/control/crdt_admission/mod.rs new file mode 100644 index 000000000..d88ec1843 --- /dev/null +++ b/nodedb/src/control/crdt_admission/mod.rs @@ -0,0 +1,25 @@ +// SPDX-License-Identifier: BUSL-1.1 + +//! Serialized preview, policy, and fenced CRDT apply admission. + +pub(crate) mod apply; +mod classify; +mod fence; +mod legs; +mod policy; +mod request; +mod restore; +#[cfg(test)] +mod test_support; +mod workflow; + +pub use apply::{ + dispatch_authorized_crdt_apply_admitted, dispatch_authorized_crdt_apply_admitted_outcome, +}; +pub use classify::changes_crdt_frontier; +pub use policy::{CrdtPostImagePolicy, TrustedInternalCrdtPolicy}; +pub use request::{ + AuthorizedCrdtApplyAdmissionRequest, CrdtAdmissionOutcome, CrdtApplyAdmissionRequest, + CrdtRestoreAdmissionRequest, +}; +pub(crate) use restore::dispatch_crdt_restore_admitted; diff --git a/nodedb/src/control/crdt_admission/policy.rs b/nodedb/src/control/crdt_admission/policy.rs new file mode 100644 index 000000000..c4ddea0ff --- /dev/null +++ b/nodedb/src/control/crdt_admission/policy.rs @@ -0,0 +1,18 @@ +// SPDX-License-Identifier: BUSL-1.1 + +//! The post-image policy an admission evaluates before it proposes. + +use nodedb_types::CrdtPreviewResult; + +pub trait CrdtPostImagePolicy: Send + Sync { + fn evaluate(&self, preview: &CrdtPreviewResult) -> crate::Result<()>; +} + +/// Explicit policy for trusted internal callers. +pub struct TrustedInternalCrdtPolicy; + +impl CrdtPostImagePolicy for TrustedInternalCrdtPolicy { + fn evaluate(&self, _preview: &CrdtPreviewResult) -> crate::Result<()> { + Ok(()) + } +} diff --git a/nodedb/src/control/crdt_admission/request.rs b/nodedb/src/control/crdt_admission/request.rs new file mode 100644 index 000000000..4b80d2549 --- /dev/null +++ b/nodedb/src/control/crdt_admission/request.rs @@ -0,0 +1,73 @@ +// SPDX-License-Identifier: BUSL-1.1 + +//! What a caller hands a CRDT admission, and what it gets back. + +use crate::bridge::envelope::PhysicalPlan; +use crate::control::server::shared::authorization::AuthorizedTask; +use crate::control::server::shared::session::StatementDeadline; +use crate::event::EventSource; +use crate::types::{DatabaseId, TenantId}; + +use super::policy::CrdtPostImagePolicy; + +pub struct AuthorizedCrdtApplyAdmissionRequest<'a> { + pub authorized: AuthorizedTask, + /// The db-qualified collection (`QualifiedCollection::as_str`), the same + /// string the plan's `CrdtOp::Apply` carries. The preview and the apply + /// address the Data Plane by it, and the catalog lookup de-qualifies it. + pub collection: &'a str, + /// The deadline every phase of the admission ends at. + pub deadline: StatementDeadline, + pub event_source: EventSource, + pub policy: &'a dyn CrdtPostImagePolicy, +} + +pub struct CrdtApplyAdmissionRequest<'a> { + pub tenant_id: TenantId, + pub database_id: DatabaseId, + /// The db-qualified collection, equal to the plan's `CrdtOp::Apply` + /// collection. + pub collection: &'a str, + pub plan: PhysicalPlan, + /// The deadline every phase of the admission ends at. + pub deadline: StatementDeadline, + pub event_source: EventSource, + pub policy: &'a dyn CrdtPostImagePolicy, +} + +pub struct CrdtAdmissionOutcome { + pub payload: Vec, + /// The versions the apply stamped: the entry's log position on the + /// written vShard. + pub write_versions: crate::types::ReadVersions, + /// Operations the admitted delta encoded that the target document already + /// knew, measured by the preview that fenced this apply. Distinguishes a + /// client whose writes land from one whose writes are being absorbed — + /// both otherwise produce the same successful payload. + pub trimmed_ops: u64, +} + +impl CrdtAdmissionOutcome { + /// Attach the trim count measured by the preview that fenced this apply. + pub(super) fn with_trimmed_ops(self, trimmed_ops: u64) -> Self { + Self { + trimmed_ops, + ..self + } + } +} + +pub struct CrdtRestoreAdmissionRequest<'a> { + pub tenant_id: TenantId, + pub database_id: DatabaseId, + /// The db-qualified collection the generated restore ops address. + pub collection: &'a str, + pub document_id: &'a str, + pub target_version_json: &'a str, + pub surrogate: nodedb_types::Surrogate, + pub peer_id: u64, + /// The deadline every phase of the admission ends at. + pub deadline: StatementDeadline, + pub event_source: EventSource, + pub policy: &'a dyn CrdtPostImagePolicy, +} diff --git a/nodedb/src/control/crdt_admission/restore.rs b/nodedb/src/control/crdt_admission/restore.rs new file mode 100644 index 000000000..a0cd29533 --- /dev/null +++ b/nodedb/src/control/crdt_admission/restore.rs @@ -0,0 +1,201 @@ +// SPDX-License-Identifier: BUSL-1.1 + +//! Admit a restore delta generated inside the vShard's admission slot. + +use nodedb_physical::physical_plan::CrdtOp; + +use crate::bridge::envelope::{ErrorCode, PhysicalPlan}; +use crate::control::state::SharedState; + +use super::fence::{apply_fenced, stamp_fence}; +use super::legs::{generate_restore_delta, preview}; +use super::request::{CrdtAdmissionOutcome, CrdtRestoreAdmissionRequest}; +use super::workflow::{CrdtAdmissionWorkflow, FRONTIER_RETRY_LIMIT}; + +/// Generate a restore delta and admit it without releasing the vShard slot. +/// A stale apply fence regenerates the delta from authoritative state before +/// retrying, so neither historical projection nor policy output is reused. +pub(crate) async fn dispatch_crdt_restore_admitted( + state: &SharedState, + request: CrdtRestoreAdmissionRequest<'_>, +) -> crate::Result> { + let CrdtRestoreAdmissionRequest { + tenant_id, + database_id, + collection, + document_id, + target_version_json, + surrogate, + peer_id, + deadline, + event_source, + policy, + } = request; + // The restore writes a delta like any apply; the lease gates it on a + // drained collection and lives until its outcome. + let _lease = crate::control::server::shared::clone_write::collections_write_lease( + state, + tenant_id, + database_id, + [collection.to_owned()], + ) + .await?; + let workflow = CrdtAdmissionWorkflow::new( + state, + tenant_id, + database_id, + collection, + deadline, + event_source, + policy, + )?; + workflow + .run(|| { + restore_locked( + &workflow, + document_id, + target_version_json, + surrogate, + peer_id, + ) + }) + .await +} + +/// Generate, preview and propose the restore delta, inside the slot. +async fn restore_locked( + workflow: &CrdtAdmissionWorkflow<'_>, + document_id: &str, + target_version_json: &str, + surrogate: nodedb_types::Surrogate, + peer_id: u64, +) -> crate::Result> { + for _attempt in 0..FRONTIER_RETRY_LIMIT { + let delta = + generate_restore_delta(workflow, document_id, target_version_json, surrogate).await?; + if delta.is_empty() { + return Ok(None); + } + let plan = PhysicalPlan::Crdt(CrdtOp::Apply { + collection: nodedb_types::QualifiedCollection::from_stored( + workflow.collection.to_owned(), + ), + document_id: document_id.to_owned(), + delta: delta.clone(), + peer_id, + mutation_id: 0, + surrogate, + provenance: None, + constraint_version_required: 0, + expected_frontier_digest: None, + }); + let preview = preview(workflow, document_id, &delta).await?; + workflow.policy.evaluate(&preview)?; + match apply_fenced(workflow, stamp_fence(plan, preview.frontier_digest)?).await { + Err(crate::Error::DataPlane(ErrorCode::CrdtFrontierMismatch { .. })) => {} + result => return result.map(Some), + } + } + Err(crate::Error::CrdtAdmissionRetriesExhausted { + vshard_id: workflow.vshard_id, + attempts: FRONTIER_RETRY_LIMIT, + }) +} + +#[cfg(test)] +mod tests { + use std::sync::atomic::{AtomicUsize, Ordering}; + use std::sync::{Arc, Mutex}; + use std::time::Duration; + + use nodedb_types::{CrdtPreviewResult, Surrogate}; + + use super::*; + use crate::control::crdt_admission::test_support::{ + RecordingPolicy, fixture, respond_n, response, + }; + use crate::control::server::shared::session::StatementDeadline; + use crate::event::EventSource; + use crate::types::{DatabaseId, TenantId}; + + #[tokio::test] + async fn stale_restore_regenerates_before_retrying_admission() { + let (state, side, _directory) = fixture(); + let delta = vec![0x91, 0x02]; + let preview_payload = zerompk::to_msgpack_vec(&CrdtPreviewResult { + post_image_msgpack: vec![0xc0], + imported_ops: 1, + trimmed_ops: 0, + frontier_digest: [0x84; 32], + }) + .expect("preview payload"); + let request_index = Arc::new(AtomicUsize::new(0)); + let index = Arc::clone(&request_index); + let responder = tokio::spawn(respond_n(Arc::clone(&state), side, 4, move |request| { + let is_restore = matches!( + &request.plan, + PhysicalPlan::Crdt(CrdtOp::RestoreToVersion { .. }) + ); + match index.fetch_add(1, Ordering::SeqCst) { + 0 | 2 => assert!(is_restore), + 1 | 3 => assert!(matches!( + &request.plan, + PhysicalPlan::Crdt(CrdtOp::PreviewApply { .. }) + )), + _ => unreachable!("responder only handles four requests"), + } + let payload = if is_restore { + delta.clone() + } else { + preview_payload.clone() + }; + response(request.request_id, payload) + })); + let attempts = Arc::new(AtomicUsize::new(0)); + let raw: Arc = { + let attempts = Arc::clone(&attempts); + Arc::new(move |_shard, _key, _bytes, _deadline| { + let attempts = Arc::clone(&attempts); + Box::pin(async move { + if attempts.fetch_add(1, Ordering::SeqCst) == 0 { + Err(crate::Error::DataPlane(ErrorCode::CrdtFrontierMismatch { + expected: [0x84; 32], + actual: [0; 32], + })) + } else { + Ok((Vec::new(), crate::types::ReadVersions::new())) + } + }) + }) + }; + crate::control::vshard_admission::install_async_raft_proposer( + &state, + crate::control::vshard_admission::applying_submit(raw), + ) + .expect("install proposer"); + let policy = RecordingPolicy { + seen: Arc::new(Mutex::new(Vec::new())), + reject: false, + }; + dispatch_crdt_restore_admitted( + &state, + CrdtRestoreAdmissionRequest { + tenant_id: TenantId::new(1), + database_id: DatabaseId::DEFAULT, + collection: "docs", + document_id: "doc-1", + target_version_json: "{}", + surrogate: Surrogate::new(1), + peer_id: 1, + deadline: StatementDeadline::after(Duration::from_secs(1)), + event_source: EventSource::User, + policy: &policy, + }, + ) + .await + .expect("second restore admission succeeds"); + responder.await.expect("responder completes"); + assert_eq!(request_index.load(Ordering::SeqCst), 4); + assert_eq!(attempts.load(Ordering::SeqCst), 2); + } +} diff --git a/nodedb/src/control/crdt_admission/test_support.rs b/nodedb/src/control/crdt_admission/test_support.rs new file mode 100644 index 000000000..7a4d6c94f --- /dev/null +++ b/nodedb/src/control/crdt_admission/test_support.rs @@ -0,0 +1,149 @@ +// SPDX-License-Identifier: BUSL-1.1 + +//! Fixtures shared by the admission tests: a one-core state, a fake Data +//! Plane responder, and a recording policy. + +use std::sync::{Arc, Mutex}; +use std::time::{Duration, Instant}; + +use nodedb_physical::physical_plan::CrdtOp; +use nodedb_types::{CrdtPreviewResult, QualifiedCollection, Surrogate}; + +use crate::bridge::dispatch::{BridgeResponse, CoreChannelDataSide, Dispatcher}; +use crate::bridge::envelope::{PhysicalPlan, Response, Status}; +use crate::control::server::shared::session::StatementDeadline; +use crate::control::state::SharedState; +use crate::event::EventSource; +use crate::types::{DatabaseId, Lsn, RequestId, TenantId}; +use crate::wal::WalManager; + +use super::apply::dispatch_crdt_apply_admitted_outcome; +use super::policy::CrdtPostImagePolicy; +use super::request::CrdtApplyAdmissionRequest; + +/// Payload-only wrapper over the trusted-internal admission. +pub(super) async fn dispatch_crdt_apply_admitted( + state: &SharedState, + request: CrdtApplyAdmissionRequest<'_>, +) -> crate::Result> { + Ok(dispatch_crdt_apply_admitted_outcome(state, request) + .await? + .payload) +} + +pub(super) fn admission_request<'a>( + policy: &'a dyn CrdtPostImagePolicy, +) -> CrdtApplyAdmissionRequest<'a> { + admission_request_until(policy, StatementDeadline::after(Duration::from_secs(1))) +} + +pub(super) fn admission_request_until<'a>( + policy: &'a dyn CrdtPostImagePolicy, + deadline: StatementDeadline, +) -> CrdtApplyAdmissionRequest<'a> { + CrdtApplyAdmissionRequest { + tenant_id: TenantId::new(1), + database_id: DatabaseId::DEFAULT, + collection: "docs", + plan: apply_plan(), + deadline, + event_source: EventSource::User, + policy, + } +} + +pub(super) fn apply_plan() -> PhysicalPlan { + PhysicalPlan::Crdt(CrdtOp::Apply { + collection: QualifiedCollection::new(DatabaseId::DEFAULT, "docs"), + document_id: "doc-1".into(), + delta: vec![0x91, 0x01], + peer_id: 7, + mutation_id: 9, + surrogate: Surrogate::new(1), + provenance: None, + constraint_version_required: 0, + expected_frontier_digest: None, + }) +} + +pub(super) fn fixture() -> (Arc, CoreChannelDataSide, tempfile::TempDir) { + let directory = tempfile::tempdir().expect("temporary WAL directory"); + let wal = Arc::new( + WalManager::open_for_testing(&directory.path().join("admission.wal")).expect("test WAL"), + ); + let (dispatcher, mut sides) = Dispatcher::new(1, 64); + let side = sides.pop().expect("one data side"); + let state = SharedState::new(dispatcher, wal).expect("shared state"); + (state, side, directory) +} + +pub(super) fn response(request_id: RequestId, payload: Vec) -> Response { + Response { + request_id, + status: Status::Ok, + attempt: 1, + partial: false, + payload: payload.into(), + watermark_lsn: Lsn::ZERO, + error_code: None, + stage_vote: None, + read_versions: crate::types::ReadVersions::new(), + write_set: Vec::new(), + } +} + +/// An encoded preview answer with frontier `digest`. +pub(super) fn preview_payload(digest: [u8; 32]) -> Vec { + zerompk::to_msgpack_vec(&CrdtPreviewResult { + post_image_msgpack: vec![0xc0], + imported_ops: 1, + trimmed_ops: 0, + frontier_digest: digest, + }) + .expect("preview payload") +} + +/// Answer `n` Data-Plane requests with `f`, within two seconds. +pub(super) async fn respond_n( + state: Arc, + mut side: CoreChannelDataSide, + n: usize, + mut f: F, +) where + F: FnMut(crate::bridge::envelope::Request) -> Response + Send + 'static, +{ + let deadline = Instant::now() + Duration::from_secs(2); + let mut handled = 0; + while handled < n && Instant::now() < deadline { + if let Ok(request) = side.request_rx.try_pop() { + let response = f(request.inner); + side.response_tx + .try_push(BridgeResponse { inner: response }) + .expect("fake data-plane response queue has capacity"); + handled += 1; + } + state.poll_and_route_responses(); + tokio::task::yield_now().await; + } + assert_eq!(handled, n, "fake data-plane received expected requests"); + state.poll_and_route_responses(); +} + +pub(super) struct RecordingPolicy { + pub(super) seen: Arc>>>, + pub(super) reject: bool, +} + +impl CrdtPostImagePolicy for RecordingPolicy { + fn evaluate(&self, preview: &CrdtPreviewResult) -> crate::Result<()> { + self.seen + .lock() + .expect("test policy lock") + .push(preview.post_image_msgpack.clone()); + if self.reject { + Err(crate::Error::CrdtAdmissionCallerFence) + } else { + Ok(()) + } + } +} diff --git a/nodedb/src/control/crdt_admission/workflow.rs b/nodedb/src/control/crdt_admission/workflow.rs new file mode 100644 index 000000000..4861de843 --- /dev/null +++ b/nodedb/src/control/crdt_admission/workflow.rs @@ -0,0 +1,129 @@ +// SPDX-License-Identifier: BUSL-1.1 + +//! One CRDT admission and the deadline it runs under. +//! +//! Every phase ends at one deadline: the slot wait, the catch-up, the +//! preview and the proposal. A deadline that passes with no proposal in +//! flight refuses the delta, and nothing applied. A deadline that passes +//! while a proposal is in flight leaves the outcome unknown. + +use std::future::Future; +use std::sync::atomic::{AtomicBool, Ordering}; + +use crate::control::server::shared::session::StatementDeadline; +use crate::control::state::SharedState; +use crate::event::EventSource; +use crate::types::{DatabaseId, RequestId, TenantId, VShardId}; + +use super::policy::CrdtPostImagePolicy; + +/// Frontier mismatches an admission retries before it gives up. +pub(super) const FRONTIER_RETRY_LIMIT: usize = 8; + +pub(super) struct CrdtAdmissionWorkflow<'a> { + pub(super) state: &'a SharedState, + pub(super) tenant_id: TenantId, + pub(super) database_id: DatabaseId, + pub(super) vshard_id: VShardId, + /// The plan's database-qualified collection name. + pub(super) collection: &'a str, + pub(super) deadline: StatementDeadline, + pub(super) event_source: EventSource, + pub(super) policy: &'a dyn CrdtPostImagePolicy, + /// The budget left when the admission started, for the refusal message. + budget_ms: u64, + /// Set while a proposal is in flight. + proposing: AtomicBool, +} + +impl<'a> CrdtAdmissionWorkflow<'a> { + pub(super) fn new( + state: &'a SharedState, + tenant_id: TenantId, + database_id: DatabaseId, + collection: &'a str, + deadline: StatementDeadline, + event_source: EventSource, + policy: &'a dyn CrdtPostImagePolicy, + ) -> crate::Result { + let vshard_id = + nodedb_types::CollectionKey::from_qualified_str(database_id, collection)?.vshard(); + Ok(Self { + state, + tenant_id, + database_id, + vshard_id, + collection, + deadline, + event_source, + policy, + budget_ms: u64::try_from(deadline.remaining().as_millis()).unwrap_or(u64::MAX), + proposing: AtomicBool::new(false), + }) + } + + /// Run `operation` in the vShard's admission slot, once this node applied + /// every write admitted before it. The whole run ends at the deadline. + pub(super) async fn run(&self, operation: F) -> crate::Result + where + F: FnOnce() -> Fut, + Fut: Future>, + { + let admission = self.state.vshard_admission_sequencer.run_after_proposed( + self.vshard_id, + |last_proposed| async move { + crate::control::vshard_admission::await_admitted_applies( + self.state, + self.vshard_id, + last_proposed, + self.deadline.instant(), + ) + .await?; + operation().await + }, + ); + match tokio::time::timeout_at(self.deadline.tokio_instant(), admission).await { + Ok(result) => result, + Err(_) => Err(self.expired()), + } + } + + /// Await `proposal` with the in-flight mark set. A deadline that already + /// passed refuses before anything is proposed. + pub(super) async fn propose( + &self, + proposal: impl Future>, + ) -> crate::Result { + if self.deadline.remaining().is_zero() { + return Err(self.expired()); + } + self.proposing.store(true, Ordering::Release); + let outcome = proposal.await; + self.proposing.store(false, Ordering::Release); + outcome + } + + /// The error of a passed deadline. With no proposal in flight it is a + /// definite refusal. With one in flight the outcome is unknown. + pub(super) fn expired(&self) -> crate::Error { + if self.proposing.load(Ordering::Acquire) { + crate::Error::DeadlineExceeded { + request_id: RequestId::new(0), + } + } else { + crate::Error::CrdtAdmissionTimeout { + vshard_id: self.vshard_id, + timeout_ms: self.budget_ms, + } + } + } + + /// `error` of a dispatch that runs before any proposal. A passed deadline + /// there refuses the delta. + pub(super) fn before_propose(&self, error: crate::Error) -> crate::Error { + match error { + crate::Error::DeadlineExceeded { .. } => self.expired(), + other => other, + } + } +} diff --git a/nodedb/src/control/fail_gate.rs b/nodedb/src/control/fail_gate.rs index 8688aa721..62d87f273 100644 --- a/nodedb/src/control/fail_gate.rs +++ b/nodedb/src/control/fail_gate.rs @@ -63,9 +63,16 @@ pub(crate) async fn before_dispatch(node_id: u64, plan: &PhysicalPlan, wal_lsn: /// The execute gate of node `node_id`, `exec::before_execute`: a received /// `ExecuteRequest` or `ExecuteStreamRequest` parks here before it is checked -/// or run. A test arms it so one node takes a request and never answers. +/// or run. Armed with `wait_file()`, one node takes a request and +/// never answers. Armed with `sleep()`, it runs the request late. pub(crate) async fn before_execute(node_id: u64) { - wait(FailScope::Node(node_id), "exec::before_execute").await; + let scope = FailScope::Node(node_id); + let name = "exec::before_execute"; + if let Some(FailAction::Sleep(pause)) = lookup(scope, name) { + tokio::time::sleep(pause).await; + return; + } + wait(scope, name).await; } /// The Calvin submit gate of node `node_id`: after a transaction's collection diff --git a/nodedb/src/control/gateway/cluster_error.rs b/nodedb/src/control/gateway/cluster_error.rs index 84f6790b1..1be61531b 100644 --- a/nodedb/src/control/gateway/cluster_error.rs +++ b/nodedb/src/control/gateway/cluster_error.rs @@ -76,42 +76,91 @@ pub(super) fn map_typed_cluster_error(err: TypedClusterError, vshard_id: u64) -> } } +/// One `ExecuteRequest` or `ExecuteStreamRequest` send, as its failure +/// mapping needs it. +#[derive(Debug, Clone, Copy)] +pub(super) struct RemoteSend { + pub node_id: u64, + pub vshard_id: u64, + /// A second run of the plan is harmless: the plan only reads. + pub resend_safe: bool, + /// The statement's deadline for this send. + pub deadline: tokio::time::Instant, +} + /// The error of a failed `ExecuteRequest` or `ExecuteStreamRequest` send. /// /// A request the target never received, and a lost answer to a read, retry as /// `NotLeader` with `leader_node = 0`. The unreachable node is then not kept as -/// leader, and the next try routes locally to the real leader. A lost answer -/// to any other plan is an unknown outcome: the plan can have run, and a -/// second send would run it again. -pub(super) fn send_error( - e: ClusterError, - resend_safe: bool, - node_id: u64, - vshard_id: u64, -) -> Error { - if e.is_lost_answer() && !resend_safe { - tracing::warn!( - node_id, - vshard_id, - error = %e, - "gateway: a remote execute reached its deadline with no answer" - ); - return Error::DeadlineExceeded { - request_id: crate::types::RequestId::new(0), - }; +/// leader, and the next try routes locally to the real leader. +/// +/// These end the statement as `DeadlineExceeded` instead: +/// - a lost answer to any other plan: it can have run, and a second send +/// would run it again; +/// - a lost answer once the deadline passed: the wait used the whole budget, +/// and no retry has time left; +/// - a `ShardTimeout`: the wait used the whole budget. +pub(super) fn send_error(e: ClusterError, send: &RemoteSend) -> Error { + if matches!(e, ClusterError::ShardTimeout { .. }) { + return deadline_exceeded(send, &e); + } + if e.is_lost_answer() && (!send.resend_safe || tokio::time::Instant::now() >= send.deadline) { + return deadline_exceeded(send, &e); } Error::NotLeader { - vshard_id: VShardId::new((vshard_id % VShardId::COUNT as u64) as u32), + vshard_id: VShardId::new((send.vshard_id % VShardId::COUNT as u64) as u32), leader_node: 0, - leader_addr: format!("node-{node_id} (transport error: {e})"), + leader_addr: format!("node-{} (transport error: {e})", send.node_id), leader_term: 0, } } +/// The error of a remote execute whose reply wait expired. +/// +/// The wait covers the statement's whole remaining budget plus the reply +/// margin, so no retry has time left: the statement ends as `DeadlineExceeded`. +pub(super) fn reply_wait_expired(send: &RemoteSend, reply_wait: std::time::Duration) -> Error { + let timed_out = ClusterError::ShardTimeout { + vshard_id: (send.vshard_id % VShardId::COUNT as u64) as u32, + elapsed_ms: u64::try_from(reply_wait.as_millis()).unwrap_or(u64::MAX), + }; + deadline_exceeded(send, &timed_out) +} + +fn deadline_exceeded(send: &RemoteSend, cause: &ClusterError) -> Error { + tracing::warn!( + node_id = send.node_id, + vshard_id = send.vshard_id, + error = %cause, + "gateway: a remote execute ended with no answer within the statement budget" + ); + Error::DeadlineExceeded { + request_id: crate::types::RequestId::new(0), + } +} + #[cfg(test)] mod tests { use super::*; + /// A send to node 2 for vShard 7 whose statement has time left. + fn send(resend_safe: bool) -> RemoteSend { + RemoteSend { + node_id: 2, + vshard_id: 7, + resend_safe, + deadline: tokio::time::Instant::now() + std::time::Duration::from_secs(30), + } + } + + /// The same send once its statement deadline passed. + fn spent(resend_safe: bool) -> RemoteSend { + RemoteSend { + deadline: tokio::time::Instant::now(), + ..send(resend_safe) + } + } + fn unanswered() -> ClusterError { ClusterError::Unanswered { node_id: 2, @@ -123,7 +172,7 @@ mod tests { #[test] fn a_lost_answer_to_a_write_is_an_unknown_outcome() { assert!(matches!( - send_error(unanswered(), false, 2, 7), + send_error(unanswered(), &send(false)), Error::DeadlineExceeded { .. } )); } @@ -132,29 +181,74 @@ mod tests { #[test] fn a_lost_answer_to_a_read_retries() { assert!(matches!( - send_error(unanswered(), true, 2, 7), + send_error(unanswered(), &send(true)), Error::NotLeader { leader_node: 0, .. } )); } - /// A send that timed out is a lost answer: a read retries, a write is an - /// unknown outcome. + /// A send that timed out used the statement's whole budget: a read and a + /// write both end as `DeadlineExceeded`, with no retry. #[test] - fn a_timed_out_send_follows_the_lost_answer_rule() { + fn a_timed_out_read_ends_the_statement() { let timed_out = || ClusterError::ShardTimeout { vshard_id: 7, elapsed_ms: 1_000, }; assert!(matches!( - send_error(timed_out(), true, 2, 7), + send_error(timed_out(), &send(true)), + Error::DeadlineExceeded { .. } + )); + assert!(matches!( + send_error(timed_out(), &send(false)), + Error::DeadlineExceeded { .. } + )); + } + + /// A live node holds a read past its budget: the transport's read timeout + /// ends the wait as a lost answer once the deadline passed. No retry has + /// time left, so the read ends as `DeadlineExceeded`, never `NotLeader`. + #[test] + fn a_lost_answer_to_a_read_past_its_deadline_ends_the_statement() { + assert!(matches!( + send_error(unanswered(), &spent(true)), + Error::DeadlineExceeded { .. } + )); + } + + /// A request that never went out leaves nothing to wait for, so it + /// retries even at the deadline. The retry loop decides whether time is left. + #[test] + fn an_unsent_read_past_its_deadline_stays_retryable() { + let unsent = ClusterError::Transport { + detail: "connect refused".into(), + }; + assert!(matches!( + send_error(unsent, &spent(true)), Error::NotLeader { leader_node: 0, .. } )); + } + + /// An expired reply wait ends the statement, never a retry. + #[test] + fn an_expired_reply_wait_is_deadline_exceeded() { assert!(matches!( - send_error(timed_out(), false, 2, 7), + reply_wait_expired(&send(true), std::time::Duration::from_secs(14)), Error::DeadlineExceeded { .. } )); } + /// A transport error is unsent: a read and a write both retry. + #[test] + fn an_unsent_read_retries() { + let unsent = ClusterError::Transport { + detail: "handshake timeout".into(), + }; + assert!(matches!( + send_error(unsent, &send(true)), + Error::NotLeader { leader_node: 0, .. } + )); + } + /// A request that never reached the target retries, read or write. #[test] fn an_unsent_request_retries() { @@ -162,7 +256,7 @@ mod tests { detail: "connect refused".into(), }; assert!(matches!( - send_error(unsent, false, 2, 7), + send_error(unsent, &send(false)), Error::NotLeader { leader_node: 0, .. } )); } diff --git a/nodedb/src/control/gateway/core.rs b/nodedb/src/control/gateway/core.rs index 5e776baf3..a47d6b5ca 100644 --- a/nodedb/src/control/gateway/core.rs +++ b/nodedb/src/control/gateway/core.rs @@ -8,7 +8,7 @@ //! version mapping). //! 2. Routes the plan via [`route_plan`] to `Local` or `Remote` task routes. //! 3. Dispatches each route (local SPSC or `ExecuteRequest` RPC) with typed -//! `NotLeader` retry (up to 3 attempts). +//! `NotLeader` retry until the statement deadline. //! 4. Handles `RetryableSchemaChanged` (descriptor cache miss) by fetching a //! fresh lease and re-planning once. //! 5. Fuses multiple vShard payloads for broadcast scans. @@ -37,7 +37,7 @@ use super::key_extractor::UnwiredKeyExtractor; use super::live_leaders::resolve_live_decision; use super::outcome::GatewayOutcome; use super::plan_cache::PlanCache; -use super::retry::retry_not_leader; +use super::retry::{RetryPacing, retry_not_leader}; use super::route::TaskRoute; use super::router::route_plan; use super::version_set::GatewayVersionSet; @@ -251,6 +251,7 @@ impl Gateway { // One deadline for every route and attempt of the statement. let deadline = StatementDeadline::of_statement(&shared).tokio_instant(); + let pacing = RetryPacing::from_tuning(&shared.tuning.cluster_transport); // Gateway-level byte ceiling: per-route `dispatch_to_data_plane` // already caps each shard's payload; this additionally caps the // scatter-gather *sum* so an N-shard fan-out can't accumulate @@ -273,7 +274,7 @@ impl Gateway { let retry_counter = Arc::clone(&self.not_leader_retry_count); let version_set_for_route = version_set.clone(); let shared_for_route = Arc::clone(&shared); - let outcome = retry_not_leader(routing_ref, deadline, move |attempt| { + let outcome = retry_not_leader(routing_ref, deadline, pacing, move |attempt| { if attempt.index > 0 { retry_counter.fetch_add(1, Ordering::Relaxed); } @@ -299,7 +300,6 @@ impl Gateway { database_id, trace_id, deadline_ms: attempt.remaining_ms(), - attempts_left: attempt.attempts_left(), version_set: &version_set, txn_id, linearizable, diff --git a/nodedb/src/control/gateway/dispatch_remote.rs b/nodedb/src/control/gateway/dispatch_remote.rs index 07cb3d112..2053e95a4 100644 --- a/nodedb/src/control/gateway/dispatch_remote.rs +++ b/nodedb/src/control/gateway/dispatch_remote.rs @@ -8,7 +8,6 @@ use std::sync::Arc; -use nodedb_cluster::ClusterError; use nodedb_cluster::rpc_codec::{ExecuteRequest, RaftRpc}; use tracing::debug; @@ -18,7 +17,7 @@ use crate::control::state::SharedState; use crate::types::{DatabaseId, Lsn, TenantId, TraceId, TxnId, VShardId}; use nodedb_physical::physical_plan::wire as plan_wire; -use super::cluster_error::{map_typed_cluster_error, send_error}; +use super::cluster_error::{RemoteSend, map_typed_cluster_error, reply_wait_expired, send_error}; use super::dispatcher::DispatchOutcome; use super::version_set::GatewayVersionSet; @@ -34,8 +33,6 @@ pub(super) struct RemoteDispatchArgs<'a> { pub trace_id: TraceId, /// Milliseconds left on the statement when this attempt starts. pub deadline_ms: u64, - /// The dispatch attempts this route has left, this one included. - pub attempts_left: usize, pub version_set: &'a GatewayVersionSet, /// Session-transaction context forwarded to the remote executor, or `None` /// for non-transactional dispatch. @@ -61,12 +58,13 @@ pub(super) async fn dispatch_remote( database_id, trace_id, deadline_ms, - attempts_left, version_set, txn_id, linearizable, read_groups, } = args; + // The statement's deadline, fixed before any wait of this attempt. + let deadline = tokio::time::Instant::now() + std::time::Duration::from_millis(deadline_ms); let transport = shared.cluster_transport.as_ref().ok_or(Error::Internal { detail: "gateway: cluster transport not available for remote dispatch".into(), })?; @@ -136,15 +134,12 @@ pub(super) async fn dispatch_remote( // A read can be sent again after a lost answer. Any other plan can have // run on the target, so a lost answer is an unknown outcome. - let resend_safe = required_permission(&plan) == Permission::Read; - // A read leaves time for the attempts after this one, so a node that - // never answers cannot spend the whole statement budget. - let attempt_ms = nodedb_cluster::rpc_codec::attempt_budget( - deadline_ms, - resend_safe, - attempts_left, - transport.rpc_timeout(), - ); + let send = RemoteSend { + node_id, + vshard_id, + resend_safe: required_permission(&plan) == Permission::Read, + deadline, + }; // Encode the plan. let plan_bytes = plan_wire::encode(&plan).map_err(|e| Error::Internal { @@ -167,7 +162,7 @@ pub(super) async fn dispatch_remote( plan_bytes, tenant_id: tenant_id.as_u64(), database_id: database_id.as_u64(), - deadline_remaining_ms: attempt_ms, + deadline_remaining_ms: deadline_ms, trace_id: trace_id.0, descriptor_versions, txn_id, @@ -182,28 +177,19 @@ pub(super) async fn dispatch_remote( "gateway: dispatching ExecuteRequest to remote node" ); - // The remote handler works within `attempt_ms`: a write can wait at the - // leader's gate or in a Calvin queue for that long. The reply wait follows - // that budget, never the transport's fixed RPC timeout. - let reply_wait = nodedb_cluster::rpc_codec::reply_wait(attempt_ms); + // The remote handler works within the statement's whole remaining budget. + // A live node gets all of it. A dead node fails the send at the QUIC idle + // timeout, and a read then retries elsewhere. + let reply_wait = nodedb_cluster::rpc_codec::reply_wait(deadline_ms); let resp_rpc = match tokio::time::timeout( reply_wait, transport.send_rpc_with_read_timeout(node_id, req, reply_wait), ) .await { - Ok(answer) => answer.map_err(|e| send_error(e, resend_safe, node_id, vshard_id))?, - Err(_) => { - return Err(send_error( - ClusterError::ShardTimeout { - vshard_id: (vshard_id % VShardId::COUNT as u64) as u32, - elapsed_ms: u64::try_from(reply_wait.as_millis()).unwrap_or(u64::MAX), - }, - resend_safe, - node_id, - vshard_id, - )); - } + Ok(answer) => answer.map_err(|e| send_error(e, &send))?, + // The wait was the statement's whole budget: no retry has time left. + Err(_) => return Err(reply_wait_expired(&send, reply_wait)), }; match resp_rpc { diff --git a/nodedb/src/control/gateway/dispatch_remote_stream.rs b/nodedb/src/control/gateway/dispatch_remote_stream.rs index 51a87b7ed..03c88a10a 100644 --- a/nodedb/src/control/gateway/dispatch_remote_stream.rs +++ b/nodedb/src/control/gateway/dispatch_remote_stream.rs @@ -15,7 +15,7 @@ use crate::control::security::identity::{Permission, required_permission}; use crate::control::server::result_stream::{ResultStream, RowBatch}; use crate::types::{Lsn, VShardId}; -use super::cluster_error::{map_typed_cluster_error, send_error}; +use super::cluster_error::{RemoteSend, map_typed_cluster_error, reply_wait_expired, send_error}; use super::dispatch_remote::{RemoteDispatchArgs, scoped_vshard}; /// Remote streaming dispatch via the multi-frame `ExecuteStreamRequest` RPC. @@ -36,9 +36,9 @@ use super::dispatch_remote::{RemoteDispatchArgs, scoped_vshard}; /// `Err` and never retried (re-running the plan duplicates the rows /// already streamed to the client). /// -/// The open and the first frame wait at most `reply_wait` of this attempt's -/// budget. A read leaves time there for the attempts after it. The remote -/// runs the stream within the statement's whole remaining budget. +/// The remote runs the stream within the statement's whole remaining budget. +/// The open and the first frame wait at most `reply_wait` of that budget. A +/// dead node fails the open at the QUIC idle timeout, and a read retries. /// /// The returned stream re-emits the buffered first batch followed by the rest. pub(super) async fn dispatch_remote_stream( @@ -53,12 +53,13 @@ pub(super) async fn dispatch_remote_stream( database_id, trace_id, deadline_ms, - attempts_left, version_set, txn_id, linearizable, read_groups, } = args; + // The statement's deadline, fixed before any wait of this attempt. + let deadline = tokio::time::Instant::now() + std::time::Duration::from_millis(deadline_ms); let transport = shared.cluster_transport.as_ref().ok_or(Error::Internal { detail: "gateway: cluster transport not available for remote stream dispatch".into(), })?; @@ -99,13 +100,12 @@ pub(super) async fn dispatch_remote_stream( // A read can be sent again after a lost answer. Any other plan can have // run on the target, so a lost answer is an unknown outcome. - let resend_safe = required_permission(&plan) == Permission::Read; - let attempt_ms = nodedb_cluster::rpc_codec::attempt_budget( - deadline_ms, - resend_safe, - attempts_left, - transport.rpc_timeout(), - ); + let send = RemoteSend { + node_id, + vshard_id, + resend_safe: required_permission(&plan) == Permission::Read, + deadline, + }; let plan_bytes = plan_wire::encode(&plan).map_err(|e| Error::Internal { detail: format!("gateway: plan encode failed: {e}"), @@ -126,8 +126,7 @@ pub(super) async fn dispatch_remote_stream( plan_bytes, tenant_id: tenant_id.as_u64(), database_id: database_id.as_u64(), - // The remote bounds the whole stream with this budget, so it keeps - // the statement's remainder. Only the first frame waits on `attempt_ms`. + // The remote bounds the whole stream with the statement's remainder. deadline_remaining_ms: deadline_ms, trace_id: trace_id.0, descriptor_versions, @@ -143,15 +142,15 @@ pub(super) async fn dispatch_remote_stream( "gateway: dispatching ExecuteStreamRequest to remote node" ); - // Open the stream and pull the FIRST frame eagerly, within this attempt's - // reply wait, so a pre-row error or a silent node is catchable and - // retryable. Any error here is a pre-row error. - let reply_wait = nodedb_cluster::rpc_codec::reply_wait(attempt_ms); + // Open the stream and pull the FIRST frame eagerly, within the reply + // wait, so a pre-row error is catchable and retryable. Any error here is + // a pre-row error. + let reply_wait = nodedb_cluster::rpc_codec::reply_wait(deadline_ms); let opened = tokio::time::timeout(reply_wait, async move { let stream = transport .send_rpc_stream(node_id, req) .await - .map_err(|e| send_error(e, resend_safe, node_id, vshard_id))?; + .map_err(|e| send_error(e, &send))?; // The `async_stream` body is `!Unpin`; pin it on the heap so the tail // stays usable for `.chain` after the first frame. let mut stream = Box::pin(stream); @@ -161,17 +160,8 @@ pub(super) async fn dispatch_remote_stream( .await; let (stream, first) = match opened { Ok(opened) => opened?, - Err(_) => { - return Err(send_error( - ClusterError::ShardTimeout { - vshard_id: (vshard_id % VShardId::COUNT as u64) as u32, - elapsed_ms: u64::try_from(reply_wait.as_millis()).unwrap_or(u64::MAX), - }, - resend_safe, - node_id, - vshard_id, - )); - } + // The wait was the statement's whole budget: no retry has time left. + Err(_) => return Err(reply_wait_expired(&send, reply_wait)), }; let first = match first { Some(Ok((payload, lsn))) => RowBatch { @@ -182,7 +172,7 @@ pub(super) async fn dispatch_remote_stream( read_versions: crate::types::ReadVersions::new(), }, Some(Err(e)) => { - return Err(map_stream_cluster_error(e, resend_safe, node_id, vshard_id)); + return Err(map_stream_cluster_error(e, &send)); } // Clean EOF with zero rows: a valid empty result. Return an empty stream. None => return Ok(Box::pin(futures::stream::empty())), @@ -216,12 +206,8 @@ pub(super) async fn dispatch_remote_stream( /// unknown outcome. Any other cluster error becomes a transport-style /// `NotLeader` (leader_node = 0), so the next attempt re-resolves routing /// rather than re-entrenching an unreachable node. -fn map_stream_cluster_error( - err: ClusterError, - resend_safe: bool, - node_id: u64, - vshard_id: u64, -) -> Error { +fn map_stream_cluster_error(err: ClusterError, send: &RemoteSend) -> Error { + let vshard_id = send.vshard_id; match err { ClusterError::StreamTerminal { error, .. } | ClusterError::ShardExecution { error, .. } => { map_typed_cluster_error(*error, vshard_id) @@ -230,7 +216,7 @@ fn map_stream_cluster_error( // repeats it, so it keeps its SQLSTATE. ClusterError::DataPlane { code } => Error::DataPlane(code.into()), // The request can have run on the remote node. - lost if lost.is_lost_answer() => send_error(lost, resend_safe, node_id, vshard_id), + lost if lost.is_lost_answer() => send_error(lost, send), other @ (ClusterError::Raft(_) | ClusterError::VShardNotMapped { .. } | ClusterError::GroupNotFound { .. } @@ -281,6 +267,15 @@ fn map_stream_cluster_error( mod tests { use super::*; + fn send(resend_safe: bool) -> RemoteSend { + RemoteSend { + node_id: 2, + vshard_id: 7, + resend_safe, + deadline: tokio::time::Instant::now() + std::time::Duration::from_secs(30), + } + } + fn unanswered() -> ClusterError { ClusterError::Unanswered { node_id: 2, @@ -292,7 +287,7 @@ mod tests { #[test] fn a_lost_stream_answer_to_a_read_retries() { assert!(matches!( - map_stream_cluster_error(unanswered(), true, 2, 7), + map_stream_cluster_error(unanswered(), &send(true)), Error::NotLeader { leader_node: 0, .. } )); } @@ -302,7 +297,7 @@ mod tests { #[test] fn a_lost_stream_answer_to_a_non_read_is_an_unknown_outcome() { assert!(matches!( - map_stream_cluster_error(unanswered(), false, 2, 7), + map_stream_cluster_error(unanswered(), &send(false)), Error::DeadlineExceeded { .. } )); let timed_out = ClusterError::ShardTimeout { @@ -310,7 +305,7 @@ mod tests { elapsed_ms: 1_000, }; assert!(matches!( - map_stream_cluster_error(timed_out, false, 2, 7), + map_stream_cluster_error(timed_out, &send(false)), Error::DeadlineExceeded { .. } )); } @@ -322,7 +317,7 @@ mod tests { detail: "connect refused".into(), }; assert!(matches!( - map_stream_cluster_error(unsent, false, 2, 7), + map_stream_cluster_error(unsent, &send(false)), Error::NotLeader { leader_node: 0, .. } )); } diff --git a/nodedb/src/control/gateway/dispatcher.rs b/nodedb/src/control/gateway/dispatcher.rs index 73e4ac17c..5398386de 100644 --- a/nodedb/src/control/gateway/dispatcher.rs +++ b/nodedb/src/control/gateway/dispatcher.rs @@ -53,9 +53,6 @@ pub struct DispatchRouteParams<'a> { pub database_id: DatabaseId, pub trace_id: TraceId, pub deadline_ms: u64, - /// The dispatch attempts this route has left, this one included. A - /// caller that never retries passes 1. - pub attempts_left: usize, pub version_set: &'a GatewayVersionSet, pub txn_id: Option, /// The route is a leg of a linearizable read (see @@ -74,7 +71,6 @@ pub(crate) async fn dispatch_route( database_id, trace_id, deadline_ms, - attempts_left, version_set, txn_id, linearizable, @@ -107,7 +103,6 @@ pub(crate) async fn dispatch_route( database_id, trace_id, deadline_ms, - attempts_left, version_set, txn_id, linearizable, @@ -144,8 +139,6 @@ pub struct DispatchRouteStreamParams<'a> { pub database_id: DatabaseId, pub trace_id: TraceId, pub deadline_ms: u64, - /// The dispatch attempts this route has left, this one included. - pub attempts_left: usize, pub version_set: &'a GatewayVersionSet, /// The route is a leg of a linearizable read. pub linearizable: bool, @@ -179,7 +172,6 @@ pub(crate) async fn dispatch_route_stream( database_id, trace_id, deadline_ms, - attempts_left, version_set, linearizable, } = args; @@ -213,7 +205,6 @@ pub(crate) async fn dispatch_route_stream( database_id, trace_id, deadline_ms, - attempts_left, version_set, // No session-transaction context crosses the streaming gateway // boundary yet (see `resolve/exchange.rs`), so `None`. diff --git a/nodedb/src/control/gateway/retry.rs b/nodedb/src/control/gateway/retry.rs index 4171a4b98..2adb75ec7 100644 --- a/nodedb/src/control/gateway/retry.rs +++ b/nodedb/src/control/gateway/retry.rs @@ -1,25 +1,27 @@ // SPDX-License-Identifier: BUSL-1.1 -//! Typed `NotLeader` retry with 3-attempt budget + 50/100/200 ms backoff. +//! Typed `NotLeader` retry that runs until the statement deadline. //! //! When a remote dispatch returns `Error::NotLeader`, the retry helper: //! 1. Extracts the hinted new leader and its term from the error. //! 2. Offers them to the routing table entry for the affected group. The //! hint moves only when the redirect's term is above the hint's term, so //! a stale redirect never undoes a newer leader. -//! 3. Sleeps for the appropriate backoff duration. +//! 3. Sleeps for the next backoff of its [`RetryPacing`]. //! 4. Re-invokes the closure. //! //! If the hinted leader is unknown (no hint), we still retry after sleep //! without updating the routing table — a subsequent routing lookup will //! re-read the table from the current routing state. //! -//! Every attempt shares the statement's deadline, so the attempts together -//! stay within one statement budget. No retry starts once it has passed. The -//! last refusal then ends the statement: it is definite, so it never becomes -//! `DeadlineExceeded`, which means a lost answer. -//! -//! After `MAX_RETRIES` attempts the final `NotLeader` error is propagated. +//! A re-election can outlast any fixed attempt count, so retries continue +//! while the statement has time left. Every attempt shares the statement's +//! deadline. When the next backoff would reach it, one last attempt starts +//! at once, and its verdict ends the statement. A lost answer to a read then +//! ends as the deadline the last send reports. No retry starts once the +//! deadline passed. The last refusal then ends the statement: it is +//! definite, so it never becomes `DeadlineExceeded`, which means a lost +//! answer. use std::future::Future; use std::sync::RwLock; @@ -28,16 +30,11 @@ use tokio::time::{Duration, Instant, sleep}; use tracing::debug; use nodedb_cluster::RoutingTable; +use nodedb_types::config::tuning::ClusterTransportTuning; use crate::Error; use crate::control::server::shared::session::deadline::hop_budget_ms; -/// Maximum number of dispatch attempts (initial + 2 retries = 3 total). -pub const MAX_RETRIES: usize = 3; - -/// Backoff durations for each retry attempt. -const BACKOFF_MS: [u64; MAX_RETRIES] = [50, 100, 200]; - /// One dispatch attempt of a statement. #[derive(Debug, Clone, Copy, PartialEq, Eq)] pub struct Attempt { @@ -48,27 +45,49 @@ pub struct Attempt { } impl Attempt { - /// The attempts left, this one included. - pub fn attempts_left(&self) -> usize { - MAX_RETRIES.saturating_sub(self.index) - } - /// The milliseconds left on the statement now, at least 1. See /// [`hop_budget_ms`]. pub fn remaining_ms(&self) -> u64 { hop_budget_ms(self.deadline.into_std()) } +} - /// The one attempt of a send that no retry follows. - pub fn sole(deadline: Instant) -> Self { - Self { - index: MAX_RETRIES - 1, - deadline, - } +/// The backoff between leader retries: doubles from `first` up to `cap`. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub struct RetryPacing { + /// The pause before the first retry. + pub first: Duration, + /// The longest pause between two retries. + pub cap: Duration, +} + +impl RetryPacing { + /// Pacing from the cluster's Raft timing. + /// + /// The first pause is one Raft tick. The cap is the minimum election + /// timeout: no new leader can appear faster than that. + pub fn from_tuning(tuning: &ClusterTransportTuning) -> Self { + let first = Duration::from_millis(tuning.raft_tick_interval_ms.max(1)); + let cap = Duration::from_millis(tuning.effective_election_timeout_min_ms()).max(first); + Self { first, cap } + } + + /// The pause after `retries` retries already ran. + fn backoff(&self, retries: u32) -> Duration { + self.first + .checked_mul(2u32.saturating_pow(retries)) + .map_or(self.cap, |d| d.min(self.cap)) } } -/// Execute `f` up to `MAX_RETRIES` times, retrying on `Error::NotLeader`. +impl Default for RetryPacing { + fn default() -> Self { + Self::from_tuning(&ClusterTransportTuning::default()) + } +} + +/// Execute `f`, retrying on `Error::NotLeader` and +/// `Error::RetryableLeaderChange` until `deadline`. /// /// `f` receives the current [`Attempt`]: its index and the statement's /// `deadline`, which every attempt shares. @@ -79,6 +98,7 @@ impl Attempt { pub async fn retry_not_leader( routing: Option<&RwLock>, deadline: Instant, + pacing: RetryPacing, f: F, ) -> Result where @@ -86,6 +106,7 @@ where Fut: Future>, { let mut attempt = 0; + let mut last_call = false; loop { let refusal = match f(Attempt { index: attempt, @@ -143,12 +164,9 @@ where group_id, log_index, }) => { - // The Raft entry the proposer was waiting on was - // overwritten by a new leader's election no-op. Re- - // propose: the next attempt will land at a fresh log - // index under the new leader's term. Same backoff - // schedule as NotLeader — bounded retries so a - // pathological cluster doesn't loop forever. + // A new leader's election no-op overwrote the entry the + // proposer waited on. The next attempt re-proposes under the + // new leader's term. debug!( attempt, group_id, log_index, "gateway: leader change overwrote entry — will re-propose" @@ -162,22 +180,23 @@ where Err(other) => return Err(other), }; - // Both refusals mean nothing applied. When no retry follows, that - // definite refusal ends the statement, never an unknown outcome. - attempt += 1; - if attempt >= MAX_RETRIES { - return Err(refusal); - } - sleep(Duration::from_millis(BACKOFF_MS[attempt - 1])).await; - // A retry starts only while the statement has time left. That keeps - // every attempt's wait within the budget plus one reply margin. - if Instant::now() >= deadline { + // Both refusals mean nothing applied. Once the deadline passed, or + // the last call ran, that definite refusal ends the statement. + let now = Instant::now(); + if last_call || now >= deadline { debug!( attempt, - "gateway: statement deadline passed — no further retry" + "gateway: statement deadline reached — no further retry" ); return Err(refusal); } + let pause = pacing.backoff(u32::try_from(attempt).unwrap_or(u32::MAX)); + if now + pause >= deadline { + last_call = true; + } else { + sleep(pause).await; + } + attempt += 1; } } @@ -195,9 +214,25 @@ mod tests { Instant::now() + Duration::from_secs(30) } + fn pacing() -> RetryPacing { + RetryPacing { + first: Duration::from_millis(10), + cap: Duration::from_millis(150), + } + } + + fn not_leader(leader_node: u64, leader_term: u64) -> Error { + Error::NotLeader { + vshard_id: VShardId::new(0), + leader_node, + leader_addr: String::new(), + leader_term, + } + } + #[tokio::test] async fn success_on_first_attempt() { - let result = retry_not_leader(None, far_deadline(), |_attempt| async { + let result = retry_not_leader(None, far_deadline(), pacing(), |_attempt| async { Ok::(42) }) .await; @@ -208,17 +243,11 @@ mod tests { async fn success_on_second_attempt() { let call_count = Arc::new(AtomicUsize::new(0)); let count = Arc::clone(&call_count); - let result = retry_not_leader(None, far_deadline(), move |_attempt| { + let result = retry_not_leader(None, far_deadline(), pacing(), move |_attempt| { let c = Arc::clone(&count); async move { - let n = c.fetch_add(1, Ordering::SeqCst); - if n == 0 { - Err(Error::NotLeader { - vshard_id: VShardId::new(0), - leader_node: 2, - leader_addr: "10.0.0.2:9400".into(), - leader_term: 1, - }) + if c.fetch_add(1, Ordering::SeqCst) == 0 { + Err(not_leader(2, 1)) } else { Ok::(99) } @@ -229,25 +258,11 @@ mod tests { assert_eq!(call_count.load(Ordering::SeqCst), 2); } - #[tokio::test] - async fn exhausts_retries_returns_not_leader() { - let result = retry_not_leader(None, far_deadline(), |_| async { - Err::(Error::NotLeader { - vshard_id: VShardId::new(1), - leader_node: 0, - leader_addr: String::new(), - leader_term: 0, - }) - }) - .await; - assert!(matches!(result, Err(Error::NotLeader { .. }))); - } - #[tokio::test] async fn non_not_leader_error_propagates_immediately() { let call_count = Arc::new(AtomicUsize::new(0)); let count = Arc::clone(&call_count); - let result = retry_not_leader(None, far_deadline(), move |_| { + let result = retry_not_leader(None, far_deadline(), pacing(), move |_| { let c = Arc::clone(&count); async move { c.fetch_add(1, Ordering::SeqCst); @@ -261,6 +276,59 @@ mod tests { assert_eq!(call_count.load(Ordering::SeqCst), 1); } + /// A re-election longer than a few fixed attempts is retried until the + /// new leader answers. + #[tokio::test(start_paused = true)] + async fn a_leader_change_past_350_ms_is_retried_until_it_resolves() { + let started = Instant::now(); + let resolves_at = started + Duration::from_millis(1_000); + let calls = Arc::new(AtomicUsize::new(0)); + let count = Arc::clone(&calls); + let result = retry_not_leader(None, far_deadline(), pacing(), move |_| { + let c = Arc::clone(&count); + async move { + c.fetch_add(1, Ordering::SeqCst); + if Instant::now() < resolves_at { + Err(not_leader(0, 0)) + } else { + Ok::(7) + } + } + }) + .await; + assert_eq!(result.unwrap(), 7); + assert!(calls.load(Ordering::SeqCst) > 3, "more than three attempts"); + let waited = started.elapsed(); + assert!( + waited < Duration::from_millis(1_000) + pacing().cap, + "the retry follows the resolution within one capped backoff: {waited:?}" + ); + } + + /// Retries stop when the next backoff reaches the deadline. The last + /// refusal stays the answer, never `DeadlineExceeded`. + #[tokio::test(start_paused = true)] + async fn a_passing_deadline_stops_retries_with_the_last_refusal() { + let started = Instant::now(); + let deadline = started + Duration::from_millis(600); + let result = retry_not_leader(None, deadline, pacing(), |attempt| async move { + Err::<(), Error>(not_leader(7, attempt.index as u64 + 1)) + }) + .await; + match result { + Err(Error::NotLeader { + leader_node: 7, + leader_term, + .. + }) => assert!(leader_term > 1, "the last refusal returns"), + other => panic!("expected the last NotLeader, got {other:?}"), + } + assert!( + Instant::now() <= deadline, + "no retry runs past the deadline" + ); + } + #[tokio::test] async fn routing_table_updated_on_not_leader_hint() { let table = RoutingTable::uniform(1, &[1, 2], 2); @@ -270,17 +338,11 @@ mod tests { let call_count = Arc::new(AtomicUsize::new(0)); let count = Arc::clone(&call_count); - let _ = retry_not_leader(Some(&*rt_clone), far_deadline(), move |_| { + let _ = retry_not_leader(Some(&*rt_clone), far_deadline(), pacing(), move |_| { let c = Arc::clone(&count); async move { - let n = c.fetch_add(1, Ordering::SeqCst); - if n == 0 { - Err(Error::NotLeader { - vshard_id: VShardId::new(0), - leader_node: 2, - leader_addr: "addr".into(), - leader_term: 4, - }) + if c.fetch_add(1, Ordering::SeqCst) == 0 { + Err(not_leader(2, 4)) } else { Ok::<(), Error>(()) } @@ -300,14 +362,9 @@ mod tests { assert!(table.observe_leader(group_id, 3, 7)); let rt = RwLock::new(table); - let _ = retry_not_leader(Some(&rt), far_deadline(), |attempt| async move { + let _ = retry_not_leader(Some(&rt), far_deadline(), pacing(), |attempt| async move { if attempt.index == 0 { - Err(Error::NotLeader { - vshard_id: VShardId::new(0), - leader_node: 2, - leader_addr: "addr".into(), - leader_term: 5, - }) + Err(not_leader(2, 5)) } else { Ok::<(), Error>(()) } @@ -323,14 +380,9 @@ mod tests { #[tokio::test] async fn a_term_less_redirect_never_replaces_a_termed_hint() { async fn redirect_to(rt: &RwLock, leader_node: u64) { - let _ = retry_not_leader(Some(rt), far_deadline(), |attempt| async move { + let _ = retry_not_leader(Some(rt), far_deadline(), pacing(), |attempt| async move { if attempt.index == 0 { - Err(Error::NotLeader { - vshard_id: VShardId::new(0), - leader_node, - leader_addr: String::new(), - leader_term: 0, - }) + Err(not_leader(leader_node, 0)) } else { Ok::<(), Error>(()) } @@ -356,44 +408,27 @@ mod tests { /// Every attempt carries the one statement deadline, so the budget left /// shrinks across attempts and never restarts. - #[tokio::test] + #[tokio::test(start_paused = true)] async fn the_statement_deadline_is_shared_across_attempts() { - let deadline = far_deadline(); + let deadline = Instant::now() + Duration::from_millis(500); let seen = Arc::new(std::sync::Mutex::new(Vec::new())); let record = Arc::clone(&seen); - let result = retry_not_leader(None, deadline, move |attempt| { + let result = retry_not_leader(None, deadline, pacing(), move |attempt| { let record = Arc::clone(&record); async move { - record.lock().unwrap().push(( - attempt, - attempt.attempts_left(), - attempt.remaining_ms(), - )); - Err::<(), Error>(Error::NotLeader { - vshard_id: VShardId::new(0), - leader_node: 0, - leader_addr: String::new(), - leader_term: 0, - }) + record.lock().unwrap().push(attempt); + Err::<(), Error>(not_leader(0, 0)) } }) .await; assert!(matches!(result, Err(Error::NotLeader { .. }))); let seen = seen.lock().unwrap(); - assert_eq!(seen.len(), MAX_RETRIES); - for (index, (attempt, attempts_left, _)) in seen.iter().enumerate() { + assert!(seen.len() > 1); + for (index, attempt) in seen.iter().enumerate() { assert_eq!(attempt.index, index); assert_eq!(attempt.deadline, deadline); - assert_eq!(*attempts_left, MAX_RETRIES - index); } - let first_remaining = seen[0].2; - let last_remaining = seen[MAX_RETRIES - 1].2; - let backoff: u64 = BACKOFF_MS[..MAX_RETRIES - 1].iter().sum(); - assert!( - last_remaining <= first_remaining - backoff, - "the last attempt has {last_remaining} ms, the first had {first_remaining} ms" - ); } /// A retry never starts once the statement's deadline has passed. The @@ -402,16 +437,11 @@ mod tests { async fn no_retry_starts_past_the_deadline() { let calls = Arc::new(AtomicUsize::new(0)); let count = Arc::clone(&calls); - let result = retry_not_leader(None, Instant::now(), move |_| { + let result = retry_not_leader(None, Instant::now(), pacing(), move |_| { let c = Arc::clone(&count); async move { c.fetch_add(1, Ordering::SeqCst); - Err::<(), Error>(Error::NotLeader { - vshard_id: VShardId::new(0), - leader_node: 7, - leader_addr: String::new(), - leader_term: 3, - }) + Err::<(), Error>(not_leader(7, 3)) } }) .await; @@ -429,11 +459,59 @@ mod tests { assert_eq!(calls.load(Ordering::SeqCst), 1); } + /// A lost answer to a read just before the deadline gets one last call + /// at once. Its verdict, the deadline, ends the statement. + #[tokio::test(start_paused = true)] + async fn the_last_call_before_the_deadline_decides_the_verdict() { + let deadline = Instant::now() + Duration::from_millis(5); + let calls = Arc::new(AtomicUsize::new(0)); + let count = Arc::clone(&calls); + let result = retry_not_leader(None, deadline, pacing(), move |_| { + let c = Arc::clone(&count); + async move { + if c.fetch_add(1, Ordering::SeqCst) == 0 { + Err::<(), Error>(not_leader(0, 0)) + } else { + Err(Error::DeadlineExceeded { + request_id: crate::types::RequestId::new(0), + }) + } + } + }) + .await; + assert!( + matches!(result, Err(Error::DeadlineExceeded { .. })), + "got {result:?}" + ); + assert_eq!(calls.load(Ordering::SeqCst), 2, "one last call, no more"); + } + + /// A refusal from the last call ends the statement: no call follows it. + #[tokio::test(start_paused = true)] + async fn a_refused_last_call_ends_the_retries() { + let deadline = Instant::now() + Duration::from_millis(5); + let calls = Arc::new(AtomicUsize::new(0)); + let count = Arc::clone(&calls); + let result = retry_not_leader(None, deadline, pacing(), move |_| { + let c = Arc::clone(&count); + async move { + c.fetch_add(1, Ordering::SeqCst); + Err::<(), Error>(not_leader(0, 0)) + } + }) + .await; + assert!( + matches!(result, Err(Error::NotLeader { .. })), + "got {result:?}" + ); + assert_eq!(calls.load(Ordering::SeqCst), 2); + } + /// An overwritten entry was never applied. Past the deadline it stays a /// definite refusal, never an unknown outcome. #[tokio::test] async fn an_overwritten_entry_past_the_deadline_stays_definite() { - let result = retry_not_leader(None, Instant::now(), |_| async { + let result = retry_not_leader(None, Instant::now(), pacing(), |_| async { Err::<(), Error>(Error::RetryableLeaderChange { group_id: 4, log_index: 9, @@ -452,19 +530,29 @@ mod tests { ); } - /// A sole attempt leaves no retry behind it. + /// The backoff doubles from one Raft tick and stops at the minimum + /// election timeout. #[test] - fn a_sole_attempt_has_one_attempt_left() { - let deadline = far_deadline(); - let attempt = Attempt::sole(deadline); - assert_eq!(attempt.attempts_left(), 1); - assert_eq!(attempt.deadline, deadline); + fn pacing_follows_the_raft_timing() { + let tuning = ClusterTransportTuning { + raft_tick_interval_ms: 10, + election_timeout_min_ms: 150, + ..ClusterTransportTuning::default() + }; + let pacing = RetryPacing::from_tuning(&tuning); + assert_eq!(pacing.backoff(0), Duration::from_millis(10)); + assert_eq!(pacing.backoff(1), Duration::from_millis(20)); + assert_eq!(pacing.backoff(4), Duration::from_millis(150)); + assert_eq!(pacing.backoff(u32::MAX), Duration::from_millis(150)); } /// A spent deadline still sends a 1 ms hop, so the receiver checks it. #[tokio::test] async fn remaining_ms_never_reaches_zero() { - let attempt = Attempt::sole(Instant::now()); + let attempt = Attempt { + index: 0, + deadline: Instant::now(), + }; assert_eq!(attempt.remaining_ms(), 1); } } diff --git a/nodedb/src/control/gateway/stream.rs b/nodedb/src/control/gateway/stream.rs index 03d81642e..1f8ffaaba 100644 --- a/nodedb/src/control/gateway/stream.rs +++ b/nodedb/src/control/gateway/stream.rs @@ -17,7 +17,7 @@ use nodedb_physical::physical_plan::PhysicalPlan; use super::core::{Gateway, QueryContext, authorized_plan_for_context}; use super::dispatcher::{DispatchRouteStreamParams, dispatch_route_stream}; use super::live_leaders::resolve_live_decision; -use super::retry::retry_not_leader; +use super::retry::{RetryPacing, retry_not_leader}; use super::route::TaskRoute; use crate::control::server::shared::session::StatementDeadline; @@ -66,6 +66,7 @@ impl Gateway { // One deadline for every route and attempt of the statement. let deadline = StatementDeadline::of_statement(&shared).tokio_instant(); + let pacing = RetryPacing::from_tuning(&shared.tuning.cluster_transport); let mut per_route: Vec = Vec::with_capacity(routes.len()); for route in routes { @@ -80,7 +81,7 @@ impl Gateway { // attempt re-resolves the routing decision and re-opens the stream; // a pre-row NotLeader / DescriptorMismatch is retryable, anything // after the first frame is terminal (handled inside the stream). - let stream = retry_not_leader(routing_ref, deadline, move |attempt| { + let stream = retry_not_leader(routing_ref, deadline, pacing, move |attempt| { if attempt.index > 0 { retry_counter.fetch_add(1, Ordering::Relaxed); } @@ -105,7 +106,6 @@ impl Gateway { database_id, trace_id, deadline_ms: attempt.remaining_ms(), - attempts_left: attempt.attempts_left(), version_set: &version_set, linearizable, }) diff --git a/nodedb/src/control/maintenance/clone_materializer/dispatch.rs b/nodedb/src/control/maintenance/clone_materializer/dispatch.rs index 43040674f..757dd3b0c 100644 --- a/nodedb/src/control/maintenance/clone_materializer/dispatch.rs +++ b/nodedb/src/control/maintenance/clone_materializer/dispatch.rs @@ -7,13 +7,13 @@ //! runtime. use std::sync::atomic::Ordering; -use std::time::{Duration, Instant}; use nodedb_types::{DatabaseId, TenantId}; use crate::bridge::envelope::{ErrorCode, Priority, Request, Response}; use crate::control::gateway::core::QueryContext; use crate::control::server::dispatch_utils::{OwnedRead, OwnedReadScope, route_owned_read}; +use crate::control::server::shared::session::StatementDeadline; use crate::control::server::shared::write_admission::plan_is_write; use crate::control::state::SharedState; use crate::types::{ReadConsistency, RequestId, TraceId, TxnId, VShardId}; @@ -157,15 +157,14 @@ pub(crate) async fn dispatch_on_this_node( txn_id: Option, ) -> crate::Result { let req_id = RequestId::new(state.request_id_counter.fetch_add(1, Ordering::Relaxed)); - let deadline_secs = state.tuning.network.default_deadline_secs; - let deadline_dur = Duration::from_secs(deadline_secs); + let deadline = StatementDeadline::of_statement(state); let req = Request { request_id: req_id, tenant_id, vshard_id, database_id, plan, - deadline: Instant::now() + deadline_dur, + deadline: deadline.instant(), priority: Priority::Normal, trace_id: TraceId::ZERO, consistency: ReadConsistency::Strong, @@ -188,7 +187,7 @@ pub(crate) async fn dispatch_on_this_node( Ok(mut d) => d.dispatch(req)?, Err(p) => p.into_inner().dispatch(req)?, } - tokio::time::timeout(deadline_dur, rx.recv()) + tokio::time::timeout_at(deadline.tokio_instant(), rx.recv()) .await .map_err(|_| crate::Error::DeadlineExceeded { request_id: req_id })? .ok_or(crate::Error::Dispatch { diff --git a/nodedb/src/control/metadata_proposer/catalog.rs b/nodedb/src/control/metadata_proposer/catalog.rs index 58aeed322..3657d229e 100644 --- a/nodedb/src/control/metadata_proposer/catalog.rs +++ b/nodedb/src/control/metadata_proposer/catalog.rs @@ -45,6 +45,8 @@ pub async fn propose_catalog_entry_async( group_id: METADATA_GROUP_ID, through: log_index, }], + crate::control::server::shared::session::StatementDeadline::of_statement(shared) + .instant(), ) .await?; } diff --git a/nodedb/src/control/metadata_proposer/catalog_batch.rs b/nodedb/src/control/metadata_proposer/catalog_batch.rs index dff28c3b7..42f551f6a 100644 --- a/nodedb/src/control/metadata_proposer/catalog_batch.rs +++ b/nodedb/src/control/metadata_proposer/catalog_batch.rs @@ -85,6 +85,8 @@ pub async fn propose_catalog_batch_async( group_id: METADATA_GROUP_ID, through: log_index, }], + crate::control::server::shared::session::StatementDeadline::of_statement(shared) + .instant(), ) .await?; } diff --git a/nodedb/src/control/planner/calvin/dependent_recon_finish.rs b/nodedb/src/control/planner/calvin/dependent_recon_finish.rs index 5f6b210f2..a71d0f170 100644 --- a/nodedb/src/control/planner/calvin/dependent_recon_finish.rs +++ b/nodedb/src/control/planner/calvin/dependent_recon_finish.rs @@ -28,7 +28,12 @@ pub(super) async fn finish_committed( .any(|collection| sources.is_source_collection(collection)) }); if binds_authorization { - crate::control::security::auth_lease::calvin_write_barrier(state).await?; + crate::control::security::auth_lease::calvin_write_barrier( + state, + crate::control::server::shared::session::StatementDeadline::of_statement(state) + .instant(), + ) + .await?; } // Completion fired: the scheduler deposited the applied Response (with any diff --git a/nodedb/src/control/planner/calvin/submit/local.rs b/nodedb/src/control/planner/calvin/submit/local.rs index 4ed9979aa..f05c08281 100644 --- a/nodedb/src/control/planner/calvin/submit/local.rs +++ b/nodedb/src/control/planner/calvin/submit/local.rs @@ -198,7 +198,12 @@ pub(crate) async fn submit_prepared_and_await( AttemptOutcome::Aborted { reason } => return Err(calvin_abort_error(reason)), } if binds_authorization { - crate::control::security::auth_lease::calvin_write_barrier(state).await?; + crate::control::security::auth_lease::calvin_write_barrier( + state, + crate::control::server::shared::session::StatementDeadline::of_statement(state) + .instant(), + ) + .await?; } // Completion fired: the scheduler deposited the applied Response (with any diff --git a/nodedb/src/control/security/auth_lease/barrier.rs b/nodedb/src/control/security/auth_lease/barrier.rs index f0cdfae8c..17b2b7bfa 100644 --- a/nodedb/src/control/security/auth_lease/barrier.rs +++ b/nodedb/src/control/security/auth_lease/barrier.rs @@ -8,8 +8,6 @@ //! lease holder too, so its own Event Plane lag closes the same way. A //! single-node cluster runs the same barrier against its one lease. -use std::time::{Duration, Instant}; - use nodedb_cluster::calvin::SEQUENCER_GROUP_ID; use nodedb_cluster::{AuthBarrierOutcome, AuthBarrierRequest, GroupCoverage, RaftRpc}; @@ -22,7 +20,7 @@ use super::leadership::{metadata_leader, send_to_leader}; /// /// `targets` name the change: per group, the index it committed at. The /// change itself is committed; an error means only that the barrier did not -/// release within the request deadline. +/// release by `deadline`, the caller's statement deadline. /// /// Latency: in a cluster, every authorization-bearing write waits about one /// renewal interval (the Raft heartbeat) before it is acknowledged. Each @@ -35,15 +33,16 @@ use super::leadership::{metadata_leader, send_to_leader}; pub async fn authorization_barrier( state: &SharedState, targets: Vec, + deadline: std::time::Instant, ) -> crate::Result<()> { - let deadline_secs = state.tuning.network.default_deadline_secs; - let deadline = Instant::now() + Duration::from_secs(deadline_secs); + let remaining_until = || deadline.saturating_duration_since(std::time::Instant::now()); + let budget_ms = remaining_until().as_millis(); let timing = lease_timing(state)?; loop { - let remaining = deadline.saturating_duration_since(Instant::now()); + let remaining = remaining_until(); if remaining.is_zero() { return Err(committed_but_pending(format!( - "the barrier did not release within {deadline_secs}s" + "the barrier did not release within the statement deadline ({budget_ms} ms)" ))); } let request = AuthBarrierRequest { @@ -86,7 +85,7 @@ pub async fn authorization_barrier( // leader again. A new leader holds the barrier to its own floors. Some(AuthBarrierOutcome::NotLeader { .. }) | Some(AuthBarrierOutcome::Timeout { .. }) - | None => tokio::time::sleep(timing.renew_every).await, + | None => tokio::time::sleep(timing.renew_every.min(remaining_until())).await, } } } @@ -96,8 +95,12 @@ pub async fn authorization_barrier( /// /// Its completion acks sit in the sequencer log, at or below the sequencer /// group's commit index now. A node covers that index only once its own -/// replicas applied every acknowledged transaction below it. -pub async fn calvin_write_barrier(state: &SharedState) -> crate::Result<()> { +/// replicas applied every acknowledged transaction below it. The wait ends +/// at `deadline`, the caller's statement deadline. +pub async fn calvin_write_barrier( + state: &SharedState, + deadline: std::time::Instant, +) -> crate::Result<()> { lease_timing(state)?; let commit_index = state .raft_status_fn @@ -115,6 +118,7 @@ pub async fn calvin_write_barrier(state: &SharedState) -> crate::Result<()> { group_id: SEQUENCER_GROUP_ID, through: commit_index, }], + deadline, ) .await } diff --git a/nodedb/src/control/server/exchange/all_cores/read_cut.rs b/nodedb/src/control/server/exchange/all_cores/read_cut.rs index a66df9898..879d20548 100644 --- a/nodedb/src/control/server/exchange/all_cores/read_cut.rs +++ b/nodedb/src/control/server/exchange/all_cores/read_cut.rs @@ -22,6 +22,7 @@ use std::time::Duration; use nodedb_cluster::calvin::SequencerEntry; use nodedb_physical::physical_plan::{GraphOp, PhysicalPlan}; +use crate::control::server::shared::session::StatementDeadline; use crate::control::state::SharedState; /// How long a node waits for the marker before it proposes it again. A @@ -81,8 +82,9 @@ async fn resolve_read_cut(state: &SharedState, marker: u64) -> crate::Result crate::Result( database_id, trace_id: TraceId::ZERO, deadline_ms, - attempts_left: 1, version_set: &version_set, txn_id: read.txn_id, linearizable: read.linearizable, diff --git a/nodedb/src/control/server/graph_dispatch/match_scatter/round_zero.rs b/nodedb/src/control/server/graph_dispatch/match_scatter/round_zero.rs index 286b3a640..e134bf6fe 100644 --- a/nodedb/src/control/server/graph_dispatch/match_scatter/round_zero.rs +++ b/nodedb/src/control/server/graph_dispatch/match_scatter/round_zero.rs @@ -114,7 +114,6 @@ pub(super) async fn scatter_round_zero( database_id, trace_id: TraceId::ZERO, deadline_ms, - attempts_left: 1, version_set: &version_set, txn_id, linearizable, diff --git a/nodedb/src/control/server/graph_dispatch/rag_fusion/stages.rs b/nodedb/src/control/server/graph_dispatch/rag_fusion/stages.rs index 50c52f31e..bf4d6e093 100644 --- a/nodedb/src/control/server/graph_dispatch/rag_fusion/stages.rs +++ b/nodedb/src/control/server/graph_dispatch/rag_fusion/stages.rs @@ -73,7 +73,6 @@ pub(super) async fn export_legs( database_id, trace_id: TraceId::ZERO, deadline_ms: StatementDeadline::of_statement(state).hop_budget_ms(), - attempts_left: 1, version_set: &GatewayVersionSet::from_pairs(Vec::new()), txn_id: None, linearizable, diff --git a/nodedb/src/control/server/http/routes/crdt.rs b/nodedb/src/control/server/http/routes/crdt.rs index 607eb2703..da6b0ce05 100644 --- a/nodedb/src/control/server/http/routes/crdt.rs +++ b/nodedb/src/control/server/http/routes/crdt.rs @@ -175,8 +175,8 @@ pub async fn crdt_apply( crate::control::crdt_admission::AuthorizedCrdtApplyAdmissionRequest { authorized, collection: &collection, - timeout: std::time::Duration::from_secs( - state.shared.tuning.network.default_deadline_secs, + deadline: crate::control::server::shared::session::StatementDeadline::of_statement( + &state.shared, ), event_source: crate::event::EventSource::User, policy: &policy, diff --git a/nodedb/src/control/server/native/dispatch/raw_dispatch.rs b/nodedb/src/control/server/native/dispatch/raw_dispatch.rs index 93a95aab2..87c87348b 100644 --- a/nodedb/src/control/server/native/dispatch/raw_dispatch.rs +++ b/nodedb/src/control/server/native/dispatch/raw_dispatch.rs @@ -164,7 +164,9 @@ async fn dispatch_external_crdt_apply( crate::control::crdt_admission::AuthorizedCrdtApplyAdmissionRequest { authorized, collection: collection.as_str(), - timeout: std::time::Duration::from_secs(ctx.state.tuning.network.default_deadline_secs), + deadline: crate::control::server::shared::session::StatementDeadline::of_statement( + ctx.state, + ), event_source: crate::event::EventSource::User, policy: &policy, }, diff --git a/nodedb/src/control/server/shared/clone_write/probes.rs b/nodedb/src/control/server/shared/clone_write/probes.rs index 341083029..72eb7a4c7 100644 --- a/nodedb/src/control/server/shared/clone_write/probes.rs +++ b/nodedb/src/control/server/shared/clone_write/probes.rs @@ -4,8 +4,6 @@ //! write-interception paths: presence probes on the target collection, and //! source-row/value fetches for copy-up. -use std::time::Duration; - use nodedb_types::{DatabaseId, QualifiedCollection, Surrogate, TenantId}; use crate::bridge::envelope::{Priority, Request, Response, Status}; @@ -269,15 +267,14 @@ pub(super) async fn dispatch_data_plane_raw( .request_id_counter .fetch_add(1, std::sync::atomic::Ordering::Relaxed), ); - let deadline_secs = state.tuning.network.default_deadline_secs; - let deadline_dur = Duration::from_secs(deadline_secs); + let deadline = crate::control::server::shared::session::StatementDeadline::of_statement(state); let req = Request { request_id: req_id, tenant_id, vshard_id, database_id, plan, - deadline: std::time::Instant::now() + deadline_dur, + deadline: deadline.instant(), priority: Priority::Normal, trace_id: TraceId::ZERO, consistency: ReadConsistency::Strong, @@ -300,7 +297,7 @@ pub(super) async fn dispatch_data_plane_raw( Ok(mut d) => d.dispatch(req)?, Err(p) => p.into_inner().dispatch(req)?, } - tokio::time::timeout(deadline_dur, rx.recv()) + tokio::time::timeout_at(deadline.tokio_instant(), rx.recv()) .await .map_err(|_| crate::Error::DeadlineExceeded { request_id: req_id })? .ok_or(crate::Error::Dispatch { diff --git a/nodedb/src/control/server/shared/ddl/engine_apply.rs b/nodedb/src/control/server/shared/ddl/engine_apply.rs index 6aaef0f89..c841a6859 100644 --- a/nodedb/src/control/server/shared/ddl/engine_apply.rs +++ b/nodedb/src/control/server/shared/ddl/engine_apply.rs @@ -7,14 +7,13 @@ //! refused configuration as a successful statement. Index DDL routes its //! engine registration through here so the refusal always reaches the client. -use std::time::Duration; - use crate::bridge::envelope::{ErrorCode, PhysicalPlan}; +use crate::control::server::shared::session::StatementDeadline; use crate::control::state::SharedState; use crate::types::{DatabaseId, TenantId}; use super::result::DdlError; -use super::sync_dispatch::{SystemReason, SystemTask, dispatch_system}; +use super::sync_dispatch::{SystemReason, SystemTask, dispatch_system_by}; /// Dispatch `plan` for `collection` and translate any Data-Plane refusal into /// a [`DdlError`] with `context` before its message. @@ -29,8 +28,7 @@ pub(crate) async fn apply_in_engine( plan: PhysicalPlan, context: &str, ) -> Result<(), DdlError> { - let timeout = Duration::from_secs(state.tuning.network.default_deadline_secs); - dispatch_system( + dispatch_system_by( state, SystemTask::new( SystemReason::DdlApply, @@ -38,7 +36,7 @@ pub(crate) async fn apply_in_engine( nodedb_types::CollectionKey::from_bare(database_id, collection), plan, ), - timeout, + StatementDeadline::of_statement(state), ) .await .map(|_| ()) @@ -63,7 +61,6 @@ pub(crate) async fn refuse_materialized_vector_index( field_name: &str, context: &str, ) -> Result<(), DdlError> { - let timeout = Duration::from_secs(state.tuning.network.default_deadline_secs); let plan = PhysicalPlan::Vector(nodedb_physical::physical_plan::VectorOp::QueryStats { collection: nodedb_types::QualifiedCollection::new(database_id, collection), field_name: field_name.to_string(), @@ -76,7 +73,7 @@ pub(crate) async fn refuse_materialized_vector_index( nodedb_types::CollectionKey::from_bare(database_id, collection), plan, ), - timeout, + StatementDeadline::of_statement(state), crate::event::EventSource::User, ) .await diff --git a/nodedb/src/control/server/shared/ddl/neutral/convert/driver.rs b/nodedb/src/control/server/shared/ddl/neutral/convert/driver.rs index 59e6c98ac..2ced86de2 100644 --- a/nodedb/src/control/server/shared/ddl/neutral/convert/driver.rs +++ b/nodedb/src/control/server/shared/ddl/neutral/convert/driver.rs @@ -18,6 +18,7 @@ use crate::control::server::pgwire::types::error_to_sqlstate; use crate::control::server::shared::ddl::sync_dispatch::{ SystemReason, SystemTask, dispatch_system_response_with_source, }; +use crate::control::server::shared::session::StatementDeadline; use crate::control::state::SharedState; use nodedb_physical::physical_plan::MetaOp; @@ -114,7 +115,7 @@ pub async fn convert_collection( nodedb_types::CollectionKey::from_bare(database_id, &collection), plan, ), - Duration::from_secs(60), + StatementDeadline::after(Duration::from_secs(60)), event_source, ) .await diff --git a/nodedb/src/control/server/shared/ddl/neutral/crdt_ops.rs b/nodedb/src/control/server/shared/ddl/neutral/crdt_ops.rs index e310bf3e3..fd2a00760 100644 --- a/nodedb/src/control/server/shared/ddl/neutral/crdt_ops.rs +++ b/nodedb/src/control/server/shared/ddl/neutral/crdt_ops.rs @@ -8,8 +8,6 @@ //! Handlers build [`DdlResult`](super::super::result::DdlResult) directly and //! carry no pgwire types. -use std::time::Duration; - use serde_json::{Map, Value as JsonValue}; use crate::bridge::envelope::PhysicalPlan; @@ -20,6 +18,7 @@ use crate::control::security::identity::{AuthenticatedIdentity, Permission}; use crate::control::server::response_shape::types::ShapedRows; use crate::control::server::shared::authorization::{authorize_collection, authorize_task_set}; use crate::control::server::shared::ddl::sql_parse::hex_decode; +use crate::control::server::shared::session::StatementDeadline; use crate::control::state::SharedState; use crate::types::DatabaseId; use nodedb_physical::physical_plan::CrdtOp; @@ -87,7 +86,7 @@ pub async fn crdt_state( database_id, collection, plan, - timeout: Duration::from_secs(state.tuning.network.default_deadline_secs), + deadline: StatementDeadline::of_statement(state), // This handler receives no session or peer information; the // request was admitted at its own transport entry, and // `AlreadyAdmitted` carries no address precisely so there is @@ -228,7 +227,7 @@ pub async fn crdt_apply( crate::control::crdt_admission::AuthorizedCrdtApplyAdmissionRequest { authorized, collection: &qualified_collection, - timeout: Duration::from_secs(state.tuning.network.default_deadline_secs), + deadline: StatementDeadline::of_statement(state), event_source: crate::event::EventSource::User, policy: &policy, }, diff --git a/nodedb/src/control/server/shared/ddl/neutral/dsl/crdt_merge.rs b/nodedb/src/control/server/shared/ddl/neutral/dsl/crdt_merge.rs index 3929ad00a..0f2f1051a 100644 --- a/nodedb/src/control/server/shared/ddl/neutral/dsl/crdt_merge.rs +++ b/nodedb/src/control/server/shared/ddl/neutral/dsl/crdt_merge.rs @@ -2,13 +2,12 @@ //! `CRDT MERGE INTO` DSL handler. -use std::time::Duration; - use crate::bridge::envelope::PhysicalPlan; use crate::control::crdt_post_image_policy::ExternalCrdtPostImagePolicy; use crate::control::security::audit::ArcAuditEmitter; use crate::control::security::identity::{AuthenticatedIdentity, Permission}; use crate::control::server::shared::authorization::{authorize_collection, authorize_task_set}; +use crate::control::server::shared::session::StatementDeadline; use crate::control::state::SharedState; use crate::types::DatabaseId; use nodedb_physical::physical_plan::CrdtOp; @@ -80,7 +79,7 @@ pub async fn crdt_merge( database_id, collection, plan: source_plan, - timeout: Duration::from_secs(state.tuning.network.default_deadline_secs), + deadline: StatementDeadline::of_statement(state), // Admitted at this request's own transport entry; this handler // has no session or peer information of its own. admission: crate::control::server::shared::ddl::user_dispatch::RequestAdmission::AlreadyAdmitted, @@ -165,7 +164,7 @@ pub async fn crdt_merge( crate::control::crdt_admission::AuthorizedCrdtApplyAdmissionRequest { authorized, collection: &qualified_collection, - timeout: Duration::from_secs(state.tuning.network.default_deadline_secs), + deadline: StatementDeadline::of_statement(state), event_source: crate::event::EventSource::User, policy: &policy, }, diff --git a/nodedb/src/control/server/shared/ddl/neutral/graph_ops/rag_fusion.rs b/nodedb/src/control/server/shared/ddl/neutral/graph_ops/rag_fusion.rs index dad1e6fa2..779564f12 100644 --- a/nodedb/src/control/server/shared/ddl/neutral/graph_ops/rag_fusion.rs +++ b/nodedb/src/control/server/shared/ddl/neutral/graph_ops/rag_fusion.rs @@ -7,8 +7,6 @@ //! this single function, so caps and defaults cannot drift between the //! two surfaces. -use std::time::Duration; - use serde_json::{Map, Value as JsonValue}; use nodedb_sql::ddl_ast::FusionParams; @@ -18,6 +16,7 @@ use crate::bridge::envelope::PhysicalPlan; use crate::control::security::identity::AuthenticatedIdentity; use crate::control::server::response_shape::types::ShapedRows; use crate::control::server::shared::ddl::user_dispatch; +use crate::control::server::shared::session::StatementDeadline; use crate::control::state::SharedState; use crate::data::executor::response_codec; use crate::engine::graph::edge_store::Direction; @@ -127,7 +126,7 @@ pub async fn rag_fusion( database_id, collection: &collection, plan, - timeout: Duration::from_secs(state.tuning.network.default_deadline_secs), + deadline: StatementDeadline::of_statement(state), // Admitted at this request's own transport entry; this handler has no // session or peer information of its own. admission: user_dispatch::RequestAdmission::AlreadyAdmitted, diff --git a/nodedb/src/control/server/shared/ddl/neutral/last_value.rs b/nodedb/src/control/server/shared/ddl/neutral/last_value.rs index 4583e3c94..c9416c129 100644 --- a/nodedb/src/control/server/shared/ddl/neutral/last_value.rs +++ b/nodedb/src/control/server/shared/ddl/neutral/last_value.rs @@ -18,6 +18,7 @@ use serde_json::{Map, Value as JsonValue}; use crate::bridge::envelope::PhysicalPlan; use crate::control::security::identity::AuthenticatedIdentity; use crate::control::server::response_shape::types::{DdlColType, ShapedRows}; +use crate::control::server::shared::session::StatementDeadline; use crate::control::state::SharedState; use crate::types::DatabaseId; use nodedb_physical::physical_plan::MetaOp; @@ -44,7 +45,7 @@ pub async fn query_last_values( database_id, collection, plan, - timeout: Duration::from_secs(5), + deadline: StatementDeadline::after(Duration::from_secs(5)), // Admitted at this request's own transport entry; this handler // has no session or peer information of its own. admission: crate::control::server::shared::ddl::user_dispatch::RequestAdmission::AlreadyAdmitted, @@ -116,7 +117,7 @@ pub async fn query_last_value( database_id, collection, plan, - timeout: Duration::from_secs(5), + deadline: StatementDeadline::after(Duration::from_secs(5)), // Admitted at this request's own transport entry; this handler // has no session or peer information of its own. admission: crate::control::server::shared::ddl::user_dispatch::RequestAdmission::AlreadyAdmitted, diff --git a/nodedb/src/control/server/shared/ddl/neutral/permission_tree.rs b/nodedb/src/control/server/shared/ddl/neutral/permission_tree.rs index c967830fd..2b9f7fa2e 100644 --- a/nodedb/src/control/server/shared/ddl/neutral/permission_tree.rs +++ b/nodedb/src/control/server/shared/ddl/neutral/permission_tree.rs @@ -150,7 +150,12 @@ async fn source_group_barrier(state: &SharedState, sources: &[TreeKey]) -> crate .await?; targets.push(nodedb_cluster::GroupCoverage { group_id, through }); } - crate::control::security::auth_lease::authorization_barrier(state, targets).await + crate::control::security::auth_lease::authorization_barrier( + state, + targets, + crate::control::server::shared::session::StatementDeadline::of_statement(state).instant(), + ) + .await } /// ALTER COLLECTION DROP PERMISSION_TREE diff --git a/nodedb/src/control/server/shared/ddl/neutral/version_history/checkpoint.rs b/nodedb/src/control/server/shared/ddl/neutral/version_history/checkpoint.rs index c0abbaf98..0144c5097 100644 --- a/nodedb/src/control/server/shared/ddl/neutral/version_history/checkpoint.rs +++ b/nodedb/src/control/server/shared/ddl/neutral/version_history/checkpoint.rs @@ -7,8 +7,6 @@ //! DROP CHECKPOINT 'launch-ready' ON documents WHERE id = 'doc-123'; //! ``` -use std::time::Duration; - use nodedb_sql::parser::preprocess::lex::find_ascii_case_insensitive; use crate::bridge::envelope::PhysicalPlan; @@ -16,8 +14,9 @@ use crate::control::security::catalog::types::{CheckpointDoc, CheckpointRecord}; use crate::control::security::identity::AuthenticatedIdentity; use crate::control::server::shared::ddl::sql_parse::parse_ident_token; use crate::control::server::shared::ddl::sync_dispatch::{ - SystemReason, SystemTask, dispatch_system, + SystemReason, SystemTask, dispatch_system_by, }; +use crate::control::server::shared::session::StatementDeadline; use crate::control::state::SharedState; use crate::types::DatabaseId; use nodedb_physical::physical_plan::CrdtOp; @@ -42,8 +41,7 @@ pub async fn create_checkpoint( let plan = PhysicalPlan::Crdt(CrdtOp::GetVersionVector { collection: nodedb_types::QualifiedCollection::new(database_id, &collection), }); - let timeout = Duration::from_secs(state.tuning.network.default_deadline_secs); - let vv_bytes = dispatch_system( + let vv_bytes = dispatch_system_by( state, SystemTask::new( SystemReason::CatalogMaintenance, @@ -51,7 +49,7 @@ pub async fn create_checkpoint( nodedb_types::CollectionKey::from_bare(database_id, &collection), plan, ), - timeout, + StatementDeadline::of_statement(state), ) .await .map_err(|e| DdlError::from_error_in_context("dispatch", &e))?; diff --git a/nodedb/src/control/server/shared/ddl/neutral/version_history/dispatch.rs b/nodedb/src/control/server/shared/ddl/neutral/version_history/dispatch.rs index fa6387830..f18d1c361 100644 --- a/nodedb/src/control/server/shared/ddl/neutral/version_history/dispatch.rs +++ b/nodedb/src/control/server/shared/ddl/neutral/version_history/dispatch.rs @@ -10,8 +10,6 @@ //! these, so they mint a capability instead: the plan that reaches storage is //! the plan authorization approved. -use std::time::Duration; - use nodedb_physical::physical_task::{PhysicalTask, PostSetOp}; use crate::bridge::envelope::PhysicalPlan; @@ -21,6 +19,7 @@ use crate::control::server::shared::clone_write::{ }; use crate::control::server::shared::ddl::sync_dispatch::dispatch_authorized; use crate::control::server::shared::response_payload::payload_or_typed_error; +use crate::control::server::shared::session::StatementDeadline; use crate::control::state::SharedState; use crate::types::DatabaseId; @@ -64,7 +63,7 @@ pub(super) async fn dispatch_authorized_read( .await .map_err(gate_error)?; - let timeout = Duration::from_secs(state.tuning.network.default_deadline_secs); + let deadline = StatementDeadline::of_statement(state); match outcome { // The clone-read hook merged the target with its source and produced the // answer itself; the payload is flattened the same way the Data Plane's @@ -76,7 +75,7 @@ pub(super) async fn dispatch_authorized_read( // No session reaches this handler: the read takes the strong // default (see `DmlTxnCtx::linearizable_reads`). let linearizable = true; - dispatch_authorized(state, checked, collection, timeout, linearizable) + dispatch_authorized(state, checked, collection, deadline, linearizable) .await .map_err(|e| DdlError::from_error_in_context("dispatch", &e)) } diff --git a/nodedb/src/control/server/shared/ddl/neutral/version_history/restore.rs b/nodedb/src/control/server/shared/ddl/neutral/version_history/restore.rs index ae7ea0dba..9517806c6 100644 --- a/nodedb/src/control/server/shared/ddl/neutral/version_history/restore.rs +++ b/nodedb/src/control/server/shared/ddl/neutral/version_history/restore.rs @@ -2,8 +2,6 @@ //! RESTORE collection SET VERSION = 'checkpoint' WHERE id = 'doc-id' -use std::time::Duration; - #[cfg(test)] use crate::bridge::envelope::PhysicalPlan; use crate::control::crdt_post_image_policy::ExternalCrdtPostImagePolicy; @@ -11,6 +9,7 @@ use crate::control::security::audit::ArcAuditEmitter; use crate::control::security::identity::{AuthenticatedIdentity, Permission}; use crate::control::server::shared::authorization::authorize_collection; use crate::control::server::shared::ddl::sql_parse::parse_ident_token; +use crate::control::server::shared::session::StatementDeadline; use crate::control::state::SharedState; use crate::types::DatabaseId; #[cfg(test)] @@ -72,7 +71,7 @@ pub async fn restore_version( .await .map_err(|e| DdlError::from_error_in_context("surrogate assign", &e))?; - let timeout = Duration::from_secs(state.tuning.network.default_deadline_secs); + let deadline = StatementDeadline::of_statement(state); // RLS write policies are stored keyed by `db_qualified(database_id, // collection)`, so the policy is handed that same key or it silently // misses a policy on a non-default database. The admission workflow @@ -98,7 +97,7 @@ pub async fn restore_version( target_version_json: &vv_json, surrogate, peer_id: identity.user_id, - timeout, + deadline, event_source: crate::event::EventSource::User, policy: &policy, }, @@ -169,14 +168,14 @@ async fn persist_restore_delta( expected_frontier_digest: None, }); - let outcome = crate::control::crdt_admission::dispatch_crdt_apply_admitted_outcome( + let outcome = crate::control::crdt_admission::apply::dispatch_crdt_apply_admitted_outcome( state, crate::control::crdt_admission::CrdtApplyAdmissionRequest { tenant_id, database_id, collection: qualified.as_str(), plan, - timeout: Duration::from_secs(state.tuning.network.default_deadline_secs), + deadline: StatementDeadline::of_statement(state), event_source: crate::event::EventSource::User, policy: &crate::control::crdt_admission::TrustedInternalCrdtPolicy, }, @@ -431,7 +430,7 @@ mod tests { target_version_json: "{}", surrogate: Surrogate(1), peer_id: 42, - timeout: Duration::from_secs(2), + deadline: StatementDeadline::after(std::time::Duration::from_secs(2)), event_source: crate::event::EventSource::User, policy: &crate::control::crdt_admission::TrustedInternalCrdtPolicy, }, diff --git a/nodedb/src/control/server/shared/ddl/sync_dispatch/dispatch.rs b/nodedb/src/control/server/shared/ddl/sync_dispatch/dispatch.rs index 09cf73d83..a321b3632 100644 --- a/nodedb/src/control/server/shared/ddl/sync_dispatch/dispatch.rs +++ b/nodedb/src/control/server/shared/ddl/sync_dispatch/dispatch.rs @@ -7,7 +7,7 @@ use std::time::{Duration, Instant}; use crate::bridge::envelope::{PhysicalPlan, Priority, Request, Response}; use crate::control::server::shared::clone_write::CloneCheckedTask; use crate::control::server::shared::response_payload::payload_or_typed_error; -use crate::control::server::shared::session::statement_deadline; +use crate::control::server::shared::session::{StatementDeadline, statement_deadline}; use crate::control::server::shared::write_admission::plan_is_write; use crate::control::state::SharedState; use crate::types::{DatabaseId, ReadConsistency, TenantId, TraceId, VShardId}; @@ -23,13 +23,25 @@ use super::system_task::SystemTask; /// own [`crate::bridge::envelope::ErrorCode`]. Each protocol renders that code /// through its SQLSTATE or status table. Only a refusal with no code is /// `crate::Error::Internal`. +/// +/// `timeout` bounds work the server starts on its own. Work inside a statement +/// passes the statement's deadline to [`dispatch_system_by`]. pub(crate) async fn dispatch_system( state: &SharedState, task: SystemTask<'_>, timeout: Duration, +) -> crate::Result> { + dispatch_system_by(state, task, StatementDeadline::after(timeout)).await +} + +/// [`dispatch_system`] bounded by `deadline`. +pub(crate) async fn dispatch_system_by( + state: &SharedState, + task: SystemTask<'_>, + deadline: StatementDeadline, ) -> crate::Result> { let event_source = task.reason.event_source(); - let resp = dispatch_system_response_with_source(state, task, timeout, event_source).await?; + let resp = dispatch_system_response_with_source(state, task, deadline, event_source).await?; // A system task never advances the tenant's observed write-HLC. Its // `SystemReason` states that no client asked for the work, and RESTORE's @@ -49,7 +61,7 @@ pub(crate) async fn dispatch_system( pub(crate) async fn dispatch_system_response_with_source( state: &SharedState, task: SystemTask<'_>, - timeout: Duration, + deadline: StatementDeadline, event_source: crate::event::EventSource, ) -> crate::Result { let vshard_id = task.collection.vshard(); @@ -65,7 +77,7 @@ pub(crate) async fn dispatch_system_response_with_source( database_id: task.collection.database_id(), vshard_id, plan: task.plan, - timeout, + deadline, event_source, }, ) @@ -94,7 +106,7 @@ pub(crate) async fn dispatch_authorized( state: &SharedState, checked: CloneCheckedTask, collection: &str, - timeout: Duration, + deadline: StatementDeadline, linearizable: bool, ) -> crate::Result> { // The write lease lives until the dispatch returns its outcome. @@ -143,7 +155,7 @@ pub(crate) async fn dispatch_authorized( database_id: task.database_id, vshard_id, plan, - timeout, + deadline, event_source: crate::event::EventSource::User, }, ) @@ -158,7 +170,8 @@ struct PlanDispatch { database_id: DatabaseId, vshard_id: VShardId, plan: PhysicalPlan, - timeout: Duration, + /// The caller's own bound, never past the running statement's deadline. + deadline: StatementDeadline, event_source: crate::event::EventSource, } @@ -169,17 +182,17 @@ async fn dispatch_plan(state: &SharedState, dispatch: PlanDispatch) -> crate::Re database_id, vshard_id, plan, - timeout, + deadline: cap, event_source, } = dispatch; let request_id = state.next_request_id(); // Whichever comes first: the running statement's deadline, or the caller's - // own timeout. A DDL statement a session bounded with `statement_timeout` + // own bound. A DDL statement a session bounded with `statement_timeout` // stops with the statement; a background caller with no statement scope - // keeps exactly the timeout it asked for. - let deadline = statement_deadline(state.tuning.network.default_deadline_secs) - .min(Instant::now() + timeout); + // keeps exactly the bound it asked for. + let deadline = + statement_deadline(state.tuning.network.default_deadline_secs).min(cap.instant()); let request = Request { request_id, diff --git a/nodedb/src/control/server/shared/ddl/sync_dispatch/mod.rs b/nodedb/src/control/server/shared/ddl/sync_dispatch/mod.rs index 34c852c8f..2ea05d459 100644 --- a/nodedb/src/control/server/shared/ddl/sync_dispatch/mod.rs +++ b/nodedb/src/control/server/shared/ddl/sync_dispatch/mod.rs @@ -11,6 +11,6 @@ mod dispatch; mod system_task; pub(crate) use dispatch::{ - dispatch_authorized, dispatch_system, dispatch_system_response_with_source, + dispatch_authorized, dispatch_system, dispatch_system_by, dispatch_system_response_with_source, }; pub(crate) use system_task::{SystemReason, SystemTask}; diff --git a/nodedb/src/control/server/shared/ddl/user_dispatch.rs b/nodedb/src/control/server/shared/ddl/user_dispatch.rs index a507c446d..3e9475439 100644 --- a/nodedb/src/control/server/shared/ddl/user_dispatch.rs +++ b/nodedb/src/control/server/shared/ddl/user_dispatch.rs @@ -8,8 +8,6 @@ //! this instead of the system door, which exists only for work no user asked //! for. -use std::time::Duration; - use crate::bridge::envelope::PhysicalPlan; use crate::control::security::audit::ArcAuditEmitter; use crate::control::security::identity::AuthenticatedIdentity; @@ -21,6 +19,7 @@ use crate::control::server::shared::metering::{ PlanMeteringInfo, meter_dispatch, operation_for_plan, }; use crate::control::server::shared::response_payload::payload_or_typed_error; +use crate::control::server::shared::session::StatementDeadline; use crate::control::state::SharedState; use crate::types::DatabaseId; use nodedb_physical::physical_task::{PhysicalTask, PostSetOp}; @@ -62,7 +61,7 @@ pub(crate) enum RequestAdmission<'a> { /// Parameters for [`dispatch_for_identity`]. /// /// Grouped into a struct rather than passed positionally because the -/// argument count (state, identity, database, collection, plan, timeout, +/// argument count (state, identity, database, collection, plan, deadline, /// admission) exceeds what a positional call stays readable at. pub(crate) struct DispatchRequest<'a> { pub state: &'a SharedState, @@ -70,7 +69,8 @@ pub(crate) struct DispatchRequest<'a> { pub database_id: DatabaseId, pub collection: &'a str, pub plan: PhysicalPlan, - pub timeout: Duration, + /// The bound of the dispatch, never past the running statement. + pub deadline: StatementDeadline, /// Whether this request still has to pass the admission gate, and — when /// it does — the real remote address it is admitted against. pub admission: RequestAdmission<'a>, @@ -90,7 +90,7 @@ pub(crate) async fn dispatch_for_identity(req: DispatchRequest<'_>) -> crate::Re database_id, collection, plan, - timeout, + deadline, admission, linearizable, } = req; @@ -115,7 +115,7 @@ pub(crate) async fn dispatch_for_identity(req: DispatchRequest<'_>) -> crate::Re // caller cannot tell the two apart by error shape. CloneCheckedOutcome::Handled(response) => payload_or_typed_error(response), CloneCheckedOutcome::Proceed(checked) => { - dispatch_authorized(state, checked, collection, timeout, linearizable).await + dispatch_authorized(state, checked, collection, deadline, linearizable).await } }; if result.is_ok() { @@ -534,7 +534,7 @@ mod tests { database_id: DatabaseId::DEFAULT, collection: "widgets", plan: trivial_kv_get_plan(), - timeout: Duration::from_secs(5), + deadline: StatementDeadline::after(Duration::from_secs(5)), admission: RequestAdmission::AlreadyAdmitted, linearizable: false, }) @@ -570,7 +570,7 @@ mod tests { database_id: DatabaseId::DEFAULT, collection: "widgets", plan: trivial_kv_get_plan(), - timeout: Duration::from_secs(5), + deadline: StatementDeadline::after(Duration::from_secs(5)), admission: RequestAdmission::NotYetAdmitted { peer_addr: "127.0.0.1:9", }, @@ -605,7 +605,7 @@ mod tests { database_id: DatabaseId::DEFAULT, collection: "widgets", plan: trivial_kv_get_plan(), - timeout: Duration::from_secs(5), + deadline: StatementDeadline::after(Duration::from_secs(5)), admission: RequestAdmission::AlreadyAdmitted, linearizable: false, }) @@ -635,7 +635,7 @@ mod tests { database_id: DatabaseId::DEFAULT, collection: "widgets", plan: trivial_kv_get_plan(), - timeout: Duration::from_secs(5), + deadline: StatementDeadline::after(Duration::from_secs(5)), admission: RequestAdmission::AlreadyAdmitted, linearizable: false, }) diff --git a/nodedb/src/control/server/shared/session/array_fanout_stage.rs b/nodedb/src/control/server/shared/session/array_fanout_stage.rs index 5e5d9bab6..7fdab3986 100644 --- a/nodedb/src/control/server/shared/session/array_fanout_stage.rs +++ b/nodedb/src/control/server/shared/session/array_fanout_stage.rs @@ -23,7 +23,6 @@ use futures::future::join_all; use crate::bridge::envelope::{PhysicalPlan, Response, Status}; use crate::control::gateway::RouteDecision; -use crate::control::gateway::retry::Attempt; use crate::control::security::request_scope::RequestAuthScope; use crate::control::server::shared::metering::{PlanMeteringInfo, meter_staged_write}; use crate::control::server::shared::quota_admission::admit_quota_for_dispatch; @@ -124,9 +123,8 @@ where // the INNER write, so the un-wrapped plan rides alongside. remote => { let version_plan = inner.clone(); - let attempt = - Attempt::sole(StatementDeadline::of_statement(state).tokio_instant()); - forward_to_leader(state, remote, wrap(inner), &version_plan, attempt).await + let deadline = StatementDeadline::of_statement(state).tokio_instant(); + forward_to_leader(state, remote, wrap(inner), &version_plan, deadline).await } } } diff --git a/nodedb/src/control/server/shared/session/commit/read_validation.rs b/nodedb/src/control/server/shared/session/commit/read_validation.rs index de0b3e946..008c395b9 100644 --- a/nodedb/src/control/server/shared/session/commit/read_validation.rs +++ b/nodedb/src/control/server/shared/session/commit/read_validation.rs @@ -294,7 +294,6 @@ async fn current_versions( database_id, trace_id: TraceId::ZERO, deadline_ms: StatementDeadline::of_statement(shared).hop_budget_ms(), - attempts_left: 1, version_set: &GatewayVersionSet::from_pairs(Vec::new()), txn_id: None, // The leader proves its leadership itself: it answers each diff --git a/nodedb/src/control/server/shared/session/commit/run.rs b/nodedb/src/control/server/shared/session/commit/run.rs index b0b861dc2..5470310cc 100644 --- a/nodedb/src/control/server/shared/session/commit/run.rs +++ b/nodedb/src/control/server/shared/session/commit/run.rs @@ -369,9 +369,9 @@ pub async fn run_commit( // longer reach the overlay) cannot un-commit it, so it is // surfaced at ERROR and the remaining vShards are still reaped // rather than aborting a committed transaction. `drop_txn_overlay` - // already retries a transient remote failure a bounded number of - // times internally (see `retry_not_leader`); a drop that still - // fails after that budget strands a bounded, invisible (the + // already retries a transient remote failure until its cleanup + // deadline (see `retry_not_leader`); a drop that still fails + // after that budget strands a bounded, invisible (the // `txn_id` is never reused) overlay on the unreachable former // leader, cleared on that node's restart and visible meanwhile // via `active_txn_overlays`. diff --git a/nodedb/src/control/server/shared/session/ddl_authorization.rs b/nodedb/src/control/server/shared/session/ddl_authorization.rs index 4be804b1f..f752cc6d9 100644 --- a/nodedb/src/control/server/shared/session/ddl_authorization.rs +++ b/nodedb/src/control/server/shared/session/ddl_authorization.rs @@ -41,6 +41,7 @@ pub(super) async fn barrier_at(state: &SharedState, log_index: u64) -> crate::Re group_id: METADATA_GROUP_ID, through: log_index, }], + super::StatementDeadline::of_statement(state).instant(), ) .await } diff --git a/nodedb/src/control/server/shared/session/deadline.rs b/nodedb/src/control/server/shared/session/deadline.rs index eedd1d3d9..56e290646 100644 --- a/nodedb/src/control/server/shared/session/deadline.rs +++ b/nodedb/src/control/server/shared/session/deadline.rs @@ -152,6 +152,11 @@ impl StatementDeadline { pub fn hop_budget_ms(self) -> u64 { hop_budget_ms(self.0) } + + /// Time left until the deadline, zero once it passed. + pub fn remaining(self) -> Duration { + self.0.saturating_duration_since(Instant::now()) + } } /// Install a fresh node-default deadline for cleanup work, such as an overlay diff --git a/nodedb/src/control/server/shared/session/leader_forward.rs b/nodedb/src/control/server/shared/session/leader_forward.rs index 848e7f30a..3a522e335 100644 --- a/nodedb/src/control/server/shared/session/leader_forward.rs +++ b/nodedb/src/control/server/shared/session/leader_forward.rs @@ -36,9 +36,9 @@ use crate::bridge::envelope::{Payload, PhysicalPlan, Response}; use crate::control::gateway::dispatcher::{DispatchRouteParams, dispatch_route}; use crate::control::gateway::live_leaders::resolve_live_decision; -use crate::control::gateway::retry::Attempt; use crate::control::gateway::{GatewayVersionSet, RouteDecision, TaskRoute}; use crate::control::server::graph_dispatch::cluster_resolve::gateway_shared; +use crate::control::server::shared::session::deadline::hop_budget_ms; use crate::control::server::shared::write_admission::bare_ok_response; use crate::control::state::SharedState; use crate::types::{DatabaseId, RequestId, TenantId, TraceId}; @@ -68,15 +68,14 @@ pub(crate) fn resolve_leader(task: &PhysicalTask, state: &SharedState) -> RouteD /// remote write does and is not spuriously rejected; a `DropTxnOverlay` touches /// no user collection, so its version set is empty. /// -/// `attempt` carries the deadline and the attempts left. A caller inside a -/// retry loop passes the loop's attempt. A one-shot caller passes -/// [`Attempt::sole`]. +/// `deadline` is the statement's deadline. The forward carries what is left +/// of it. pub(crate) async fn forward_to_leader( state: &SharedState, decision: RouteDecision, forward_task: PhysicalTask, version_plan: &PhysicalPlan, - attempt: Attempt, + deadline: tokio::time::Instant, ) -> crate::Result { // The remote dispatch primitive needs an owned `Arc`; upgrade // the gateway's back-reference (loud typed error if the gateway is not wired @@ -103,8 +102,7 @@ pub(crate) async fn forward_to_leader( tenant_id: forward_task.tenant_id, database_id: forward_task.database_id, trace_id: TraceId::ZERO, - deadline_ms: attempt.remaining_ms(), - attempts_left: attempt.attempts_left(), + deadline_ms: hop_budget_ms(deadline.into_std()), version_set: &version_set, // The leader's session-less staging handler reads THIS to key the // per-transaction overlay; forwarding it is the whole point. diff --git a/nodedb/src/control/server/shared/session/overlay_drop.rs b/nodedb/src/control/server/shared/session/overlay_drop.rs index 0b32ae45e..9427a77e6 100644 --- a/nodedb/src/control/server/shared/session/overlay_drop.rs +++ b/nodedb/src/control/server/shared/session/overlay_drop.rs @@ -7,7 +7,7 @@ use nodedb_physical::physical_plan::meta::MetaOp; use nodedb_physical::physical_task::{PhysicalTask, PostSetOp}; use crate::control::gateway::RouteDecision; -use crate::control::gateway::retry::retry_not_leader; +use crate::control::gateway::retry::{RetryPacing, retry_not_leader}; use crate::control::state::SharedState; use super::deadline::{self, StatementDeadline}; @@ -46,13 +46,17 @@ pub(super) async fn drop_txn_overlay( let routing_ref = shared.cluster_routing.as_deref(); // Reads the cleanup deadline installed above. let cleanup_deadline = StatementDeadline::of_statement(&shared).tokio_instant(); - retry_not_leader(routing_ref, cleanup_deadline, |attempt| { + let pacing = RetryPacing::from_tuning(&shared.tuning.cluster_transport); + retry_not_leader(routing_ref, cleanup_deadline, pacing, |attempt| { let task = task.clone(); let drop_plan = drop_plan.clone(); async move { match resolve_leader(&task, state) { RouteDecision::Local => dp.dispatch_no_wal(task).await, - remote => forward_to_leader(state, remote, task, &drop_plan, attempt).await, + remote => { + forward_to_leader(state, remote, task, &drop_plan, attempt.deadline) + .await + } } } }) diff --git a/nodedb/src/control/server/shared/session/staging_gate.rs b/nodedb/src/control/server/shared/session/staging_gate.rs index 3758aa460..70a8a0005 100644 --- a/nodedb/src/control/server/shared/session/staging_gate.rs +++ b/nodedb/src/control/server/shared/session/staging_gate.rs @@ -25,7 +25,6 @@ use std::future::Future; use crate::bridge::envelope::{ErrorCode, PhysicalPlan, Response, Status}; use crate::control::gateway::RouteDecision; -use crate::control::gateway::retry::Attempt; use crate::control::security::request_scope::RequestAuthScope; use crate::control::server::shared::metering::{PlanMeteringInfo, meter_staged_write}; use crate::control::server::shared::quota_admission::admit_quota_for_dispatch; @@ -378,7 +377,7 @@ where remote, stage_task, &task.plan, - Attempt::sole(StatementDeadline::of_statement(state).tokio_instant()), + StatementDeadline::of_statement(state).tokio_instant(), ) .await .map_err(StagingGateError::Dispatch)?, diff --git a/nodedb/src/control/server/sync/async_dispatch/delta/apply.rs b/nodedb/src/control/server/sync/async_dispatch/delta/apply.rs index aa88a8616..f39d15561 100644 --- a/nodedb/src/control/server/sync/async_dispatch/delta/apply.rs +++ b/nodedb/src/control/server/sync/async_dispatch/delta/apply.rs @@ -3,8 +3,6 @@ //! Dispatch a CRDT delta to the Data Plane and hand its outcome to the frame //! builder. -use std::time::Duration; - use tracing::warn; use nodedb_types::sync::wire::{EngineKind, SyncProvenance, stream_id_for}; @@ -12,6 +10,7 @@ use nodedb_types::sync::wire::{EngineKind, SyncProvenance, stream_id_for}; use crate::control::security::audit::ArcAuditEmitter; use crate::control::security::identity::AuthenticatedIdentity; use crate::control::server::shared::metering::{PlanMeteringInfo, meter_dispatch}; +use crate::control::server::shared::session::StatementDeadline; use crate::control::state::SharedState; use crate::types::DatabaseId; @@ -81,6 +80,8 @@ pub(crate) async fn apply_delta_and_finalize( epoch: session_epoch, peer_addr, } = session; + // Pinned once: preview, propose and apply all end at this instant. + let deadline = StatementDeadline::of_statement(shared); use crate::bridge::envelope::PhysicalPlan; use nodedb_physical::physical_plan::CrdtOp; @@ -354,7 +355,7 @@ pub(crate) async fn apply_delta_and_finalize( shared, &delta_msg.collection, authorized, - Duration::from_secs(10), + deadline, crate::event::EventSource::CrdtSync, &policy, ) diff --git a/nodedb/src/control/server/sync/async_dispatch/shape/snapshot.rs b/nodedb/src/control/server/sync/async_dispatch/shape/snapshot.rs index e47060626..d594d6b32 100644 --- a/nodedb/src/control/server/sync/async_dispatch/shape/snapshot.rs +++ b/nodedb/src/control/server/sync/async_dispatch/shape/snapshot.rs @@ -2,8 +2,6 @@ //! Shape snapshot production: plan construction, RLS, and predicate filtering. -use std::time::Duration; - use tracing::{info, warn}; use nodedb_types::sync::shape::{ShapeDefinition, ShapeType}; @@ -185,7 +183,9 @@ async fn document_snapshot(req: DocumentSnapshot<'_>) -> Option crate::Result> { let RecordOwner { tenant_id, @@ -63,7 +65,13 @@ pub(crate) async fn propose_sync_plan( ), }, )?; - propose_sync_write(state, entry.with_event_source(event_source), proposer).await + propose_sync_write( + state, + entry.with_event_source(event_source), + proposer, + deadline, + ) + .await } /// Propose a `ReplicatedEntry` through Raft and block until the entry is @@ -84,6 +92,7 @@ pub(crate) async fn propose_sync_write( state: &SharedState, mut entry: ReplicatedEntry, proposer: &Arc, + deadline: StatementDeadline, ) -> crate::Result> { if let Some(response) = crate::control::planner::calvin::sequence_replicated_edge_write(state, &entry).await? @@ -100,8 +109,7 @@ pub(crate) async fn propose_sync_write( let idempotency_key = entry.idempotency_key; let data = entry.encode()?; let vshard_id = entry.vshard_id; - let deadline = tokio::time::Instant::now() - + Duration::from_secs(state.tuning.network.default_deadline_secs); + let deadline = deadline.tokio_instant(); const BACKOFF_MS: [u64; 5] = [10, 25, 50, 100, 200]; let mut payload: Option> = None; diff --git a/nodedb/src/control/server/sync/raft_dispatch/response.rs b/nodedb/src/control/server/sync/raft_dispatch/response.rs index 8be27caf8..1bb90b5d7 100644 --- a/nodedb/src/control/server/sync/raft_dispatch/response.rs +++ b/nodedb/src/control/server/sync/raft_dispatch/response.rs @@ -8,6 +8,7 @@ use crate::bridge::envelope::{PhysicalPlan, Response, Status}; use crate::control::server::dispatch_utils::RecordOwner; use crate::control::server::shared::authorization::AuthorizedTask; +use crate::control::server::shared::session::StatementDeadline; use crate::control::state::SharedState; use crate::event::EventSource; use crate::types::{DatabaseId, Lsn, TenantId, VShardId}; @@ -58,7 +59,14 @@ async fn sync_write( event_source: EventSource, ) -> crate::Result> { reject_unadmitted_crdt_apply(plan)?; - propose_sync_plan(state, owner, plan, event_source).await + propose_sync_plan( + state, + owner, + plan, + event_source, + StatementDeadline::of_statement(state), + ) + .await } /// The applied payload wrapped in `Status::Ok`. A non-`Ok` status means a diff --git a/nodedb/src/control/server/sync/raft_dispatch/write.rs b/nodedb/src/control/server/sync/raft_dispatch/write.rs index 000b91b5a..f410ca4da 100644 --- a/nodedb/src/control/server/sync/raft_dispatch/write.rs +++ b/nodedb/src/control/server/sync/raft_dispatch/write.rs @@ -2,11 +2,10 @@ //! Sync dispatch that returns raw payload bytes, used by the CRDT delta path. -use std::time::Duration; - use crate::bridge::envelope::PhysicalPlan; use crate::control::server::dispatch_utils::RecordOwner; use crate::control::server::shared::authorization::AuthorizedTask; +use crate::control::server::shared::session::StatementDeadline; use crate::control::state::SharedState; use crate::event::EventSource; @@ -21,7 +20,7 @@ pub async fn dispatch_sync_bytes( state: &SharedState, collection: &str, authorized: AuthorizedTask, - timeout: Duration, + deadline: StatementDeadline, event_source: EventSource, policy: &dyn crate::control::crdt_admission::CrdtPostImagePolicy, ) -> crate::Result { @@ -39,7 +38,7 @@ pub async fn dispatch_sync_bytes( crate::control::crdt_admission::AuthorizedCrdtApplyAdmissionRequest { authorized, collection, - timeout, + deadline, event_source, policy, }, @@ -51,7 +50,7 @@ pub async fn dispatch_sync_bytes( }); } // The Raft entry's apply owns durability. - dispatch_write_replicated(state, collection, authorized, event_source) + dispatch_write_replicated(state, collection, authorized, deadline, event_source) .await .map(SyncDispatchOutcome::untrimmed) } @@ -63,6 +62,7 @@ pub(crate) async fn dispatch_write_replicated( state: &SharedState, collection: &str, authorized: AuthorizedTask, + deadline: StatementDeadline, event_source: EventSource, ) -> crate::Result> { let task = authorized.into_physical_task(); @@ -91,7 +91,7 @@ pub(crate) async fn dispatch_write_replicated( ) .await?; // The apply's write funnel records the write's mark on every replica. - propose_sync_plan(state, owner, &plan, event_source).await + propose_sync_plan(state, owner, &plan, event_source, deadline).await } #[cfg(test)] @@ -100,6 +100,7 @@ mod tests { COLLECTION, applying_proposer, authorized_write, fixture, }; use super::dispatch_write_replicated; + use crate::control::server::shared::session::StatementDeadline; use crate::event::EventSource; /// A proposed write answers with the applied entry's payload. @@ -113,10 +114,15 @@ mod tests { .expect("install proposer"); let authorized = authorized_write(&state); - let payload = - dispatch_write_replicated(&state, COLLECTION, authorized, EventSource::CrdtSync) - .await - .expect("the proposal applies"); + let payload = dispatch_write_replicated( + &state, + COLLECTION, + authorized, + StatementDeadline::of_statement(&state), + EventSource::CrdtSync, + ) + .await + .expect("the proposal applies"); assert_eq!(payload, b"applied".to_vec()); } @@ -128,8 +134,14 @@ mod tests { let (state, _side, _directory) = fixture(); let authorized = authorized_write(&state); - let result = - dispatch_write_replicated(&state, COLLECTION, authorized, EventSource::CrdtSync).await; + let result = dispatch_write_replicated( + &state, + COLLECTION, + authorized, + StatementDeadline::of_statement(&state), + EventSource::CrdtSync, + ) + .await; assert!( matches!(result, Err(crate::Error::Internal { .. })), diff --git a/nodedb/src/control/server/sync/timeseries_handler.rs b/nodedb/src/control/server/sync/timeseries_handler.rs index 5f0feccdc..58db02a8e 100644 --- a/nodedb/src/control/server/sync/timeseries_handler.rs +++ b/nodedb/src/control/server/sync/timeseries_handler.rs @@ -138,6 +138,7 @@ impl<'a> TimeseriesDispatcher for SharedStateTimeseriesDispatcher<'a> { self.shared, &collection, authorized, + crate::control::server::shared::session::StatementDeadline::of_statement(self.shared), crate::event::EventSource::CrdtSync, ) .await?; diff --git a/nodedb/tests/inproc/cases/system_task_call_sites.rs b/nodedb/tests/inproc/cases/system_task_call_sites.rs index d7ff0042a..7e21a4895 100644 --- a/nodedb/tests/inproc/cases/system_task_call_sites.rs +++ b/nodedb/tests/inproc/cases/system_task_call_sites.rs @@ -59,7 +59,7 @@ const ALLOWED: &[&str] = &[ // Event Plane rules dispatched back through the Control Plane. "event/alert/executor.rs", // Legs of a request whose capability was consumed at the entry point. - "control/crdt_admission.rs", + "control/crdt_admission/legs.rs", // `sync/raft_dispatch/write.rs` is absent: a sync write proposes its // plan through Raft, and the entry's apply dispatches it on each replica. ];