diff --git a/.config/nextest.toml b/.config/nextest.toml index 26689fbb1..a24eff7de 100644 --- a/.config/nextest.toml +++ b/.config/nextest.toml @@ -212,18 +212,8 @@ server-process-serial = { max-threads = 1 } retries = { backoff = "fixed", count = 1, delay = "2s" } fail-fast = false -# One cluster test is excluded from the CI profile ONLY. It still runs under the -# default profile, which is what a local `cargo nextest run` uses. -# -# `learner_caught_up_via_real_install_snapshot` fails on CI runners about once in -# six runs. A newly elected sequencer leader whose log was compacted refuses to -# propose ("sequencer log was truncated below every retained epoch"), so the -# sequencer stalls. The joining node then never catches up on the sequencer -# group, and its authorization lease is withheld. Re-enable it once a leader can -# seed its epoch from the installed sequencer snapshot. NEVER widen a timeout. -default-filter = """ -not test(learner_caught_up_via_real_install_snapshot) -""" +# CI runs every test, including the two binaries the default profile excludes. +default-filter = "all()" # CI-only cluster headroom. On CI runners (~2× slower per-core than a dev # workstation) the cluster tests' in-test `wait_for` budgets are proportionally diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 75f4aa99d..652651b20 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -153,7 +153,7 @@ NodeDB routes all DML through `EngineRules` in `nodedb-sql`. This is the single **Integration tests** — place in `nodedb/tests/` or the relevant crate's `tests/` directory. Test public API only. Use `tests/common/mod.rs` for shared helpers (not `tests/common.rs`). -**Failpoint tests** — for fault injection and panic recovery, use the `failpoints` feature flag and `FailGuard` from `nodedb::fail_point`. Compile with `--features failpoints`. +**Failpoint tests** — for fault injection and panic recovery, use the `failpoints` feature flag and `FailGuard` from `nodedb::fail_point`. Compile with `--features failpoints`. An in-process cluster shares one fail-point registry across its nodes. Arm a point on one node with `FailGuard::for_node(node_id, name, action)`, never by putting the node in the name. A spawned server takes `NODEDB_FAILPOINTS='name=action'` for every node, or `name@node=action` for node `N` only. **Cluster tests** — 3-node integration tests live in `nodedb/tests/` and are annotated with the `cluster` nextest group. They serialize automatically via `.config/nextest.toml` — don't run them with `cargo test`. diff --git a/Cargo.lock b/Cargo.lock index 55f8abdb3..1fa939bb9 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -4594,6 +4594,7 @@ name = "nodedb-raft" version = "0.5.0" dependencies = [ "crc32c", + "nodedb-types", "rand 0.10.2", "rkyv", "serde", diff --git a/docs/getting-started.md b/docs/getting-started.md index 2ef35d74d..c8ffa0233 100644 --- a/docs/getting-started.md +++ b/docs/getting-started.md @@ -589,6 +589,48 @@ a crashed node's descriptor leases block DDL until they expire. All three must be positive. `scope_expiry_interval_secs` has a floor of `10`. Below that the sweep costs more than the resolution it buys. +**Calvin scheduler settings** (config file only): + +| Config field | Default | +| ------------------------------------------------ | ------- | +| `tuning.calvin.channel_capacity` | `512` | +| `tuning.calvin.txn_deadline_multiplier` | `3` | +| `tuning.calvin.dependent_read_passive_timeout_ms` | `5000` | +| `tuning.calvin.verdict_stall_warn_ms` | `5000` | +| `tuning.calvin.max_inflight_backlog` | `1024` | +| `tuning.calvin.catch_up_window` | `512` | +| `tuning.calvin.restage_attempts` | `5` | +| `tuning.calvin.restage_backoff_ms` | `100` | + +Each hosted vShard runs one Calvin scheduler with these settings. + +- `channel_capacity` sizes each scheduler's bounded input and completion + channels. +- `txn_deadline_multiplier` sets a transaction deadline in sequencer epochs. +- `dependent_read_passive_timeout_ms` bounds the wait of a dependent-read + barrier for passive read results. Past it, the vShard's data-group leader + proposes a timeout entry, and every replica aborts the barrier alike. +- `verdict_stall_warn_ms` spaces the stall warnings of a transaction waiting + on its global verdict. The transaction keeps waiting and never aborts. +- `max_inflight_backlog` is the in-flight backlog at which a scheduler stops + taking new input. It also bounds the dependent-read barrier entries a + vShard holds in memory for transactions its scheduler has not taken yet. + Entries past it wait in the system catalog alone. +- `catch_up_window` is the most sequencer log entries one catch-up pass + replays. +- `restage_attempts` is the most times a data-group leader stages a + committed transaction again after its own stage failed. The scheduler + halts once they run out. +- `restage_backoff_ms` is the wait before the first restage. Each later + restage waits twice as long, at most 60 seconds. + +Every value must be positive. `verdict_stall_warn_ms` has a floor of `4`. + +After a restart, a scheduler re-applies epochs up to the highest epoch it +applied before. The client gateway opens only after every scheduler on the +node reaches that epoch. The wait shares the data-group recovery bound, and +startup fails when the bound expires. + **Observability settings:** | Config field | Environment variable | Default | diff --git a/nodedb-array/src/lib.rs b/nodedb-array/src/lib.rs index 0e49d0148..4a4d0daaa 100644 --- a/nodedb-array/src/lib.rs +++ b/nodedb-array/src/lib.rs @@ -36,7 +36,8 @@ pub use sync::{ SnapshotHeader, SnapshotSink, TileSnapshot, }; pub use tile::{ - AttrStats, DenseTile, SparseTile, TileMBR, tile_id_for_cell, tile_indices_for_cell, + AttrStats, DenseTile, SparseTile, TileMBR, cell_tile_prefix, tile_id_for_cell, + tile_indices_for_cell, tile_prefix, tiles_per_dim, }; pub use types::{ArrayId, CellValue, Coord, Domain, TileId}; pub use wal::ArrayWalRecord; diff --git a/nodedb-array/src/tile/layout.rs b/nodedb-array/src/tile/layout.rs index 6c1ff722e..cb61577d3 100644 --- a/nodedb-array/src/tile/layout.rs +++ b/nodedb-array/src/tile/layout.rs @@ -8,10 +8,10 @@ //! same tile share a [`TileId::hilbert_prefix`]. use crate::coord::encode::encode_tile_prefix_with_order; -use crate::coord::normalize::bits_per_dim; +use crate::coord::normalize::{bits_per_dim, normalize_coord}; use crate::error::{ArrayError, ArrayResult}; use crate::schema::ArraySchema; -use crate::schema::dim_spec::DimType; +use crate::schema::dim_spec::{DimSpec, DimType}; use crate::types::TileId; use crate::types::coord::value::CoordValue; use crate::types::domain::DomainBound; @@ -76,11 +76,93 @@ pub fn tile_id_for_cell( system_from_ms: i64, ) -> ArrayResult { let tile_indices = tile_indices_for_cell(schema, coord)?; - let bits = bits_per_dim(schema.arity()); - let prefix = encode_tile_prefix_with_order(schema, &tile_indices, bits)?; + let prefix = tile_prefix(schema, &tile_indices)?; Ok(TileId::new(prefix, system_from_ms)) } +/// The space-filling-curve key of the tile at `tile_indices`. +/// +/// The memtable, the segment index, the transaction overlay and retention +/// group cells by this key, and a cluster routes a tile by its top bits. +/// Every cell inside a group stays addressed by its full coordinate: a +/// lookup matches the coordinate within the group +/// (`TileBuffer::get_cell_bytes`, `extract_cell_bytes`), and a tile's MBR +/// is built from the cells it holds. Two tiles that share a key therefore +/// share one group and lose no cell. +/// +/// A dimension whose tile count fits its share of the key bits scales each +/// index over that count to the full bit width. Its tiles keep distinct, +/// ordered keys that spread over the top bits. A dimension with more tiles +/// than its bits can number, such as one with the default full-range +/// domain, keeps the low bits of the index instead. Nearby tiles then keep +/// distinct keys, and tiles `2^bits` apart share one. +pub fn tile_prefix(schema: &ArraySchema, tile_indices: &[u64]) -> ArrayResult { + if tile_indices.len() != schema.arity() { + return Err(ArrayError::CoordArityMismatch { + array: schema.name.clone(), + expected: schema.arity(), + got: tile_indices.len(), + }); + } + let bits = bits_per_dim(schema.arity()); + let scaled: Vec = tile_indices + .iter() + .zip(schema.dims.iter().zip(schema.tile_extents.iter())) + .map(|(index, (dim, extent))| scale_tile_index(*index, tiles_per_dim(dim, *extent), bits)) + .collect(); + encode_tile_prefix_with_order(schema, &scaled, bits) +} + +/// The key of the tile that holds the cell at `coord`: the key every +/// cluster path routes the cell by. Fails when `coord` lies outside the +/// schema's domain. +pub fn cell_tile_prefix(schema: &ArraySchema, coord: &[CoordValue]) -> ArrayResult { + normalize_coord(schema, coord, bits_per_dim(schema.arity()))?; + tile_prefix(schema, &tile_indices_for_cell(schema, coord)?) +} + +/// The number of tiles along `dim` for tile extent `extent`. Mirrors how +/// [`tile_indices_for_cell`] numbers them: a cell in the domain has a tile +/// index below this count. +pub fn tiles_per_dim(dim: &DimSpec, extent: u64) -> u128 { + let extent = u128::from(extent.max(1)); + match (dim.dtype, &dim.domain.lo, &dim.domain.hi) { + (DimType::Int64, DomainBound::Int64(lo), DomainBound::Int64(hi)) + | (DimType::TimestampMs, DomainBound::TimestampMs(lo), DomainBound::TimestampMs(hi)) => { + let span = ((*hi as i128) - (*lo as i128)).max(0) as u128; + span / extent + 1 + } + (DimType::Float64, DomainBound::Float64(lo), DomainBound::Float64(hi)) => { + if hi.is_finite() && lo.is_finite() && hi > lo { + u128::from((hi - lo) as u64) / extent + 1 + } else { + 1 + } + } + (DimType::String, DomainBound::String(_), DomainBound::String(_)) => extent, + _ => 1, + } +} + +/// Map tile index `index` of `tiles` tiles to `[0, 2^bits)`. +/// +/// With `tiles <= 2^bits` the index is scaled over the tile count, so +/// distinct indices keep distinct, ordered values. An index past the last +/// tile, a cell above the domain, takes the top value. With more tiles the +/// index keeps its low `bits` bits. +fn scale_tile_index(index: u64, tiles: u128, bits: u32) -> u64 { + let limit: u128 = 1u128 << bits.min(64); + let max = limit - 1; + if tiles > limit { + return u64::try_from(u128::from(index) & max).unwrap_or(u64::MAX); + } + if tiles <= 1 { + return if index == 0 { 0 } else { max as u64 }; + } + let scaled = (u128::from(index) * max / (tiles - 1)).min(max); + u64::try_from(scaled).unwrap_or(u64::MAX) +} + fn out_of_domain(schema: &ArraySchema, dim: &str, detail: &str) -> ArrayResult { Err(ArrayError::CoordOutOfDomain { array: schema.name.clone(), @@ -138,6 +220,99 @@ mod tests { assert_ne!(t1.hilbert_prefix, t2.hilbert_prefix); } + #[test] + fn tile_keys_spread_over_the_top_bits() { + let s = schema(); + let tops: std::collections::BTreeSet = (0..10) + .flat_map(|x| (0..10).map(move |y| (x * 10, y * 10))) + .map(|(x, y)| { + tile_id_for_cell(&s, &[CoordValue::Int64(x), CoordValue::Int64(y)], 0) + .unwrap() + .hilbert_prefix + >> 56 + }) + .collect(); + assert!( + tops.len() > 1, + "the tiles of one array route to more than one bucket: {tops:?}" + ); + } + + #[test] + fn every_tile_keeps_a_distinct_key() { + let s = schema(); + let keys: std::collections::BTreeSet = (0..10) + .flat_map(|x| (0..10).map(move |y| (x * 10, y * 10))) + .map(|(x, y)| { + tile_id_for_cell(&s, &[CoordValue::Int64(x), CoordValue::Int64(y)], 0) + .unwrap() + .hilbert_prefix + }) + .collect(); + assert_eq!(keys.len(), 100); + } + + #[test] + fn a_cell_routes_by_its_tile_key() { + let s = schema(); + let coord = [CoordValue::Int64(37), CoordValue::Int64(81)]; + assert_eq!( + cell_tile_prefix(&s, &coord).unwrap(), + tile_id_for_cell(&s, &coord, 0).unwrap().hilbert_prefix + ); + assert_eq!( + cell_tile_prefix(&s, &[CoordValue::Int64(30), CoordValue::Int64(89)]).unwrap(), + cell_tile_prefix(&s, &coord).unwrap(), + "cells of one tile route together" + ); + } + + #[test] + fn a_cell_outside_the_domain_has_no_route() { + let s = schema(); + assert!(cell_tile_prefix(&s, &[CoordValue::Int64(100), CoordValue::Int64(0)]).is_err()); + } + + #[test] + fn a_cell_above_the_domain_takes_the_last_key() { + let s = schema(); + let above = tile_id_for_cell(&s, &[CoordValue::Int64(150), CoordValue::Int64(0)], 0) + .unwrap() + .hilbert_prefix; + let last = tile_id_for_cell(&s, &[CoordValue::Int64(99), CoordValue::Int64(0)], 0) + .unwrap() + .hilbert_prefix; + assert_eq!(above, last); + } + + #[test] + fn nearby_tiles_of_a_full_range_dimension_keep_distinct_keys() { + let s = ArraySchemaBuilder::new("open") + .dim(DimSpec::new( + "x", + DimType::Int64, + Domain::new(DomainBound::Int64(i64::MIN), DomainBound::Int64(i64::MAX)), + )) + .dim(DimSpec::new( + "y", + DimType::Int64, + Domain::new(DomainBound::Int64(i64::MIN), DomainBound::Int64(i64::MAX)), + )) + .attr(AttrSpec::new("v", AttrType::Int64, false)) + .tile_extents(vec![10, 10]) + .build() + .unwrap(); + let keys: std::collections::BTreeSet = (0..10) + .flat_map(|x| (0..10).map(move |y| (x * 10, y * 10))) + .map(|(x, y)| { + tile_id_for_cell(&s, &[CoordValue::Int64(x), CoordValue::Int64(y)], 0) + .unwrap() + .hilbert_prefix + }) + .collect(); + assert_eq!(keys.len(), 100, "a schema with any domain builds"); + } + #[test] fn system_from_ms_propagates() { let s = schema(); diff --git a/nodedb-array/src/tile/mod.rs b/nodedb-array/src/tile/mod.rs index 9ef2272fc..f2a285abf 100644 --- a/nodedb-array/src/tile/mod.rs +++ b/nodedb-array/src/tile/mod.rs @@ -11,6 +11,8 @@ pub use cell_payload::{ is_cell_gdpr_erasure, is_cell_sentinel, is_cell_tombstone, }; pub use dense_tile::DenseTile; -pub use layout::{tile_id_for_cell, tile_indices_for_cell}; +pub use layout::{ + cell_tile_prefix, tile_id_for_cell, tile_indices_for_cell, tile_prefix, tiles_per_dim, +}; pub use mbr::{AttrStats, TileMBR}; pub use sparse_tile::SparseTile; diff --git a/nodedb-cluster-tests/Cargo.toml b/nodedb-cluster-tests/Cargo.toml index f36ab2553..bf5bc827f 100644 --- a/nodedb-cluster-tests/Cargo.toml +++ b/nodedb-cluster-tests/Cargo.toml @@ -12,7 +12,11 @@ path = "src/lib.rs" [features] # Arms the in-process fail points the cluster tests park writes at. -failpoints = ["nodedb/failpoints", "nodedb-types/failpoints"] +failpoints = [ + "nodedb/failpoints", + "nodedb-types/failpoints", + "nodedb-test-support/failpoints", +] [dependencies] diff --git a/nodedb-cluster-tests/tests/cluster_suite/cases/calvin_e2e_ollp.rs b/nodedb-cluster-tests/tests/cluster_suite/cases/calvin_e2e_ollp.rs index f71e163d5..216ba73c8 100644 --- a/nodedb-cluster-tests/tests/cluster_suite/cases/calvin_e2e_ollp.rs +++ b/nodedb-cluster-tests/tests/cluster_suite/cases/calvin_e2e_ollp.rs @@ -13,7 +13,7 @@ //! surrogates currently matching the predicate ("predicted surrogates"). //! 2. The TxClass is built with the predicted surrogates for the OLLP //! collection and static surrogates for any additional static-key writes -//! in the same transaction (via `build_dependent_tx_class`), then submitted +//! in the same transaction (via `build_predicted_tx_class`), then submitted //! to the sequencer inbox. //! 3. The sequencer admits the txn and fans it out to all participant vshards. //! 4. The Data Plane executor verifies the predicted set against the actual @@ -71,7 +71,7 @@ fn two_distinct_vshard_collections() -> (String, String) { /// - An OLLP BulkUpdate on `col_ollp` with `predicted_surrogates` as the /// write set (since the exact set is not known statically). /// -/// This mirrors what `build_dependent_tx_class` produces on the Control Plane +/// This mirrors what `build_predicted_tx_class` produces on the Control Plane /// for a cross-shard transaction that contains a value-dependent predicate write. fn make_ollp_tx_class( col_static: &str, @@ -94,7 +94,7 @@ fn make_ollp_tx_class( write_set, // Empty plan bytes — this test exercises the sequencer admission and // fan-out path only. The actual BulkUpdate plan bytes are embedded by - // `build_dependent_tx_class` in production. + // `build_predicted_tx_class` in production. vec![], TenantId::new(1), None, diff --git a/nodedb-cluster-tests/tests/common_suite/cases/backup_schedule_failover/lagging_catalog.rs b/nodedb-cluster-tests/tests/common_suite/cases/backup_schedule_failover/lagging_catalog.rs index 6f5a71565..bc69e9048 100644 --- a/nodedb-cluster-tests/tests/common_suite/cases/backup_schedule_failover/lagging_catalog.rs +++ b/nodedb-cluster-tests/tests/common_suite/cases/backup_schedule_failover/lagging_catalog.rs @@ -4,8 +4,8 @@ //! instead of rerunning the due minute. use nodedb::control::backup::schedule::envelope_name; -use nodedb::control::cluster::metadata_applier::backup_mark_fail_point; -use nodedb_types::fail_point::FailGuard; +use nodedb::control::cluster::metadata_applier::BACKUP_MARK_FAIL_POINT; +use nodedb_test_support::fail_point::{FailAction, FailGuard}; use super::fixture::{DATABASE, Fixture}; @@ -26,9 +26,10 @@ async fn a_new_leader_whose_catalog_lags_never_reruns_the_due_minute() { .iter() .filter(|node| node.node_id != leader) .map(|node| { - FailGuard::fail( - &backup_mark_fail_point(node.node_id), - "held back by the test", + FailGuard::for_node( + node.node_id, + BACKUP_MARK_FAIL_POINT, + FailAction::Fail("held back by the test".to_string()), ) }) .collect(); diff --git a/nodedb-cluster-tests/tests/common_suite/cases/calvin_backup_cut_order.rs b/nodedb-cluster-tests/tests/common_suite/cases/calvin_backup_cut_order.rs new file mode 100644 index 000000000..78831172e --- /dev/null +++ b/nodedb-cluster-tests/tests/common_suite/cases/calvin_backup_cut_order.rs @@ -0,0 +1,307 @@ +// SPDX-License-Identifier: BUSL-1.1 + +//! A database backup taken while cross-shard Calvin transactions commit +//! restores whole transactions: no transaction is in the backup on one data +//! group only. +//! +//! 1. A 3-node cluster with one backup root holds `accts`, whose `balance` is +//! the materialized sum of `entries.amount`, and `entries`, in distinct +//! data groups. Every account starts at 100. +//! 2. The fail gate `calvin::before_redo_propose::` holds the +//! entries slice of every transaction after its resolve, on the entries +//! group's leader. Transaction `held` adds 25 to `acc-0`: its balance +//! slice installs, its entries slice waits. +//! 3. Writers commit more cross-shard transactions on the other accounts. +//! `BACKUP DATABASE default` starts meanwhile. Its cut's marker follows +//! `held` in the sequencer log, so the entries group's barrier must follow +//! `held`'s entries redo, which is still unproposed. The backup stays +//! unfinished. +//! 4. The test releases the gate. The held redo installs, the barriers +//! follow, and the writers keep committing while the backup finishes. +//! 5. A fresh cluster restores the backup. `held`'s entry is in it, and every +//! account's balance is 100 plus the amounts of its restored entries. +//! +//! Requires `--features failpoints`. + +#![cfg(feature = "failpoints")] + +use std::sync::Arc; +use std::sync::atomic::{AtomicBool, Ordering}; +use std::time::Duration; + +use nodedb_test_support::fail_point::{FailAction, FailGuard}; + +use super::calvin_multishard_fixture::Fixture; +use super::calvin_replica_content::{run_retrying, strict_session, sum_ddl, vshard_of}; +use crate::common::cluster_harness::shared_steps::{db_detail, name_where}; +use crate::common::cluster_harness::{TestCluster, wait_for, wait_for_async}; + +/// Every account, each starting at [`BASE`]. +const ACCOUNTS: [&str; 4] = ["acc-0", "acc-1", "acc-2", "acc-3"]; + +/// The balance every account starts at. +const BASE: i64 = 100; + +/// The amount the held transaction adds to `acc-0`. +const HELD_AMOUNT: i64 = 25; + +/// How long each step gets. +const STEP_DEADLINE: Duration = Duration::from_secs(30); + +/// Pause between two polls of a step. +const STEP_POLL: Duration = Duration::from_millis(20); + +/// How long the backup must stay unfinished while the held redo waits. +const PARKED_FOR: Duration = Duration::from_millis(1500); + +/// Most transactions one writer commits. +const MAX_WRITES: u32 = 40; + +/// The backup object, under the shared backup root. +const OBJECT: &str = "calvin_cut/default.ndbb"; + +/// Every row `sql` returns through `client`, each column as text. +async fn rows(client: &tokio_postgres::Client, sql: &str) -> Vec> { + client + .simple_query(sql) + .await + .unwrap_or_else(|e| panic!("`{sql}`: {}", db_detail(&e))) + .into_iter() + .filter_map(|msg| match msg { + tokio_postgres::SimpleQueryMessage::Row(row) => Some( + (0..row.len()) + .map(|idx| row.get(idx).unwrap_or_default().to_owned()) + .collect(), + ), + _ => None, + }) + .collect() +} + +/// Commit cross-shard transactions that add to `account`, one at a time, +/// until `stop` or [`MAX_WRITES`]. Returns how many committed. +async fn write_until( + client: tokio_postgres::Client, + entries: String, + account: &'static str, + stop: Arc, +) -> u32 { + let mut written = 0; + while !stop.load(Ordering::Relaxed) && written < MAX_WRITES { + let sql = format!( + "BEGIN; \ + INSERT INTO {entries} (id, account_id, amount) \ + VALUES ('{account}-w{written}', '{account}', '{}'); \ + COMMIT", + written % 7 + 1 + ); + run_retrying(&client, "a concurrent Calvin commit", &sql).await; + written += 1; + } + written +} + +/// Run `sql` on node 0 of `cluster`. Panics with the server's error. +async fn exec(cluster: &TestCluster, sql: &str) { + cluster.nodes[0] + .client + .simple_query(sql) + .await + .unwrap_or_else(|e| panic!("`{sql}`: {}", db_detail(&e))); +} + +/// Read `accts` and `entries` on node 1 of `cluster`, and check that each +/// account's balance is [`BASE`] plus the amounts of its entries, and that +/// the held transaction's entry is there. +async fn assert_whole_transactions(cluster: &TestCluster, accts: &str, entries: &str) { + let client = &cluster.nodes[1].client; + let balances = rows(client, &format!("SELECT id, balance FROM {accts}")).await; + let restored = rows( + client, + &format!("SELECT id, account_id, amount FROM {entries}"), + ) + .await; + assert!( + restored.iter().any(|row| row[0] == "held"), + "the held transaction was sequenced before the backup's cut, so its entry is in \ + the backup; restored entries: {restored:?}" + ); + assert_eq!( + balances.len(), + ACCOUNTS.len(), + "every account is restored: {balances:?}" + ); + for row in &balances { + let account = &row[0]; + let balance: i64 = row[1] + .parse() + .unwrap_or_else(|e| panic!("{account}'s balance '{}': {e}", row[1])); + let added: i64 = restored + .iter() + .filter(|entry| entry[1] == *account) + .map(|entry| { + entry[2] + .parse::() + .unwrap_or_else(|e| panic!("entry {}'s amount '{}': {e}", entry[0], entry[2])) + }) + .sum(); + assert_eq!( + balance, + BASE + added, + "{account}: a transaction is in the backup on one data group only. Balance \ + {balance}, base {BASE} plus restored entries {added}" + ); + } +} + +#[tokio::test(flavor = "multi_thread", worker_threads = 8)] +async fn a_backup_during_calvin_commits_restores_whole_transactions() { + let root_dir = tempfile::tempdir().expect("backup root"); + let root = root_dir + .path() + .canonicalize() + .expect("canonical backup root"); + let uri = format!("file://{}/{OBJECT}", root.display()); + let source = TestCluster::spawn_three_with_backup_root(root.clone()) + .await + .expect("source cluster"); + let fx = Fixture::from_cluster(source).await; + let probe = &fx.cluster.nodes[0]; + let entries = name_where("cut_order_entries", |name| { + probe.group_id_for_collection(name).is_some() + }); + let group = probe + .group_id_for_collection(&entries) + .expect("the entries vShard maps to a group"); + let accts = name_where("cut_order_accts", |name| { + vshard_of(name) != vshard_of(&entries) + && probe + .group_id_for_collection(name) + .is_some_and(|other| other != group) + }); + let [accts_ddl, entries_ddl, sum] = sum_ddl(&entries, &accts); + fx.create(&[accts_ddl, entries_ddl]).await; + fx.cluster + .exec_ddl_on_any_leader(&sum) + .await + .unwrap_or_else(|e| panic!("{sum}: {e}")); + for collection in [&accts, &entries] { + fx.wait_group_mounted(collection).await; + } + for account in ACCOUNTS { + run_retrying( + &fx.cluster.nodes[0].client, + "seed an account", + &format!( + "INSERT INTO {accts} (id, owner, balance) VALUES ('{account}', 'seed', '{BASE}')" + ), + ) + .await; + } + fx.converge().await; + + let gate_dir = tempfile::tempdir().expect("gate directory"); + let release = gate_dir.path().join("propose"); + let parked = gate_dir.path().join("propose.parked"); + let gate = FailGuard::install( + &format!("calvin::before_redo_propose::{entries}"), + FailAction::WaitForFile(release.clone()), + ); + + // The held transaction: its balance slice installs, its entries slice + // waits at the gate. + let held_client = strict_session(fx.coordinator()).await; + let held_sql = format!( + "BEGIN; \ + INSERT INTO {entries} (id, account_id, amount) VALUES ('held', 'acc-0', '{HELD_AMOUNT}'); \ + COMMIT" + ); + let held = tokio::spawn(async move { + held_client + .simple_query(&held_sql) + .await + .map(|_| ()) + .map_err(|e| db_detail(&e)) + }); + wait_for( + "the entries leader holds the held slice's redo", + STEP_DEADLINE, + STEP_POLL, + || parked.exists(), + ) + .await; + let coordinator = fx.coordinator(); + let accts_name = accts.as_str(); + let held_balance = (BASE + HELD_AMOUNT).to_string(); + wait_for_async( + "the held transaction's balance slice installs", + STEP_DEADLINE, + STEP_POLL, + || { + let held_balance = held_balance.clone(); + async move { + rows( + &coordinator.client, + &format!("SELECT balance FROM {accts_name} WHERE id = 'acc-0'"), + ) + .await + == vec![vec![held_balance]] + } + }, + ) + .await; + + // Writers on the other accounts, before and after the backup's cut. + let stop = Arc::new(AtomicBool::new(false)); + let mut writers = Vec::new(); + for (idx, account) in ACCOUNTS.into_iter().skip(1).enumerate() { + let node = &fx.cluster.nodes[idx % fx.cluster.nodes.len()]; + let client = strict_session(node).await; + writers.push(tokio::spawn(write_until( + client, + entries.clone(), + account, + Arc::clone(&stop), + ))); + } + + let backup_client = strict_session(&fx.cluster.nodes[0]).await; + let backup_sql = format!("BACKUP DATABASE default TO '{uri}'"); + let backup = tokio::spawn(async move { + backup_client + .simple_query(&backup_sql) + .await + .map(|_| ()) + .map_err(|e| db_detail(&e)) + }); + tokio::time::sleep(PARKED_FOR).await; + assert!( + !backup.is_finished(), + "the backup finished while a transaction sequenced before its cut was not installed" + ); + + std::fs::write(&release, b"release").expect("release the held redo"); + held.await + .expect("held transaction task") + .unwrap_or_else(|e| panic!("the held transaction: {e}")); + backup + .await + .expect("backup task") + .unwrap_or_else(|e| panic!("BACKUP DATABASE: {e}")); + stop.store(true, Ordering::Relaxed); + for writer in writers { + let written = writer.await.expect("writer task"); + assert!(written > 0, "every writer committed while the backup ran"); + } + drop(gate); + assert!(root.join(OBJECT).exists(), "the backup wrote its object"); + fx.cluster.shutdown().await; + + let target = TestCluster::spawn_three_with_backup_root(root.clone()) + .await + .expect("target cluster"); + exec(&target, &format!("RESTORE DATABASE default FROM '{uri}'")).await; + target.wait_for_full_apply_convergence(STEP_DEADLINE).await; + assert_whole_transactions(&target, &accts, &entries).await; + target.shutdown().await; +} diff --git a/nodedb-cluster-tests/tests/common_suite/cases/calvin_cluster_array_read_set.rs b/nodedb-cluster-tests/tests/common_suite/cases/calvin_cluster_array_read_set.rs new file mode 100644 index 000000000..a1415884f --- /dev/null +++ b/nodedb-cluster-tests/tests/common_suite/cases/calvin_cluster_array_read_set.rs @@ -0,0 +1,142 @@ +// SPDX-License-Identifier: BUSL-1.1 + +//! A transaction's cluster array read joins its read set. +//! +//! A 3-node cluster holds the array `grid`, whose cells spread over the tile +//! vShards of several nodes, and the keyed collection `log`. A transaction +//! reads an `ARRAY_SLICE` that spans the tiles of several shards, then +//! writes `log`. Each shard leg reports its vShard's write version, even +//! when it matched no cell, and COMMIT validates the read on every covered +//! vShard. +//! +//! - No write to `grid` after the read: the COMMIT succeeds. +//! - A cell written to a covered tile after the read moves that vShard's +//! version past the read's, so the COMMIT aborts with SQLSTATE 40001 and +//! the transaction's write is not visible. + +use super::calvin_multishard_fixture::{Fixture, data_rows, keyed_ddl, row_count}; +use super::calvin_replica_content::{run_retrying, strict_session}; +use crate::common::cluster_harness::shared_steps::db_detail; + +/// SQLSTATE of a transaction aborted by a conflict. +const SERIALIZATION_FAILURE: &str = "40001"; + +/// The slice the transaction reads: three tiles, one per `chr`. +const SLICE: &str = + "SELECT * FROM ARRAY_SLICE('grid', '{chr: [0, 2], pos: [0, 99]}', ['qual'], 100)"; + +/// A 3-node cluster holding the array `grid`, seeded with three cells per +/// tile on `chr` 0 to 2, and the keyed collection `log`. +async fn seeded(log: &str) -> Fixture { + let fx = Fixture::spawn(&[keyed_ddl(log)]).await; + fx.wait_group_mounted(log).await; + fx.cluster + .exec_ddl_on_any_leader( + "CREATE ARRAY grid \ + DIMS (chr INT64 [0..9], pos INT64 [0..99]) \ + ATTRS (qual FLOAT64) \ + TILE_EXTENTS (1, 100) \ + CELL_ORDER HILBERT", + ) + .await + .unwrap_or_else(|e| panic!("CREATE ARRAY grid: {e}")); + run_retrying( + &fx.cluster.nodes[0].client, + "seed the array", + "INSERT INTO ARRAY grid \ + COORDS (0, 10) VALUES (1.0), \ + COORDS (0, 20) VALUES (2.0), \ + COORDS (1, 10) VALUES (10.0), \ + COORDS (1, 20) VALUES (20.0), \ + COORDS (2, 10) VALUES (100.0), \ + COORDS (2, 20) VALUES (200.0)", + ) + .await; + fx.converge().await; + fx +} + +/// Open a transaction on the coordinator that reads the slice and buffers a +/// write to `log`, and return its session. +async fn read_slice_then_buffer_write(fx: &Fixture, log: &str) -> tokio_postgres::Client { + let session = strict_session(fx.coordinator()).await; + session + .simple_query("BEGIN") + .await + .unwrap_or_else(|e| panic!("BEGIN: {}", db_detail(&e))); + let rows = session + .simple_query(SLICE) + .await + .unwrap_or_else(|e| panic!("in-transaction slice: {}", db_detail(&e))); + assert_eq!(data_rows(&rows), 6, "the slice reads every seeded cell"); + session + .simple_query(&format!("INSERT INTO {log} (id, v) VALUES ('w1', 'txn')")) + .await + .unwrap_or_else(|e| panic!("buffer the write: {}", db_detail(&e))); + session +} + +#[tokio::test(flavor = "multi_thread", worker_threads = 6)] +async fn a_current_cluster_array_read_commits() { + let log = "car_ok_log"; + let fx = seeded(log).await; + let session = read_slice_then_buffer_write(&fx, log).await; + + session.simple_query("COMMIT").await.unwrap_or_else(|e| { + panic!( + "no write reached the slice's tiles, so the commit succeeds: {}", + db_detail(&e) + ) + }); + fx.converge().await; + assert_eq!( + row_count( + &fx.coordinator().client, + &format!("SELECT id FROM {log} WHERE id = 'w1'") + ) + .await, + 1, + "the committed transaction's write is visible" + ); + + fx.cluster.shutdown().await; +} + +#[tokio::test(flavor = "multi_thread", worker_threads = 6)] +async fn a_write_to_a_covered_tile_aborts_the_cluster_array_read() { + let log = "car_stale_log"; + let fx = seeded(log).await; + let session = read_slice_then_buffer_write(&fx, log).await; + + let writer = (fx.coordinator + 1) % fx.cluster.nodes.len(); + run_retrying( + &fx.cluster.nodes[writer].client, + "a conflicting write to a covered tile", + "INSERT INTO ARRAY grid COORDS (1, 50) VALUES (5.0)", + ) + .await; + fx.converge().await; + + let error = session + .simple_query("COMMIT") + .await + .expect_err("a cell written to a covered tile after the read makes it stale"); + assert_eq!( + error.as_db_error().map(|db| db.code().code()), + Some(SERIALIZATION_FAILURE), + "the stale array read aborts the commit with a serialization failure: {}", + db_detail(&error) + ); + fx.converge().await; + assert_eq!( + row_count( + &fx.coordinator().client, + &format!("SELECT id FROM {log} WHERE id = 'w1'") + ) + .await, + 0, + "the aborted transaction's write is not visible" + ); + + fx.cluster.shutdown().await; +} diff --git a/nodedb-cluster-tests/tests/common_suite/cases/calvin_cluster_pgwire_e2e.rs b/nodedb-cluster-tests/tests/common_suite/cases/calvin_cluster_pgwire_e2e.rs index cb4625379..35f5385da 100644 --- a/nodedb-cluster-tests/tests/common_suite/cases/calvin_cluster_pgwire_e2e.rs +++ b/nodedb-cluster-tests/tests/common_suite/cases/calvin_cluster_pgwire_e2e.rs @@ -165,7 +165,7 @@ async fn calvin_multishard_write_in_explicit_block_commits() { before={admitted_before} after={admitted_after}" ); - // Both rows are readable after the commit applied. The Calvin flush lands + // Both rows are readable after the commit applied. The redo install lands // asynchronously after the completion ack, so poll for visibility. let deadline = std::time::Instant::now() + Duration::from_secs(10); loop { diff --git a/nodedb-cluster-tests/tests/common_suite/cases/calvin_dependent_read_commit.rs b/nodedb-cluster-tests/tests/common_suite/cases/calvin_dependent_read_commit.rs new file mode 100644 index 000000000..58dd8c80d --- /dev/null +++ b/nodedb-cluster-tests/tests/common_suite/cases/calvin_dependent_read_commit.rs @@ -0,0 +1,56 @@ +// SPDX-License-Identifier: BUSL-1.1 + +//! A cross-shard write whose value depends on a row of another vShard +//! commits on every replica as one dependent-read Calvin transaction. +//! +//! `SELECT TRANSFER_ITEM(src, dst, 'sword', 'alice', 'bob')` moves an item +//! between key-value collections on two vShards. The source vShard reads the +//! item under the transaction's locks and broadcasts it through the +//! data-group log of both active vShards. Every replica of the source then +//! lacks the item, and every replica of the destination holds the bytes the +//! source held. A second move of the same item finds it gone. + +use super::calvin_dependent_read_fixture::{ItemMove, assert_no_apply_halt}; +use super::calvin_replica_content::strict_session; +use crate::common::cluster_harness::shared_steps::db_detail; + +#[tokio::test(flavor = "multi_thread", worker_threads = 6)] +async fn a_cross_shard_item_move_commits_on_every_replica() { + let mv = ItemMove::spawn("dep_commit").await; + let source_bytes = mv.source_bytes().await; + let session = strict_session(mv.fx.coordinator()).await; + + let reply = session + .simple_query(&mv.transfer_sql()) + .await + .unwrap_or_else(|e| panic!("the move commits: {}", db_detail(&e))); + let text: Vec = reply + .into_iter() + .filter_map(|msg| match msg { + tokio_postgres::SimpleQueryMessage::Row(row) => row.get(0).map(str::to_owned), + _ => None, + }) + .collect(); + assert_eq!(text.len(), 1, "the move answers one row"); + assert!( + text[0].contains("bob:sword") && text[0].contains(&mv.dest), + "the answer names the item's new home: {}", + text[0] + ); + + mv.wait_replicas_agree(Some(&source_bytes)).await; + + let again = session + .simple_query(&mv.transfer_sql()) + .await + .expect_err("the item left the source"); + let detail = db_detail(&again).to_lowercase(); + assert!( + detail.contains("not found") || detail.contains("not_found") || detail.contains("22023"), + "a second move reports the item missing, got: {detail}" + ); + + assert_no_apply_halt(mv.cluster()); + let ItemMove { fx, .. } = mv; + fx.cluster.shutdown().await; +} diff --git a/nodedb-cluster-tests/tests/common_suite/cases/calvin_dependent_read_early_result.rs b/nodedb-cluster-tests/tests/common_suite/cases/calvin_dependent_read_early_result.rs new file mode 100644 index 000000000..e34fc8b4e --- /dev/null +++ b/nodedb-cluster-tests/tests/common_suite/cases/calvin_dependent_read_early_result.rs @@ -0,0 +1,124 @@ +// SPDX-License-Identifier: BUSL-1.1 + +//! A read result that reaches a vShard before the vShard granted its txn is +//! kept, and the barrier the grant opens completes from it. +//! +//! 1. A transaction `P` writes `bob:sword` at the destination and a row at +//! the source. The fail gate `calvin::before_redo_propose::` holds +//! its destination redo, so every destination replica keeps `P`'s lock +//! on `bob:sword`. +//! 2. The item move `T` queues behind that lock on the destination. Its +//! source vShard holds no conflicting lock, reads the item, and +//! broadcasts it. The broadcast applies on every destination replica +//! before the replica granted `T`. It waits in the replica's read-result +//! buffer and in its stored barrier row. +//! 3. Released, `P` commits and frees the lock. Each replica grants `T` and +//! takes the buffered result. The destination's leader stages at once, +//! and `T` commits with the item's bytes on every replica. Each replica +//! then drops the txn's buffered entries and its row. +//! +//! Requires `--features failpoints`. + +#![cfg(feature = "failpoints")] + +use std::time::Duration; + +use nodedb_test_support::fail_point::{FailAction, FailGuard}; + +use super::calvin_dependent_read_fixture::{ + ItemMove, assert_no_apply_halt, stored_barrier_rows, waiting_barrier_txns, +}; +use super::calvin_replica_content::strict_session; +use crate::common::cluster_harness::shared_steps::db_detail; +use crate::common::cluster_harness::wait_for; + +/// How long the test waits for a held step. +const HOLD_DEADLINE: Duration = Duration::from_secs(30); + +#[tokio::test(flavor = "multi_thread", worker_threads = 6)] +async fn a_read_result_before_the_grant_is_applied_not_dropped() { + let mv = ItemMove::spawn("dep_early").await; + let source_bytes = mv.source_bytes().await; + let dest_group = mv.group_of(&mv.dest); + let dest_vshard = mv.dest_vshard(); + let holder = strict_session(mv.fx.coordinator()).await; + let mover = strict_session(mv.fx.coordinator()).await; + + let gate_dir = tempfile::tempdir().expect("gate directory"); + let release = gate_dir.path().join("release"); + let parked = gate_dir.path().join("release.parked"); + let gate = FailGuard::install( + &format!("calvin::before_redo_propose::{}", mv.dest), + FailAction::WaitForFile(release.clone()), + ); + + let hold_lock = format!( + "BEGIN; \ + INSERT INTO {dest} (key, owner, note) VALUES ('bob:sword', 'bob', 'old'); \ + INSERT INTO {source} (key, owner, note) VALUES ('carol:shield', 'carol', 'x'); \ + COMMIT", + dest = mv.dest, + source = mv.source, + ); + let transfer = mv.transfer_sql(); + let (held, moved, ()) = tokio::join!( + holder.simple_query(&hold_lock), + async { + wait_for( + "the destination leader holds P's redo, and with it the lock", + HOLD_DEADLINE, + Duration::from_millis(20), + || parked.exists(), + ) + .await; + mover.simple_query(&transfer).await + }, + async { + wait_for( + "every destination replica buffers the read result before its grant", + HOLD_DEADLINE, + Duration::from_millis(20), + || { + parked.exists() + && mv + .cluster() + .nodes + .iter() + .filter(|node| node.replicates_data_group(dest_group)) + .all(|node| { + waiting_barrier_txns(node, dest_vshard) >= 1 + && stored_barrier_rows(node, dest_vshard) >= 1 + }) + }, + ) + .await; + std::fs::write(&release, b"release").expect("release P's redo"); + }, + ); + held.unwrap_or_else(|e| panic!("P commits: {}", db_detail(&e))); + moved.unwrap_or_else(|e| { + panic!( + "the move commits from the buffered result: {}", + db_detail(&e) + ) + }); + drop(gate); + + mv.wait_replicas_agree(Some(&source_bytes)).await; + wait_for( + "every node drops the barrier entries and rows of the finished move", + HOLD_DEADLINE, + Duration::from_millis(20), + || { + mv.cluster().nodes.iter().all(|node| { + waiting_barrier_txns(node, dest_vshard) == 0 + && stored_barrier_rows(node, dest_vshard) == 0 + }) + }, + ) + .await; + assert_no_apply_halt(mv.cluster()); + + let ItemMove { fx, .. } = mv; + fx.cluster.shutdown().await; +} diff --git a/nodedb-cluster-tests/tests/common_suite/cases/calvin_dependent_read_fixture.rs b/nodedb-cluster-tests/tests/common_suite/cases/calvin_dependent_read_fixture.rs new file mode 100644 index 000000000..623d4a558 --- /dev/null +++ b/nodedb-cluster-tests/tests/common_suite/cases/calvin_dependent_read_fixture.rs @@ -0,0 +1,285 @@ +// SPDX-License-Identifier: BUSL-1.1 + +//! Shared fixture for the Calvin dependent-read barrier cases. +//! +//! `SELECT TRANSFER_ITEM(src, dst, 'sword', 'alice', 'bob')` between two +//! key-value collections on distinct vShards is a read-dependent Calvin +//! transaction. The source vShard reads `alice:sword` under the +//! transaction's locks and broadcasts it, through the data-group log of each +//! active vShard, to the delete at the source and the put at the +//! destination. +//! +//! Each replica's content is read from its own Data-Plane cores, so a check +//! sees what that replica installed, never a row a gateway fetched. + +use std::collections::BTreeMap; +use std::time::Duration; + +use nodedb::types::TenantId; + +use super::calvin_multishard_fixture::Fixture; +use super::calvin_replica_content::names_on_distinct_vshards; +#[cfg(feature = "failpoints")] +use super::calvin_replica_content::vshard_of; +use crate::common::cluster_harness::shared_steps::{db_detail, leader_of}; +use crate::common::cluster_harness::{TestCluster, TestClusterNode, wait_for_async}; + +/// The tenant the `nodedb` pgwire user writes as. +const TENANT: u64 = 1; + +/// The item's key at the source. +pub(super) const SOURCE_KEY: &[u8] = b"alice:sword"; + +/// The item's key at the destination. +pub(super) const DEST_KEY: &[u8] = b"bob:sword"; + +/// How long the replicas get to converge on the move's outcome. +const CONVERGE: Duration = Duration::from_secs(30); + +/// A cluster with a source and a destination key-value collection on +/// distinct vShards, the item stored at the source. +pub(super) struct ItemMove { + pub(super) fx: Fixture, + pub(super) source: String, + pub(super) dest: String, +} + +impl ItemMove { + /// Spawn the cluster, create both collections, and store the item. + pub(super) async fn spawn(prefix: &str) -> Self { + let source_prefix = format!("{prefix}_src"); + let dest_prefix = format!("{prefix}_dst"); + let [source, dest] = + names_on_distinct_vshards([source_prefix.as_str(), dest_prefix.as_str()]); + let fx = Fixture::spawn(&[kv_ddl(&source), kv_ddl(&dest)]).await; + fx.wait_group_mounted(&source).await; + fx.wait_group_mounted(&dest).await; + // Collection creation installs each collection's constraint set on + // every replica in the background. The install is one more write of + // the collection, so the cases start their txns once it applied. + wait_constraints_installed(&fx.cluster, &source).await; + wait_constraints_installed(&fx.cluster, &dest).await; + fx.coordinator() + .client + .simple_query(&format!( + "INSERT INTO {source} (key, owner, note) VALUES ('alice:sword', 'alice', 'forged')" + )) + .await + .unwrap_or_else(|e| panic!("store the item: {}", db_detail(&e))); + fx.converge().await; + Self { fx, source, dest } + } + + /// The statement that moves the item from the source to the + /// destination. + pub(super) fn transfer_sql(&self) -> String { + format!( + "SELECT TRANSFER_ITEM('{}', '{}', 'sword', 'alice', 'bob')", + self.source, self.dest + ) + } + + pub(super) fn cluster(&self) -> &TestCluster { + &self.fx.cluster + } + + /// The vShard of the source collection. + #[cfg(feature = "failpoints")] + pub(super) fn source_vshard(&self) -> u32 { + vshard_of(&self.source) + } + + /// The vShard of the destination collection. + #[cfg(feature = "failpoints")] + pub(super) fn dest_vshard(&self) -> u32 { + vshard_of(&self.dest) + } + + /// The data group of `collection`, as the first node routes it. + pub(super) fn group_of(&self, collection: &str) -> u64 { + self.cluster().nodes[0] + .group_id_for_collection(collection) + .unwrap_or_else(|| panic!("no data group for {collection}")) + } + + /// Wait until every replica agrees on the move's outcome. With + /// `moved = None`, every source replica holds the item and no + /// destination replica holds it. With `moved = Some(bytes)`, no source + /// replica holds the item and every destination replica holds `bytes` + /// for it: the bytes the source held, not merely some row at the key. + pub(super) async fn wait_replicas_agree(&self, moved: Option<&[u8]>) { + let source_group = self.group_of(&self.source); + let dest_group = self.group_of(&self.dest); + let deadline = std::time::Instant::now() + CONVERGE; + loop { + let mut agree = true; + let mut views = Vec::new(); + for node in &self.cluster().nodes { + let source_holds = node.replicates_data_group(source_group) + && local_kv_rows(node, &self.source) + .await + .contains_key(SOURCE_KEY); + let dest_bytes = if node.replicates_data_group(dest_group) { + local_kv_rows(node, &self.dest).await.remove(DEST_KEY) + } else { + moved.map(<[u8]>::to_vec) + }; + if (node.replicates_data_group(source_group) && source_holds == moved.is_some()) + || dest_bytes.as_deref() != moved + { + agree = false; + } + views.push(format!( + "node {}: source holds the item = {source_holds}, destination bytes match = \ + {}, source group applied {} (leader {}), destination group applied {} \ + (leader {})", + node.node_id, + dest_bytes.as_deref() == moved, + node.shared.applied_index_watcher(source_group).current(), + leader_of(node, source_group), + node.shared.applied_index_watcher(dest_group).current(), + leader_of(node, dest_group), + )); + } + if agree { + return; + } + assert!( + std::time::Instant::now() < deadline, + "the replicas disagree on the move (moved = {}) after {CONVERGE:?}:\n{}", + moved.is_some(), + views.join("\n") + ); + tokio::time::sleep(Duration::from_millis(200)).await; + } + } + + /// The bytes the source's replicas hold for the item. Panics when two + /// replicas disagree or none holds it. + pub(super) async fn source_bytes(&self) -> Vec { + let source_group = self.group_of(&self.source); + let mut seen: Option> = None; + for node in &self.cluster().nodes { + if !node.replicates_data_group(source_group) { + continue; + } + let bytes = local_kv_rows(node, &self.source) + .await + .remove(SOURCE_KEY) + .unwrap_or_else(|| panic!("node {} does not hold the item", node.node_id)); + if let Some(first) = &seen { + assert_eq!(first, &bytes, "the source's replicas disagree"); + } + seen = Some(bytes); + } + seen.expect("the source group has a replica") + } +} + +/// Assert no node halted a Calvin scheduler. +pub(super) fn assert_no_apply_halt(cluster: &TestCluster) { + for node in &cluster.nodes { + let halt = node.shared.sequencer_halt.apply_halt().report(); + assert!( + halt.is_none(), + "node {} halted a Calvin scheduler: {halt:?}", + node.node_id + ); + } +} + +/// The txns of `vshard` whose barrier entries `node` holds and no scheduler +/// took, in memory or in their stored row. +#[cfg(feature = "failpoints")] +pub(super) fn waiting_barrier_txns(node: &TestClusterNode, vshard: u32) -> usize { + node.shared.calvin.read_results.waiting_txns(vshard) +} + +/// The barrier rows `node` stores for `vshard`: one per txn whose barrier +/// entry applied there and did not finish. +#[cfg(feature = "failpoints")] +pub(super) fn stored_barrier_rows(node: &TestClusterNode, vshard: u32) -> usize { + node.shared + .credentials + .catalog() + .load_calvin_barrier_logs(Some(&std::collections::BTreeSet::from([vshard]))) + .expect("the barrier rows load") + .len() +} + +/// The `(group, log index)` of the latest read-result entries `node` +/// applied for `vshard` in its current life. +#[cfg(feature = "failpoints")] +pub(super) fn read_entries(node: &TestClusterNode, vshard: u32) -> Vec<(u64, u64)> { + node.shared + .calvin + .read_results + .vshard(vshard) + .read_entries() +} + +/// The read-result entries `node` applied for `vshard`. +#[cfg(feature = "failpoints")] +pub(super) fn reads_applied(node: &TestClusterNode, vshard: u32) -> u64 { + node.shared + .calvin + .read_results + .vshard(vshard) + .reads_applied() +} + +/// Wait until every replica of `collection`'s data group installed the +/// collection's constraint set in its CRDT validator. The set of a key-value +/// collection names its key `NOT NULL`, so it is never empty. +async fn wait_constraints_installed(cluster: &TestCluster, collection: &str) { + let group = cluster.nodes[0] + .group_id_for_collection(collection) + .unwrap_or_else(|| panic!("no data group for {collection}")); + wait_for_async( + &format!("every replica of {collection} installed its constraint set"), + CONVERGE, + Duration::from_millis(100), + || async move { + for node in &cluster.nodes { + if node.replicates_data_group(group) + && node + .crdt_constraints(TenantId::new(TENANT), collection) + .await + .is_empty() + { + return false; + } + } + true + }, + ) + .await; +} + +fn kv_ddl(collection: &str) -> String { + format!( + "CREATE COLLECTION {collection} (key TEXT PRIMARY KEY, owner TEXT, note TEXT) \ + WITH (engine='kv')" + ) +} + +/// Every key-value row of `collection` as `node` stores it: key to stored +/// bytes, read from each of its Data-Plane cores. +async fn local_kv_rows(node: &TestClusterNode, collection: &str) -> BTreeMap, Vec> { + let table = format!("0:{TENANT}:{collection}"); + let mut rows = BTreeMap::new(); + for core in 0..node.num_cores() { + let snapshot = node + .tenant_snapshot_on_core(core, TenantId::new(TENANT)) + .await; + for (key, bytes) in snapshot.kv_tables { + if key != table { + continue; + } + let entries: Vec<(Vec, Vec, u64, u32)> = + zerompk::from_msgpack(&bytes).expect("KV table snapshot decodes"); + rows.extend(entries.into_iter().map(|(key, value, _, _)| (key, value))); + } + } + rows +} diff --git a/nodedb-cluster-tests/tests/common_suite/cases/calvin_dependent_read_follower_promotion.rs b/nodedb-cluster-tests/tests/common_suite/cases/calvin_dependent_read_follower_promotion.rs new file mode 100644 index 000000000..d7b886470 --- /dev/null +++ b/nodedb-cluster-tests/tests/common_suite/cases/calvin_dependent_read_follower_promotion.rs @@ -0,0 +1,148 @@ +// SPDX-License-Identifier: BUSL-1.1 + +//! A follower that buffered a dependent-read result before its grant does +//! not halt once it leads: the barrier decides from the log, not a clock. +//! +//! 1. Node `F` leads neither the source's nor the destination's data group. +//! Its metadata apply is held, and a DDL through another node moves the +//! metadata log past it. The item move `T` carries the coordinator's +//! metadata floor, so `F`'s schedulers hold `T` ungranted. +//! 2. The source vShard reads the item and broadcasts it. The broadcast +//! applies on `F` before `F` granted `T`. The destination's leader stages +//! `T` and votes commit, and the fail gate +//! `calvin::before_redo_propose::` holds its redo. +//! 3. `F` is released and grants `T` as a follower, and takes the buffered +//! result. Leadership of the destination's group moves to `F`. `F` opens +//! the barrier from the result, stages `T`, and proposes its redo once +//! the gate is released. +//! 4. `T` commits on every replica, and no node halts a scheduler with +//! `LocalStageFailed`. +//! +//! Requires `--features failpoints`. + +#![cfg(feature = "failpoints")] + +use std::time::Duration; + +use nodedb::control::cluster::metadata_applier::METADATA_APPLY_HOLD_POINT; +use nodedb_test_support::fail_point::{FailAction, FailGuard}; + +use super::calvin_dependent_read_fixture::{ItemMove, assert_no_apply_halt, waiting_barrier_txns}; +use super::calvin_replica_content::strict_session; +use crate::common::cluster_harness::shared_steps::{db_detail, leader_of}; +use crate::common::cluster_harness::wait_for; + +/// How long the test waits for a held step. +const HOLD_DEADLINE: Duration = Duration::from_secs(30); + +#[tokio::test(flavor = "multi_thread", worker_threads = 6)] +async fn a_promoted_follower_commits_the_move_without_halting() { + let mv = ItemMove::spawn("dep_promote").await; + mv.cluster().wait_for_preferred_leaders(HOLD_DEADLINE).await; + let source_bytes = mv.source_bytes().await; + let source_group = mv.group_of(&mv.source); + let dest_group = mv.group_of(&mv.dest); + let dest_vshard = mv.dest_vshard(); + let nodes = &mv.cluster().nodes; + let source_leader = leader_of(&nodes[0], source_group); + let dest_leader = leader_of(&nodes[0], dest_group); + let follower = nodes + .iter() + .position(|node| node.node_id != source_leader && node.node_id != dest_leader) + .expect("three nodes hold a node that leads neither group"); + let follower_id = nodes[follower].node_id; + let coordinator = (0..nodes.len()) + .find(|idx| *idx != follower) + .expect("a coordinator besides the follower"); + let session = strict_session(&nodes[coordinator]).await; + + // Hold the follower's metadata apply and move the metadata log past it. + let meta_dir = tempfile::tempdir().expect("metadata gate directory"); + let meta_release = meta_dir.path().join("release"); + let meta_parked = meta_dir.path().join("release.parked"); + let _meta_hold = FailGuard::for_node( + follower_id, + METADATA_APPLY_HOLD_POINT, + FailAction::WaitForFile(meta_release.clone()), + ); + nodes[coordinator] + .exec("CREATE TOPIC dep_promote_lag") + .await + .unwrap_or_else(|e| panic!("the DDL commits without the follower: {e}")); + let metadata = nodedb_cluster::METADATA_GROUP_ID; + wait_for( + "the follower's metadata apply lags the coordinator's", + HOLD_DEADLINE, + Duration::from_millis(20), + || { + meta_parked.exists() + && nodes[follower] + .shared + .applied_index_watcher(metadata) + .current() + < nodes[coordinator] + .shared + .applied_index_watcher(metadata) + .current() + }, + ) + .await; + + let redo_dir = tempfile::tempdir().expect("redo gate directory"); + let redo_release = redo_dir.path().join("release"); + let redo_parked = redo_dir.path().join("release.parked"); + let redo_gate = FailGuard::install( + &format!("calvin::before_redo_propose::{}", mv.dest), + FailAction::WaitForFile(redo_release.clone()), + ); + + let transfer = mv.transfer_sql(); + let (moved, ()) = tokio::join!(session.simple_query(&transfer), async { + // Two waits, so a timeout names the half that never held. + wait_for( + "the follower buffered the read result before its grant", + HOLD_DEADLINE, + Duration::from_millis(20), + || waiting_barrier_txns(&nodes[follower], dest_vshard) >= 1, + ) + .await; + wait_for( + "the destination leader voted commit and holds the redo", + HOLD_DEADLINE, + Duration::from_millis(20), + || redo_parked.exists(), + ) + .await; + assert!( + nodes[follower] + .shared + .applied_index_watcher(metadata) + .current() + < nodes[coordinator] + .shared + .applied_index_watcher(metadata) + .current(), + "the follower's metadata apply still lags, so its grant waits" + ); + std::fs::write(&meta_release, b"release").expect("release the follower's metadata"); + wait_for( + "the follower granted the move and took its buffered result", + HOLD_DEADLINE, + Duration::from_millis(20), + || waiting_barrier_txns(&nodes[follower], dest_vshard) == 0, + ) + .await; + mv.cluster() + .transfer_leadership(dest_group, follower_id) + .await; + std::fs::write(&redo_release, b"release").expect("release the redo"); + }); + moved.unwrap_or_else(|e| panic!("the move commits under the new leader: {}", db_detail(&e))); + drop(redo_gate); + + mv.wait_replicas_agree(Some(&source_bytes)).await; + assert_no_apply_halt(mv.cluster()); + + let ItemMove { fx, .. } = mv; + fx.cluster.shutdown().await; +} diff --git a/nodedb-cluster-tests/tests/common_suite/cases/calvin_dependent_read_restart.rs b/nodedb-cluster-tests/tests/common_suite/cases/calvin_dependent_read_restart.rs new file mode 100644 index 000000000..87b832840 --- /dev/null +++ b/nodedb-cluster-tests/tests/common_suite/cases/calvin_dependent_read_restart.rs @@ -0,0 +1,192 @@ +// SPDX-License-Identifier: BUSL-1.1 + +//! A replica that restarts after a dependent-read result applied, and before +//! the txn finished, reaches the barrier outcome its peers reached: the +//! result's stored row survives the restart. +//! +//! 1. Node `R` leads neither the source's nor the destination's data group. +//! The fail gate `calvin::before_redo_propose::` holds the +//! destination redo of the item move `T`, so `T` does not finish on the +//! destination. +//! 2. The source vShard's read result applies on `R` and lands in its stored +//! barrier row. The test pins the result's entry by its log index. A +//! later write to the destination group moves `R`'s durable applied +//! floor past that index, so a restart never delivers the entry again. +//! 3. `R` stops and starts again, and applies no pinned entry again. A +//! duplicate entry at a later index can apply. Leadership of the +//! destination's group moves to `R`. `R` opens `T`'s barrier from the +//! stored row, stages `T`, and proposes its redo once the gate is +//! released. +//! 4. `T` commits on every replica, and no node halts a scheduler. +//! +//! Requires `--features failpoints`. + +#![cfg(feature = "failpoints")] + +use std::time::Duration; + +use nodedb_test_support::fail_point::{FailAction, FailGuard}; + +use super::calvin_dependent_read_fixture::{ + ItemMove, assert_no_apply_halt, read_entries, stored_barrier_rows, waiting_barrier_txns, +}; +use super::calvin_replica_content::strict_session; +use crate::common::cluster_harness::shared_steps::{db_detail, leader_of}; +use crate::common::cluster_harness::wait_for; + +/// How long the test waits for a held step. +const HOLD_DEADLINE: Duration = Duration::from_secs(30); + +#[tokio::test(flavor = "multi_thread", worker_threads = 6)] +async fn a_restarted_replica_reaches_the_barrier_outcome_from_its_stored_row() { + let mut mv = ItemMove::spawn("dep_restart").await; + mv.cluster().wait_for_preferred_leaders(HOLD_DEADLINE).await; + let source_bytes = mv.source_bytes().await; + let source_group = mv.group_of(&mv.source); + let dest_group = mv.group_of(&mv.dest); + let dest_vshard = mv.dest_vshard(); + let (restarted, restarted_id, coordinator) = { + let nodes = &mv.cluster().nodes; + let source_leader = leader_of(&nodes[0], source_group); + let dest_leader = leader_of(&nodes[0], dest_group); + let restarted = nodes + .iter() + .position(|node| { + node.node_id != source_leader + && node.node_id != dest_leader + && node.replicates_data_group(dest_group) + }) + .expect("three nodes hold a destination replica that leads neither group"); + let coordinator = (0..nodes.len()) + .find(|idx| *idx != restarted) + .expect("a coordinator besides the restarted node"); + (restarted, nodes[restarted].node_id, coordinator) + }; + let session = strict_session(&mv.cluster().nodes[coordinator]).await; + let filler = strict_session(&mv.cluster().nodes[coordinator]).await; + + let redo_dir = tempfile::tempdir().expect("redo gate directory"); + let redo_release = redo_dir.path().join("release"); + let redo_parked = redo_dir.path().join("release.parked"); + let redo_gate = FailGuard::install( + &format!("calvin::before_redo_propose::{}", mv.dest), + FailAction::WaitForFile(redo_release.clone()), + ); + + let transfer = mv.transfer_sql(); + let filler_sql = format!( + "INSERT INTO {} (key, owner, note) VALUES ('dave:axe', 'dave', 'filler')", + mv.dest + ); + let (moved, ()) = tokio::join!(session.simple_query(&transfer), async { + { + let node = &mv.cluster().nodes[restarted]; + wait_for( + "the destination leader holds the redo, and the replica stored the read result", + HOLD_DEADLINE, + Duration::from_millis(20), + || { + redo_parked.exists() + && !read_entries(node, dest_vshard).is_empty() + && stored_barrier_rows(node, dest_vshard) >= 1 + }, + ) + .await; + } + // The read-result entries the replica applied before it stops. The + // first is the one the barrier counts. + let applied_before = read_entries(&mv.cluster().nodes[restarted], dest_vshard); + let pinned = applied_before + .iter() + .map(|(_, index)| *index) + .max() + .expect("the replica applied a read result"); + filler + .simple_query(&filler_sql) + .await + .unwrap_or_else(|e| panic!("the filler write commits: {}", db_detail(&e))); + let dest_leader_applied = { + let nodes = &mv.cluster().nodes; + let leader = leader_of(&nodes[coordinator], dest_group); + nodes + .iter() + .find(|node| node.node_id == leader) + .expect("the destination leader runs") + .shared + .applied_index_watcher(dest_group) + .current() + }; + { + let node = &mv.cluster().nodes[restarted]; + wait_for( + "the replica's durable applied floor passes the read result's entry", + HOLD_DEADLINE, + Duration::from_millis(20), + || node.shared.pitr.durable_applied(dest_group) >= dest_leader_applied.max(pinned), + ) + .await; + } + + let stopped = mv + .fx + .cluster + .stop_member(restarted) + .await + .expect("stop the replica"); + mv.fx + .cluster + .restart_member(stopped) + .await + .expect("restart the replica"); + { + let node = &mv.cluster().nodes[restarted]; + assert_eq!(node.node_id, restarted_id, "the replica keeps its index"); + // Boot resumes delivery above the durable floor, which passed + // every entry applied before the stop. A duplicate read-result + // entry the log holds later can apply now: it sits at another + // index, and the barrier counts the first copy alone. + let again: Vec<(u64, u64)> = read_entries(node, dest_vshard) + .into_iter() + .filter(|entry| applied_before.contains(entry)) + .collect(); + assert!( + again.is_empty(), + "the restart delivers again the read-result entries {again:?}" + ); + assert_eq!( + stored_barrier_rows(node, dest_vshard), + 1, + "the stored row survives the restart" + ); + } + mv.cluster() + .transfer_leadership(dest_group, restarted_id) + .await; + std::fs::write(&redo_release, b"release").expect("release the redo"); + }); + moved.unwrap_or_else(|e| { + panic!( + "the move commits under the restarted leader: {}", + db_detail(&e) + ) + }); + drop(redo_gate); + + mv.wait_replicas_agree(Some(&source_bytes)).await; + wait_for( + "every node drops the barrier entries and rows of the finished move", + HOLD_DEADLINE, + Duration::from_millis(20), + || { + mv.cluster().nodes.iter().all(|node| { + waiting_barrier_txns(node, dest_vshard) == 0 + && stored_barrier_rows(node, dest_vshard) == 0 + }) + }, + ) + .await; + assert_no_apply_halt(mv.cluster()); + + let ItemMove { fx, .. } = mv; + fx.cluster.shutdown().await; +} diff --git a/nodedb-cluster-tests/tests/common_suite/cases/calvin_dependent_read_timeout.rs b/nodedb-cluster-tests/tests/common_suite/cases/calvin_dependent_read_timeout.rs new file mode 100644 index 000000000..c8eb74ecf --- /dev/null +++ b/nodedb-cluster-tests/tests/common_suite/cases/calvin_dependent_read_timeout.rs @@ -0,0 +1,105 @@ +// SPDX-License-Identifier: BUSL-1.1 + +//! A dependent-read barrier whose read result never arrives aborts on every +//! replica alike, through the timeout entry its leader puts in the log. +//! +//! 1. The fail gate `calvin::before_read_result_propose::` holds the +//! source vShard's broadcast of the item it read. +//! 2. The active vShards' barriers wait past +//! `tuning.calvin.dependent_read_passive_timeout_ms`. Each data-group +//! leader proposes the barrier's timeout entry, and every replica folds +//! it ahead of the read result. +//! 3. The move fails. No replica moved the item, and no scheduler halted. +//! 4. Released, the stale broadcast reaches both logs after the txn +//! finished. Every replica applies it and drops it, so no buffer and no +//! stored row keeps it. +//! 5. A second move commits: the aborted txn left no lock behind. +//! +//! Requires `--features failpoints`. + +#![cfg(feature = "failpoints")] + +use std::time::Duration; + +use nodedb_test_support::fail_point::{FailAction, FailGuard}; + +use super::calvin_dependent_read_fixture::{ + ItemMove, assert_no_apply_halt, reads_applied, stored_barrier_rows, waiting_barrier_txns, +}; +use super::calvin_replica_content::strict_session; +use crate::common::cluster_harness::shared_steps::db_detail; +use crate::common::cluster_harness::wait_for; + +#[tokio::test(flavor = "multi_thread", worker_threads = 6)] +async fn a_read_result_held_past_the_timeout_aborts_on_every_replica() { + let mv = ItemMove::spawn("dep_timeout").await; + let source_bytes = mv.source_bytes().await; + let session = strict_session(mv.fx.coordinator()).await; + + let gate_dir = tempfile::tempdir().expect("gate directory"); + let release = gate_dir.path().join("release"); + let _gate = FailGuard::install( + &format!("calvin::before_read_result_propose::{}", mv.source), + FailAction::WaitForFile(release.clone()), + ); + + let aborted = session + .simple_query(&mv.transfer_sql()) + .await + .expect_err("the move aborts once its barrier times out in the log"); + assert!( + gate_dir.path().join("release.parked").exists(), + "the source vShard read the item and held its broadcast: {}", + db_detail(&aborted) + ); + + mv.wait_replicas_agree(None).await; + assert_no_apply_halt(mv.cluster()); + + let source_group = mv.group_of(&mv.source); + let dest_group = mv.group_of(&mv.dest); + let source_vshard = mv.source_vshard(); + let dest_vshard = mv.dest_vshard(); + for node in &mv.cluster().nodes { + assert_eq!( + reads_applied(node, source_vshard) + reads_applied(node, dest_vshard), + 0, + "node {} applied no read result while the gate held the broadcast", + node.node_id + ); + } + std::fs::write(&release, b"release").expect("release the held broadcast"); + // The released broadcast reaches both data-group logs. Every replica of + // each group applies its entry for the finished txn and keeps neither a + // buffered event nor a row of it. + wait_for( + "every replica applies the stale broadcast and drops it", + Duration::from_secs(30), + Duration::from_millis(50), + || { + mv.cluster().nodes.iter().all(|node| { + let source_done = !node.replicates_data_group(source_group) + || reads_applied(node, source_vshard) >= 1; + let dest_done = !node.replicates_data_group(dest_group) + || reads_applied(node, dest_vshard) >= 1; + source_done + && dest_done + && [source_vshard, dest_vshard].iter().all(|vshard| { + waiting_barrier_txns(node, *vshard) == 0 + && stored_barrier_rows(node, *vshard) == 0 + }) + }) + }, + ) + .await; + + session + .simple_query(&mv.transfer_sql()) + .await + .unwrap_or_else(|e| panic!("the second move commits: {}", db_detail(&e))); + mv.wait_replicas_agree(Some(&source_bytes)).await; + assert_no_apply_halt(mv.cluster()); + + let ItemMove { fx, .. } = mv; + fx.cluster.shutdown().await; +} diff --git a/nodedb-cluster-tests/tests/common_suite/cases/calvin_duplicate_redo_copy.rs b/nodedb-cluster-tests/tests/common_suite/cases/calvin_duplicate_redo_copy.rs new file mode 100644 index 000000000..a384be6a2 --- /dev/null +++ b/nodedb-cluster-tests/tests/common_suite/cases/calvin_duplicate_redo_copy.rs @@ -0,0 +1,198 @@ +// SPDX-License-Identifier: BUSL-1.1 + +//! A Calvin slice whose redo installed, while its scheduler has not heard +//! of the install, gets its redo proposed again. The second copy also +//! commits, and installs nothing: every replica applies each slice once. +//! +//! 1. A 3-node cluster holds `accts`, whose `balance` is the materialized +//! sum of `entries.amount`, and `entries`, on distinct vShards. `acc-0` +//! starts at 100. +//! 2. The fail gates `calvin::before_redo_applied_push::` and +//! `calvin::before_redo_applied_push::` withhold the `RedoApplied` +//! event of each slice's install. The apply concludes, so each data group +//! applies past the redo with no install heard, and the leader's stall +//! tick proposes the redo again. +//! 3. Once every node skipped a second copy, the test releases the gates. +//! 4. The client's COMMIT succeeds once. `acc-0` reads 140, not the 180 of a +//! second install. Every node installed each slice it replicates once, +//! and every replica holds the same content. +//! +//! Requires `--features failpoints`. + +#![cfg(feature = "failpoints")] + +use std::sync::atomic::Ordering; +use std::time::Duration; + +use nodedb_test_support::fail_point::{FailAction, FailGuard}; + +use super::calvin_multishard_fixture::{Fixture, tags}; +use super::calvin_replica_content::{ + first_column, names_on_distinct_vshards, run_retrying, sum_ddl, wait_replicas_agree, +}; +use crate::common::cluster_harness::{TestClusterNode, wait_for}; + +/// How long the test waits for the first install, and for every node to +/// skip a second copy. +const HOLD_DEADLINE: Duration = Duration::from_secs(30); + +/// How long the replicas get to report their installs after the release. +const INSTALL_DEADLINE: Duration = Duration::from_secs(15); + +/// Committed Calvin slices whose stamped redo installed on `node`. +fn installs(node: &TestClusterNode) -> u64 { + node.shared + .calvin + .counters + .write_versions_recorded + .load(Ordering::Relaxed) +} + +/// Stamped redo copies `node` applied that installed nothing. +fn skipped_copies(node: &TestClusterNode) -> u64 { + node.shared + .calvin + .counters + .redo_copies_skipped + .load(Ordering::Relaxed) +} + +/// How many of `collections` `node` replicates: each is one slice of the +/// transaction. +fn slices_on(node: &TestClusterNode, collections: &[&str]) -> u64 { + let hosted = collections + .iter() + .filter(|collection| { + node.group_id_for_collection(collection) + .is_some_and(|group| node.replicates_data_group(group)) + }) + .count(); + u64::try_from(hosted).unwrap_or(u64::MAX) +} + +#[tokio::test(flavor = "multi_thread", worker_threads = 6)] +async fn a_second_redo_copy_that_commits_installs_nothing() { + let [accts, entries] = names_on_distinct_vshards(["dup_redo_accts", "dup_redo_entries"]); + let collections = [accts.as_str(), entries.as_str()]; + let [accts_ddl, entries_ddl, sum] = sum_ddl(&entries, &accts); + let fx = Fixture::spawn(&[accts_ddl, entries_ddl]).await; + fx.cluster + .exec_ddl_on_any_leader(&sum) + .await + .unwrap_or_else(|e| panic!("{sum}: {e}")); + for collection in collections { + fx.wait_group_mounted(collection).await; + } + run_retrying( + &fx.cluster.nodes[0].client, + "seed the account", + &format!("INSERT INTO {accts} (id, owner, balance) VALUES ('acc-0', 'seed', '100')"), + ) + .await; + fx.converge().await; + + let installs_before: Vec = fx.cluster.nodes.iter().map(installs).collect(); + let skipped_before: Vec = fx.cluster.nodes.iter().map(skipped_copies).collect(); + let mut session = fx.raw().await; + let txn = format!( + "BEGIN; \ + INSERT INTO {entries} (id, account_id, amount) VALUES ('e1', 'acc-0', '25'); \ + INSERT INTO {entries} (id, account_id, amount) VALUES ('e2', 'acc-0', '15'); \ + COMMIT" + ); + + let gate_dir = tempfile::tempdir().expect("gate directory"); + let release = gate_dir.path().join("release"); + let parked = gate_dir.path().join("release.parked"); + let committed = { + let _gates = collections.map(|collection| { + FailGuard::install( + &format!("calvin::before_redo_applied_push::{collection}"), + FailAction::WaitForFile(release.clone()), + ) + }); + let commit = tags(&mut session, &txn); + let copy_again = async { + wait_for( + "a replica installs a slice and withholds its RedoApplied", + HOLD_DEADLINE, + Duration::from_millis(20), + || parked.exists(), + ) + .await; + wait_for( + "every node applies a second redo copy that installs nothing", + HOLD_DEADLINE, + Duration::from_millis(50), + || { + fx.cluster + .nodes + .iter() + .zip(&skipped_before) + .all(|(node, before)| skipped_copies(node) > *before) + }, + ) + .await; + std::fs::write(&release, b"release").expect("release the withheld events"); + }; + let (committed, ()) = tokio::join!(commit, copy_again); + committed + }; + + assert_eq!( + committed + .iter() + .filter(|tag| tag.as_str() == "COMMIT") + .count(), + 1, + "the client's COMMIT answers once, found tags {committed:?}" + ); + assert_eq!( + committed.last().map(String::as_str), + Some("COMMIT"), + "the transaction ends committed, found tags {committed:?}" + ); + + fx.converge().await; + wait_for( + "every node reports the install of each slice it replicates", + INSTALL_DEADLINE, + Duration::from_millis(50), + || { + fx.cluster + .nodes + .iter() + .zip(&installs_before) + .all(|(node, before)| installs(node) - before >= slices_on(node, &collections)) + }, + ) + .await; + for (node, before) in fx.cluster.nodes.iter().zip(&installs_before) { + assert_eq!( + installs(node) - before, + slices_on(node, &collections), + "node {} installs each slice it replicates once, whatever copies its log holds", + node.node_id + ); + } + + wait_replicas_agree(&fx.cluster, &collections).await; + let balance = first_column( + &fx.coordinator().client, + &format!("SELECT balance FROM {accts} WHERE id = 'acc-0'"), + ) + .await; + assert_eq!( + balance, + vec!["140".to_owned()], + "100 + 25 + 15 lands once; a second install of the balance slice reads 180" + ); + let rows = first_column( + &fx.coordinator().client, + &format!("SELECT id FROM {entries}"), + ) + .await; + assert_eq!(rows.len(), 2, "the entries slice wrote its two rows once"); + + fx.cluster.shutdown().await; +} diff --git a/nodedb-cluster-tests/tests/common_suite/cases/calvin_leader_change_mid_txn.rs b/nodedb-cluster-tests/tests/common_suite/cases/calvin_leader_change_mid_txn.rs new file mode 100644 index 000000000..a96679bd9 --- /dev/null +++ b/nodedb-cluster-tests/tests/common_suite/cases/calvin_leader_change_mid_txn.rs @@ -0,0 +1,124 @@ +// SPDX-License-Identifier: BUSL-1.1 + +//! A Calvin transaction whose data-group leader dies between its vote and +//! its redo commits once on every replica. +//! +//! 1. A 3-node cluster holds two collections on distinct vShards. A session +//! on a node that does not lead `a`'s data group runs +//! `BEGIN; INSERT a; INSERT b; COMMIT` in strict cross-shard mode. +//! 2. The fail gate `calvin::before_redo_propose::` holds `a`'s slice +//! after its vote and resolve, before its leader proposes the redo. +//! 3. The leader of `a`'s data group is killed while the redo is held. +//! 4. Released, the new leader stages the held txn again, resolves it, and +//! proposes its redo. Each collection holds its row exactly once on every +//! surviving node. +//! +//! The session's node submits the txn to the sequencer leader and waits for +//! its answer. When the victim also leads the sequencer group, that answer +//! is lost with it: the COMMIT reports an unknown outcome (`57014`) and the +//! txn still commits once. Otherwise the COMMIT succeeds. +//! +//! Requires `--features failpoints`. + +#![cfg(feature = "failpoints")] + +use std::time::Duration; + +use nodedb_test_support::fail_point::{FailAction, FailGuard}; + +use super::calvin_multishard_fixture::{Fixture, keyed_ddl}; +use super::vshard_names::distinct_vshard_collections; +use crate::common::cluster_harness::shared_steps::{kill_node, leader_index_of, leader_of}; +use crate::common::cluster_harness::{TestClusterNode, wait_for}; + +/// How long the test waits for the leader to hold the redo. +const HOLD_DEADLINE: Duration = Duration::from_secs(15); + +/// A fresh session on `node` in strict cross-shard mode. +async fn strict_session(node: &TestClusterNode) -> tokio_postgres::Client { + let (client, connection) = tokio_postgres::connect( + &format!( + "host=127.0.0.1 port={} user=nodedb dbname=default", + node.pg_addr.port() + ), + tokio_postgres::NoTls, + ) + .await + .expect("connect a session"); + tokio::spawn(async move { + let _ = connection.await; + }); + client + .simple_query("SET cross_shard_txn = 'strict'") + .await + .expect("SET cross_shard_txn = strict"); + client +} + +#[tokio::test(flavor = "multi_thread", worker_threads = 6)] +async fn a_txn_whose_leader_dies_between_vote_and_redo_commits_once() { + let (col_a, col_b) = + distinct_vshard_collections("calvin_leader_change_0", "calvin_leader_change"); + let mut fx = Fixture::spawn(&[keyed_ddl(&col_a), keyed_ddl(&col_b)]).await; + fx.wait_group_mounted(&col_a).await; + fx.wait_group_mounted(&col_b).await; + + let victim = leader_index_of(&fx.cluster, &col_a); + let session_node = (0..fx.cluster.nodes.len()) + .find(|idx| *idx != victim) + .expect("a 3-node cluster has a node besides the leader"); + let victim_leads_sequencer = leader_of( + &fx.cluster.nodes[session_node], + nodedb_cluster::calvin::SEQUENCER_GROUP_ID, + ) == fx.cluster.nodes[victim].node_id; + let session = strict_session(&fx.cluster.nodes[session_node]).await; + let txn = format!( + "BEGIN; \ + INSERT INTO {col_a} (id, v) VALUES ('k1', 'a'); \ + INSERT INTO {col_b} (id, v) VALUES ('k2', 'b'); \ + COMMIT" + ); + + let gate_dir = tempfile::tempdir().expect("gate directory"); + let release = gate_dir.path().join("release"); + let parked = gate_dir.path().join("release.parked"); + let committed = { + let _gate = FailGuard::install( + &format!("calvin::before_redo_propose::{col_a}"), + FailAction::WaitForFile(release.clone()), + ); + let commit = session.simple_query(&txn); + let change_leader = async { + wait_for( + "the leader holds the slice's redo after its vote", + HOLD_DEADLINE, + Duration::from_millis(20), + || parked.exists(), + ) + .await; + kill_node(&mut fx.cluster, victim).await; + std::fs::write(&release, b"release").expect("release the held redo"); + }; + let (committed, ()) = tokio::join!(commit, change_leader); + committed + }; + + if victim_leads_sequencer { + let error = committed.expect_err("the submit's answer died with the sequencer leader"); + let code = error.code().map(|code| code.code().to_owned()); + assert_eq!( + code.as_deref(), + Some("57014"), + "a lost submit answer reports an unknown outcome, got: {error:?}" + ); + } else { + committed.expect("the transaction commits under the new leader"); + } + fx.converge().await; + for coll in [&col_a, &col_b] { + fx.wait_rows_on_every_node(&format!("SELECT id FROM {coll}"), 1) + .await; + } + + fx.cluster.shutdown().await; +} diff --git a/nodedb-cluster-tests/tests/common_suite/cases/calvin_multi_shard_bitemporal_best_effort_restart.rs b/nodedb-cluster-tests/tests/common_suite/cases/calvin_multi_shard_bitemporal_best_effort_restart.rs index 650e59af1..a3a051e47 100644 --- a/nodedb-cluster-tests/tests/common_suite/cases/calvin_multi_shard_bitemporal_best_effort_restart.rs +++ b/nodedb-cluster-tests/tests/common_suite/cases/calvin_multi_shard_bitemporal_best_effort_restart.rs @@ -175,7 +175,7 @@ async fn calvin_multi_shard_bitemporal_best_effort_commit_survives_wal_only_rest // ONE `simple_query` call carrying the whole transaction: the two INSERTs // are buffered during the block, and on COMMIT `classify_dispatch` sees // the full task set spanning two vShards → MultiShard → best-effort fans - // out into two independent single-vShard Calvin flushes. + // out into two independent single-vShard Calvin transactions. let txn_sql = format!( "BEGIN; \ INSERT INTO {coll_a} (id, value) VALUES ('a1', 'va'); \ @@ -199,7 +199,7 @@ async fn calvin_multi_shard_bitemporal_best_effort_commit_survives_wal_only_rest .await; // Pre-restart: both rows are visible and each carries exactly one - // system-time version. The Calvin flush lands asynchronously after the + // system-time version. The redo install lands asynchronously after the // completion ack, so poll. let (stamp_a, stamp_b); let deadline = std::time::Instant::now() + Duration::from_secs(10); diff --git a/nodedb-cluster-tests/tests/common_suite/cases/calvin_multi_shard_bitemporal_restart.rs b/nodedb-cluster-tests/tests/common_suite/cases/calvin_multi_shard_bitemporal_restart.rs index 67d690f1c..3214c4a7a 100644 --- a/nodedb-cluster-tests/tests/common_suite/cases/calvin_multi_shard_bitemporal_restart.rs +++ b/nodedb-cluster-tests/tests/common_suite/cases/calvin_multi_shard_bitemporal_restart.rs @@ -171,8 +171,8 @@ async fn calvin_multi_shard_bitemporal_commit_survives_wal_only_restart() { // ONE `simple_query` call carrying the whole transaction: the two INSERTs // are buffered during the block, and on COMMIT `classify_dispatch` sees - // the full task set spanning two vShards → MultiShard → leader-routed - // Calvin flush. + // the full task set spanning two vShards → MultiShard → one Calvin + // transaction. let txn_sql = format!( "BEGIN; \ INSERT INTO {coll_a} (id, value) VALUES ('a1', 'va'); \ @@ -193,7 +193,7 @@ async fn calvin_multi_shard_bitemporal_commit_survives_wal_only_restart() { .await; // Pre-restart: both rows are visible and each carries exactly one - // system-time version. The Calvin flush lands asynchronously after the + // system-time version. The redo install lands asynchronously after the // completion ack, so poll. let (stamp_a, stamp_b); let deadline = std::time::Instant::now() + Duration::from_secs(10); diff --git a/nodedb-cluster-tests/tests/common_suite/cases/calvin_multi_shard_redo_restart.rs b/nodedb-cluster-tests/tests/common_suite/cases/calvin_multi_shard_redo_restart.rs index 921409a28..c0a212ac2 100644 --- a/nodedb-cluster-tests/tests/common_suite/cases/calvin_multi_shard_redo_restart.rs +++ b/nodedb-cluster-tests/tests/common_suite/cases/calvin_multi_shard_redo_restart.rs @@ -180,7 +180,7 @@ async fn calvin_multi_shard_write_in_explicit_block_commits_and_survives_restart .await; // Pre-restart: both rows are visible and the vector is in the live HNSW. - // The Calvin flush lands asynchronously after the completion ack, so poll. + // The redo install lands asynchronously after the completion ack, so poll. let deadline = std::time::Instant::now() + Duration::from_secs(10); loop { let kv_v = read_once_a_leader_exists( diff --git a/nodedb-cluster-tests/tests/common_suite/cases/calvin_multishard_pk_read.rs b/nodedb-cluster-tests/tests/common_suite/cases/calvin_multishard_pk_read.rs index cfbe7fd9b..6790a08e1 100644 --- a/nodedb-cluster-tests/tests/common_suite/cases/calvin_multishard_pk_read.rs +++ b/nodedb-cluster-tests/tests/common_suite/cases/calvin_multishard_pk_read.rs @@ -43,7 +43,7 @@ async fn cross_shard_transaction_commit_reports_commit_and_rows_land() { assert_eq!( tags(&mut conn, "COMMIT").await, vec!["COMMIT"], - "the Calvin flush answers exactly one COMMIT tag" + "the Calvin commit answers exactly one COMMIT tag" ); fx.converge().await; diff --git a/nodedb-cluster-tests/tests/common_suite/cases/calvin_old_term_redo_copy.rs b/nodedb-cluster-tests/tests/common_suite/cases/calvin_old_term_redo_copy.rs new file mode 100644 index 000000000..ac5efb88c --- /dev/null +++ b/nodedb-cluster-tests/tests/common_suite/cases/calvin_old_term_redo_copy.rs @@ -0,0 +1,308 @@ +// SPDX-License-Identifier: BUSL-1.1 + +//! A Calvin slice's redo that its leader proposed, and that commits only in +//! the next term under a new leader, installs once. The new leader stages +//! nothing for that position. +//! +//! 1. A 3-node cluster holds `accts`, whose `balance` is the materialized +//! sum of `entries.amount`, and `entries`, in distinct data groups. +//! `acc-0` starts at 100. Every data group is led by its preferred +//! leader, so the leader balance moves nothing by itself. +//! 2. The fail gate `calvin::before_redo_propose::` holds the +//! entries slice after its vote and resolve, on the leader `L` of the +//! entries group `G`. The balance slice installs meanwhile. +//! 3. The test arms `raft::hold_commit::group` on node `L` only, then +//! releases the redo. `L` proposes it, the entry replicates to every replica, and +//! `L` commits nothing. +//! 4. The test moves `G`'s leadership to another replica `N`. The entry of +//! the old term commits with `N`'s first entry of the new term. `N` +//! opens its stage gate only once that entry applied, so it finds the +//! position applied and stages nothing. +//! 5. The client's COMMIT succeeds once. No node resolved a commit or +//! restaged after the hold, no node applied a second copy, each replica +//! of `G` installed the slice once, and `acc-0` reads 125, the value of +//! a single install. +//! +//! Requires `--features failpoints`. + +#![cfg(feature = "failpoints")] + +use std::sync::atomic::Ordering; +use std::time::Duration; + +use nodedb_test_support::fail_point::{FailAction, FailGuard}; + +use super::calvin_multishard_fixture::{Fixture, tags}; +use super::calvin_replica_content::{ + first_column, run_retrying, sum_ddl, vshard_of, wait_replicas_agree, +}; +use crate::common::cluster_harness::shared_steps::{group_members, group_status, name_where}; +use crate::common::cluster_harness::{ + GroupLeader, TestCluster, TestClusterNode, wait_for, wait_for_async, +}; + +/// How long each step of the hold gets. +const STEP_DEADLINE: Duration = Duration::from_secs(30); + +/// Pause between two polls of a step. +const STEP_POLL: Duration = Duration::from_millis(20); + +/// One node's Calvin counters. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +struct Counters { + /// Committed slices this node's leaders resolved. + commits: u64, + /// Restages this node's leaders dispatched. + restages: u64, + /// Stamped redo copies this node applied that installed nothing. + skipped_copies: u64, + /// Stamped redo installs on this node. + installs: u64, +} + +impl Counters { + fn of(node: &TestClusterNode) -> Self { + let counters = &node.shared.calvin.counters; + Self { + commits: counters.commits_flushed.load(Ordering::Relaxed), + restages: counters.stage_restages.load(Ordering::Relaxed), + skipped_copies: counters.redo_copies_skipped.load(Ordering::Relaxed), + installs: counters.write_versions_recorded.load(Ordering::Relaxed), + } + } +} + +/// The served balance of `acc-0`, `None` while no leader serves it. +async fn balance(node: &TestClusterNode, accts: &str) -> Option { + let sql = format!("SELECT balance FROM {accts} WHERE id = 'acc-0'"); + let rows = node.client.simple_query(&sql).await.ok()?; + rows.into_iter().find_map(|msg| match msg { + tokio_postgres::SimpleQueryMessage::Row(row) => row.get(0).map(str::to_owned), + _ => None, + }) +} + +/// The index in `cluster.nodes` of node `node_id`. +fn index_of(cluster: &TestCluster, node_id: u64) -> usize { + cluster + .nodes + .iter() + .position(|node| node.node_id == node_id) + .unwrap_or_else(|| panic!("node {node_id} is live")) +} + +#[tokio::test(flavor = "multi_thread", worker_threads = 6)] +async fn an_old_term_redo_copy_committed_by_the_new_leader_installs_once() { + let fx = Fixture::from_cluster(TestCluster::spawn_three().await.expect("3-node cluster")).await; + let probe = &fx.cluster.nodes[0]; + let entries = name_where("old_term_entries", |name| { + probe.group_id_for_collection(name).is_some() + }); + let group = probe + .group_id_for_collection(&entries) + .expect("the entries vShard maps to a group"); + let accts = name_where("old_term_accts", |name| { + vshard_of(name) != vshard_of(&entries) + && probe + .group_id_for_collection(name) + .is_some_and(|other| other != group) + }); + let [accts_ddl, entries_ddl, sum] = sum_ddl(&entries, &accts); + fx.create(&[accts_ddl, entries_ddl]).await; + fx.cluster + .exec_ddl_on_any_leader(&sum) + .await + .unwrap_or_else(|e| panic!("{sum}: {e}")); + for collection in [&accts, &entries] { + fx.wait_group_mounted(collection).await; + } + run_retrying( + &fx.cluster.nodes[0].client, + "seed the account", + &format!("INSERT INTO {accts} (id, owner, balance) VALUES ('acc-0', 'seed', '100')"), + ) + .await; + fx.converge().await; + fx.cluster.wait_for_preferred_leaders(STEP_DEADLINE).await; + + let mut session = fx.raw().await; + let txn = format!( + "BEGIN; \ + INSERT INTO {entries} (id, account_id, amount) VALUES ('e1', 'acc-0', '25'); \ + COMMIT" + ); + let gate_dir = tempfile::tempdir().expect("gate directory"); + let propose_release = gate_dir.path().join("propose"); + let propose_parked = gate_dir.path().join("propose.parked"); + let commit_release = gate_dir.path().join("commit"); + let (committed, before) = { + let _propose_gate = FailGuard::install( + &format!("calvin::before_redo_propose::{entries}"), + FailAction::WaitForFile(propose_release.clone()), + ); + let commit = tags(&mut session, &txn); + let move_leader = async { + wait_for( + "the entries leader holds the slice's redo after its resolve", + STEP_DEADLINE, + STEP_POLL, + || propose_parked.exists(), + ) + .await; + let old_leader = fx + .cluster + .wait_for_group_leader(group, GroupLeader::Any, STEP_DEADLINE) + .await; + let old = index_of(&fx.cluster, old_leader); + // The balance slice installs on every replica while the entries + // slice is held. Every count below then belongs to the held slice. + let coordinator = fx.coordinator(); + let accts = accts.as_str(); + wait_for_async( + "the balance slice installs", + STEP_DEADLINE, + STEP_POLL, + || async move { balance(coordinator, accts).await.as_deref() == Some("125") }, + ) + .await; + fx.converge().await; + let before: Vec = fx.cluster.nodes.iter().map(Counters::of).collect(); + let held_status = group_status(&fx.cluster.nodes[old], group) + .expect("the old leader hosts the entries group"); + + let _commit_gate = FailGuard::for_node( + old_leader, + &format!("raft::hold_commit::group{group}"), + FailAction::WaitForFile(commit_release.clone()), + ); + std::fs::write(&propose_release, b"release").expect("release the held redo"); + let new_leader = group_members(&fx.cluster.nodes[0], group) + .into_iter() + .find(|member| *member != old_leader) + .expect("the entries group has a replica besides its leader"); + let new = index_of(&fx.cluster, new_leader); + let mut redo_index = 0; + wait_for( + "the old leader appends the redo, and the new leader holds it, uncommitted", + STEP_DEADLINE, + STEP_POLL, + || { + let (Some(old_status), Some(new_status)) = ( + group_status(&fx.cluster.nodes[old], group), + group_status(&fx.cluster.nodes[new], group), + ) else { + return false; + }; + redo_index = old_status.last_log_index; + old_status.last_log_index > held_status.last_log_index + && old_status.commit_index < old_status.last_log_index + && new_status.last_log_index >= old_status.last_log_index + }, + ) + .await; + + fx.cluster.transfer_leadership(group, new_leader).await; + let new_status = group_status(&fx.cluster.nodes[new], group) + .expect("the new leader hosts the entries group"); + assert!( + new_status.term > held_status.term, + "the new leader leads a later term" + ); + std::fs::write(&commit_release, b"release").expect("release the commit gate"); + wait_for( + "the redo of the old term commits under the new leader", + STEP_DEADLINE, + STEP_POLL, + || { + group_status(&fx.cluster.nodes[new], group) + .is_some_and(|status| status.commit_index >= redo_index) + }, + ) + .await; + before + }; + tokio::join!(commit, move_leader) + }; + + assert_eq!( + committed + .iter() + .filter(|tag| tag.as_str() == "COMMIT") + .count(), + 1, + "the client's COMMIT answers once, found tags {committed:?}" + ); + assert_eq!( + committed.last().map(String::as_str), + Some("COMMIT"), + "the transaction ends committed, found tags {committed:?}" + ); + + fx.converge().await; + let replicas: Vec = fx + .cluster + .nodes + .iter() + .map(|node| node.replicates_data_group(group)) + .collect(); + wait_for( + "every replica of the entries group installs the slice", + STEP_DEADLINE, + STEP_POLL, + || { + fx.cluster + .nodes + .iter() + .zip(&before) + .zip(&replicas) + .all(|((node, before), replica)| { + !replica || Counters::of(node).installs > before.installs + }) + }, + ) + .await; + for ((node, before), replica) in fx.cluster.nodes.iter().zip(&before).zip(&replicas) { + let after = Counters::of(node); + assert_eq!( + after.installs - before.installs, + u64::from(*replica), + "node {} installs the slice once when it replicates the entries group", + node.node_id + ); + assert_eq!( + after.skipped_copies, before.skipped_copies, + "node {} applies no second copy of the redo", + node.node_id + ); + assert_eq!( + after.commits, before.commits, + "node {} resolves no commit after the hold: no leader staged the position again", + node.node_id + ); + assert_eq!( + after.restages, before.restages, + "node {} restages nothing", + node.node_id + ); + } + + let collections = [accts.as_str(), entries.as_str()]; + wait_replicas_agree(&fx.cluster, &collections).await; + let client = &fx.coordinator().client; + assert_eq!( + first_column( + client, + &format!("SELECT balance FROM {accts} WHERE id = 'acc-0'") + ) + .await, + vec!["125".to_owned()], + "100 + 25 lands once; a second install reads 150" + ); + assert_eq!( + first_column(client, &format!("SELECT id FROM {entries}")).await, + vec!["e1".to_owned()], + "the entries slice wrote its row once" + ); + + fx.cluster.shutdown().await; +} diff --git a/nodedb-cluster-tests/tests/common_suite/cases/calvin_read_version_leader_move.rs b/nodedb-cluster-tests/tests/common_suite/cases/calvin_read_version_leader_move.rs new file mode 100644 index 000000000..41fb4f3d1 --- /dev/null +++ b/nodedb-cluster-tests/tests/common_suite/cases/calvin_read_version_leader_move.rs @@ -0,0 +1,211 @@ +// SPDX-License-Identifier: BUSL-1.1 + +//! A transaction's read validates at whichever replica leads its vShard at +//! COMMIT. +//! +//! A transaction reads a row of `reads`, then writes `writes` on another +//! vShard, so its COMMIT runs through Calvin with the read's vShard as a +//! read-only participant. The read's version is the data-group log position +//! of the last write to `reads`, the same on every replica. +//! +//! The read runs while a non-preferred replica leads the read's data group. +//! Before the COMMIT, leadership moves back to the preferred replica. The +//! leader balance never moves a group away from its preferred leader, so that +//! replica, never the one that served the read, validates it. +//! +//! - No write to `reads` after the read: the validating leader finds the +//! read current, and the COMMIT succeeds. +//! - A write to the read row committed after the read: its log position is +//! above the read's version, so the validating leader aborts the COMMIT +//! with SQLSTATE 40001. + +use std::time::Duration; + +use super::calvin_multishard_fixture::{Fixture, data_rows, keyed_ddl, row_count}; +use super::calvin_replica_content::{names_on_distinct_vshards, run_retrying, strict_session}; +use crate::common::cluster_harness::GroupLeader; +use crate::common::cluster_harness::shared_steps::{db_detail, group_members, group_of}; + +/// SQLSTATE of a transaction aborted by a conflict. +const SERIALIZATION_FAILURE: &str = "40001"; + +/// How long a group gets to show a leader. +const LEADER_WAIT: Duration = Duration::from_secs(30); + +/// Attempts at a read served by the non-preferred leader. The leader balance +/// can move the group back between the move and the read. The attempt then +/// rolls back and starts again. +const READ_ATTEMPTS: usize = 10; + +/// A 3-node cluster holding `reads` and `writes` on distinct vShards, with +/// row `r1` committed in `reads` and replicated everywhere. +async fn seeded(prefixes: [&str; 2]) -> (Fixture, String, String) { + let [reads, writes] = names_on_distinct_vshards(prefixes); + let fx = Fixture::spawn(&[keyed_ddl(&reads), keyed_ddl(&writes)]).await; + for collection in [&reads, &writes] { + fx.wait_group_mounted(collection).await; + } + run_retrying( + &fx.cluster.nodes[0].client, + "seed the read row", + &format!("INSERT INTO {reads} (id, v) VALUES ('r1', 'seed')"), + ) + .await; + fx.converge().await; + (fx, reads, writes) +} + +/// The leaders a case moves `reads`' data group between. +struct Leaders { + group: u64, + /// The preferred leader, which validates the read at COMMIT. + validating: u64, + /// Another replica, which serves the read. + serving: u64, +} + +/// Settle every group on its preferred leader and pick the two leaders of +/// `collection`'s data group. +async fn leaders_of(fx: &Fixture, collection: &str) -> Leaders { + fx.cluster.wait_for_preferred_leaders(LEADER_WAIT).await; + let group = group_of(&fx.cluster.nodes[0], collection); + let validating = fx + .cluster + .wait_for_group_leader(group, GroupLeader::Any, LEADER_WAIT) + .await; + let serving = group_members(&fx.cluster.nodes[0], group) + .into_iter() + .find(|member| *member != validating) + .unwrap_or_else(|| panic!("group {group} has a replica besides its leader {validating}")); + Leaders { + group, + validating, + serving, + } +} + +/// Open a transaction on the coordinator whose read of `r1` from `reads` +/// the serving leader answered, and buffer a write to `writes`. Return its +/// session. +async fn read_on_serving_leader( + fx: &Fixture, + leaders: &Leaders, + reads: &str, + writes: &str, +) -> tokio_postgres::Client { + let session = strict_session(fx.coordinator()).await; + for _ in 0..READ_ATTEMPTS { + fx.cluster + .transfer_leadership(leaders.group, leaders.serving) + .await; + session + .simple_query("BEGIN") + .await + .unwrap_or_else(|e| panic!("BEGIN: {}", db_detail(&e))); + let rows = session + .simple_query(&format!("SELECT v FROM {reads} WHERE id = 'r1'")) + .await + .unwrap_or_else(|e| panic!("in-transaction read: {}", db_detail(&e))); + assert_eq!(data_rows(&rows), 1, "the transaction reads the seeded row"); + if fx.cluster.group_leader(leaders.group) != Some(leaders.serving) { + session + .simple_query("ROLLBACK") + .await + .unwrap_or_else(|e| panic!("ROLLBACK: {}", db_detail(&e))); + continue; + } + session + .simple_query(&format!( + "INSERT INTO {writes} (id, v) VALUES ('w1', 'txn')" + )) + .await + .unwrap_or_else(|e| panic!("buffer the write: {}", db_detail(&e))); + return session; + } + panic!( + "node {} did not hold the leadership of group {} across a read in {READ_ATTEMPTS} attempts", + leaders.serving, leaders.group + ); +} + +/// Move the leadership back to the preferred leader, which then validates +/// the read at COMMIT. +async fn hand_back(fx: &Fixture, leaders: &Leaders) { + fx.cluster + .transfer_leadership(leaders.group, leaders.validating) + .await; + fx.cluster + .wait_for_group_leader( + leaders.group, + GroupLeader::Node(leaders.validating), + LEADER_WAIT, + ) + .await; +} + +#[tokio::test(flavor = "multi_thread", worker_threads = 6)] +async fn a_current_read_commits_after_its_leader_moves() { + let (fx, reads, writes) = seeded(["lvm_ok_reads", "lvm_ok_writes"]).await; + let leaders = leaders_of(&fx, &reads).await; + let session = read_on_serving_leader(&fx, &leaders, &reads, &writes).await; + hand_back(&fx, &leaders).await; + + session.simple_query("COMMIT").await.unwrap_or_else(|e| { + panic!( + "a read the serving leader answered is current at the validating \ + leader, so the commit succeeds: {}", + db_detail(&e) + ) + }); + fx.converge().await; + assert_eq!( + row_count( + &fx.coordinator().client, + &format!("SELECT id FROM {writes} WHERE id = 'w1'") + ) + .await, + 1, + "the committed transaction's write is visible" + ); + + fx.cluster.shutdown().await; +} + +#[tokio::test(flavor = "multi_thread", worker_threads = 6)] +async fn a_write_after_the_read_aborts_the_commit_after_its_leader_moves() { + let (fx, reads, writes) = seeded(["lvm_stale_reads", "lvm_stale_writes"]).await; + let leaders = leaders_of(&fx, &reads).await; + let session = read_on_serving_leader(&fx, &leaders, &reads, &writes).await; + hand_back(&fx, &leaders).await; + let writer = (fx.coordinator + 1) % fx.cluster.nodes.len(); + run_retrying( + &fx.cluster.nodes[writer].client, + "a conflicting write to the read row", + &format!("UPDATE {reads} SET v = 'moved' WHERE id = 'r1'"), + ) + .await; + fx.converge().await; + + let error = session + .simple_query("COMMIT") + .await + .expect_err("a write committed after the read makes the read stale"); + assert_eq!( + error.as_db_error().map(|db| db.code().code()), + Some(SERIALIZATION_FAILURE), + "the stale read aborts the commit with a serialization failure: {}", + db_detail(&error) + ); + fx.converge().await; + assert_eq!( + row_count( + &fx.coordinator().client, + &format!("SELECT id FROM {writes} WHERE id = 'w1'") + ) + .await, + 0, + "the aborted transaction's write is not visible" + ); + + fx.cluster.shutdown().await; +} diff --git a/nodedb-cluster-tests/tests/common_suite/cases/calvin_remote_participant_report.rs b/nodedb-cluster-tests/tests/common_suite/cases/calvin_remote_participant_report.rs index 929a7bafe..b252c95f9 100644 --- a/nodedb-cluster-tests/tests/common_suite/cases/calvin_remote_participant_report.rs +++ b/nodedb-cluster-tests/tests/common_suite/cases/calvin_remote_participant_report.rs @@ -33,8 +33,8 @@ use crate::common; use common::cluster_harness::shared_steps::{fail_stopped, sequencer_admitted}; use common::cluster_harness::{TestCluster, TestClusterNode, wait_for}; +use nodedb_test_support::fail_point::{FailAction, FailGuard}; use nodedb_test_support::native_harness::send_sql; -use nodedb_types::fail_point::{FailAction, FailGuard}; use nodedb_types::{DatabaseId, TenantId}; use super::ts_native_ingest::{ diff --git a/nodedb-cluster-tests/tests/common_suite/cases/calvin_replica_content.rs b/nodedb-cluster-tests/tests/common_suite/cases/calvin_replica_content.rs new file mode 100644 index 000000000..2fc5faec6 --- /dev/null +++ b/nodedb-cluster-tests/tests/common_suite/cases/calvin_replica_content.rs @@ -0,0 +1,320 @@ +// SPDX-License-Identifier: BUSL-1.1 + +//! Per-replica stored content for the Calvin redo convergence cases. +//! +//! A replica's content comes from the tenant snapshot of each Data-Plane +//! core of its node. The read never leaves the node, so it shows the rows +//! that replica installed, never a row a gateway fetched from a leader. +//! +//! A collection homes to one vShard. The content of a vShard is every +//! document and index entry of its collections: storage key and stored +//! bytes, in key order. Two replicas that installed the same entries in the +//! same order hold equal content. + +use std::collections::{BTreeMap, BTreeSet}; +use std::time::Duration; + +use nodedb::types::{DatabaseId, TenantId}; + +use crate::common::cluster_harness::shared_steps::{db_detail, key_collection, name_where}; +use crate::common::cluster_harness::{TestCluster, TestClusterNode, is_no_serving_leader}; + +/// The tenant the `nodedb` pgwire user writes as. +const TENANT: u64 = 1; + +/// SQLSTATE of a transaction aborted by a conflict. It wrote nothing. +const SERIALIZATION_FAILURE: &str = "40001"; + +/// How long a refused statement is retried. +const RETRY_DEADLINE: Duration = Duration::from_secs(30); + +/// Pause between two attempts of a refused statement. +const RETRY_BACKOFF: Duration = Duration::from_millis(50); + +/// How long the replicas get to agree after the cluster quiesced. +const AGREE_DEADLINE: Duration = Duration::from_secs(30); + +/// Pause between two content reads while the replicas disagree. +const AGREE_POLL: Duration = Duration::from_millis(200); + +/// The most differing keys a disagreement report names per vShard. +const REPORT_KEYS: usize = 10; + +/// One vShard's stored content on one replica, keyed by table and storage +/// key. +pub(super) type VShardContent = BTreeMap>; + +/// The vShard `collection` in the default database homes to. +pub(super) fn vshard_of(collection: &str) -> u32 { + nodedb_cluster::routing::vshard_for_collection(nodedb_types::CollectionKey::from_bare( + DatabaseId::DEFAULT, + collection, + )) +} + +/// One collection name per prefix, `{prefix}_{i}` for the lowest `i` whose +/// vShard differs from the vShard of every name picked before it. +pub(super) fn names_on_distinct_vshards(prefixes: [&str; N]) -> [String; N] { + let mut taken: Vec = Vec::with_capacity(N); + prefixes.map(|prefix| { + let name = name_where(prefix, |candidate| !taken.contains(&vshard_of(candidate))); + taken.push(vshard_of(&name)); + name + }) +} + +/// A fresh session on `node` in strict cross-shard mode. +pub(super) async fn strict_session(node: &TestClusterNode) -> tokio_postgres::Client { + let (client, connection) = tokio_postgres::connect( + &format!( + "host=127.0.0.1 port={} user=nodedb dbname=default", + node.pg_addr.port() + ), + tokio_postgres::NoTls, + ) + .await + .expect("connect a session"); + tokio::spawn(async move { + let _ = connection.await; + }); + client + .simple_query("SET cross_shard_txn = 'strict'") + .await + .expect("SET cross_shard_txn = strict"); + client +} + +/// Run `sql` through `client` as one request. A `40001` refusal aborted the +/// transaction and wrote nothing, and a missing leader refused it before it +/// ran: either runs again until [`RETRY_DEADLINE`]. Any other error panics +/// with `what`, the SQLSTATE and the detail. +pub(super) async fn run_retrying(client: &tokio_postgres::Client, what: &str, sql: &str) { + let deadline = tokio::time::Instant::now() + RETRY_DEADLINE; + loop { + let error = match client.simple_query(sql).await { + Ok(_) => return, + Err(error) => error, + }; + let retryable = is_no_serving_leader(&error) + || error + .as_db_error() + .is_some_and(|db| db.code().code() == SERIALIZATION_FAILURE); + if !retryable || tokio::time::Instant::now() >= deadline { + panic!("{what}: {}", db_detail(&error)); + } + // A failed block can leave the session in an aborted transaction. + // Outside one, ROLLBACK only warns, so its result does not matter. + let _ = client.simple_query("ROLLBACK").await; + tokio::time::sleep(RETRY_BACKOFF).await; + } +} + +/// Create `target` and `source` as strict collections and declare +/// `target.balance` as the materialized sum of `source.amount` over +/// `source.account_id = target.id`. +pub(super) fn sum_ddl(source: &str, target: &str) -> [String; 3] { + [ + format!( + "CREATE COLLECTION {target} (id TEXT PRIMARY KEY, owner TEXT) \ + WITH (engine='document_strict')" + ), + format!( + "CREATE COLLECTION {source} (id TEXT PRIMARY KEY, account_id TEXT, amount TEXT) \ + WITH (engine='document_strict')" + ), + format!( + "ALTER COLLECTION {target} ADD COLUMN balance TEXT \ + MATERIALIZED_SUM SOURCE {source} \ + ON {source}.account_id = {target}.id VALUE {source}.amount" + ), + ] +} + +/// The first column of every row `sql` returns through `client`. +pub(super) async fn first_column(client: &tokio_postgres::Client, sql: &str) -> Vec { + client + .simple_query(sql) + .await + .unwrap_or_else(|e| panic!("`{sql}`: {}", db_detail(&e))) + .into_iter() + .filter_map(|msg| match msg { + tokio_postgres::SimpleQueryMessage::Row(row) => row.get(0).map(str::to_owned), + _ => None, + }) + .collect() +} + +/// A served number column as `f64`. +fn number(text: &str, what: &str) -> f64 { + text.parse() + .unwrap_or_else(|e| panic!("{what} `{text}` is not a number: {e}")) +} + +/// Each of the `accounts` rows `acc-0` to `acc-{accounts - 1}` of `accts` +/// serves a balance equal to the sum of its served `entries` amounts. +pub(super) async fn assert_balances_match_entries( + client: &tokio_postgres::Client, + accts: &str, + entries: &str, + accounts: usize, +) { + for account in 0..accounts { + let balances = first_column( + client, + &format!("SELECT balance FROM {accts} WHERE id = 'acc-{account}'"), + ) + .await; + let [balance] = balances.as_slice() else { + panic!("acc-{account} has one row, found balances {balances:?}"); + }; + let amounts = first_column( + client, + &format!("SELECT amount FROM {entries} WHERE account_id = 'acc-{account}'"), + ) + .await; + let expected: f64 = amounts.iter().map(|a| number(a, "amount")).sum(); + assert_eq!( + number(balance, "balance"), + expected, + "acc-{account}'s balance is the sum of its entries {amounts:?}" + ); + } +} + +/// The content of each vShard `collections` home to, as `node` stores it. +pub(super) async fn local_content( + node: &TestClusterNode, + collections: &[&str], +) -> BTreeMap { + let mut by_vshard: BTreeMap = collections + .iter() + .map(|collection| (vshard_of(collection), VShardContent::new())) + .collect(); + for core in 0..node.num_cores() { + let snapshot = node + .tenant_snapshot_on_core(core, TenantId::new(TENANT)) + .await; + for (table, entries) in [("doc", snapshot.documents), ("idx", snapshot.indexes)] { + for (key, value) in entries { + let Some(collection) = key_collection(&key) else { + continue; + }; + if !collections.contains(&collection) { + continue; + } + let vshard = vshard_of(collection); + if let Some(content) = by_vshard.get_mut(&vshard) { + content.insert(format!("{table}:{key}"), value); + } + } + } + } + by_vshard +} + +/// The indexes in `cluster.nodes` of the replicas of `collection`'s data +/// group. Panics when fewer than two nodes replicate it: one replica agrees +/// with itself and proves nothing. +fn replicas_of(cluster: &TestCluster, collection: &str) -> Vec { + let replicas: Vec = cluster + .nodes + .iter() + .enumerate() + .filter(|(_, node)| { + node.group_id_for_collection(collection) + .is_some_and(|group| node.replicates_data_group(group)) + }) + .map(|(idx, _)| idx) + .collect(); + assert!( + replicas.len() >= 2, + "{collection}'s data group has {} replica(s); the comparison needs two or more", + replicas.len() + ); + replicas +} + +/// Wait until every replica of each vShard `collections` home to holds the +/// same content, and return that content per vShard. Panics at the deadline +/// with the node ids and the keys that differ. +pub(super) async fn wait_replicas_agree( + cluster: &TestCluster, + collections: &[&str], +) -> BTreeMap { + let replicas: BTreeMap> = collections + .iter() + .map(|collection| (vshard_of(collection), replicas_of(cluster, collection))) + .collect(); + let deadline = tokio::time::Instant::now() + AGREE_DEADLINE; + loop { + let mut per_node = Vec::with_capacity(cluster.nodes.len()); + for node in &cluster.nodes { + per_node.push(local_content(node, collections).await); + } + let disagreements: Vec = replicas + .iter() + .filter_map(|(vshard, nodes)| disagreement(cluster, &per_node, *vshard, nodes)) + .collect(); + if disagreements.is_empty() { + return replicas + .iter() + .map(|(vshard, nodes)| { + let content = per_node[nodes[0]].get(vshard).cloned().unwrap_or_default(); + (*vshard, content) + }) + .collect(); + } + if tokio::time::Instant::now() >= deadline { + panic!( + "replicas still hold different content after quiesce:\n{}", + disagreements.join("\n") + ); + } + tokio::time::sleep(AGREE_POLL).await; + } +} + +/// A report of how `vshard`'s replicas `nodes` differ, `None` when they +/// agree. A key is listed when one replica lacks it or stores other bytes. +fn disagreement( + cluster: &TestCluster, + per_node: &[BTreeMap], + vshard: u32, + nodes: &[usize], +) -> Option { + let empty = VShardContent::new(); + let content = |idx: usize| per_node[idx].get(&vshard).unwrap_or(&empty); + let reference = content(nodes[0]); + let reference_id = cluster.nodes[nodes[0]].node_id; + let differing: Vec = nodes[1..] + .iter() + .filter(|idx| content(**idx) != reference) + .flat_map(|idx| { + let other = content(*idx); + let other_id = cluster.nodes[*idx].node_id; + let keys: BTreeSet<&String> = reference.keys().chain(other.keys()).collect(); + keys.into_iter() + .filter(|key| reference.get(*key) != other.get(*key)) + .map(|key| format!("node {reference_id} vs node {other_id}: {key}")) + .collect::>() + }) + .take(REPORT_KEYS) + .collect(); + (!differing.is_empty()).then(|| { + let sizes: Vec = nodes + .iter() + .map(|idx| { + format!( + "node {}: {} entries", + cluster.nodes[*idx].node_id, + content(*idx).len() + ) + }) + .collect(); + format!( + "vShard {vshard} ({}): {}", + sizes.join(", "), + differing.join("; ") + ) + }) +} diff --git a/nodedb-cluster-tests/tests/common_suite/cases/calvin_replica_convergence.rs b/nodedb-cluster-tests/tests/common_suite/cases/calvin_replica_convergence.rs new file mode 100644 index 000000000..78d9e5d09 --- /dev/null +++ b/nodedb-cluster-tests/tests/common_suite/cases/calvin_replica_convergence.rs @@ -0,0 +1,216 @@ +// SPDX-License-Identifier: BUSL-1.1 + +//! Concurrent fast-path writes and cross-shard Calvin transactions on the +//! same rows leave every replica of every vShard with the same content. +//! +//! A 3-node cluster holds four collections, each on its own vShard: +//! - `accts`: strict, its `balance` is the materialized sum of +//! `entries.amount` per account, +//! - `entries`: strict, the sum's source, +//! - `notes` and `scratch`: keyed. +//! +//! Three sessions on three nodes run at once: +//! - fast path: single-shard autocommit writes to `accts`, `notes` and +//! `scratch`, +//! - Calvin: transactions that insert and delete `entries` rows and update +//! the same `notes` rows, +//! - Calvin with TRUNCATE: transactions that truncate `scratch`, and once +//! `entries`, next to a `notes` insert. +//! +//! After quiesce, each vShard's documents and index entries are byte-equal +//! on every replica, read from each node's own cores. Each account's served +//! balance equals the sum of its served entries. + +use super::calvin_multishard_fixture::{Fixture, keyed_ddl}; +use super::calvin_replica_content::{ + assert_balances_match_entries, names_on_distinct_vshards, run_retrying, strict_session, + sum_ddl, vshard_of, wait_replicas_agree, +}; + +/// Rounds each session runs. +const ROUNDS: usize = 20; + +/// Accounts in `accts`, `acc-0` to `acc-{ACCOUNTS - 1}`. +const ACCOUNTS: usize = 3; + +/// Seeded `notes` rows, `n0` to `n{NOTES - 1}`. +const NOTES: usize = 5; + +/// The round whose TRUNCATE names `entries`. Every other round truncates +/// `scratch`. +const ENTRIES_TRUNCATE_ROUND: usize = ROUNDS / 2; + +/// The four collections of the case. +struct Names { + accts: String, + entries: String, + notes: String, + scratch: String, +} + +impl Names { + fn all(&self) -> [&str; 4] { + [ + self.accts.as_str(), + self.entries.as_str(), + self.notes.as_str(), + self.scratch.as_str(), + ] + } +} + +/// Single-shard autocommit writes: an `accts` column the sum leaves alone, +/// a `notes` row the Calvin session also updates, and a new `scratch` row. +async fn fast_path_writes(client: &tokio_postgres::Client, names: &Names) { + for k in 0..ROUNDS { + let account = k % ACCOUNTS; + let note = k % NOTES; + let statements = [ + format!( + "UPDATE {} SET owner = 'fp{k}' WHERE id = 'acc-{account}'", + names.accts + ), + format!( + "UPDATE {} SET v = 'fp{k}' WHERE id = 'n{note}'", + names.notes + ), + format!( + "INSERT INTO {} (id, v) VALUES ('fp{k}', 'fp')", + names.scratch + ), + ]; + for sql in &statements { + run_retrying(client, &format!("fast path round {k}"), sql).await; + } + } +} + +/// Cross-shard transactions: each inserts an entry, which moves its +/// account's balance on another vShard, and updates a `notes` row. Every +/// third round also deletes an earlier entry. +async fn calvin_writes(client: &tokio_postgres::Client, names: &Names) { + for k in 0..ROUNDS { + let account = k % ACCOUNTS; + let note = k % NOTES; + let amount = k + 1; + let delete = if k >= 3 && k % 3 == 0 { + format!("DELETE FROM {} WHERE id = 'e{}'; ", names.entries, k - 3) + } else { + String::new() + }; + let txn = format!( + "BEGIN; \ + INSERT INTO {entries} (id, account_id, amount) \ + VALUES ('e{k}', 'acc-{account}', '{amount}'); \ + UPDATE {notes} SET v = 'cv{k}' WHERE id = 'n{note}'; \ + {delete}\ + COMMIT", + entries = names.entries, + notes = names.notes, + ); + run_retrying(client, &format!("calvin round {k}"), &txn).await; + } +} + +/// Cross-shard transactions that truncate a collection and insert a +/// `notes` row. Round [`ENTRIES_TRUNCATE_ROUND`] truncates `entries`, which +/// takes every truncated entry back off its account's balance. +async fn truncating_writes(client: &tokio_postgres::Client, names: &Names) { + for k in 0..ROUNDS { + let truncated = if k == ENTRIES_TRUNCATE_ROUND { + &names.entries + } else { + &names.scratch + }; + let txn = format!( + "BEGIN; \ + TRUNCATE {truncated}; \ + INSERT INTO {notes} (id, v) VALUES ('t{k}', 'tr'); \ + COMMIT", + notes = names.notes, + ); + run_retrying(client, &format!("truncate round {k}"), &txn).await; + } +} + +#[tokio::test(flavor = "multi_thread", worker_threads = 6)] +async fn replicas_converge_under_concurrent_fast_path_and_calvin_writes() { + let [accts, entries, notes, scratch] = + names_on_distinct_vshards(["conv_accts", "conv_entries", "conv_notes", "conv_scratch"]); + let names = Names { + accts, + entries, + notes, + scratch, + }; + let [accts_ddl, entries_ddl, sum] = sum_ddl(&names.entries, &names.accts); + let fx = Fixture::spawn(&[ + accts_ddl, + entries_ddl, + keyed_ddl(&names.notes), + keyed_ddl(&names.scratch), + ]) + .await; + fx.cluster + .exec_ddl_on_any_leader(&sum) + .await + .unwrap_or_else(|e| panic!("{sum}: {e}")); + for collection in names.all() { + fx.wait_group_mounted(collection).await; + } + + let seeder = &fx.cluster.nodes[0].client; + for account in 0..ACCOUNTS { + run_retrying( + seeder, + "seed an account", + &format!( + "INSERT INTO {} (id, owner, balance) VALUES ('acc-{account}', 'seed', '0')", + names.accts + ), + ) + .await; + } + for note in 0..NOTES { + run_retrying( + seeder, + "seed a note", + &format!( + "INSERT INTO {} (id, v) VALUES ('n{note}', 'seed')", + names.notes + ), + ) + .await; + } + fx.converge().await; + + let calvin = strict_session(&fx.cluster.nodes[1]).await; + let truncating = strict_session(&fx.cluster.nodes[2]).await; + tokio::join!( + fast_path_writes(&fx.cluster.nodes[0].client, &names), + calvin_writes(&calvin, &names), + truncating_writes(&truncating, &names), + ); + + fx.converge().await; + let content = wait_replicas_agree(&fx.cluster, &names.all()).await; + // The `entries` truncate can run after the last insert, so only the + // seeded collections are sure to hold rows. + for collection in [&names.accts, &names.notes] { + assert!( + content + .get(&vshard_of(collection)) + .is_some_and(|stored| !stored.is_empty()), + "{collection} holds rows after the workload; equal empty replicas prove nothing" + ); + } + assert_balances_match_entries( + &fx.coordinator().client, + &names.accts, + &names.entries, + ACCOUNTS, + ) + .await; + + fx.cluster.shutdown().await; +} diff --git a/nodedb-cluster-tests/tests/common_suite/cases/calvin_replica_held_apply.rs b/nodedb-cluster-tests/tests/common_suite/cases/calvin_replica_held_apply.rs new file mode 100644 index 000000000..5dc0bc526 --- /dev/null +++ b/nodedb-cluster-tests/tests/common_suite/cases/calvin_replica_held_apply.rs @@ -0,0 +1,317 @@ +// SPDX-License-Identifier: BUSL-1.1 + +//! A replica whose apply is held while fast-path writes and Calvin +//! transactions hit the same rows converges byte-equal once released, and +//! the hold touches that replica only. +//! +//! A 3-node cluster holds three collections, each on its own vShard: +//! - `accts`: strict, its `balance` is the materialized sum of +//! `entries.amount` per account, +//! - `entries`: strict, the sum's source, +//! - `notes`: keyed. +//! +//! The fail gate `funnel::before_dispatch::` is armed for one +//! follower `F` of the `notes` group, through a node-scoped guard. Every +//! node shares the one fail-point registry, so the scope alone keeps the +//! other nodes running. +//! +//! - A probe insert into `notes` commits. Every other replica of the group +//! installs it, and `F` still stores the `notes` content it had before. +//! - Two sessions on the other two nodes run at once: single-shard +//! autocommit writes to `accts` and `notes`, and Calvin transactions that +//! insert `entries` rows and update the same `notes` rows. Both finish +//! while `F` is held, and `F` still stores its old `notes` content. +//! - The gate releases. Each vShard's documents and index entries become +//! byte-equal on every replica, read from each node's own cores. Each +//! account's served balance equals the sum of its served entries, and no +//! node halted its Calvin apply. +//! +//! Requires `--features failpoints`. + +#![cfg(feature = "failpoints")] + +use std::path::Path; +use std::time::Duration; + +use nodedb_test_support::fail_point::{FailAction, FailGuard}; + +use super::calvin_dependent_read_fixture::assert_no_apply_halt; +use super::calvin_multishard_fixture::{Fixture, keyed_ddl}; +use super::calvin_replica_content::{ + VShardContent, assert_balances_match_entries, local_content, names_on_distinct_vshards, + run_retrying, strict_session, sum_ddl, vshard_of, wait_replicas_agree, +}; +use crate::common::cluster_harness::shared_steps::group_members; +use crate::common::cluster_harness::{GroupLeader, TestCluster, TestClusterNode, wait_for_async}; + +/// Rounds each session runs. +const ROUNDS: usize = 10; + +/// Accounts in `accts`, `acc-0` to `acc-{ACCOUNTS - 1}`. +const ACCOUNTS: usize = 3; + +/// Seeded `notes` rows, `n0` to `n{NOTES - 1}`. +const NOTES: usize = 4; + +/// How long each step around the hold gets. +const STEP_DEADLINE: Duration = Duration::from_secs(30); + +/// Pause between two polls of a step. +const STEP_POLL: Duration = Duration::from_millis(50); + +/// How long the workload gets to finish while `F` is held. +const HOLD_WINDOW: Duration = Duration::from_secs(90); + +/// The three collections of the case. +struct Names { + accts: String, + entries: String, + notes: String, +} + +impl Names { + fn all(&self) -> [&str; 3] { + [ + self.accts.as_str(), + self.entries.as_str(), + self.notes.as_str(), + ] + } +} + +/// Single-shard autocommit writes: an `accts` column the sum leaves alone, +/// and a `notes` row the Calvin session also updates. +async fn fast_path_writes(client: &tokio_postgres::Client, names: &Names) { + for k in 0..ROUNDS { + let account = k % ACCOUNTS; + let note = k % NOTES; + let statements = [ + format!( + "UPDATE {} SET owner = 'fp{k}' WHERE id = 'acc-{account}'", + names.accts + ), + format!( + "UPDATE {} SET v = 'fp{k}' WHERE id = 'n{note}'", + names.notes + ), + ]; + for sql in &statements { + run_retrying(client, &format!("fast path round {k}"), sql).await; + } + } +} + +/// Cross-shard transactions: each inserts an entry, which moves its +/// account's balance on another vShard, and updates a `notes` row. +async fn calvin_writes(client: &tokio_postgres::Client, names: &Names) { + for k in 0..ROUNDS { + let account = k % ACCOUNTS; + let note = k % NOTES; + let amount = k + 1; + let txn = format!( + "BEGIN; \ + INSERT INTO {entries} (id, account_id, amount) \ + VALUES ('e{k}', 'acc-{account}', '{amount}'); \ + UPDATE {notes} SET v = 'cv{k}' WHERE id = 'n{note}'; \ + COMMIT", + entries = names.entries, + notes = names.notes, + ); + run_retrying(client, &format!("calvin round {k}"), &txn).await; + } +} + +/// `node`'s stored content of the vShard `collection` homes to. +async fn vshard_content(node: &TestClusterNode, collection: &str) -> VShardContent { + local_content(node, &[collection]) + .await + .remove(&vshard_of(collection)) + .unwrap_or_default() +} + +/// The index in `cluster.nodes` of node `node_id`. +fn index_of(cluster: &TestCluster, node_id: u64) -> usize { + cluster + .nodes + .iter() + .position(|node| node.node_id == node_id) + .unwrap_or_else(|| panic!("node {node_id} is live")) +} + +/// Wait until the held apply parked at the gate, so `parked` exists, and +/// every node of `replicas` stores `collection` content other than `before`. +async fn wait_others_moved_on( + cluster: &TestCluster, + replicas: &[usize], + collection: &str, + before: &VShardContent, + parked: &Path, +) { + wait_for_async( + "the held follower parks, and every other replica installs the probe", + STEP_DEADLINE, + STEP_POLL, + || async move { + if !parked.exists() { + return false; + } + for idx in replicas { + if vshard_content(&cluster.nodes[*idx], collection).await == *before { + return false; + } + } + true + }, + ) + .await; +} + +#[tokio::test(flavor = "multi_thread", worker_threads = 6)] +async fn a_replica_held_under_fast_path_and_calvin_writes_converges_once_released() { + let [accts, entries, notes] = + names_on_distinct_vshards(["held_accts", "held_entries", "held_notes"]); + let names = Names { + accts, + entries, + notes, + }; + let [accts_ddl, entries_ddl, sum] = sum_ddl(&names.entries, &names.accts); + let fx = Fixture::spawn(&[accts_ddl, entries_ddl, keyed_ddl(&names.notes)]).await; + fx.cluster + .exec_ddl_on_any_leader(&sum) + .await + .unwrap_or_else(|e| panic!("{sum}: {e}")); + for collection in names.all() { + fx.wait_group_mounted(collection).await; + } + + let seeder = &fx.cluster.nodes[0].client; + for account in 0..ACCOUNTS { + run_retrying( + seeder, + "seed an account", + &format!( + "INSERT INTO {} (id, owner, balance) VALUES ('acc-{account}', 'seed', '0')", + names.accts + ), + ) + .await; + } + for note in 0..NOTES { + run_retrying( + seeder, + "seed a note", + &format!( + "INSERT INTO {} (id, v) VALUES ('n{note}', 'seed')", + names.notes + ), + ) + .await; + } + fx.converge().await; + // Stable leaders: a leadership move onto the held follower would make + // the workload wait for the hold. + fx.cluster.wait_for_preferred_leaders(STEP_DEADLINE).await; + + let group = fx.cluster.nodes[0] + .group_id_for_collection(&names.notes) + .expect("the notes data group"); + let leader = fx + .cluster + .wait_for_group_leader(group, GroupLeader::Any, STEP_DEADLINE) + .await; + let held_id = group_members(&fx.cluster.nodes[0], group) + .into_iter() + .find(|member| *member != leader) + .expect("the notes group has a follower"); + let held = index_of(&fx.cluster, held_id); + let others: Vec = (0..fx.cluster.nodes.len()) + .filter(|idx| *idx != held) + .collect(); + let &[fast_node, calvin_node] = others.as_slice() else { + panic!("a 3-node cluster has two nodes besides the held one, found {others:?}"); + }; + let replicas: Vec = others + .iter() + .copied() + .filter(|idx| fx.cluster.nodes[*idx].replicates_data_group(group)) + .collect(); + assert!( + !replicas.is_empty(), + "the notes group has a replica besides the held follower" + ); + let before = vshard_content(&fx.cluster.nodes[held], &names.notes).await; + + let gate_dir = tempfile::tempdir().expect("gate directory"); + let release = gate_dir.path().join("release"); + let parked = gate_dir.path().join("release.parked"); + let gate = FailGuard::for_node( + held_id, + &format!("funnel::before_dispatch::{}", names.notes), + FailAction::WaitForFile(release.clone()), + ); + + let fast = &fx.cluster.nodes[fast_node].client; + run_retrying( + fast, + "probe insert", + &format!("INSERT INTO {} (id, v) VALUES ('probe', 'p')", names.notes), + ) + .await; + wait_others_moved_on(&fx.cluster, &replicas, &names.notes, &before, &parked).await; + let held_after_probe = vshard_content(&fx.cluster.nodes[held], &names.notes).await; + + let calvin = strict_session(&fx.cluster.nodes[calvin_node]).await; + let mut workload = Box::pin(async { + tokio::join!( + fast_path_writes(fast, &names), + calvin_writes(&calvin, &names), + ) + }); + let finished_while_held = tokio::time::timeout(HOLD_WINDOW, workload.as_mut()) + .await + .is_ok(); + let held_after_workload = vshard_content(&fx.cluster.nodes[held], &names.notes).await; + + // Release before any assertion, so a failed one leaves no apply parked. + std::fs::write(&release, b"release").expect("release the held apply"); + drop(gate); + if !finished_while_held { + workload.as_mut().await; + } + drop(workload); + assert!( + held_after_probe == before && held_after_workload == before, + "node {held_id} installed a notes write while its apply was held: {} entries before, \ + {} after the probe, {} after the workload", + before.len(), + held_after_probe.len(), + held_after_workload.len() + ); + assert!( + finished_while_held, + "the workload did not finish within {HOLD_WINDOW:?} while only node {held_id}'s \ + notes apply was held" + ); + + fx.converge().await; + let content = wait_replicas_agree(&fx.cluster, &names.all()).await; + for collection in names.all() { + assert!( + content + .get(&vshard_of(collection)) + .is_some_and(|stored| !stored.is_empty()), + "{collection} holds rows after the workload; equal empty replicas prove nothing" + ); + } + assert_balances_match_entries( + &fx.coordinator().client, + &names.accts, + &names.entries, + ACCOUNTS, + ) + .await; + assert_no_apply_halt(&fx.cluster); + + fx.cluster.shutdown().await; +} diff --git a/nodedb-cluster-tests/tests/common_suite/cases/calvin_split_leader.rs b/nodedb-cluster-tests/tests/common_suite/cases/calvin_split_leader.rs new file mode 100644 index 000000000..a0a12b8fe --- /dev/null +++ b/nodedb-cluster-tests/tests/common_suite/cases/calvin_split_leader.rs @@ -0,0 +1,195 @@ +// SPDX-License-Identifier: BUSL-1.1 + +//! Concurrent cross-shard Calvin transactions commit exactly once while the +//! data-group leadership of their vShards moves between nodes. +//! +//! 1. A 3-node cluster holds `accts`, whose `balance` is the materialized +//! sum of `entries.amount`, and `entries`, on distinct vShards. Each +//! account starts at 0. +//! 2. Two sessions on two nodes run transactions at once. Each inserts one +//! entry, which moves its account's balance on the other vShard. +//! 3. Meanwhile the test moves the leadership of each participant data +//! group to another replica, several times. A moved leader drops what it +//! staged, and the new leader stages its held transactions again. +//! 4. After quiesce, every entry a COMMIT answered for is stored once, +//! every replica holds the same content, and each balance equals both +//! the sum of its entries and the sum the client committed. + +use std::collections::{BTreeMap, BTreeSet}; +use std::time::Duration; + +use super::calvin_multishard_fixture::Fixture; +use super::calvin_replica_content::{ + first_column, names_on_distinct_vshards, run_retrying, strict_session, sum_ddl, + wait_replicas_agree, +}; +use crate::common::cluster_harness::shared_steps::{group_members, group_of}; +use crate::common::cluster_harness::{GroupLeader, TestCluster}; + +/// Transactions each session runs. +const ROUNDS: usize = 12; + +/// Sessions that run transactions at once, one per node. +const SESSIONS: usize = 2; + +/// Accounts in `accts`, `acc-0` to `acc-{ACCOUNTS - 1}`. +const ACCOUNTS: usize = 3; + +/// Leadership moves of each participant data group. +const MOVES: usize = 6; + +/// Pause between two rounds of moves, so transactions run under each +/// leader. +const MOVE_PAUSE: Duration = Duration::from_millis(250); + +/// How long a group gets to show a leader before a move. +const LEADER_WAIT: Duration = Duration::from_secs(30); + +/// The account transaction `k` of session `session` books to. +fn account(session: usize, k: usize) -> usize { + (session + k) % ACCOUNTS +} + +/// The amount transaction `k` of session `session` books. Every amount is +/// distinct, so a sum names the entries it counts. +fn amount(session: usize, k: usize) -> usize { + session * 100 + k + 1 +} + +/// Run session `session`'s transactions through `client`, one at a time. +/// Each returns only once it committed. +async fn book_entries(client: &tokio_postgres::Client, entries: &str, session: usize) { + for k in 0..ROUNDS { + let txn = format!( + "BEGIN; \ + INSERT INTO {entries} (id, account_id, amount) \ + VALUES ('s{session}e{k}', 'acc-{}', '{}'); \ + COMMIT", + account(session, k), + amount(session, k), + ); + run_retrying(client, &format!("session {session} txn {k}"), &txn).await; + } +} + +/// Move the leadership of each of `groups` to another of its replicas, +/// [`MOVES`] times, cycling through the replicas. +async fn move_leaders(cluster: &TestCluster, groups: &[u64]) { + for round in 0..MOVES { + for &group in groups { + let current = cluster + .wait_for_group_leader(group, GroupLeader::Any, LEADER_WAIT) + .await; + let others: Vec = group_members(&cluster.nodes[0], group) + .into_iter() + .filter(|member| *member != current) + .collect(); + assert!( + !others.is_empty(), + "group {group} has a replica besides its leader node {current}" + ); + let target = others[round % others.len()]; + cluster.transfer_leadership(group, target).await; + } + tokio::time::sleep(MOVE_PAUSE).await; + } +} + +/// A served number column as an integer. +fn number(text: &str, what: &str) -> usize { + text.parse::() + .map(|value| value as usize) + .unwrap_or_else(|e| panic!("{what} `{text}` is not a number: {e}")) +} + +#[tokio::test(flavor = "multi_thread", worker_threads = 6)] +async fn calvin_txns_commit_once_while_data_group_leaders_move() { + let [accts, entries] = names_on_distinct_vshards(["split_accts", "split_entries"]); + let [accts_ddl, entries_ddl, sum] = sum_ddl(&entries, &accts); + let fx = Fixture::spawn(&[accts_ddl, entries_ddl]).await; + fx.cluster + .exec_ddl_on_any_leader(&sum) + .await + .unwrap_or_else(|e| panic!("{sum}: {e}")); + for collection in [&accts, &entries] { + fx.wait_group_mounted(collection).await; + } + for account in 0..ACCOUNTS { + run_retrying( + &fx.cluster.nodes[0].client, + "seed an account", + &format!( + "INSERT INTO {accts} (id, owner, balance) VALUES ('acc-{account}', 'seed', '0')" + ), + ) + .await; + } + fx.converge().await; + + let groups: Vec = [&accts, &entries] + .into_iter() + .map(|collection| group_of(&fx.cluster.nodes[0], collection)) + .collect::>() + .into_iter() + .collect(); + let sessions = [ + strict_session(&fx.cluster.nodes[1]).await, + strict_session(&fx.cluster.nodes[2]).await, + ]; + tokio::join!( + book_entries(&sessions[0], &entries, 0), + book_entries(&sessions[1], &entries, 1), + move_leaders(&fx.cluster, &groups), + ); + + fx.converge().await; + let collections = [accts.as_str(), entries.as_str()]; + wait_replicas_agree(&fx.cluster, &collections).await; + + let client = &fx.coordinator().client; + let mut stored = first_column(client, &format!("SELECT id FROM {entries}")).await; + stored.sort(); + let mut expected_ids: Vec = (0..SESSIONS) + .flat_map(|session| (0..ROUNDS).map(move |k| format!("s{session}e{k}"))) + .collect(); + expected_ids.sort(); + assert_eq!( + stored, expected_ids, + "each committed transaction's entry is stored once" + ); + + let mut committed: BTreeMap = BTreeMap::new(); + for session in 0..SESSIONS { + for k in 0..ROUNDS { + *committed.entry(account(session, k)).or_default() += amount(session, k); + } + } + for account in 0..ACCOUNTS { + let balances = first_column( + client, + &format!("SELECT balance FROM {accts} WHERE id = 'acc-{account}'"), + ) + .await; + let [balance] = balances.as_slice() else { + panic!("acc-{account} has one row, found balances {balances:?}"); + }; + let amounts = first_column( + client, + &format!("SELECT amount FROM {entries} WHERE account_id = 'acc-{account}'"), + ) + .await; + let summed: usize = amounts.iter().map(|a| number(a, "amount")).sum(); + let balance = number(balance, "balance"); + assert_eq!( + balance, summed, + "acc-{account}'s balance is the sum of its entries {amounts:?}" + ); + assert_eq!( + balance, + committed.get(&account).copied().unwrap_or_default(), + "acc-{account}'s balance counts each committed transaction once" + ); + } + + fx.cluster.shutdown().await; +} diff --git a/nodedb-cluster-tests/tests/common_suite/cases/calvin_superseded_collection_cluster.rs b/nodedb-cluster-tests/tests/common_suite/cases/calvin_superseded_collection_cluster.rs index f2e21a287..14e4c54ee 100644 --- a/nodedb-cluster-tests/tests/common_suite/cases/calvin_superseded_collection_cluster.rs +++ b/nodedb-cluster-tests/tests/common_suite/cases/calvin_superseded_collection_cluster.rs @@ -24,7 +24,7 @@ use std::time::Duration; -use nodedb_types::fail_point::{FailAction, FailGuard}; +use nodedb_test_support::fail_point::{FailAction, FailGuard}; use tokio_postgres::error::SqlState; use super::calvin_multishard_fixture::{Fixture, keyed_ddl}; diff --git a/nodedb-cluster-tests/tests/common_suite/cases/cluster_backup_remote_cut.rs b/nodedb-cluster-tests/tests/common_suite/cases/cluster_backup_remote_cut.rs index 3ce94ab65..e9d9f39ac 100644 --- a/nodedb-cluster-tests/tests/common_suite/cases/cluster_backup_remote_cut.rs +++ b/nodedb-cluster-tests/tests/common_suite/cases/cluster_backup_remote_cut.rs @@ -4,8 +4,8 @@ //! //! The backup's coordinator snapshots every vShard from the leader of its //! group. The test parks a write on the collection's group leader only, at -//! the fail gate `funnel::before_dispatch::node::`, between -//! its record and its core. The coordinator is another node, so its own +//! the fail gate `funnel::before_dispatch::` armed for that node, +//! between its record and its core. The coordinator is another node, so its own //! replica applies the write and its own cut passes. The leader must take the //! same cut before it snapshots: the backup waits until the parked write //! applies there, and the envelope holds the row. After the purge settles on @@ -19,10 +19,10 @@ use std::time::Duration; use bytes::Bytes; use futures::{SinkExt, StreamExt}; +use nodedb_test_support::fail_point::{FailAction, FailGuard}; use nodedb_types::backup_envelope::{ DEFAULT_MAX_TOTAL_BYTES, DatabaseDataSection, parse_encrypted, }; -use nodedb_types::fail_point::{FailAction, FailGuard}; use crate::common; use common::cluster_harness::wait::wait_for; @@ -192,8 +192,9 @@ async fn a_backup_waits_for_a_write_held_on_a_remote_source_node() { // Park the write on the source node only. let gate_dir = tempfile::tempdir().expect("gate tempdir"); let release = gate_dir.path().join("release-source-apply"); - let _gate = FailGuard::install( - &format!("funnel::before_dispatch::node{source}::{COLLECTION}"), + let _gate = FailGuard::for_node( + source, + &format!("funnel::before_dispatch::{COLLECTION}"), FailAction::WaitForFile(release.clone()), ); diff --git a/nodedb-cluster-tests/tests/common_suite/cases/cluster_pitr_restore_point.rs b/nodedb-cluster-tests/tests/common_suite/cases/cluster_pitr_restore_point.rs index 7ab977aca..5a9f7fbe5 100644 --- a/nodedb-cluster-tests/tests/common_suite/cases/cluster_pitr_restore_point.rs +++ b/nodedb-cluster-tests/tests/common_suite/cases/cluster_pitr_restore_point.rs @@ -215,7 +215,7 @@ async fn a_cluster_restores_every_group_to_a_restore_point() { #[cfg(feature = "failpoints")] #[tokio::test(flavor = "multi_thread", worker_threads = 4)] async fn a_ddl_after_the_watermark_is_absent_though_it_precedes_the_point() { - use nodedb_types::fail_point::{FailAction, FailGuard}; + use nodedb_test_support::fail_point::{FailAction, FailGuard}; let root = tempfile::tempdir().expect("storage root"); let pitr = PitrStorage::create(root.path()).expect("PITR storage"); diff --git a/nodedb-cluster-tests/tests/common_suite/cases/cluster_restore_retry.rs b/nodedb-cluster-tests/tests/common_suite/cases/cluster_restore_retry.rs index 93b1612cc..35623482b 100644 --- a/nodedb-cluster-tests/tests/common_suite/cases/cluster_restore_retry.rs +++ b/nodedb-cluster-tests/tests/common_suite/cases/cluster_restore_retry.rs @@ -16,7 +16,7 @@ use std::collections::HashSet; use std::time::Duration; -use nodedb_types::fail_point::FailGuard; +use nodedb_test_support::fail_point::FailGuard; use crate::common::cluster_harness::shared_steps::{db_detail, drain_backup, try_push_restore}; use crate::common::cluster_harness::{TestCluster, wait_for, wait_for_async}; diff --git a/nodedb-cluster-tests/tests/common_suite/cases/committed_publish_failover.rs b/nodedb-cluster-tests/tests/common_suite/cases/committed_publish_failover.rs index f4737d4c1..15127427e 100644 --- a/nodedb-cluster-tests/tests/common_suite/cases/committed_publish_failover.rs +++ b/nodedb-cluster-tests/tests/common_suite/cases/committed_publish_failover.rs @@ -27,8 +27,8 @@ use std::path::Path; use std::time::Duration; use nodedb::event::cdc::CdcOffset; +use nodedb_test_support::fail_point::{FailAction, FailGuard}; use nodedb_types::DatabaseId; -use nodedb_types::fail_point::{FailAction, FailGuard}; const TOPIC: &str = "committed_failover_feed"; const SRC: &str = "committed_failover_src"; @@ -184,8 +184,9 @@ async fn a_publish_committed_before_a_leader_kill_is_delivered_once() { // Never created: the leader stays parked until it dies. let release = gate_dir.path().join("release"); let parked = gate_dir.path().join("release.parked"); - let _gate = FailGuard::install( - &format!("publish::before_delivery::node{leader_id}"), + let _gate = FailGuard::for_node( + leader_id, + "publish::before_delivery", FailAction::WaitForFile(release), ); wait_parked(&parked, "the leader's delivery parks").await; @@ -223,8 +224,9 @@ async fn a_publish_sent_before_a_leader_kill_is_not_appended_twice() { let hold_dir = tempfile::tempdir().expect("gate directory"); let hold_release = hold_dir.path().join("release"); let hold_parked = hold_dir.path().join("release.parked"); - let _hold = FailGuard::install( - &format!("publish::before_delivery::node{leader_id}"), + let _hold = FailGuard::for_node( + leader_id, + "publish::before_delivery", FailAction::WaitForFile(hold_release.clone()), ); wait_parked(&hold_parked, "the leader's delivery parks").await; @@ -241,8 +243,9 @@ async fn a_publish_sent_before_a_leader_kill_is_not_appended_twice() { // file of the second gate is never created: the leader dies parked. let commit_dir = tempfile::tempdir().expect("gate directory"); let commit_parked = commit_dir.path().join("release.parked"); - let _commit = FailGuard::install( - &format!("publish::before_cursor_commit::node{leader_id}"), + let _commit = FailGuard::for_node( + leader_id, + "publish::before_cursor_commit", FailAction::WaitForFile(commit_dir.path().join("release")), ); std::fs::write(&hold_release, b"release").expect("release the delivery"); diff --git a/nodedb-cluster-tests/tests/common_suite/cases/committed_publish_metadata_lag.rs b/nodedb-cluster-tests/tests/common_suite/cases/committed_publish_metadata_lag.rs index cefee65c5..5fb632be1 100644 --- a/nodedb-cluster-tests/tests/common_suite/cases/committed_publish_metadata_lag.rs +++ b/nodedb-cluster-tests/tests/common_suite/cases/committed_publish_metadata_lag.rs @@ -20,10 +20,10 @@ use common::cluster_harness::{TestCluster, TestClusterNode, wait_for}; use std::sync::atomic::Ordering; use std::time::{Duration, Instant}; -use nodedb::control::cluster::metadata_applier::metadata_apply_hold_point; +use nodedb::control::cluster::metadata_applier::METADATA_APPLY_HOLD_POINT; use nodedb::event::cdc::CdcOffset; +use nodedb_test_support::fail_point::{FailAction, FailGuard}; use nodedb_types::DatabaseId; -use nodedb_types::fail_point::{FailAction, FailGuard}; const TOPIC: &str = "metadata_lag_feed"; const SRC: &str = "metadata_lag_src"; @@ -141,8 +141,9 @@ async fn a_lagging_lease_holder_waits_for_the_topic_and_delivers_once() { let gate_dir = tempfile::tempdir().expect("gate directory"); let release = gate_dir.path().join("release"); let parked = gate_dir.path().join("release.parked"); - let _hold = FailGuard::install( - &metadata_apply_hold_point(b_id), + let _hold = FailGuard::for_node( + b_id, + METADATA_APPLY_HOLD_POINT, FailAction::WaitForFile(release.clone()), ); exec_on(&cluster.nodes[a], &format!("CREATE TOPIC {TOPIC}")).await; diff --git a/nodedb-cluster-tests/tests/common_suite/cases/cross_node_read_occ_abort.rs b/nodedb-cluster-tests/tests/common_suite/cases/cross_node_read_occ_abort.rs index 3c3818403..f87bd2781 100644 --- a/nodedb-cluster-tests/tests/common_suite/cases/cross_node_read_occ_abort.rs +++ b/nodedb-cluster-tests/tests/common_suite/cases/cross_node_read_occ_abort.rs @@ -8,11 +8,10 @@ //! An interactive transaction can WRITE several collections and READ another. To //! exercise the cross-shard OCC read-validation path, the transaction must go //! through the multi-participant Calvin barrier (`run_commit_calvin`) rather than -//! the single-shard local-WAL `si_conflict_abort` fast path. That barrier is the -//! only place that revalidates a read slice on its owning vShard using the real -//! per-shard `read_lsn`, by dispatching a validate-only task to every read-only -//! participant. If a read-only participant is never validated, a stale read -//! commits silently — a non-serializable execution. +//! the single-shard path. The barrier revalidates a read slice on its owning +//! vShard at the read's real version, by dispatching a validate-only task to +//! every read-only participant. If a read-only participant is never validated, +//! a stale read commits silently — a non-serializable execution. //! //! ## Why a single node reproduces it //! @@ -230,7 +229,7 @@ async fn cross_shard_read_occ_abort_on_stale_read() { // writes, leaving the transaction OPEN (no COMMIT). The two INSERTs buffer on // the coordinator, so at COMMIT the write set spans two vShards → participant // floor >= 2 → the multi-participant Calvin barrier; the SELECT registers - // bread's vShard as a read-only participant with a captured read_lsn. + // bread's vShard as a read-only participant with a captured read version. let block = format!( "BEGIN; \ SELECT * FROM {bread}; \ @@ -284,8 +283,8 @@ async fn cross_shard_read_occ_abort_on_stale_read() { pg_detail(&err) ); - // Discriminator: `si_conflict_abort`'s single-shard fast path raises this same - // SQLSTATE without ever admitting the batch to a sequencer epoch, so the 40001 + // Discriminator: the single-shard commit check raises this same SQLSTATE + // without ever admitting the batch to a sequencer epoch, so the 40001 // assertion above alone cannot tell "aborted by the multi-participant Calvin // barrier revalidating bread's read-only-participant slice" apart from "aborted // for some unrelated single-shard reason". Only the barrier path admits the @@ -318,7 +317,7 @@ async fn cross_shard_read_occ_abort_on_stale_read() { /// The precision control: the SAME cross-shard shape (read `bread`, write /// `w1`/`w2`) with NO concurrent write to `bread` must COMMIT SUCCESSFULLY. The -/// barrier validates `bread`'s read at its real `read_lsn` and finds it still +/// barrier validates `bread`'s read at its real read version and finds it still /// current — proving the abort above is caused by the genuine version advance, /// not a blanket cross-shard abort. #[tokio::test(flavor = "multi_thread", worker_threads = 4)] @@ -355,7 +354,7 @@ async fn cross_shard_read_occ_commits_when_read_still_current() { ) .await; - // Both committed writes become visible (the Calvin flush lands asynchronously + // Both committed writes become visible (the redo install lands asynchronously // after the completion ack, so poll). wait_for_async( "both committed writes visible after COMMIT", diff --git a/nodedb-cluster-tests/tests/common_suite/cases/ddl_prepare_lease_reclaim.rs b/nodedb-cluster-tests/tests/common_suite/cases/ddl_prepare_lease_reclaim.rs index 6282f3e7d..a932af4e5 100644 --- a/nodedb-cluster-tests/tests/common_suite/cases/ddl_prepare_lease_reclaim.rs +++ b/nodedb-cluster-tests/tests/common_suite/cases/ddl_prepare_lease_reclaim.rs @@ -22,7 +22,8 @@ const DEAD_TOKEN: u64 = 0x00dd_1ea5; fn owner_token(node: &TestClusterNode) -> Option { node.shared - .metadata_ddl_owner + .metadata_ddl + .owner .lock() .unwrap_or_else(|p| p.into_inner()) .map(|owner| owner.token) @@ -124,7 +125,8 @@ async fn a_dead_ddl_lease_owner_is_reclaimed_before_its_lease_runs_out() { assert_ne!( survivor .shared - .metadata_ddl_applied_token + .metadata_ddl + .applied_token .load(std::sync::atomic::Ordering::Acquire), DEAD_TOKEN, "a reclaimed token's finalize applies nothing" diff --git a/nodedb-cluster-tests/tests/common_suite/cases/gather_join_in_txn_occ.rs b/nodedb-cluster-tests/tests/common_suite/cases/gather_join_in_txn_occ.rs index 593526524..6c9c4c574 100644 --- a/nodedb-cluster-tests/tests/common_suite/cases/gather_join_in_txn_occ.rs +++ b/nodedb-cluster-tests/tests/common_suite/cases/gather_join_in_txn_occ.rs @@ -11,7 +11,7 @@ //! where the left side is scanned locally, and GATHERS the RIGHT (build) //! collection across all vShards (`gather_join_build_side`) to inline it as a //! `ProviderScan`. Each of those two gathers observes its own collection's -//! `coll_write_lsn` at read time. +//! write version at read time. //! //! The GATHER path must not have a serializability HOLE where only the plan's //! collapsed left collection reaches the read-set (`extract_collection` of an @@ -28,7 +28,7 @@ //! * `commits_when_neither_side_concurrently_written` — neither join side is //! written during the txn, so both recorded real versions still validate and //! COMMIT must SUCCEED (proves the captures carry the sound per-collection -//! `coll_write_lsn`, not an inflated global watermark that would over-abort). +//! write version, not an inflated global watermark that would over-abort). //! * `occ_aborts_on_stale_probe_read` — a confirmed-visible concurrent write to //! the LEFT/probe collection advances it past the captured probe version, so //! COMMIT must abort with 40001. @@ -183,7 +183,7 @@ async fn spawn_node_with_collections() -> ( .await; // Seed left and right with three matching join keys so the join returns real - // rows AND each side's committed `coll_write_lsn` is non-zero (gives the + // rows AND each side's committed write version is non-zero (gives the // stale-read cases a real baseline version to advance past, and the // commits-clean case a non-zero version that must still validate). node.client @@ -267,7 +267,7 @@ async fn begin_join_read_and_buffer_writes( /// Neither join side is concurrently written, so both recorded real read versions /// still validate at the barrier and COMMIT must SUCCEED. Proves the captured -/// read-versions are the sound per-collection `coll_write_lsn` — an inflated +/// read-versions are the sound per-collection write versions — an inflated /// global watermark would over-abort this clean commit. #[tokio::test(flavor = "multi_thread", worker_threads = 4)] async fn gather_join_commits_when_neither_side_concurrently_written() { @@ -301,7 +301,7 @@ async fn gather_join_commits_when_neither_side_concurrently_written() { ) .await; - // Both committed writes become visible (Calvin flush lands asynchronously). + // Both committed writes become visible (the redo install lands asynchronously). wait_for_async( "both committed writes visible after COMMIT", Duration::from_secs(10), diff --git a/nodedb-cluster-tests/tests/common_suite/cases/graph_wcc_core_error_cross_node.rs b/nodedb-cluster-tests/tests/common_suite/cases/graph_wcc_core_error_cross_node.rs index eb64b42a3..c48325062 100644 --- a/nodedb-cluster-tests/tests/common_suite/cases/graph_wcc_core_error_cross_node.rs +++ b/nodedb-cluster-tests/tests/common_suite/cases/graph_wcc_core_error_cross_node.rs @@ -15,7 +15,7 @@ use std::collections::HashSet; use std::time::Duration; -use nodedb_types::fail_point::FailGuard; +use nodedb_test_support::fail_point::FailGuard; use crate::common::cluster_harness::{TestCluster, wait_for}; diff --git a/nodedb-cluster-tests/tests/common_suite/cases/install_snapshot_multi_core_fault.rs b/nodedb-cluster-tests/tests/common_suite/cases/install_snapshot_multi_core_fault.rs index 0f73d47cc..22bf34134 100644 --- a/nodedb-cluster-tests/tests/common_suite/cases/install_snapshot_multi_core_fault.rs +++ b/nodedb-cluster-tests/tests/common_suite/cases/install_snapshot_multi_core_fault.rs @@ -4,8 +4,8 @@ //! and a re-install of the same bytes converges to every row on its owning //! core. //! -//! The fail point `snapshot_install::node::core1` makes core 1 of node N -//! report its share as failed after it installed it: the other core holds the +//! The fail point `snapshot_install::core1`, armed for node N, makes core 1 +//! of node N report its share as failed after it installed it: the other core holds the //! snapshot, this one does not count as settled. The re-install clears every //! core before it installs, so no row lands twice or off its core. //! @@ -21,7 +21,7 @@ use nodedb::control::cluster::snapshot_builder::DataPlaneSnapshotBuilder; use nodedb::control::cluster::snapshot_install::SnapshotInstallError; use nodedb::types::TenantId; use nodedb_cluster::SnapshotBuilder; -use nodedb_types::fail_point::FailGuard; +use nodedb_test_support::fail_point::{FailAction, FailGuard}; use crate::common::cluster_harness::shared_steps::key_collection; use crate::common::cluster_harness::{TestCluster, wait_for}; @@ -97,9 +97,10 @@ async fn failed_core_install_is_retryable_and_reinstall_converges() { let applier = DataPlaneSnapshotApplier::new(target.shared.clone()); { - let _fault = FailGuard::fail( - &format!("snapshot_install::node{}::core1", target.node_id), - "injected core install fault", + let _fault = FailGuard::for_node( + target.node_id, + "snapshot_install::core1", + FailAction::Fail("injected core install fault".to_string()), ); let err = applier .install(gid, &bytes) diff --git a/nodedb-cluster-tests/tests/common_suite/cases/mod.rs b/nodedb-cluster-tests/tests/common_suite/cases/mod.rs index d40f209e2..1e7a13aa4 100644 --- a/nodedb-cluster-tests/tests/common_suite/cases/mod.rs +++ b/nodedb-cluster-tests/tests/common_suite/cases/mod.rs @@ -11,9 +11,19 @@ mod array_vshard_migration; mod assign_surrogate_cross_node; mod backup_schedule_failover; mod bitemporal_array_cluster; +mod calvin_backup_cut_order; mod calvin_cdc_net_kinds; +mod calvin_cluster_array_read_set; mod calvin_cluster_pgwire_e2e; +mod calvin_dependent_read_commit; +mod calvin_dependent_read_early_result; +mod calvin_dependent_read_fixture; +mod calvin_dependent_read_follower_promotion; +mod calvin_dependent_read_restart; +mod calvin_dependent_read_timeout; +mod calvin_duplicate_redo_copy; mod calvin_joiner_after_sequencer_compaction; +mod calvin_leader_change_mid_txn; mod calvin_multi_shard_bitemporal_best_effort_restart; mod calvin_multi_shard_bitemporal_restart; mod calvin_multi_shard_redo_restart; @@ -21,10 +31,16 @@ mod calvin_multishard_dml_tag_fold; mod calvin_multishard_fixture; mod calvin_multishard_pk_read; mod calvin_multishard_txn_staging_cross_node; +mod calvin_old_term_redo_copy; mod calvin_ollp_cross_node; mod calvin_ollp_pk_delete; mod calvin_ollp_update; +mod calvin_read_version_leader_move; mod calvin_remote_participant_report; +mod calvin_replica_content; +mod calvin_replica_convergence; +mod calvin_replica_held_apply; +mod calvin_split_leader; mod calvin_submit_routed_cross_node; mod calvin_superseded_collection_cluster; mod catalog_put_if_absent; @@ -163,3 +179,4 @@ mod vshard_names; mod webhook_sink_partial_replication; mod webhook_sink_single_owner; mod write_admission_concurrent_same_key_replay; +mod write_admission_leader_gate_cluster; diff --git a/nodedb-cluster-tests/tests/common_suite/cases/move_tenant_reissue_retry.rs b/nodedb-cluster-tests/tests/common_suite/cases/move_tenant_reissue_retry.rs index c21a882c7..98a9b4e40 100644 --- a/nodedb-cluster-tests/tests/common_suite/cases/move_tenant_reissue_retry.rs +++ b/nodedb-cluster-tests/tests/common_suite/cases/move_tenant_reissue_retry.rs @@ -15,7 +15,7 @@ use std::time::Duration; -use nodedb_types::fail_point::FailGuard; +use nodedb_test_support::fail_point::FailGuard; use crate::common; use common::cluster_harness::TestCluster; diff --git a/nodedb-cluster-tests/tests/common_suite/cases/native_gather_join_in_txn_occ.rs b/nodedb-cluster-tests/tests/common_suite/cases/native_gather_join_in_txn_occ.rs index a77327353..6536f34a3 100644 --- a/nodedb-cluster-tests/tests/common_suite/cases/native_gather_join_in_txn_occ.rs +++ b/nodedb-cluster-tests/tests/common_suite/cases/native_gather_join_in_txn_occ.rs @@ -15,13 +15,13 @@ //! //! Cases bounding the fixed behavior: //! * `commits_when_neither_side_concurrently_written` — clean commit MUST SUCCEED -//! (captures carry the sound `coll_write_lsn`, not an over-aborting watermark). +//! (captures carry the sound collection write version, not an over-aborting watermark). //! * `occ_aborts_on_stale_probe_read` — a stale LEFT/probe read MUST abort. //! * `occ_aborts_on_stale_build_read` — a stale RIGHT/build read (pgwire rival) //! MUST abort: the native build-side capture hole is closed on the Gather path. //! * `occ_aborts_on_native_writer_stale_read` — the build-side rival is a NATIVE //! autocommit INSERT. Pre-fix the gateway leader-local autocommit path never -//! proposed a Raft entry nor bumped `coll_write_lsn`, so the read never went +//! proposed a Raft entry nor bumped the collection write version, so the read never went //! stale and COMMIT silently succeeded; routing native writes through Raft //! closes that lost-update cell. //! @@ -177,7 +177,7 @@ async fn spawn_node_with_collections() -> ( .await; // Seed left and right with three matching join keys so the join returns real - // rows AND each side's committed `coll_write_lsn` is non-zero (gives the + // rows AND each side's committed write version is non-zero (gives the // stale-read cases a real baseline version to advance past, and the // commits-clean case a non-zero version that must still validate). node.client @@ -300,7 +300,7 @@ async fn concurrent_write_and_confirm(node: &TestClusterNode, coll: &str) { /// NATIVE concurrent writer: the rival autocommit INSERT rides a SEPARATE native /// client, exercising the gateway leader-local autocommit-write path that must now -/// PROPOSE through Raft and bump `coll_write_lsn`. Confirmed visible before COMMIT. +/// PROPOSE through Raft and bump the collection write version. Confirmed visible before COMMIT. async fn native_concurrent_write_and_confirm(node: &TestClusterNode, coll: &str) { let writer = pinned_native_client(node); writer @@ -314,7 +314,7 @@ async fn native_concurrent_write_and_confirm(node: &TestClusterNode, coll: &str) /// Neither join side is concurrently written, so both recorded real read versions /// still validate at the barrier and COMMIT must SUCCEED. Proves the captured -/// read-versions are the sound per-collection `coll_write_lsn` — an inflated +/// read-versions are the sound per-collection write versions — an inflated /// global watermark would over-abort this clean commit. #[tokio::test(flavor = "multi_thread", worker_threads = 4)] async fn native_gather_join_commits_when_neither_side_concurrently_written() { @@ -344,7 +344,7 @@ async fn native_gather_join_commits_when_neither_side_concurrently_written() { ) .await; - // Both committed writes become visible (Calvin flush lands asynchronously). + // Both committed writes become visible (the redo install lands asynchronously). wait_for_async( "both committed writes visible after COMMIT", Duration::from_secs(10), @@ -464,7 +464,7 @@ async fn native_gather_join_occ_aborts_on_stale_build_read() { /// rival autocommit INSERT rides a SEPARATE native client, driving the gateway's /// leader-local autocommit-write path. Pre-fix that path applied the row via a /// leader-local SPSC dispatch that never proposed a Raft entry nor bumped the build -/// collection's `coll_write_lsn`, so the captured build version never went stale +/// collection's write version, so the captured build version never went stale /// and the COMMIT SILENTLY succeeded (undetected lost-update). Routing native /// autocommit writes through Raft advances the build side, so COMMIT must abort. #[tokio::test(flavor = "multi_thread", worker_threads = 4)] diff --git a/nodedb-cluster-tests/tests/common_suite/cases/shuffle_aggregate_in_txn_occ.rs b/nodedb-cluster-tests/tests/common_suite/cases/shuffle_aggregate_in_txn_occ.rs index 3ee2bdb35..0c4e37698 100644 --- a/nodedb-cluster-tests/tests/common_suite/cases/shuffle_aggregate_in_txn_occ.rs +++ b/nodedb-cluster-tests/tests/common_suite/cases/shuffle_aggregate_in_txn_occ.rs @@ -8,16 +8,16 @@ //! An interactive transaction can WRITE several collections and READ another via //! a distributed GROUP BY that the planner lowers to `Exchange{ShuffleAggregate}`. //! The coordinator fans partial-state PRODUCERs to the source collection's -//! owner(s); each producer scans the collection and — with the producer-side -//! read-version fix — reports the scanned collection's `coll_write_lsn` at read -//! time back on its `ShuffleProduceResponse`. The coordinator max-folds those and -//! records the aggregate's read-set entry with that version. +//! owner(s); each producer scans the collection and reports the scanned +//! collection's write version at read time back on its +//! `ShuffleProduceResponse`. The coordinator max-folds those and records the +//! aggregate's read-set entry with that version. //! -//! The shuffle-aggregate resolver must not record `Lsn::ZERO` as the +//! The shuffle-aggregate resolver must not record `WriteVersion::ZERO` as the //! read version. At the cross-shard COMMIT barrier the read-only participant is -//! revalidated with `coll_write_lsn(coll) <= read_version`; with `read_version = -//! 0` and any prior write to the collection (`coll_write_lsn > 0`) the comparison -//! is `>0 <= 0` → false → a SPURIOUS serialization abort (SQLSTATE 40001) for a +//! revalidated with `write_version(coll) <= read_version`. With a zero read +//! version and any prior write to the collection, the write version is above +//! zero, so the comparison is false → a SPURIOUS serialization abort (SQLSTATE 40001) for a //! read that was never concurrently written. This suite bounds the behavior from //! BOTH sides: //! @@ -148,7 +148,7 @@ async fn spawn_node_with_collections() .await; // Seed metrics with committed low-cardinality GROUP BY rows so the aggregate - // read observes real data AND metrics' committed `coll_write_lsn` is non-zero + // read observes real data AND metrics' committed write version is non-zero // (the exact precondition that made the pre-fix ZERO read-version abort). node.client .simple_query(&format!( @@ -197,8 +197,8 @@ async fn spawn_node_with_collections() /// version still validates at the barrier and COMMIT must SUCCEED. This is the /// direction the producer-side ZERO→real read-version fix unblocks: before it, /// the shuffle-aggregate recorded read version `0`, and the barrier's -/// `coll_write_lsn(metrics) <= 0` check (metrics has committed writes, so -/// `coll_write_lsn > 0`) spuriously aborted the COMMIT with 40001. +/// `write_version(metrics) <= 0` check (metrics has committed writes, so its +/// write version is above zero) spuriously aborted the COMMIT with 40001. #[tokio::test(flavor = "multi_thread", worker_threads = 4)] async fn shuffle_aggregate_read_commits_when_not_concurrently_written() { let (node, _data_dir, metrics, w1, w2) = spawn_node_with_collections().await; @@ -246,7 +246,7 @@ async fn shuffle_aggregate_read_commits_when_not_concurrently_written() { ) .await; - // Both committed writes become visible (Calvin flush lands asynchronously). + // Both committed writes become visible (the redo install lands asynchronously). wait_for_async( "both committed writes visible after COMMIT", Duration::from_secs(10), @@ -273,7 +273,7 @@ async fn shuffle_aggregate_read_commits_when_not_concurrently_written() { /// The precision control: the SAME shape, but a confirmed-visible concurrent write /// advances `metrics` past the captured aggregate read version. COMMIT must STILL /// abort with SQLSTATE 40001 — the barrier revalidates metrics' read-only slice -/// at its real `read_lsn` and finds it stale. Guards the fix against becoming a +/// at its real read version and finds it stale. Guards the fix against becoming a /// blanket "always valid". #[tokio::test(flavor = "multi_thread", worker_threads = 4)] async fn shuffle_aggregate_read_occ_aborts_on_stale_read() { diff --git a/nodedb-cluster-tests/tests/common_suite/cases/shuffle_join_in_txn_occ.rs b/nodedb-cluster-tests/tests/common_suite/cases/shuffle_join_in_txn_occ.rs index 4a96ac84c..1aad293fe 100644 --- a/nodedb-cluster-tests/tests/common_suite/cases/shuffle_join_in_txn_occ.rs +++ b/nodedb-cluster-tests/tests/common_suite/cases/shuffle_join_in_txn_occ.rs @@ -10,7 +10,7 @@ //! a `HashJoin`. The coordinator fans PRODUCERs to EACH side's owner: the probe //! (left) side scans `left_collection`, the build (right) side scans //! `right_collection`. Each producer reports the scanned collection's -//! `coll_write_lsn` at read time on its `ShuffleProduceResponse`; the coordinator +//! write version at read time on its `ShuffleProduceResponse`; the coordinator //! records ONE read-set entry PER SIDE, each stamped with that side's real //! max-folded read version. //! @@ -159,7 +159,7 @@ async fn spawn_node_with_collections() -> ( .await; // Seed left and right with three matching join keys so the join returns real - // rows AND each side's committed `coll_write_lsn` is non-zero (the exact + // rows AND each side's committed write version is non-zero (the exact // precondition that made the pre-fix ZERO probe read-version spuriously abort, // and gives the stale-read cases a real baseline version to advance past). node.client @@ -239,8 +239,8 @@ async fn begin_join_read_and_buffer_writes( /// Neither join side is concurrently written, so both recorded real read versions /// still validate at the barrier and COMMIT must SUCCEED. This is the direction /// the fix unblocks: before it, the probe (left) side recorded read version `0` -/// and the barrier's `coll_write_lsn(left) <= 0` check (left has committed writes, -/// so `coll_write_lsn > 0`) spuriously aborted the COMMIT with 40001. +/// and the barrier's `write_version(left) <= 0` check (left has committed writes, +/// so its write version is above zero) spuriously aborted the COMMIT with 40001. #[tokio::test(flavor = "multi_thread", worker_threads = 4)] async fn shuffle_join_commits_when_neither_side_concurrently_written() { let (node, _data_dir, left, right, w1, w2) = spawn_node_with_collections().await; @@ -273,7 +273,7 @@ async fn shuffle_join_commits_when_neither_side_concurrently_written() { ) .await; - // Both committed writes become visible (Calvin flush lands asynchronously). + // Both committed writes become visible (the redo install lands asynchronously). wait_for_async( "both committed writes visible after COMMIT", Duration::from_secs(10), diff --git a/nodedb-cluster-tests/tests/common_suite/cases/single_node_calvin_two_phase.rs b/nodedb-cluster-tests/tests/common_suite/cases/single_node_calvin_two_phase.rs index 8c7653806..2a1e3c145 100644 --- a/nodedb-cluster-tests/tests/common_suite/cases/single_node_calvin_two_phase.rs +++ b/nodedb-cluster-tests/tests/common_suite/cases/single_node_calvin_two_phase.rs @@ -1,15 +1,15 @@ // SPDX-License-Identifier: BUSL-1.1 -//! A cross-shard, Calvin-committed write goes through the stage/flush seam: -//! the scheduler STAGES the transaction (validate + buffer, no base mutation), -//! then — because the local commit vote is valid — FLUSHES the staged buffer to -//! base. The write is durable and visible only after the flush. +//! A cross-shard, Calvin-committed write goes through the stage/install seam: +//! the scheduler stages the transaction (validate + buffer, no base mutation). +//! On a COMMIT verdict the leader proposes the slice's stamped redo to the data +//! group, and its apply installs the staged writes. The write is visible only +//! after the install. //! -//! The staged buffer + verdict-driven flush/drop live on the `!Send` Data-Plane -//! core, so this asserts the flush FIRED via the node-global -//! `calvin.counters.commits_flushed` counter (incremented once per staged apply the -//! per-vShard scheduler resolved to commit) plus the functional proof that the -//! flushed write is visible. +//! The staged buffer lives on the `!Send` Data-Plane core, so this asserts the +//! commit fired via the node-global `calvin.counters.commits_flushed` counter +//! (incremented once per staged txn the scheduler resolved to commit) plus the +//! functional proof that the installed write is visible. use crate::common; diff --git a/nodedb-cluster-tests/tests/common_suite/cases/single_node_calvin_write_versions.rs b/nodedb-cluster-tests/tests/common_suite/cases/single_node_calvin_write_versions.rs index b4ee53f00..bf72e193c 100644 --- a/nodedb-cluster-tests/tests/common_suite/cases/single_node_calvin_write_versions.rs +++ b/nodedb-cluster-tests/tests/common_suite/cases/single_node_calvin_write_versions.rs @@ -5,13 +5,10 @@ //! no version at all (fast-path writes and reads already do). //! //! The version index itself lives on the `!Send` Data-Plane core and its -//! readers are test-only, so this asserts the recording FIRED via the -//! node-global `calvin.counters.write_versions_recorded` counter, which the per-vShard -//! scheduler increments once per committed Calvin apply for which it dispatched -//! a write-version record op (at the CalvinApplied WAL LSN). The counter is the -//! standalone-observable proof that a cross-shard-committed write now advances -//! the version index; the end-to-end serializability regression (via read-set -//! validation) is covered separately once validation is enforcing. +//! readers are test-only, so this asserts the recording fired via the +//! node-global `calvin.counters.write_versions_recorded` counter. The data +//! group's apply loop increments it once per committed slice whose stamped +//! redo installed, and the install records the slice's write versions. use crate::common; @@ -62,8 +59,8 @@ fn distinct_vshard_node_keys() -> (String, String) { } /// A cross-shard (dual-home edge) Calvin-committed write increments the -/// node-global write-version-recorded counter — i.e. its keys' versions are -/// recorded at the CalvinApplied WAL LSN instead of being silently dropped. +/// node-global write-version-recorded counter: the stamped redo install of +/// each slice records its keys' versions at the install's WAL LSN. #[tokio::test(flavor = "multi_thread", worker_threads = 4)] async fn cross_shard_calvin_write_records_write_version() { // 4 Data-Plane cores so distinct vShards land on distinct cores — a genuine @@ -99,7 +96,8 @@ async fn cross_shard_calvin_write_records_write_version() { // AUTOCOMMIT cross-shard edge insert: because the endpoints home to distinct // vShards, the edge dual-homes atomically through the Calvin scheduler // (`submit_calvin_routed`), not the single-home fast path. On commit each - // participating vShard's scheduler records its slice's write versions. + // participating vShard's data group installs its slice's stamped redo, + // and the install records the slice's write versions. node.client .simple_query(&format!( "GRAPH INSERT EDGE IN 'sncalvin_wv_graph' FROM '{src}' TO '{dst}' TYPE 'l'" @@ -107,8 +105,8 @@ async fn cross_shard_calvin_write_records_write_version() { .await .expect("cross-shard edge insert must commit via the single-node Calvin path"); - // Recording is dispatched fire-and-forget after the CalvinApplied WAL LSN is - // obtained, so wait for the counter to advance. + // Each participant's data-group apply loop raises the counter on its own + // schedule, so wait for it to advance. wait_for( "calvin write-version recording fired for the cross-shard commit", Duration::from_secs(10), diff --git a/nodedb-cluster-tests/tests/common_suite/cases/timeseries_ring_drop_catch_up.rs b/nodedb-cluster-tests/tests/common_suite/cases/timeseries_ring_drop_catch_up.rs index c71430df4..0942bec47 100644 --- a/nodedb-cluster-tests/tests/common_suite/cases/timeseries_ring_drop_catch_up.rs +++ b/nodedb-cluster-tests/tests/common_suite/cases/timeseries_ring_drop_catch_up.rs @@ -41,8 +41,8 @@ use std::path::{Path, PathBuf}; use std::time::{Duration, Instant}; use nodedb::event::cdc::consume::{ConsumeError, ConsumeParams, consume_local}; +use nodedb_test_support::fail_point::{FailAction, FailGuard}; use nodedb_types::DatabaseId; -use nodedb_types::fail_point::{FailAction, FailGuard}; use super::ts_native_ingest::{ ingest_native, ingest_until_accepted, native_session, rejection_warnings, @@ -75,8 +75,9 @@ impl ApplyGate { fn install(directory: &Path, node_id: u64) -> Self { let release = directory.join(format!("release-node{node_id}")); let arrival = directory.join(format!("release-node{node_id}.parked")); - let guard = FailGuard::install( - &format!("funnel::before_dispatch::node{node_id}::{COLLECTION}"), + let guard = FailGuard::for_node( + node_id, + &format!("funnel::before_dispatch::{COLLECTION}"), FailAction::WaitForFile(release.clone()), ); Self { diff --git a/nodedb-cluster-tests/tests/common_suite/cases/trigger_firing_failover.rs b/nodedb-cluster-tests/tests/common_suite/cases/trigger_firing_failover.rs index ca37ff11e..9b51574d3 100644 --- a/nodedb-cluster-tests/tests/common_suite/cases/trigger_firing_failover.rs +++ b/nodedb-cluster-tests/tests/common_suite/cases/trigger_firing_failover.rs @@ -31,7 +31,7 @@ use std::path::{Path, PathBuf}; use std::time::Duration; use nodedb::event::cdc::CdcOffset; -use nodedb_types::fail_point::{FailAction, FailGuard}; +use nodedb_test_support::fail_point::{FailAction, FailGuard}; const SRC: &str = "firing_failover_src"; const ROWS: [&str; 3] = ["r1", "r2", "r3"]; @@ -81,8 +81,9 @@ impl Gate { fn install(dir: &Path, point: &str, node_id: u64) -> Self { let release = dir.join(format!("{point}-node{node_id}")); let parked = dir.join(format!("{point}-node{node_id}.parked")); - let guard = FailGuard::install( - &format!("trigger::{point}::node{node_id}"), + let guard = FailGuard::for_node( + node_id, + &format!("trigger::{point}"), FailAction::WaitForFile(release.clone()), ); Self { diff --git a/nodedb-cluster-tests/tests/common_suite/cases/write_admission_leader_gate_cluster.rs b/nodedb-cluster-tests/tests/common_suite/cases/write_admission_leader_gate_cluster.rs new file mode 100644 index 000000000..e399a1f94 --- /dev/null +++ b/nodedb-cluster-tests/tests/common_suite/cases/write_admission_leader_gate_cluster.rs @@ -0,0 +1,129 @@ +// SPDX-License-Identifier: BUSL-1.1 + +//! An autocommit write and a Calvin cross-shard transaction on one row of a +//! replicated vShard leave the same final row on every replica. +//! +//! The data-group leader admits every replicated write through its write +//! gate before the propose. The gate takes the write's row key and its +//! collection `Intent` on the vShard's Calvin lock table, so a Calvin +//! transaction on the same row orders before or after the write on the +//! leader, never between its propose and its apply start. Followers apply +//! the committed entry in log order. +//! +//! Each round races a cross-shard transaction that writes the row against an +//! autocommit write of the row from another node. After the last round every +//! replica that hosts the row's group must hold the same image of it. + +use nodedb::types::{TenantDataSnapshot, TenantId}; + +use super::calvin_multishard_fixture::{Fixture, schemaless_ddl}; +use super::vshard_names::distinct_vshard_collections; +use crate::common::cluster_harness::TestClusterNode; +use crate::common::cluster_harness::shared_steps::key_collection; +use crate::common::pgwire_harness::raw_pgwire::RawPgConn; + +const TENANT: u64 = 1; +const ROUNDS: usize = 8; + +/// Whether the server answered `messages` with an error. +fn failed(messages: &[(u8, Vec)]) -> bool { + messages.iter().any(|(tag, _)| *tag == b'E') +} + +/// Run one cross-shard transaction that writes the row `r` of `rows` and the +/// row `s` of `other`. A refused statement or commit rolls the transaction +/// back: the race may abort it, and convergence is the property under test. +async fn calvin_round(conn: &mut RawPgConn, rows: &str, other: &str, round: usize) { + let statements = [ + "BEGIN".to_owned(), + format!("UPDATE {rows} SET v = 'calvin_{round}' WHERE id = 'r'"), + format!("UPDATE {other} SET v = 'calvin_{round}' WHERE id = 's'"), + "COMMIT".to_owned(), + ]; + for sql in &statements { + if failed(&conn.simple_query(sql).await) { + conn.simple_query("ROLLBACK").await; + return; + } + } +} + +/// Every stored row image of `collection` on `node`, from its own tenant +/// snapshot. +async fn local_row_images(node: &TestClusterNode, collection: &str) -> Vec { + let bytes = node.create_tenant_snapshot(TenantId::new(TENANT)).await; + let snapshot: TenantDataSnapshot = zerompk::from_msgpack(&bytes).unwrap_or_default(); + snapshot + .documents + .into_iter() + .filter(|(key, _)| key_collection(key) == Some(collection)) + .map(|(key, value)| { + nodedb_types::json_from_msgpack(&value) + .unwrap_or_else(|e| panic!("stored row {key} decodes: {e}")) + }) + .collect() +} + +#[tokio::test(flavor = "multi_thread", worker_threads = 6)] +async fn autocommit_and_calvin_writes_of_one_row_converge_on_every_replica() { + let (rows, other) = distinct_vshard_collections("wag_rows", "wag_other"); + let fx = Fixture::spawn(&[schemaless_ddl(&rows), schemaless_ddl(&other)]).await; + fx.wait_group_mounted(&rows).await; + fx.wait_group_mounted(&other).await; + + let mut seed = fx.raw().await; + for sql in [ + format!("INSERT INTO {rows} {{ id: 'r', v: 'seed' }}"), + format!("INSERT INTO {other} {{ id: 's', v: 'seed' }}"), + ] { + assert!(!failed(&seed.simple_query(&sql).await), "seed: {sql}"); + } + fx.converge().await; + + // The autocommit writer runs on a node other than the transaction's + // coordinator, so its write reaches the row's leader on its own route. + let writer = (fx.coordinator + 1) % fx.cluster.nodes.len(); + let mut txn_conn = fx.raw().await; + for round in 0..ROUNDS { + let autocommit = async { + // A refused autocommit leaves the row as the transaction wrote + // it; convergence still holds. + let _ = fx.cluster.nodes[writer] + .client + .simple_query(&format!( + "UPDATE {rows} SET v = 'auto_{round}' WHERE id = 'r'" + )) + .await; + }; + tokio::join!( + calvin_round(&mut txn_conn, &rows, &other, round), + autocommit + ); + } + fx.converge().await; + + let group_id = fx.cluster.nodes[0] + .group_id_for_collection(&rows) + .expect("the row's collection maps to a data group"); + let mut images = Vec::new(); + for (idx, node) in fx.cluster.nodes.iter().enumerate() { + if !node.hosts_data_group(group_id) { + continue; + } + let node_images = local_row_images(node, &rows).await; + assert_eq!( + node_images.len(), + 1, + "node {idx} holds exactly the one row: {node_images:?}" + ); + images.push((idx, node_images)); + } + assert!(images.len() > 1, "the row's group has several replicas"); + let (first_idx, first) = &images[0]; + for (idx, image) in &images[1..] { + assert_eq!( + image, first, + "replica {idx} diverged from replica {first_idx}" + ); + } +} diff --git a/nodedb-cluster-tests/tests/misc_suite/cases/transport_security.rs b/nodedb-cluster-tests/tests/misc_suite/cases/transport_security.rs index 95910aa4c..8ae10a2e1 100644 --- a/nodedb-cluster-tests/tests/misc_suite/cases/transport_security.rs +++ b/nodedb-cluster-tests/tests/misc_suite/cases/transport_security.rs @@ -118,7 +118,7 @@ impl RaftRpcHandler for EchoHandler { ) -> nodedb_cluster::rpc_codec::ShuffleProduceResponse { nodedb_cluster::rpc_codec::ShuffleProduceResponse { error: None, - read_version_lsn: 0, + read_versions: Vec::new(), } } diff --git a/nodedb-cluster-tests/tests/misc_suite/cases/wire_version_handshake.rs b/nodedb-cluster-tests/tests/misc_suite/cases/wire_version_handshake.rs index 22efa1c23..ed7e01bf5 100644 --- a/nodedb-cluster-tests/tests/misc_suite/cases/wire_version_handshake.rs +++ b/nodedb-cluster-tests/tests/misc_suite/cases/wire_version_handshake.rs @@ -84,7 +84,7 @@ impl RaftRpcHandler for EchoHandler { ) -> nodedb_cluster::rpc_codec::ShuffleProduceResponse { nodedb_cluster::rpc_codec::ShuffleProduceResponse { error: None, - read_version_lsn: 0, + read_versions: Vec::new(), } } @@ -224,7 +224,7 @@ impl RaftRpcHandler for SentinelHandler { code: 0, message: "sentinel: unexpected produce dispatch".into(), }), - read_version_lsn: 0, + read_versions: Vec::new(), } } diff --git a/nodedb-cluster-tests/tests/sql_cluster_cross_node_dml_tests/calvin_read_failover_cross_node.rs b/nodedb-cluster-tests/tests/sql_cluster_cross_node_dml_tests/calvin_read_failover_cross_node.rs index 7686335ea..7c997a033 100644 --- a/nodedb-cluster-tests/tests/sql_cluster_cross_node_dml_tests/calvin_read_failover_cross_node.rs +++ b/nodedb-cluster-tests/tests/sql_cluster_cross_node_dml_tests/calvin_read_failover_cross_node.rs @@ -1,15 +1,19 @@ // SPDX-License-Identifier: BUSL-1.1 -//! A read-write transaction's read is never validated against another node's -//! WAL. +//! A read made before a leader change validates at the new leader. //! //! A transaction reads a row of a collection whose home group leader is `A`, -//! so `A` serves the read and its version is a position in `A`'s WAL. `A` is -//! then cut off from both other nodes, they elect `B`, and a write to the read -//! row commits at `B`. The partition heals. The transaction also writes, so -//! its COMMIT goes through Calvin, and `B` validates the read. `B`'s versions -//! do not compare with `A`'s, so the commit must abort with a retryable -//! serialization error. +//! so `A` serves the read. Its version is the data-group log position of the +//! last write to the collection, the same on every replica. `A` is then cut +//! off from both other nodes, they elect `B`, and a write to the read row +//! commits at `B` at a later log position. The partition heals. The +//! transaction also writes, so its COMMIT goes through Calvin, and `B` +//! validates the read. `B`'s version of the read row is above the read's, +//! so the commit must abort with a retryable serialization error. +//! +//! The control case runs the same leader change with no write to the read +//! row. Its commit must succeed, so the leader change alone never aborts a +//! transaction, and the abort above comes from the write. use std::time::Duration; @@ -55,8 +59,12 @@ fn link(cluster: &TestCluster, a: usize, b: usize, severed: bool) { } } -#[tokio::test(flavor = "multi_thread", worker_threads = 6)] -async fn a_read_served_by_a_deposed_leader_aborts_the_calvin_commit() { +/// Read the seeded row in a transaction served by the old leader, move the +/// group's leadership by a partition, write the read row at the new leader +/// when `conflict` is set, heal, and return the transaction's COMMIT result. +async fn commit_after_a_leader_change( + conflict: bool, +) -> (TestCluster, nodedb_types::error::NodeDbResult<()>) { let cluster = TestCluster::spawn_three().await.expect("3-node cluster"); for coll in [DOCS, LOG] { cluster @@ -140,34 +148,48 @@ async fn a_read_served_by_a_deposed_leader_aborts_the_calvin_commit() { }, ) .await; - let conflicting = format!("UPDATE {DOCS} SET value = '2' WHERE id = 'a'"); - wait_for_async( - "the conflicting write commits at the new leader", - Duration::from_secs(30), - Duration::from_millis(200), - || async { - cluster.nodes[writer] - .client - .simple_query(&conflicting) - .await - .is_ok() - }, - ) - .await; + if conflict { + let conflicting = format!("UPDATE {DOCS} SET value = '2' WHERE id = 'a'"); + wait_for_async( + "the conflicting write commits at the new leader", + Duration::from_secs(30), + Duration::from_millis(200), + || async { + cluster.nodes[writer] + .client + .simple_query(&conflicting) + .await + .is_ok() + }, + ) + .await; + } link(&cluster, old, reader, false); link(&cluster, old, writer, false); cluster .wait_for_full_apply_convergence(Duration::from_secs(30)) .await; - let err = driver - .commit() - .await - .expect_err("a read the deposed leader served must not validate at the new leader"); + let committed = driver.commit().await; + (cluster, committed) +} + +#[tokio::test(flavor = "multi_thread", worker_threads = 6)] +async fn a_write_after_the_read_aborts_the_commit_across_a_leader_change() { + let (cluster, committed) = commit_after_a_leader_change(true).await; + let err = committed + .expect_err("a write committed after the read must abort the commit at the new leader"); assert!( err.message().contains(SERIALIZATION_ABORT), "expected a retryable serialization abort, got: {err}" ); + cluster.shutdown().await; +} +#[tokio::test(flavor = "multi_thread", worker_threads = 6)] +async fn a_current_read_commits_across_a_leader_change() { + let (cluster, committed) = commit_after_a_leader_change(false).await; + committed + .unwrap_or_else(|e| panic!("a read no write moved commits after the leader change: {e}")); cluster.shutdown().await; } diff --git a/nodedb-cluster/Cargo.toml b/nodedb-cluster/Cargo.toml index a3bf676cd..eb0b0403a 100644 --- a/nodedb-cluster/Cargo.toml +++ b/nodedb-cluster/Cargo.toml @@ -19,6 +19,8 @@ default = [] # `faultbox::init`; a no-op on wasm32 regardless. Off by default so the crate # stays dependency-light for embedders that report failures their own way. diagnostics = ["dep:faultbox"] +# Arms the Raft fail gates cluster tests use. Off by default. +failpoints = ["nodedb-raft/failpoints"] [dependencies] # Workspace crates diff --git a/nodedb-cluster/src/bootstrap/join.rs b/nodedb-cluster/src/bootstrap/join.rs index 7247608c4..4a7424f3e 100644 --- a/nodedb-cluster/src/bootstrap/join.rs +++ b/nodedb-cluster/src/bootstrap/join.rs @@ -581,7 +581,7 @@ mod tests { ) -> crate::rpc_codec::ShuffleProduceResponse { crate::rpc_codec::ShuffleProduceResponse { error: None, - read_version_lsn: 0, + read_versions: Vec::new(), } } diff --git a/nodedb-cluster/src/calvin/completion.rs b/nodedb-cluster/src/calvin/completion.rs index a6e24fa46..af3b6f708 100644 --- a/nodedb-cluster/src/calvin/completion.rs +++ b/nodedb-cluster/src/calvin/completion.rs @@ -6,7 +6,7 @@ use std::time::{Duration, Instant}; use tokio::sync::{mpsc, oneshot}; -use crate::calvin::completion_entry::{PendingCompletion, outcome_for}; +use crate::calvin::completion_entry::PendingCompletion; use crate::calvin::completion_waiter::{CompletionReport, CompletionWaiter}; use crate::calvin::sequencer::AbortReason; @@ -37,25 +37,15 @@ impl TxnId { /// Terminal outcome of a single Calvin transaction attempt. /// -/// Exactly one of these fires per attempt on the unified completion channel: -/// all expected vshards acked AND the global verdict was commit (`Completed`), -/// all expected vshards acked but the global cross-shard verdict was ABORT -/// (`Aborted`), the executor reported an OLLP prediction mismatch that forces a -/// retry (`Mismatch`), or the scheduler rejected the transaction's routing as -/// terminally broken (`Failed`). `Failed` is never retried. `Aborted` retries -/// only for `AbortReason::PredictionDrift`, which is the same drift as -/// `Mismatch`. +/// Exactly one fires per attempt, once the global verdict is stored and every +/// expected vShard acked. A transaction that lost its parts fires at its +/// verdict, since no participant staged it. #[derive(Clone, Debug, PartialEq, Eq)] pub enum AttemptOutcome { + /// The global verdict was COMMIT. Completed, - /// The global cross-shard verdict was ABORT. `reason` names the cause. - Aborted { - reason: AbortReason, - }, - Mismatch, - Failed { - detail: String, - }, + /// The global verdict was ABORT. `reason` names the cause. + Aborted { reason: AbortReason }, } /// One participant vShard's durable commit vote. @@ -93,10 +83,6 @@ pub struct ParticipantProgress { pub acked: bool, /// The txn's global verdict is stored. pub has_verdict: bool, - /// An `OllpMismatch` for the txn is recorded. - pub mismatched: bool, - /// A `TxnRoutingFailed` for the txn is recorded and not yet delivered. - pub routing_failed: bool, } /// `pub(crate)`: also read by the vote/verdict-tally methods in @@ -115,32 +101,47 @@ pub(crate) struct Inner { /// Terminal entries with no waiter, oldest first, with the instant each /// became one (`completion_gc`). pub(crate) waiterless: VecDeque<(Instant, TxnId)>, - /// How long a terminal entry waits for a waiter before eviction. `None` - /// takes [`super::completion_gc::DEFAULT_WAITERLESS_TTL`]. + /// Entries no `EpochBatch` apply sequenced, oldest first, with the + /// instant each was created (`completion_gc`). + pub(crate) orphans: VecDeque<(Instant, TxnId)>, + /// How long a terminal entry waits for a waiter before eviction, and how + /// long an orphan stays with no live waiter. `None` takes + /// [`super::completion_gc::DEFAULT_WAITERLESS_TTL`]. pub(crate) waiterless_ttl: Option, } impl Inner { - /// Remove `txn`'s entry and deliver `outcome` to `waiter`. `signal` names - /// the event in the warning logged when the receiver is gone. - pub(crate) fn fire( - &mut self, - txn: TxnId, - waiter: CompletionWaiter, - outcome: AttemptOutcome, - ack_results: Vec>, - signal: &'static str, - ) { - self.completions.remove(&txn); - if !waiter.send(outcome, ack_results) { - tracing::warn!( - epoch = txn.epoch, - position = txn.position, - signal, - "calvin completion receiver dropped before its outcome fired; \ - client likely timed out on completion wait" - ); + /// Deliver `txn`'s outcome to its waiter once the entry is terminal, then + /// queue or evict waiterless terminal entries. + /// + /// The entry stays after its outcome fires. Its votes, verdict, and acks + /// answer the participants on this node that still probe it. The + /// waiterless sweep removes it. + pub(crate) fn settle(&mut self, txn: TxnId) { + if let Some(entry) = self.completions.get_mut(&txn) + && entry.is_terminal() + && let Some(verdict) = entry.verdict + && let Some(waiter) = entry.completion_tx.take() + { + let results = entry.take_ack_results(); + if !waiter.send(outcome_for(verdict), results) { + tracing::warn!( + epoch = txn.epoch, + position = txn.position, + "calvin completion receiver dropped before its outcome fired; \ + client likely timed out on completion wait" + ); + } } + self.settle_waiterless(txn); + } +} + +/// The coordinator's outcome for a stored verdict. +fn outcome_for(verdict: VerdictOutcome) -> AttemptOutcome { + match verdict { + VerdictOutcome::Commit => AttemptOutcome::Completed, + VerdictOutcome::Abort(reason) => AttemptOutcome::Aborted { reason }, } } @@ -223,10 +224,8 @@ impl CalvinCompletionRegistry { client likely timed out on submit_with_retry" ); } - inner - .completions - .entry(txn) - .or_insert_with(|| PendingCompletion::new(expected_participants)); + let entry = inner.entry_mut(txn); + entry.expected_participants = entry.expected_participants.max(expected_participants); } /// Register interest in `txn`'s terminal outcome, seeding the authoritative @@ -260,46 +259,14 @@ impl CalvinCompletionRegistry { rx } + /// Store `tx` as `txn`'s waiter. An entry that is terminal already, from + /// a verdict and acks that raced ahead of registration, fires at once. fn register_waiter(&self, txn: TxnId, expected_participants: usize, tx: CompletionWaiter) { let mut inner = self.inner.lock().unwrap_or_else(|p| p.into_inner()); - let entry = inner - .completions - .entry(txn) - .or_insert_with(|| PendingCompletion::new(expected_participants)); + let entry = inner.entry_mut(txn); entry.expected_participants = entry.expected_participants.max(expected_participants); - // Routing failure takes precedence over everything else: it is terminal - // and must never be masked by a later ack or mismatch signal. - if let Some(detail) = entry.routing_failed.take() { - inner.fire( - txn, - tx, - AttemptOutcome::Failed { detail }, - Vec::new(), - "routing failure", - ); - } else if entry.abandoned { - // The entry keeps its stored verdict for the participants that - // still probe it, and parks as a waiterless one. - super::completion_parts::send_parts_lost(txn, tx); - inner.settle_waiterless(txn); - } else if entry.mismatched { - inner.fire( - txn, - tx, - AttemptOutcome::Mismatch, - Vec::new(), - "OLLP mismatch", - ); - } else if entry.is_complete() { - // Acks raced ahead of waiter registration: consult the stored - // verdict so an already-complete ABORT surfaces as `Aborted`, not a - // false `Completed`. - let outcome = outcome_for(entry); - let results = entry.take_ack_results(); - inner.fire(txn, tx, outcome, results, "all acked"); - } else { - entry.completion_tx = Some(tx); - } + entry.completion_tx = Some(tx); + inner.settle(txn); } pub fn note_completion_ack(&self, txn: TxnId, vshard_id: u32) { @@ -310,94 +277,20 @@ impl CalvinCompletionRegistry { /// it carried. An empty `result` records none. pub fn note_completion_ack_with(&self, txn: TxnId, vshard_id: u32, result: Vec) { let mut inner = self.inner.lock().unwrap_or_else(|p| p.into_inner()); - let entry = inner - .completions - .entry(txn) - .or_insert_with(|| PendingCompletion::new(0)); + let entry = inner.entry_mut(txn); entry.acked_vshards.insert(vshard_id); if !result.is_empty() { entry.ack_results.insert(vshard_id, result); } - if entry.is_complete() { - // Consult the stored global verdict: `Verdict` is applied strictly - // before this abort's `CompletionAck` in the sequencer Raft log, so - // an ABORT is `Some(false)` here and must surface as `Aborted` - // rather than a silent `Completed` (which would drop the writes and - // report COMMIT SUCCESS to the client). - // - // Only fire + evict when the coordinator's waiter is registered. If - // the final ack races AHEAD of registration, LEAVE the entry — its - // `acked_vshards` (and the stored verdict) persist, so - // `register_completion`'s `is_complete()` branch fires the outcome. - // Evicting here would strand that branch (a fresh entry created by - // the later `register_completion` never re-completes). Mirrors how - // `mismatched`/`routing_failed` persist across the same race. - if let Some(tx) = entry.completion_tx.take() { - let outcome = outcome_for(entry); - let results = entry.take_ack_results(); - inner.fire(txn, tx, outcome, results, "final ack"); - } - } - inner.settle_waiterless(txn); - } - - /// Record an OLLP prediction mismatch for `txn`, the second terminal outcome - /// of an attempt. Mismatch takes precedence over completion: a mismatched - /// attempt must retry, never falsely report success. - /// - /// If the coordinator's waiter is already registered, fire `Mismatch` and - /// evict the entry. Otherwise leave the `mismatched` flag set so a later - /// `register_completion` fires it (mirrors `acked_vshards` persistence). - pub fn note_ollp_mismatch(&self, txn: TxnId) { - let mut inner = self.inner.lock().unwrap_or_else(|p| p.into_inner()); - let entry = inner - .completions - .entry(txn) - .or_insert_with(|| PendingCompletion::new(0)); - entry.mismatched = true; - if let Some(tx) = entry.completion_tx.take() { - inner.fire( - txn, - tx, - AttemptOutcome::Mismatch, - Vec::new(), - "OLLP mismatch", - ); - } - inner.settle_waiterless(txn); - } - - /// Record a terminal, NON-retryable routing failure for `txn` — the - /// scheduler rejected the transaction's local plan routing as - /// `Unroutable`, `ControlPlaneOnly`, or `NotAWrite`. Takes precedence over - /// completion AND over an OLLP mismatch: a routing failure can never - /// converge via retry, so it must never be masked by a later ack or - /// mismatch signal. - /// - /// If the coordinator's waiter is already registered, fire `Failed` and - /// evict the entry. Otherwise leave the `routing_failed` detail set so a - /// later `register_completion` fires it (mirrors `mismatched` persistence). - pub fn note_routing_failed(&self, txn: TxnId, detail: String) { - let mut inner = self.inner.lock().unwrap_or_else(|p| p.into_inner()); - let entry = inner - .completions - .entry(txn) - .or_insert_with(|| PendingCompletion::new(0)); - entry.routing_failed = Some(detail.clone()); - if let Some(tx) = entry.completion_tx.take() { - inner.fire( - txn, - tx, - AttemptOutcome::Failed { detail }, - Vec::new(), - "routing failure", - ); - } - inner.settle_waiterless(txn); + // A participant acks only after it read the verdict, so the verdict + // is stored before the last ack applies. A waiter that registers + // after the last ack finds the terminal entry and fires then. + inner.settle(txn); } /// Set how long a terminal entry with no waiter stays for one to - /// register. The host sets it longer than its statement deadline. + /// register, and how long an orphan stays with no live waiter. The host + /// sets it longer than its statement deadline. pub fn set_waiterless_ttl(&self, ttl: Duration) { self.inner .lock() @@ -408,7 +301,7 @@ impl CalvinCompletionRegistry { /// What this registry holds for participant `vshard` of `txn`. /// /// `None` means no entry exists for `txn`. That is either a txn this node - /// never seeded, or one whose outcome already fired and evicted its entry. + /// never seeded, or a terminal one the waiterless sweep evicted. pub fn participant_progress(&self, txn: TxnId, vshard: u32) -> Option { self.inner .lock() @@ -419,13 +312,10 @@ impl CalvinCompletionRegistry { voted: entry.votes.contains_key(&vshard), acked: entry.acked_vshards.contains(&vshard), has_verdict: entry.verdict.is_some(), - mismatched: entry.mismatched, - routing_failed: entry.routing_failed.is_some(), }) } - /// Test-only: returns the number of pending completion entries. - /// Used to verify entries are removed once all acks arrive (no leak). + /// Test-only: returns the number of completion entries held. #[cfg(test)] pub fn pending_completions_len(&self) -> usize { self.inner @@ -450,31 +340,37 @@ impl CalvinCompletionRegistry { mod tests { use super::*; - /// A replica on which no coordinator waits keeps no completion entry, - /// and no ack result, past the eviction window. An entry that has a - /// waiter stays until its outcome fires. - #[tokio::test] - async fn waiterless_terminal_entries_are_evicted_with_their_results() { + /// A registry that evicts a waiterless terminal entry at its next change. + fn evicting_registry() -> Arc { let reg = CalvinCompletionRegistry::new_detached(); reg.set_waiterless_ttl(Duration::ZERO); + reg + } + + /// A replica on which no coordinator waits keeps no terminal entry, and + /// no ack result, past the eviction window. An entry that has a waiter, + /// or that is sequenced and not terminal, stays. + #[tokio::test] + async fn waiterless_sweep_evicts_only_terminal_entries() { + let reg = evicting_registry(); let acked = TxnId::new(3, 0); - reg.note_assigned(1, acked, 1); + reg.seed_expected(acked, 1); + reg.note_verdict(acked, VerdictOutcome::Commit); reg.note_completion_ack_with(acked, 5, vec![0xAB; 4096]); assert_eq!( reg.pending_completions_len(), 0, - "a complete entry nobody waits for is evicted with its result" + "a terminal entry nobody waits for is evicted with its result" ); - let mismatched = TxnId::new(3, 1); - reg.note_ollp_mismatch(mismatched); - assert_eq!(reg.pending_completions_len(), 0); - let awaited = TxnId::new(3, 2); + let undecided = TxnId::new(3, 3); let rx = reg.register_completion_report(awaited, 2); + reg.note_verdict(awaited, VerdictOutcome::Commit); reg.note_completion_ack_with(awaited, 5, b"five".to_vec()); - reg.note_completion_ack_with(TxnId::new(3, 3), 6, b"other".to_vec()); + reg.seed_expected(undecided, 2); + reg.note_completion_ack_with(undecided, 6, b"other".to_vec()); assert!( reg.participant_progress(awaited, 5).is_some(), "an entry with a waiter is never evicted" @@ -482,6 +378,12 @@ mod tests { reg.note_completion_ack_with(awaited, 6, b"six".to_vec()); let report = rx.await.expect("completion fires"); assert_eq!(report.ack_results, vec![b"five".to_vec(), b"six".to_vec()]); + assert_eq!( + reg.pending_completions_len(), + 1, + "only the undecided entry stays" + ); + assert!(reg.participant_progress(undecided, 6).is_some()); } /// The coordinator hosts no replica of either participant: it learns @@ -490,8 +392,10 @@ mod tests { /// vShard order, whether the acks land before or after it registers. #[tokio::test] async fn a_report_carries_every_participants_ack_result() { - let reg = CalvinCompletionRegistry::new_detached(); + let reg = evicting_registry(); let txn = TxnId::new(11, 4); + reg.seed_expected(txn, 3); + reg.note_verdict(txn, VerdictOutcome::Commit); reg.note_completion_ack_with(txn, 20, b"twenty".to_vec()); let rx = reg.register_completion_report(txn, 3); reg.note_completion_ack_with(txn, 10, b"ten".to_vec()); @@ -504,89 +408,80 @@ mod tests { ); assert_eq!(reg.pending_completions_len(), 0); + // A terminal entry waits the window for its waiter. + let reg = CalvinCompletionRegistry::new_detached(); let late = TxnId::new(11, 5); + reg.seed_expected(late, 1); + reg.note_verdict(late, VerdictOutcome::Commit); reg.note_completion_ack_with(late, 7, b"seven".to_vec()); let report = reg .register_completion_report(late, 1) .await - .expect("an already complete txn fires on registration"); + .expect("an already terminal txn fires on registration"); assert_eq!(report.ack_results, vec![b"seven".to_vec()]); } #[tokio::test] - async fn completion_entry_removed_after_all_acks() { - let reg = CalvinCompletionRegistry::new_detached(); + async fn completion_fires_once_the_verdict_and_every_ack_are_in() { + let reg = evicting_registry(); let txn = TxnId::new(7, 0); reg.note_assigned(1, txn, 2); - let rx = reg.register_completion(txn, 2); - assert_eq!(reg.pending_completions_len(), 1); + let mut rx = reg.register_completion(txn, 2); + reg.note_verdict(txn, VerdictOutcome::Commit); reg.note_completion_ack(txn, 10); - assert_eq!(reg.pending_completions_len(), 1); + assert_eq!(rx.try_recv(), Err(oneshot::error::TryRecvError::Empty)); reg.note_completion_ack(txn, 20); - let outcome = rx.await.expect("completion fires"); - assert_eq!(outcome, AttemptOutcome::Completed); assert_eq!( - reg.pending_completions_len(), - 0, - "entry must be evicted once all expected vshards have acked" + rx.await.expect("completion fires"), + AttemptOutcome::Completed ); + assert_eq!(reg.pending_completions_len(), 0); } #[tokio::test] - async fn completion_entry_removed_when_register_arrives_after_acks() { - let reg = CalvinCompletionRegistry::new_detached(); + async fn completion_fires_when_register_arrives_between_acks() { + let reg = evicting_registry(); let txn = TxnId::new(9, 3); - reg.note_assigned(1, txn, 2); + reg.seed_expected(txn, 2); + reg.note_verdict(txn, VerdictOutcome::Commit); reg.note_completion_ack(txn, 10); - // Entry remains: expected=2, only 1 ack received. assert_eq!(reg.pending_completions_len(), 1); let rx = reg.register_completion(txn, 2); - assert_eq!(reg.pending_completions_len(), 1); reg.note_completion_ack(txn, 20); let outcome = rx.await.expect("completion fires once both acks arrived"); assert_eq!(outcome, AttemptOutcome::Completed); - assert_eq!( - reg.pending_completions_len(), - 0, - "entry must be evicted once awaiter is signalled" - ); + assert_eq!(reg.pending_completions_len(), 0); } + /// Every ack without a stored verdict is not terminal: the waiter keeps + /// waiting, and the sweep keeps the entry. #[tokio::test] - async fn mismatch_arriving_before_register_fires_mismatch() { - let reg = CalvinCompletionRegistry::new_detached(); - let txn = TxnId::new(11, 1); + async fn all_acks_without_a_verdict_are_not_terminal() { + let reg = evicting_registry(); + let txn = TxnId::new(34, 3); reg.note_assigned(1, txn, 2); - // Mismatch observed before the coordinator registers its waiter: the - // flag must persist so a later register_completion fires it. - reg.note_ollp_mismatch(txn); + let mut rx = reg.register_completion(txn, 2); + reg.note_completion_ack(txn, 10); + reg.note_completion_ack(txn, 20); + assert_eq!(rx.try_recv(), Err(oneshot::error::TryRecvError::Empty)); assert_eq!(reg.pending_completions_len(), 1); - let rx = reg.register_completion(txn, 2); - let outcome = rx.await.expect("mismatch fires"); - assert_eq!(outcome, AttemptOutcome::Mismatch); + + reg.note_verdict(txn, VerdictOutcome::Commit); assert_eq!( - reg.pending_completions_len(), - 0, - "entry must be evicted once mismatch is signalled" + rx.await.expect("completion fires"), + AttemptOutcome::Completed ); } + /// A waiterless entry with every ack but no verdict survives the sweep. #[tokio::test] - async fn register_before_mismatch_fires_mismatch() { - let reg = CalvinCompletionRegistry::new_detached(); - let txn = TxnId::new(12, 5); - reg.note_assigned(1, txn, 2); - let rx = reg.register_completion(txn, 2); - assert_eq!(reg.pending_completions_len(), 1); - // Waiter already stored; the mismatch must wake it directly. - reg.note_ollp_mismatch(txn); - let outcome = rx.await.expect("mismatch fires"); - assert_eq!(outcome, AttemptOutcome::Mismatch); - assert_eq!( - reg.pending_completions_len(), - 0, - "entry must be evicted once mismatch is signalled" - ); + async fn a_waiterless_entry_without_a_verdict_is_never_evicted() { + let reg = evicting_registry(); + let txn = TxnId::new(34, 4); + reg.seed_expected(txn, 1); + reg.note_completion_ack(txn, 10); + reg.note_completion_ack(TxnId::new(34, 5), 10); + assert!(reg.participant_progress(txn, 10).is_some()); } #[tokio::test] @@ -595,9 +490,10 @@ mod tests { // register_completion must seed expected_participants from the assignment. // Without the seed (or with the is_complete>0 guard absent) this would // spuriously fire Completed with zero acks. - let reg = CalvinCompletionRegistry::new_detached(); + let reg = evicting_registry(); let txn = TxnId::new(21, 0); let rx = reg.register_completion(txn, 1); + reg.note_verdict(txn, VerdictOutcome::Commit); assert_eq!( reg.pending_completions_len(), 1, @@ -613,10 +509,11 @@ mod tests { async fn ack_racing_ahead_of_register_does_not_prematurely_complete() { // The replicated ack can reach a remote coordinator's registry BEFORE the // coordinator calls register_completion. With expected_participants still - // unknown (0), the ack must persist without firing/evicting; the later - // register_completion seeds the count and then completes. + // unknown (0), the ack must persist without firing or eviction within the + // window; the later register_completion seeds the count and then completes. let reg = CalvinCompletionRegistry::new_detached(); let txn = TxnId::new(22, 0); + reg.note_verdict(txn, VerdictOutcome::Commit); reg.note_completion_ack(txn, 7); assert_eq!( reg.pending_completions_len(), @@ -626,141 +523,149 @@ mod tests { let rx = reg.register_completion(txn, 1); let outcome = rx.await.expect("completion fires once participants seeded"); assert_eq!(outcome, AttemptOutcome::Completed); + reg.set_waiterless_ttl(Duration::ZERO); + reg.note_completion_ack(TxnId::new(22, 1), 7); assert_eq!(reg.pending_completions_len(), 0); } #[tokio::test] - async fn routing_failed_arriving_before_register_fires_failed() { - let reg = CalvinCompletionRegistry::new_detached(); - let txn = TxnId::new(14, 1); - reg.note_assigned(1, txn, 2); - // Routing failure observed before the coordinator registers its - // waiter: the detail must persist so a later register_completion - // fires it. - reg.note_routing_failed(txn, "unroutable plan".to_owned()); - assert_eq!(reg.pending_completions_len(), 1); + async fn abort_verdict_makes_completion_report_aborted() { + // An ABORT verdict is stored (Raft-ordered) before the acks that complete + // the entry. Completion must report `Aborted`, never a silent + // `Completed` that would drop the writes and report COMMIT success. + let reg = evicting_registry(); + let txn = TxnId::new(31, 0); + reg.seed_expected(txn, 2); + reg.note_verdict( + txn, + VerdictOutcome::Abort(AbortReason::SerializationConflict), + ); let rx = reg.register_completion(txn, 2); - let outcome = rx.await.expect("routing failure fires"); + reg.note_completion_ack(txn, 10); + reg.note_completion_ack(txn, 20); + let outcome = rx.await.expect("completion fires"); assert_eq!( outcome, - AttemptOutcome::Failed { - detail: "unroutable plan".to_owned() - } - ); - assert_eq!( - reg.pending_completions_len(), - 0, - "entry must be evicted once routing failure is signalled" + AttemptOutcome::Aborted { + reason: AbortReason::SerializationConflict + }, + "a stored ABORT verdict must surface as Aborted, not Completed" ); + assert_eq!(reg.pending_completions_len(), 0); } + /// The verdict and acks race ahead of waiter registration. The terminal + /// entry waits the window for its waiter, and registration reports + /// `Aborted` at once. The sweep evicts the entry after its window. #[tokio::test] - async fn register_before_routing_failed_fires_failed() { + async fn abort_verdict_reports_aborted_when_register_arrives_after_acks() { let reg = CalvinCompletionRegistry::new_detached(); - let txn = TxnId::new(15, 4); - reg.note_assigned(1, txn, 2); - let rx = reg.register_completion(txn, 2); - assert_eq!(reg.pending_completions_len(), 1); - // Waiter already stored; the routing failure must wake it directly. - reg.note_routing_failed(txn, "control-plane-only plan".to_owned()); - let outcome = rx.await.expect("routing failure fires"); + let txn = TxnId::new(32, 1); + reg.seed_expected(txn, 2); + reg.note_verdict(txn, VerdictOutcome::Abort(AbortReason::PlanRejected)); + reg.note_completion_ack(txn, 10); + reg.note_completion_ack(txn, 20); + let mut rx = reg.register_completion(txn, 2); assert_eq!( - outcome, - AttemptOutcome::Failed { - detail: "control-plane-only plan".to_owned() - } + rx.try_recv(), + Ok(AttemptOutcome::Aborted { + reason: AbortReason::PlanRejected + }), + "registration on a terminal entry fires at once" ); assert_eq!( reg.pending_completions_len(), - 0, - "entry must be evicted once routing failure is signalled" + 1, + "the entry waits its window" ); + + reg.set_waiterless_ttl(Duration::ZERO); + reg.note_completion_ack(TxnId::new(32, 2), 10); + assert_eq!(reg.pending_completions_len(), 0); } + /// An ack or verdict that applies after the sweep evicted its terminal + /// entry creates an orphan. The sweep evicts the orphan once its window + /// passed, so late signals never pile up. #[tokio::test] - async fn routing_failed_takes_precedence_over_pending_acks_and_mismatch() { - let reg = CalvinCompletionRegistry::new_detached(); - let txn = TxnId::new(16, 0); - reg.note_assigned(1, txn, 2); - // No waiter registered yet: an ack and a mismatch both persist onto - // the entry without firing anything. + async fn a_late_signal_for_an_evicted_txn_leaves_no_entry() { + let reg = evicting_registry(); + let txn = TxnId::new(35, 0); + reg.seed_expected(txn, 1); + reg.note_vote(txn, 10, ParticipantVote::Commit); + reg.note_verdict(txn, VerdictOutcome::Commit); reg.note_completion_ack(txn, 10); - reg.note_ollp_mismatch(txn); - assert_eq!(reg.pending_completions_len(), 1); - // The routing failure also persists (still no waiter)... - reg.note_routing_failed(txn, "non-write plan".to_owned()); - // ...and when the coordinator finally registers, it must observe the - // routing failure, not the mismatch or the ack — routing failure is - // terminal and must never be masked by either. - let rx = reg.register_completion(txn, 2); - let outcome = rx.await.expect("routing failure fires"); assert_eq!( - outcome, - AttemptOutcome::Failed { - detail: "non-write plan".to_owned() - } + reg.pending_completions_len(), + 0, + "the terminal entry is evicted" + ); + + reg.note_completion_ack(txn, 10); + assert_eq!( + reg.pending_completions_len(), + 0, + "a late ack leaves no entry" ); + reg.note_vote(txn, 10, ParticipantVote::Commit); + reg.note_verdict(txn, VerdictOutcome::Commit); assert_eq!( reg.pending_completions_len(), 0, - "entry must be evicted once routing failure is signalled" + "a late vote and verdict leave no entry" ); + assert_eq!(reg.participant_progress(txn, 10), None); } + /// An orphan a coordinator waits on stays while the coordinator listens, + /// past its window. Once the coordinator drops its receiver, the sweep + /// evicts the orphan. #[tokio::test] - async fn abort_verdict_makes_completion_report_aborted() { - // An ABORT verdict is stored (Raft-ordered) before the acks that complete - // the tally. Completion must consult it and report `Aborted`, never a - // silent `Completed` that would drop the writes and report COMMIT success. - let reg = CalvinCompletionRegistry::new_detached(); - let txn = TxnId::new(31, 0); - reg.note_assigned(1, txn, 2); - reg.note_verdict( - txn, - VerdictOutcome::Abort(AbortReason::SerializationConflict), - ); + async fn an_orphan_stays_while_its_coordinator_listens() { + let reg = evicting_registry(); + let txn = TxnId::new(36, 0); let rx = reg.register_completion(txn, 2); reg.note_completion_ack(txn, 10); - reg.note_completion_ack(txn, 20); - let outcome = rx.await.expect("completion fires"); - assert_eq!( - outcome, - AttemptOutcome::Aborted { - reason: AbortReason::SerializationConflict - }, - "a stored ABORT verdict must surface as Aborted, not Completed" - ); + reg.note_completion_ack(TxnId::new(36, 1), 10); + let progress = reg + .participant_progress(txn, 10) + .expect("a listening coordinator keeps its orphan"); + assert!(progress.acked && !progress.has_verdict); + + drop(rx); + reg.note_completion_ack(TxnId::new(36, 2), 10); + assert_eq!(reg.participant_progress(txn, 10), None); assert_eq!(reg.pending_completions_len(), 0); } + /// A sequenced entry is never an orphan. With no waiter and no verdict, + /// it stays past the window: its participants still need its tally. #[tokio::test] - async fn abort_verdict_reports_aborted_when_register_arrives_after_acks() { - // Acks (and the verdict) race ahead of waiter registration: the - // already-complete branch in `register_completion` must also consult the - // stored verdict and report `Aborted`. - let reg = CalvinCompletionRegistry::new_detached(); - let txn = TxnId::new(32, 1); + async fn a_sequenced_entry_is_never_evicted_as_an_orphan() { + let reg = evicting_registry(); + let txn = TxnId::new(37, 0); reg.note_assigned(1, txn, 2); - reg.note_verdict(txn, VerdictOutcome::Abort(AbortReason::ParticipantError)); - reg.note_completion_ack(txn, 10); - reg.note_completion_ack(txn, 20); - let rx = reg.register_completion(txn, 2); - let outcome = rx.await.expect("completion fires"); + reg.note_completion_ack(TxnId::new(37, 1), 10); assert_eq!( - outcome, - AttemptOutcome::Aborted { - reason: AbortReason::ParticipantError - } + reg.participant_progress(txn, 10), + None, + "an assignment alone does not sequence the entry" + ); + + reg.seed_expected(txn, 2); + reg.note_vote(txn, 10, ParticipantVote::Commit); + reg.note_completion_ack(TxnId::new(37, 2), 10); + assert!( + reg.participant_progress(txn, 10) + .is_some_and(|progress| progress.voted) ); - assert_eq!(reg.pending_completions_len(), 0); } #[tokio::test] async fn commit_verdict_reports_completed() { - // A COMMIT verdict must still report `Completed`. - let reg = CalvinCompletionRegistry::new_detached(); + let reg = evicting_registry(); let txn = TxnId::new(33, 2); - reg.note_assigned(1, txn, 2); + reg.seed_expected(txn, 2); reg.note_verdict(txn, VerdictOutcome::Commit); let rx = reg.register_completion(txn, 2); reg.note_completion_ack(txn, 10); @@ -770,6 +675,37 @@ mod tests { assert_eq!(reg.pending_completions_len(), 0); } + /// A fired outcome keeps the entry's votes, verdict, and acks for the + /// participants that still probe it, until the sweep evicts it. + #[tokio::test] + async fn a_fired_outcome_keeps_the_tally_until_the_sweep() { + let reg = CalvinCompletionRegistry::new_detached(); + let txn = TxnId::new(40, 2); + reg.seed_expected(txn, 1); + let rx = reg.register_completion(txn, 1); + reg.note_vote(txn, 1, ParticipantVote::Abort(AbortReason::PlanRejected)); + reg.note_verdict(txn, VerdictOutcome::Abort(AbortReason::PlanRejected)); + reg.note_completion_ack(txn, 1); + assert_eq!( + rx.await.expect("completion fires"), + AttemptOutcome::Aborted { + reason: AbortReason::PlanRejected + } + ); + + let progress = reg.participant_progress(txn, 1).expect("entry stays"); + assert!(progress.voted && progress.acked && progress.has_verdict); + assert_eq!(reg.verdict(txn), Some(false)); + assert_eq!( + reg.vote_tally(txn).and_then(|tally| tally.get(&1).copied()), + Some(ParticipantVote::Abort(AbortReason::PlanRejected)) + ); + + reg.set_waiterless_ttl(Duration::ZERO); + reg.note_completion_ack(TxnId::new(40, 3), 1); + assert_eq!(reg.participant_progress(txn, 1), None); + } + /// A dropped assignment closes the caller's channel at once and leaves no /// sender behind. A later `note_assigned` for the seq finds none. #[test] @@ -796,41 +732,6 @@ mod tests { assert_eq!(reg.pending_assignments_len(), 0); } - #[tokio::test] - async fn no_verdict_reports_completed_unchanged() { - // Single-shard / no-verdict paths (`verdict == None`) must report - // `Completed` unchanged — the fix must not stall them. - let reg = CalvinCompletionRegistry::new_detached(); - let txn = TxnId::new(34, 3); - reg.note_assigned(1, txn, 2); - let rx = reg.register_completion(txn, 2); - reg.note_completion_ack(txn, 10); - reg.note_completion_ack(txn, 20); - let outcome = rx.await.expect("completion fires"); - assert_eq!(outcome, AttemptOutcome::Completed); - assert_eq!(reg.pending_completions_len(), 0); - } - - #[tokio::test] - async fn mismatch_takes_precedence_over_pending_acks() { - let reg = CalvinCompletionRegistry::new_detached(); - let txn = TxnId::new(13, 2); - reg.note_assigned(1, txn, 2); - let rx = reg.register_completion(txn, 2); - // One ack arrives but the attempt is not yet complete (expected=2). - reg.note_completion_ack(txn, 10); - assert_eq!(reg.pending_completions_len(), 1); - // A mismatch on the same attempt must win and force a retry. - reg.note_ollp_mismatch(txn); - let outcome = rx.await.expect("mismatch fires"); - assert_eq!(outcome, AttemptOutcome::Mismatch); - assert_eq!( - reg.pending_completions_len(), - 0, - "entry must be evicted once mismatch is signalled" - ); - } - #[tokio::test] async fn participant_progress_is_none_before_any_entry_exists() { let reg = CalvinCompletionRegistry::new_detached(); @@ -844,7 +745,6 @@ mod tests { reg.seed_expected(txn, 2); let empty = reg.participant_progress(txn, 1).expect("seeded entry"); assert!(!empty.voted && !empty.acked && !empty.has_verdict); - assert!(!empty.mismatched && !empty.routing_failed); reg.note_vote(txn, 1, ParticipantVote::Commit); reg.note_completion_ack(txn, 1); @@ -857,18 +757,7 @@ mod tests { ); reg.note_verdict(txn, VerdictOutcome::Commit); - reg.note_ollp_mismatch(txn); - reg.note_routing_failed(txn, "unroutable".to_string()); let txn_wide = reg.participant_progress(txn, 2).expect("entry"); - assert!(txn_wide.has_verdict && txn_wide.mismatched && txn_wide.routing_failed); - } - - #[tokio::test] - async fn participant_progress_is_none_once_the_outcome_fired() { - let reg = CalvinCompletionRegistry::new_detached(); - let txn = TxnId::new(40, 2); - let _rx = reg.register_completion(txn, 1); - reg.note_completion_ack(txn, 1); - assert_eq!(reg.participant_progress(txn, 1), None); + assert!(txn_wide.has_verdict); } } diff --git a/nodedb-cluster/src/calvin/completion_entry.rs b/nodedb-cluster/src/calvin/completion_entry.rs index 60f64a2a6..b8b5a3ce0 100644 --- a/nodedb-cluster/src/calvin/completion_entry.rs +++ b/nodedb-cluster/src/calvin/completion_entry.rs @@ -1,11 +1,12 @@ // SPDX-License-Identifier: BUSL-1.1 //! One transaction's completion entry in the -//! [`super::completion::CalvinCompletionRegistry`], and the outcome it fires. +//! [`super::completion::CalvinCompletionRegistry`]. use std::collections::{BTreeMap, BTreeSet}; +use std::time::Instant; -use super::completion::{AttemptOutcome, ParticipantVote, VerdictOutcome}; +use super::completion::{ParticipantVote, VerdictOutcome}; use super::completion_waiter::CompletionWaiter; pub(crate) struct PendingCompletion { @@ -15,15 +16,6 @@ pub(crate) struct PendingCompletion { /// by vShard. A participant whose ack carried none has no entry. pub(crate) ack_results: BTreeMap>, pub(crate) completion_tx: Option, - /// Set when an OLLP mismatch is observed before the coordinator registers - /// its waiter, so the outcome is not lost across registration order (mirrors - /// how `acked_vshards` persists ack state regardless of registration order). - pub(crate) mismatched: bool, - /// Set when a terminal routing failure is observed before the coordinator - /// registers its waiter, mirroring `mismatched`. Takes precedence over both - /// `mismatched` and completion: a routing failure is never retried and never - /// falsely reported as success. - pub(crate) routing_failed: Option, /// Set when the multi-part transaction lost its parts /// (`SequencerEntry::TxnPartsAbandoned`). Its outcome is /// `Aborted { PartsLost }` without waiting for acks: no participant @@ -52,39 +44,29 @@ pub(crate) struct PendingCompletion { /// Queued for eviction as a terminal entry with no waiter /// (`completion_gc`). pub(crate) parked: bool, -} - -/// Choose the terminal outcome for a COMPLETED entry (all expected vshards -/// acked) by consulting the durable global verdict. -/// -/// The verdict is applied from a replicated `SequencerEntry::Verdict` that is -/// strictly ordered BEFORE the abort's `CompletionAck` in the sequencer Raft -/// log, so an ABORT verdict is always stored by the time this entry's -/// completion fires. A stored abort becomes `Aborted`, carrying the reason the -/// verdict recorded; `None` (single-shard / no-verdict paths) and a commit -/// verdict are `Completed`. We deliberately do NOT gate on `verdict.is_some()` -/// — that would stall the no-verdict completion paths. -pub(crate) fn outcome_for(entry: &PendingCompletion) -> AttemptOutcome { - match entry.verdict { - Some(VerdictOutcome::Abort(reason)) => AttemptOutcome::Aborted { reason }, - _ => AttemptOutcome::Completed, - } + /// Set when this replica applied the `EpochBatch` that sequenced the + /// txn (`seed_expected`). An entry without it is an orphan: the + /// waiterless sweep evicts it once it outlives the window with no live + /// waiter (`completion_gc`). + pub(crate) sequenced: bool, + /// When this entry was created. + pub(crate) created: Instant, } impl PendingCompletion { - pub(crate) fn new(expected_participants: usize) -> Self { + pub(crate) fn new(created: Instant) -> Self { Self { - expected_participants, + expected_participants: 0, acked_vshards: BTreeSet::new(), ack_results: BTreeMap::new(), completion_tx: None, - mismatched: false, - routing_failed: None, abandoned: false, votes: BTreeMap::new(), verdict: None, verdict_proposed: false, parked: false, + sequenced: false, + created, } } @@ -92,10 +74,20 @@ impl PendingCompletion { self.completion_tx.is_some() } - /// Whether the entry's outcome is decided: every expected participant - /// acked, or an OLLP mismatch or a routing failure is recorded. + /// Whether a waiter is registered and its receiver still listens. + pub(crate) fn has_live_waiter(&self) -> bool { + self.completion_tx + .as_ref() + .is_some_and(CompletionWaiter::is_live) + } + + /// Whether the entry's outcome is decided: the verdict is stored and + /// every expected participant acked, or the transaction lost its parts. + /// + /// Acks without a verdict are never terminal. Every participant acks + /// only after it read the verdict. pub(crate) fn is_terminal(&self) -> bool { - self.is_complete() || self.mismatched || self.routing_failed.is_some() || self.abandoned + self.abandoned || (self.verdict.is_some() && self.is_complete()) } /// Every acked participant's apply result, in vShard order. diff --git a/nodedb-cluster/src/calvin/completion_gc.rs b/nodedb-cluster/src/calvin/completion_gc.rs index 6d2eb9373..1c8d1f7a2 100644 --- a/nodedb-cluster/src/calvin/completion_gc.rs +++ b/nodedb-cluster/src/calvin/completion_gc.rs @@ -10,21 +10,83 @@ //! within its statement deadline, so a terminal entry that gained no waiter //! within the eviction window never will, and is removed with its results. //! -//! An entry becomes terminal when every expected participant acked, or when -//! an OLLP mismatch or a routing failure is recorded for it. A terminal -//! entry with no waiter joins a queue stamped with the instant it did. Each -//! later registry change sweeps the queue front, so the queue holds at most -//! the entries of one window. +//! An entry becomes terminal when its verdict is stored and every expected +//! participant acked, or when the transaction lost its parts. An entry whose +//! outcome fired stays too, for the participants that still probe its +//! verdict. A terminal entry with no waiter joins a queue stamped with the +//! instant it did. Each later registry change sweeps the queue front, so the +//! queue holds at most the entries of one window. A non-terminal sequenced +//! entry is never evicted: the scheduler that holds its txn reports the +//! stall. +//! +//! # Orphans +//! +//! The `EpochBatch` apply seeds a txn's entry on every replica, and marks it +//! sequenced, before it fans the txn out. Every vote, verdict, and ack of the +//! txn follows its `EpochBatch` in the sequencer log. A signal therefore +//! creates an unsequenced entry only in these cases: +//! +//! - It applies after the waiterless sweep evicted the terminal entry. +//! - Its `EpochBatch` applied before this replica's registry existed, in a +//! snapshot or an earlier process. +//! - A coordinator registered, or the leader assigned, before the +//! `EpochBatch` applied here, or for a batch that never commits. +//! +//! No local participant holds such a txn: a scheduler receives a txn only +//! from this replica's `EpochBatch` apply, or from a replay of an applied +//! entry. Only a coordinator can wait on an orphan. Each orphan joins a +//! second queue at creation. The sweep evicts it once the window passed +//! since its creation and no live waiter listens. A waiter that listens +//! re-queues it. +//! +//! The rule never evicts an entry a live coordinator waits on: +//! +//! - A registered waiter whose receiver still listens re-queues the entry. +//! - A coordinator registers within its statement deadline of its submit. A +//! signal of the txn exists only after the submit, so registration lands +//! inside the window that starts at the orphan's creation. The host sets +//! the window longer than the deadline. +//! +//! A late ack or verdict for an evicted txn creates an orphan with no +//! waiter. The sweep evicts it one window later, so it never accumulates. +use std::collections::btree_map::Entry; use std::time::{Duration, Instant}; use super::completion::{Inner, TxnId}; +use super::completion_entry::PendingCompletion; /// The eviction window when the host sets none. Longer than any statement /// deadline, so a coordinator's waiter always registers inside it. pub const DEFAULT_WAITERLESS_TTL: Duration = Duration::from_secs(120); impl Inner { + /// `txn`'s entry. A missing one is created unsequenced and joins the + /// orphan queue. + pub(crate) fn entry_mut(&mut self, txn: TxnId) -> &mut PendingCompletion { + match self.completions.entry(txn) { + Entry::Occupied(entry) => entry.into_mut(), + Entry::Vacant(slot) => { + // no-determinism: node-local eviction clock; never in the log. + let now = Instant::now(); + self.orphans.push_back((now, txn)); + slot.insert(PendingCompletion::new(now)) + } + } + } + + /// `txn`'s entry, marked sequenced. A missing one is created without an + /// orphan slot. + pub(crate) fn sequenced_entry_mut(&mut self, txn: TxnId) -> &mut PendingCompletion { + let entry = self + .completions + .entry(txn) + // no-determinism: node-local eviction clock; never in the log. + .or_insert_with(|| PendingCompletion::new(Instant::now())); + entry.sequenced = true; + entry + } + /// Queue `txn`'s entry when it just became a waiterless terminal entry, /// then evict every queued entry whose window passed. pub(crate) fn settle_waiterless(&mut self, txn: TxnId) { @@ -32,6 +94,44 @@ impl Inner { let now = Instant::now(); self.park_waiterless(txn, now); self.sweep_waiterless(now); + self.sweep_orphans(now); + } + + /// The eviction window. + fn window(&self) -> Duration { + self.waiterless_ttl.unwrap_or(DEFAULT_WAITERLESS_TTL) + } + + /// Evict every queued orphan that outlived the window with no live + /// waiter. An orphan whose waiter still listens is queued again. + /// + /// A queue slot is skipped when its entry is sequenced, parked as + /// terminal, or younger than the window. A younger entry is a later + /// incarnation of the txn, and it holds its own slot. + pub(crate) fn sweep_orphans(&mut self, now: Instant) { + let window = self.window(); + let mut waited_on = Vec::new(); + while let Some(&(since, txn)) = self.orphans.front() { + if now.saturating_duration_since(since) < window { + break; + } + self.orphans.pop_front(); + let Some(entry) = self.completions.get(&txn) else { + continue; + }; + if entry.sequenced + || entry.parked + || now.saturating_duration_since(entry.created) < window + { + continue; + } + if entry.has_live_waiter() { + waited_on.push((now, txn)); + continue; + } + self.completions.remove(&txn); + } + self.orphans.extend(waited_on); } /// Queue `txn`'s entry for eviction when it is terminal, has no waiter, @@ -50,7 +150,7 @@ impl Inner { /// Remove every queued entry that stayed waiterless for the whole /// window. pub(crate) fn sweep_waiterless(&mut self, now: Instant) { - let ttl = self.waiterless_ttl.unwrap_or(DEFAULT_WAITERLESS_TTL); + let ttl = self.window(); while let Some(&(since, txn)) = self.waiterless.front() { if now.saturating_duration_since(since) < ttl { break; diff --git a/nodedb-cluster/src/calvin/completion_parts.rs b/nodedb-cluster/src/calvin/completion_parts.rs index c08b128c1..06ba9390e 100644 --- a/nodedb-cluster/src/calvin/completion_parts.rs +++ b/nodedb-cluster/src/calvin/completion_parts.rs @@ -3,8 +3,7 @@ //! The completion registry's side of a multi-part transaction that lost its //! parts. -use super::completion::{AttemptOutcome, CalvinCompletionRegistry, TxnId, VerdictOutcome}; -use super::completion_entry::PendingCompletion; +use super::completion::{CalvinCompletionRegistry, TxnId, VerdictOutcome}; use super::completion_verdict::VerdictSignal; use super::sequencer::AbortReason; @@ -19,20 +18,16 @@ impl CalvinCompletionRegistry { /// waits for no ack: no participant staged the whole transaction, and /// no verdict can ever commit it. /// - /// Idempotent. A later ack of `txn` finds no entry for its outcome and - /// parks as a waiterless one. + /// Idempotent. A later ack of `txn` lands on the terminal entry, which + /// the waiterless sweep removes. pub fn note_parts_abandoned(&self, txn: TxnId) { let verdict = VerdictOutcome::Abort(AbortReason::PartsLost); let mut inner = self.inner.lock().unwrap_or_else(|p| p.into_inner()); - let waiter = { - let entry = inner - .completions - .entry(txn) - .or_insert_with(|| PendingCompletion::new(0)); + { + let entry = inner.entry_mut(txn); entry.abandoned = true; entry.verdict = Some(verdict); - entry.completion_tx.take() - }; + } let signal = VerdictSignal { epoch: txn.epoch, position: txn.position, @@ -43,31 +38,14 @@ impl CalvinCompletionRegistry { } // The entry stays: a participant that staged every part targeting // it probes the stored verdict when it parks, after this. - if let Some(waiter) = waiter { - send_parts_lost(txn, waiter); - } - inner.settle_waiterless(txn); - } -} - -/// Answer `waiter` with `Aborted { PartsLost }`, keeping `txn`'s entry and -/// its stored verdict for the participants that still probe it. -pub(crate) fn send_parts_lost(txn: TxnId, waiter: super::completion_waiter::CompletionWaiter) { - let outcome = AttemptOutcome::Aborted { - reason: AbortReason::PartsLost, - }; - if !waiter.send(outcome, Vec::new()) { - tracing::warn!( - epoch = txn.epoch, - position = txn.position, - "calvin completion receiver dropped before its parts-lost outcome fired" - ); + inner.settle(txn); } } #[cfg(test)] mod tests { use super::*; + use crate::calvin::AttemptOutcome; #[tokio::test] async fn lost_parts_abort_a_registered_waiter_without_acks() { diff --git a/nodedb-cluster/src/calvin/completion_verdict.rs b/nodedb-cluster/src/calvin/completion_verdict.rs index 1cf8845b2..8edb5ad14 100644 --- a/nodedb-cluster/src/calvin/completion_verdict.rs +++ b/nodedb-cluster/src/calvin/completion_verdict.rs @@ -12,7 +12,6 @@ use tokio::sync::mpsc; use super::TxnId; use super::completion::{CalvinCompletionRegistry, ParticipantVote, VerdictOutcome}; -use super::completion_entry::PendingCompletion; use super::sequencer::AbortReason; /// Push notification that a staged cross-shard txn's authoritative global @@ -72,10 +71,7 @@ impl CalvinCompletionRegistry { /// no-op (max with the existing value). pub fn seed_expected(&self, txn: TxnId, expected: usize) { let mut inner = self.inner.lock().unwrap_or_else(|p| p.into_inner()); - let entry = inner - .completions - .entry(txn) - .or_insert_with(|| PendingCompletion::new(0)); + let entry = inner.sequenced_entry_mut(txn); entry.expected_participants = entry.expected_participants.max(expected); } @@ -88,10 +84,7 @@ impl CalvinCompletionRegistry { /// each parked participant's flush (commit) or drop (abort). pub fn note_vote(&self, txn_id: TxnId, vshard: u32, vote: ParticipantVote) { let mut inner = self.inner.lock().unwrap_or_else(|p| p.into_inner()); - let entry = inner - .completions - .entry(txn_id) - .or_insert_with(|| PendingCompletion::new(0)); + let entry = inner.entry_mut(txn_id); entry.votes.insert(vshard, vote); // On the transition to a complete tally, emit the aggregated verdict @@ -164,10 +157,7 @@ impl CalvinCompletionRegistry { pub fn note_verdict(&self, txn: TxnId, verdict: VerdictOutcome) { let mut inner = self.inner.lock().unwrap_or_else(|p| p.into_inner()); { - let entry = inner - .completions - .entry(txn) - .or_insert_with(|| PendingCompletion::new(0)); + let entry = inner.entry_mut(txn); match entry.verdict { Some(existing) if existing.is_commit() != verdict.is_commit() => { tracing::warn!( @@ -196,6 +186,8 @@ impl CalvinCompletionRegistry { for tx in inner.verdict_signal_senders.values() { let _ = tx.try_send(signal); } + // A waiter fires here when every ack is in already. + inner.settle(txn); } /// The parked scheduler's flush/drop gate: `Some(true)` commit, `Some(false)` @@ -228,16 +220,18 @@ impl CalvinCompletionRegistry { /// commit, otherwise abort with the highest-precedence reason. /// /// Precedence: `PartsLost` outranks everything, since no participant holds -/// the whole transaction. `ParticipantError` outranks `CollectionSuperseded`, which -/// outranks `PredictionDrift`, which outranks `SerializationConflict`. A peer -/// that never staged makes every other verdict unverifiable, a superseded -/// collection makes a retry against the same read-set pointless, and a -/// drifted prediction retries with a fresh reconnaissance. The tally is +/// the whole transaction. `PlanRejected` outranks `ParticipantError`, which +/// outranks `CollectionSuperseded`, which outranks `PredictionDrift`, which +/// outranks `SerializationConflict`. Rejected plans fail the same way on every +/// resubmit, a peer that never staged makes every other verdict unverifiable, +/// a superseded collection makes a retry against the same read-set pointless, +/// and a drifted prediction retries with a fresh reconnaissance. The tally is /// order-independent. fn tally_verdict(votes: &BTreeMap) -> VerdictOutcome { fn rank(reason: AbortReason) -> u8 { match reason { - AbortReason::PartsLost => 4, + AbortReason::PartsLost => 5, + AbortReason::PlanRejected => 4, AbortReason::ParticipantError => 3, AbortReason::CollectionSuperseded => 2, AbortReason::PredictionDrift => 1, @@ -408,6 +402,36 @@ mod tests { } } + /// A rejected plan outranks every staged abort cause, in any vote order. + #[test] + fn plan_rejected_outranks_staged_abort_causes() { + let rejected = ParticipantVote::Abort(AbortReason::PlanRejected); + let tally = |votes: [&ParticipantVote; 2]| { + let votes: BTreeMap = + (1..).zip(votes.into_iter().copied()).collect(); + tally_verdict(&votes) + }; + for other in [ + AbortReason::SerializationConflict, + AbortReason::PredictionDrift, + AbortReason::CollectionSuperseded, + AbortReason::ParticipantError, + ] { + let other = ParticipantVote::Abort(other); + for votes in [[&rejected, &other], [&other, &rejected]] { + assert_eq!( + tally(votes), + VerdictOutcome::Abort(AbortReason::PlanRejected), + "{other:?}" + ); + } + } + let lost = ParticipantVote::Abort(AbortReason::PartsLost); + for votes in [[&rejected, &lost], [&lost, &rejected]] { + assert_eq!(tally(votes), VerdictOutcome::Abort(AbortReason::PartsLost)); + } + } + #[tokio::test] async fn seed_expected_is_idempotent_max_and_enables_verdict_without_note_assigned() { // Mirrors complete_all_true_tally_emits_commit_verdict_once, but seeds diff --git a/nodedb-cluster/src/calvin/completion_waiter.rs b/nodedb-cluster/src/calvin/completion_waiter.rs index 7faf0edda..31d72861a 100644 --- a/nodedb-cluster/src/calvin/completion_waiter.rs +++ b/nodedb-cluster/src/calvin/completion_waiter.rs @@ -32,6 +32,15 @@ pub(crate) enum CompletionWaiter { } impl CompletionWaiter { + /// Whether the receiver still listens. A coordinator that gave up + /// waiting dropped it. + pub(crate) fn is_live(&self) -> bool { + match self { + Self::Outcome(tx) => !tx.is_closed(), + Self::Report(tx) => !tx.is_closed(), + } + } + /// Deliver `outcome` with `ack_results`. `false` when the receiver is /// gone. pub(crate) fn send(self, outcome: AttemptOutcome, ack_results: Vec>) -> bool { diff --git a/nodedb-cluster/src/calvin/mod.rs b/nodedb-cluster/src/calvin/mod.rs index 7f72bd68d..c66504177 100644 --- a/nodedb-cluster/src/calvin/mod.rs +++ b/nodedb-cluster/src/calvin/mod.rs @@ -18,12 +18,15 @@ pub use completion::{ pub use completion_verdict::VerdictSignal; pub use completion_waiter::CompletionReport; pub use sequencer::{ - AbortReason, AdmittedTx, ConflictKey, CutInstantHook, EpochCheck, Inbox, InboxReceiver, - PartsIntake, PartsOffer, PartsOfferStatus, RejectedTx, ReservationInbox, + AbortReason, AdmittedTx, ConflictKey, CutInstantHook, EpochCheck, HistoryOrigin, Inbox, + InboxReceiver, PartsIntake, PartsOffer, PartsOfferStatus, RejectedTx, ReservationInbox, ReservationInboxReceiver, ReservationRequest, RestorePointHook, SEQUENCER_GROUP_ID, SequencerConfig, SequencerEntry, SequencerError, SequencerHalt, SequencerMetrics, SequencerReceivers, SequencerRestorePoint, SequencerService, SequencerSnapshot, SequencerStateMachine, UnrecoverableEpochHook, new_inbox, new_reservation_inbox, validate_batch, }; -pub use types::{EngineKeySet, EpochBatch, ReadWriteSet, SequencedTxn, SortedVec, TxClass}; +pub use types::{ + CutBarrierWire, CutCaptureWire, EngineKeySet, EpochBatch, ReadWriteSet, SequencedTxn, + SortedVec, TxClass, +}; diff --git a/nodedb-cluster/src/calvin/sequencer/entry.rs b/nodedb-cluster/src/calvin/sequencer/entry.rs index 63f5dd4c4..9774f2232 100644 --- a/nodedb-cluster/src/calvin/sequencer/entry.rs +++ b/nodedb-cluster/src/calvin/sequencer/entry.rs @@ -44,6 +44,10 @@ pub enum AbortReason { /// held them changed before it proposed them all. No participant staged /// the whole transaction, and the coordinator resubmits it. PartsLost, + /// A participant could not decode or route its slice of the plans, or + /// found no local work in them. Every replica rejects the same plans + /// alike, and a resubmit of them fails the same way. + PlanRejected, } /// An entry in the replicated sequencer log. @@ -83,29 +87,8 @@ pub enum SequencerEntry { /// first ack of a vShard in log order answers for it. from_node: u64, }, - /// OLLP predicate-mismatch signal. Proposed by the per-vShard scheduler when - /// the active executor returns `OllpRetryRequired`. Applied on ALL sequencer-group - /// replicas so every node's `CalvinCompletionRegistry` fires `note_ollp_mismatch`, - /// waking the coordinator's retry-loop waiter wherever it is (including remote - /// nodes). Mirrors `CompletionAck` — no `vshard_id` because mismatch is keyed - /// only by `(epoch, position)` like the registry's `TxnId`. - OllpMismatch { epoch: u64, position: u32 }, - /// Local-routing failure signal. Proposed by the per-vShard scheduler when - /// `local_calvin_plans` rejects a plan as `Unroutable`, `ControlPlaneOnly`, - /// or `NotAWrite` — a scheduler-side bug or malformed plan, never a normal - /// retry condition. Applied on ALL sequencer-group replicas so every node's - /// `CalvinCompletionRegistry` fires `note_routing_failed`, waking the - /// coordinator's completion waiter wherever it is with the terminal, - /// NON-retryable `AttemptOutcome::Failed` outcome. Mirrors `OllpMismatch` — - /// no `vshard_id` because the failure is keyed only by `(epoch, position)`. - TxnRoutingFailed { - epoch: u64, - position: u32, - detail: String, - }, /// One participant vShard's durable COMMIT vote for a staged cross-shard - /// txn: its read-set is valid. An abort vote is `AbortVote`. Unlike - /// `OllpMismatch`/`CompletionAck` which key only on `(epoch, position)`, `Vote` + /// txn: its read-set is valid. An abort vote is `AbortVote`. `Vote` /// carries `vshard` because the verdict aggregator must attribute exactly one /// vote per participant to know when the tally is complete. Vote { @@ -155,7 +138,16 @@ pub enum SequencerEntry { /// `restore_point` names the cluster restore point the cut takes, `0` for /// a backup's cut. Every replica records the sequencer's place at the /// point when it applies the marker. - CutMarker { hlc: u64, restore_point: u64 }, + /// + /// `barrier` is set when the cut places a barrier in every data group. + /// Each group's leader then proposes the barrier once its schedulers + /// passed the marker, and holds the redo of every later transaction + /// until it applied the barrier. `None` for a cut with no barrier. + CutMarker { + hlc: u64, + restore_point: u64, + barrier: Option, + }, /// The first entry of a sequencer log a cluster restore rebuilt. It sets /// the next epoch the sequencer proposes to `next_epoch`, the epoch that /// followed the restore point, so no restored epoch is minted again. It @@ -282,28 +274,6 @@ mod tests { assert_eq!(batch.epoch, 7); } - #[test] - fn txn_routing_failed_msgpack_roundtrip() { - let entry = SequencerEntry::TxnRoutingFailed { - epoch: 42, - position: 3, - detail: "unroutable plan: Vector".to_owned(), - }; - let bytes = zerompk::to_msgpack_vec(&entry).expect("encode"); - let decoded: SequencerEntry = zerompk::from_msgpack(&bytes).expect("decode"); - let SequencerEntry::TxnRoutingFailed { - epoch, - position, - detail, - } = decoded - else { - panic!("decoded wrong sequencer entry variant"); - }; - assert_eq!(epoch, 42); - assert_eq!(position, 3); - assert_eq!(detail, "unroutable plan: Vector"); - } - #[test] fn vote_msgpack_roundtrip() { let entry = SequencerEntry::Vote { diff --git a/nodedb-cluster/src/calvin/sequencer/inbox.rs b/nodedb-cluster/src/calvin/sequencer/inbox.rs index 50d9ab910..a36e76d96 100644 --- a/nodedb-cluster/src/calvin/sequencer/inbox.rs +++ b/nodedb-cluster/src/calvin/sequencer/inbox.rs @@ -740,6 +740,7 @@ mod tests { ); m }, + expected: BTreeMap::new(), }); let err = inbox.submit(tx).expect_err("should be rejected"); @@ -782,6 +783,7 @@ mod tests { } m }, + expected: BTreeMap::new(), }); let err = inbox.submit(tx).expect_err("should be rejected"); diff --git a/nodedb-cluster/src/calvin/sequencer/mod.rs b/nodedb-cluster/src/calvin/sequencer/mod.rs index 60367d229..c3b9e5c4b 100644 --- a/nodedb-cluster/src/calvin/sequencer/mod.rs +++ b/nodedb-cluster/src/calvin/sequencer/mod.rs @@ -26,7 +26,7 @@ pub use reservation_inbox::{ }; pub use service::{SequencerReceivers, SequencerService}; pub use state_machine::{ - CutInstantHook, RestorePointHook, SequencerRestorePoint, SequencerSnapshot, + CutInstantHook, HistoryOrigin, RestorePointHook, SequencerRestorePoint, SequencerSnapshot, SequencerStateMachine, }; pub use validator::validate_batch; diff --git a/nodedb-cluster/src/calvin/sequencer/replay.rs b/nodedb-cluster/src/calvin/sequencer/replay.rs index 860e67be4..8a62a6a03 100644 --- a/nodedb-cluster/src/calvin/sequencer/replay.rs +++ b/nodedb-cluster/src/calvin/sequencer/replay.rs @@ -168,8 +168,16 @@ impl SequencerStateMachine { } // A cut marker reaches every vShard, exactly as the live // `CutMarker` arm fans it out. - SequencerEntry::CutMarker { hlc, .. } => { - result.push(SchedulerInput::CutMarker { hlc }); + SequencerEntry::CutMarker { + hlc, + restore_point, + barrier, + } => { + result.push(SchedulerInput::CutMarker { + hlc, + restore_point, + barrier: barrier.map(std::sync::Arc::new), + }); } // A part reaches the vShards it targets, exactly as the live // `TxnPart` arm fans it out. The scheduler ignores a part of a @@ -216,8 +224,6 @@ impl SequencerStateMachine { // scheduler input; they are re-derived via live `apply` on every // replica's completion registry, not replayed to a scheduler. SequencerEntry::CompletionAck { .. } => {} - SequencerEntry::OllpMismatch { .. } => {} - SequencerEntry::TxnRoutingFailed { .. } => {} SequencerEntry::Vote { .. } => {} SequencerEntry::Verdict { .. } => {} SequencerEntry::AbortVote { .. } => {} diff --git a/nodedb-cluster/src/calvin/sequencer/service/core.rs b/nodedb-cluster/src/calvin/sequencer/service/core.rs index f1e7b44d6..b511db8a8 100644 --- a/nodedb-cluster/src/calvin/sequencer/service/core.rs +++ b/nodedb-cluster/src/calvin/sequencer/service/core.rs @@ -79,15 +79,17 @@ pub struct SequencerService { /// The epoch `next_reservation_position` is counting within. When it lags /// the tick's epoch, the band counter is reset before the next mint. pub(super) reservation_epoch: u64, - /// Current epoch number, or `None` until the seed has been derived. + /// The next epoch to mint and the sequencer term it was seeded in, or + /// `None` until the seed has been derived. /// /// The leader starts at the last committed epoch + 1 and increments after /// each successful proposal. The seed is NOT taken at construction — see /// [`Self::ensure_epoch_seeded`] for why it can only be read once the - /// sequencer group's log has been replayed into the state machine. On - /// leader failover, `inbox_receiver` is simply dropped (in-flight + /// sequencer group's log has been replayed into the state machine. A + /// lost leadership clears it, and a cursor from another term is derived + /// again. On leader failover, `inbox_receiver` is dropped (in-flight /// submissions are not in the log and will be retried). - pub(super) current_epoch: Option, + pub(super) current_epoch: Option, /// The `epoch_system_ms` this service last minted, or `None` before its /// first mint. A proposed batch applies later, so the next mint reads /// this as well as the state machine's applied instant. @@ -190,7 +192,7 @@ impl SequencerService { } } => { // Every replica's registry emits deterministically, but only - // the leader proposes — same leader-gate as OllpMismatch/Vote. + // the leader proposes the verdict. if let Some((txn, outcome)) = verdict && self.is_leader() && let Err(e) = self.propose_entry(&verdict_entry(txn, outcome)) @@ -245,6 +247,8 @@ impl SequencerService { // Parts held here are gone with the leadership. The next leader // abandons their transactions. self.drop_part_streams(); + // Another leader can mint epochs before this node leads again. + self.current_epoch = None; debug!( node_id = self.node_id, "not sequencer leader; discarding {discarded} inbox items \ @@ -376,6 +380,7 @@ pub(super) mod tests { use crate::calvin::sequencer::config::SequencerConfig; use crate::calvin::sequencer::inbox::{AdmittedTx, Inbox, new_inbox}; use crate::calvin::sequencer::reservation_inbox::{ReservationInbox, new_reservation_inbox}; + use crate::calvin::sequencer::service::epoch_seed::EpochCursor; use crate::calvin::sequencer::validator::validate_batch; use crate::calvin::types::{ EngineKeySet, EpochBatch, LockKeyWire, ReadWriteSet, ReleaseReason, SequencedTxn, @@ -624,7 +629,7 @@ pub(super) mod tests { } } - fn epoch_batch_bytes(epoch: u64) -> Vec { + pub(in crate::calvin::sequencer::service) fn epoch_batch_bytes(epoch: u64) -> Vec { let batch = EpochBatch { epoch, txns: vec![SequencedTxn { @@ -689,9 +694,10 @@ pub(super) mod tests { } assert_eq!(seed, 3); - // Seeding is once-only: later ticks must not re-derive and walk backwards - // over epochs this leader has already proposed. - harness.service.current_epoch = Some(9); + // Seeding is once per term: later ticks in the same term must not + // re-derive and walk backwards over epochs this leader already proposed. + let cursor = harness.service.current_epoch.expect("seeded"); + harness.service.current_epoch = Some(EpochCursor { next: 9, ..cursor }); assert_eq!(harness.service.ensure_epoch_seeded(), Some(9)); } @@ -945,6 +951,13 @@ pub(super) mod tests { .inbox .submit_with(make_rw_class(&col_b, &col_a), ®istry) .expect("submit"); + let term = harness + .multi_raft + .lock() + .unwrap_or_else(|p| p.into_inner()) + .leader_term(SEQUENCER_GROUP_ID) + .expect("leader"); + harness.service.current_epoch = Some(EpochCursor { term, next: 4 }); harness.service.mint_epoch(4); @@ -952,7 +965,7 @@ pub(super) mod tests { // Two participants: the vShard it writes and the vShard it reads. assert_eq!(winner.try_recv(), Ok((4, 0, 2))); assert_eq!(registry.pending_assignments_len(), 0); - assert_eq!(harness.service.current_epoch, Some(5)); + assert_eq!(harness.service.current_epoch.map(|c| c.next), Some(5)); } /// A batch whose proposal failed is not in the log. Its callers must fail diff --git a/nodedb-cluster/src/calvin/sequencer/service/epoch_mint.rs b/nodedb-cluster/src/calvin/sequencer/service/epoch_mint.rs index a630c9ad8..5a8446a7f 100644 --- a/nodedb-cluster/src/calvin/sequencer/service/epoch_mint.rs +++ b/nodedb-cluster/src/calvin/sequencer/service/epoch_mint.rs @@ -84,7 +84,7 @@ impl SequencerService { rejected = rejected_count, "epoch tick: all candidates rejected, no proposal" ); - self.current_epoch = Some(epoch + 1); + self.advance_epoch(epoch + 1); return; } @@ -147,7 +147,7 @@ impl SequencerService { rejected = rejected_count, "sequencer proposed epoch batch" ); - self.current_epoch = Some(epoch + 1); + self.advance_epoch(epoch + 1); } Err(e) => { warn!( diff --git a/nodedb-cluster/src/calvin/sequencer/service/epoch_seed.rs b/nodedb-cluster/src/calvin/sequencer/service/epoch_seed.rs index ee9f84b8f..448043cbf 100644 --- a/nodedb-cluster/src/calvin/sequencer/service/epoch_seed.rs +++ b/nodedb-cluster/src/calvin/sequencer/service/epoch_seed.rs @@ -17,8 +17,20 @@ use crate::multi_raft::MultiRaft; use super::core::SequencerService; +/// The next epoch this leader mints, and the sequencer term it was seeded +/// in. +/// +/// The cursor is valid only within its term. Another leader can mint epochs +/// in a later term, so a node that leads again re-derives its seed. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub(super) struct EpochCursor { + pub term: u64, + pub next: u64, +} + impl SequencerService { - /// Derive the epoch seed once, then reuse it for the life of this service. + /// Derive the epoch seed once per sequencer term, then reuse it within + /// that term. /// /// Delegates the safety gate to [`derive_epoch_seed`]. `None` means it is /// not yet safe to mint an epoch on this node and the caller must skip @@ -43,12 +55,33 @@ impl SequencerService { if self.state_machine_halted() { return None; } - if let Some(epoch) = self.current_epoch { - return Some(epoch); + // A cursor from an earlier term can trail epochs a later leader + // minted. Minting from it would send the epoch backwards. + let term = self.sequencer_current_term(); + if let Some(cursor) = self.current_epoch + && cursor.term == term + { + return Some(cursor.next); + } + self.current_epoch = None; + let cursor = derive_epoch_seed(self.node_id, &self.multi_raft, &self.state_machine)?; + self.current_epoch = Some(cursor); + Some(cursor.next) + } + + /// Record that `next` is the next epoch to mint in the cursor's term. + /// No-op without a seeded cursor. + pub(super) fn advance_epoch(&mut self, next: u64) { + if let Some(cursor) = &mut self.current_epoch { + cursor.next = next; } - let epoch = derive_epoch_seed(self.node_id, &self.multi_raft, &self.state_machine)?; - self.current_epoch = Some(epoch); - Some(epoch) + } + + /// This node's current term in the sequencer group, `0` when the group + /// is not mounted here. + fn sequencer_current_term(&self) -> u64 { + let mr = self.multi_raft.lock().unwrap_or_else(|p| p.into_inner()); + mr.group_leader_at_term(SEQUENCER_GROUP_ID).1 } /// Whether this node's sequencer state machine has stopped applying epoch @@ -91,8 +124,8 @@ impl SequencerService { } } -/// Derive — once — the first epoch this node may propose, returning `None` -/// while it is not yet safe to derive one. +/// Derive the first epoch this node may propose in its current sequencer +/// term, returning `None` while it is not yet safe to derive one. /// /// INVARIANT: **the first epoch a restarted leader proposes must be /// strictly greater than any epoch already committed to the sequencer @@ -125,24 +158,34 @@ impl SequencerService { /// the only thing that puts the first entry in the sequencer log is this /// node proposing under the very seed it is waiting for. /// -/// Returns `None` only while a replay is genuinely in flight; the caller then -/// defers minting for that tick (and only minting — every leader duty that -/// stamps no new identity still runs), so submissions stay queued rather than -/// being sequenced under a colliding epoch. +/// The state machine's history origin decides whether the seed is exact. +/// A state built from the log start, or restored from a snapshot of complete +/// history, holds every committed epoch. Its next epoch is exact, including +/// the first epoch when none committed. A state whose log start was +/// discarded with no snapshot holds only the retained entries. Epochs rise +/// along the log, so a retained epoch still bounds every discarded one. +/// With no retained epoch there is nothing to derive from, and the seed +/// refuses. +/// +/// Returns `None` while a replay is in flight or the history is unknown with +/// no retained epoch. The caller then defers minting for that tick (and only +/// minting — every leader duty that stamps no new identity still runs), so +/// submissions stay queued rather than being sequenced under a colliding +/// epoch. pub(super) fn derive_epoch_seed( node_id: u64, multi_raft: &Mutex, state_machine: &Mutex, -) -> Option { +) -> Option { // Read the Raft-side watermarks and release the lock before taking the // state machine's: the two are never held together anywhere, and this // is the only site that needs both. - let (last_applied, log_tip, first_available) = { + let (last_applied, log_tip, term) = { let mr = multi_raft.lock().unwrap_or_else(|p| p.into_inner()); ( mr.last_applied(SEQUENCER_GROUP_ID), mr.last_log_index(SEQUENCER_GROUP_ID), - mr.first_available_index(SEQUENCER_GROUP_ID), + mr.group_leader_at_term(SEQUENCER_GROUP_ID).1, ) }; let (Some(last_applied), Some(log_tip)) = (last_applied, log_tip) else { @@ -161,18 +204,16 @@ pub(super) fn derive_epoch_seed( } let state_machine = state_machine.lock().unwrap_or_else(|p| p.into_inner()); - // The retained log is the only record of committed epochs. If it starts - // above the first index, earlier entries were discarded (compaction or - // a snapshot install); when none of the retained ones carried an epoch - // there is nothing left to derive a seed from, and minting 0 would - // collide with the discarded history. Refusing to propose is a visible - // stall; minting anyway is silent loss of every batch that follows. - if first_available.unwrap_or(1) > 1 && state_machine.last_applied_epoch().is_none() { + // Refusing to propose is a visible stall. Minting 0 over discarded + // history is silent loss of every batch that follows. + let history = state_machine.history_origin(); + if !history.is_known() && state_machine.last_applied_epoch().is_none() { warn!( node_id, - first_available, - "sequencer log was truncated below every retained epoch; refusing to \ - propose rather than mint an epoch that may collide with discarded history" + ?history, + "sequencer history before the retained log is unknown and no retained \ + entry carries an epoch; refusing to propose rather than mint an epoch \ + that may collide with discarded history" ); return None; } @@ -181,7 +222,129 @@ pub(super) fn derive_epoch_seed( info!( node_id, - epoch, log_tip, "sequencer epoch seed derived from the replayed sequencer log" + epoch, + log_tip, + term, + ?history, + "sequencer epoch seed derived from the replayed sequencer log" ); - Some(epoch) + Some(EpochCursor { term, next: epoch }) +} + +#[cfg(test)] +mod tests { + use std::collections::HashMap; + + use crate::calvin::CalvinCompletionRegistry; + use crate::calvin::sequencer::config::SEQUENCER_GROUP_ID; + use crate::calvin::sequencer::entry::SequencerEntry; + use crate::calvin::sequencer::service::core::tests::{elect, epoch_batch_bytes, make_harness}; + use crate::calvin::sequencer::state_machine::{HistoryOrigin, SequencerStateMachine}; + + fn detached() -> SequencerStateMachine { + SequencerStateMachine::new(HashMap::new(), CalvinCompletionRegistry::new_detached()) + } + + /// A snapshot taken before any epoch committed holds complete history. + /// The leader that restored it mints the first epoch. + #[test] + fn seed_after_an_epoch_free_snapshot_mints_the_first_epoch() { + let mut harness = make_harness(); + let snapshot = detached().capture_snapshot(5); + { + let mut sm = harness + .state_machine + .lock() + .unwrap_or_else(|p| p.into_inner()); + sm.restore_snapshot(snapshot); + assert_eq!(sm.last_applied_epoch(), None); + assert_eq!(sm.history_origin(), HistoryOrigin::Snapshot { through: 5 }); + } + assert_eq!(harness.service.ensure_epoch_seeded(), Some(0)); + } + + /// A snapshot that holds epoch `n` seeds epoch `n + 1`. + #[test] + fn seed_after_a_snapshot_with_an_epoch_mints_the_next_epoch() { + let mut harness = make_harness(); + let mut leader = detached(); + for epoch in 0..5u64 { + leader.apply(epoch + 1, &epoch_batch_bytes(epoch)); + } + let snapshot = leader.capture_snapshot(5); + harness + .state_machine + .lock() + .unwrap_or_else(|p| p.into_inner()) + .restore_snapshot(snapshot); + assert_eq!(harness.service.ensure_epoch_seeded(), Some(5)); + } + + /// Unknown history with no retained epoch refuses the seed. A retained + /// epoch bounds every discarded one, so it seeds above it. + #[test] + fn unknown_history_without_a_retained_epoch_refuses_the_seed() { + let mut harness = make_harness(); + harness + .state_machine + .lock() + .unwrap_or_else(|p| p.into_inner()) + .mark_history_unknown(); + assert_eq!(harness.service.ensure_epoch_seeded(), None); + assert!(harness.service.current_epoch.is_none()); + + let floor = zerompk::to_msgpack_vec(&SequencerEntry::EpochFloor { + next_epoch: 8, + epoch_system_ms: 0, + }) + .expect("encode"); + harness + .state_machine + .lock() + .unwrap_or_else(|p| p.into_inner()) + .apply(1, &floor); + assert_eq!(harness.service.ensure_epoch_seeded(), Some(8)); + } + + /// A cursor seeded in an earlier term trails the epochs another leader + /// minted since. A new term derives the seed again. + #[test] + fn a_term_change_re_derives_the_seed() { + let mut harness = make_harness(); + assert_eq!(harness.service.ensure_epoch_seeded(), Some(0)); + let first_term = harness.service.current_epoch.expect("seeded").term; + + // Another leader minted epochs 0 and 1, applied here. + { + let mut sm = harness + .state_machine + .lock() + .unwrap_or_else(|p| p.into_inner()); + sm.apply(1, &epoch_batch_bytes(0)); + sm.apply(2, &epoch_batch_bytes(1)); + } + elect(&harness.multi_raft); + { + let mut mr = harness.multi_raft.lock().unwrap_or_else(|p| p.into_inner()); + let tip = mr + .last_log_index(SEQUENCER_GROUP_ID) + .expect("group is mounted"); + mr.advance_applied(SEQUENCER_GROUP_ID, tip) + .expect("advance applied"); + } + + assert_eq!(harness.service.ensure_epoch_seeded(), Some(2)); + let cursor = harness.service.current_epoch.expect("seeded"); + assert!(cursor.term > first_term); + } + + /// A tick that finds this node not leading drops the cursor. + #[test] + fn losing_leadership_drops_the_epoch_cursor() { + let mut harness = make_harness(); + assert_eq!(harness.service.ensure_epoch_seeded(), Some(0)); + assert!(harness.service.current_epoch.is_some()); + harness.service.tick(); + assert!(harness.service.current_epoch.is_none()); + } } diff --git a/nodedb-cluster/src/calvin/sequencer/state_machine/apply.rs b/nodedb-cluster/src/calvin/sequencer/state_machine/apply.rs index 94d5d3dae..0015a141c 100644 --- a/nodedb-cluster/src/calvin/sequencer/state_machine/apply.rs +++ b/nodedb-cluster/src/calvin/sequencer/state_machine/apply.rs @@ -99,21 +99,6 @@ impl SequencerStateMachine { from_node, result, ), - // Broadcast the OLLP predicate-mismatch signal to ALL replicas so the - // coordinator's registry fires wherever it lives (including remote nodes). - SequencerEntry::OllpMismatch { epoch, position } => self - .completion_registry - .note_ollp_mismatch(TxnId::new(epoch, position)), - // Broadcast the terminal routing-failure signal to ALL replicas so - // the coordinator's registry fires wherever it lives (including - // remote nodes), mirroring `OllpMismatch`. - SequencerEntry::TxnRoutingFailed { - epoch, - position, - detail, - } => self - .completion_registry - .note_routing_failed(TxnId::new(epoch, position), detail), // Durable per-participant votes for a staged cross-shard txn. The // registry tallies them per vshard. Once every participant voted, // the leader aggregates them into the global verdict that gates the @@ -176,9 +161,11 @@ impl SequencerStateMachine { next_epoch, epoch_system_ms, } => self.apply_epoch_floor(next_epoch, epoch_system_ms), - SequencerEntry::CutMarker { hlc, restore_point } => { - self.apply_cut_marker(index, hlc, restore_point) - } + SequencerEntry::CutMarker { + hlc, + restore_point, + barrier, + } => self.apply_cut_marker(index, hlc, restore_point, barrier), SequencerEntry::TxnPart { epoch, position, @@ -939,33 +926,6 @@ mod tests { assert_eq!(sm.next_epoch(), 1); } - #[tokio::test] - async fn apply_txn_routing_failed_dispatches_to_completion_registry() { - let registry = CalvinCompletionRegistry::new_detached(); - let mut sm = SequencerStateMachine::new(HashMap::new(), Arc::clone(®istry)); - - let data = encode_entry(&SequencerEntry::TxnRoutingFailed { - epoch: 5, - position: 2, - detail: "unroutable plan".to_owned(), - }); - sm.apply(1, &data); - - // The registry's waiter (registered AFTER apply) must still observe - // the failure — `note_routing_failed` persists it on the entry. - let rx = registry.register_completion(crate::calvin::TxnId::new(5, 2), 1); - let outcome = rx.await.expect("routing failure fires"); - assert_eq!( - outcome, - crate::calvin::AttemptOutcome::Failed { - detail: "unroutable plan".to_owned() - } - ); - // TxnRoutingFailed is not an EpochBatch, so it must not perturb the - // epoch counter (mirrors OllpMismatch's non-effect on last_applied_epoch). - assert_eq!(sm.last_applied_epoch(), None); - } - #[tokio::test] async fn apply_verdict_stores_decision_without_perturbing_epoch() { let registry = CalvinCompletionRegistry::new_detached(); @@ -980,8 +940,7 @@ mod tests { // The verdict is stored authoritatively on every replica. assert_eq!(registry.verdict(txn), Some(true)); - // Verdict is not an EpochBatch, so it must not perturb the epoch counter - // (mirrors OllpMismatch/TxnRoutingFailed's non-effect). + // Verdict is not an EpochBatch, so it must not perturb the epoch counter. assert_eq!(sm.last_applied_epoch(), None); } diff --git a/nodedb-cluster/src/calvin/sequencer/state_machine/core.rs b/nodedb-cluster/src/calvin/sequencer/state_machine/core.rs index 64a252010..01698afda 100644 --- a/nodedb-cluster/src/calvin/sequencer/state_machine/core.rs +++ b/nodedb-cluster/src/calvin/sequencer/state_machine/core.rs @@ -102,6 +102,8 @@ pub struct SequencerStateMachine { /// `Txn` inputs this node's schedulers received and may not have made /// durable (see [`super::undurable`]). Local to this replica. pub(super) undurable: super::undurable::UndurableInputs, + /// Where the history this state holds begins (see [`super::history`]). + pub(super) history: super::history::HistoryOrigin, } pub(super) const NOT_YET_APPLIED: u64 = u64::MAX; @@ -151,6 +153,7 @@ impl SequencerStateMachine { completion_registry, open_parts: super::parts::OpenParts::default(), undurable: super::undurable::UndurableInputs::default(), + history: super::history::HistoryOrigin::LogStart, } } diff --git a/nodedb-cluster/src/calvin/sequencer/state_machine/history.rs b/nodedb-cluster/src/calvin/sequencer/state_machine/history.rs new file mode 100644 index 000000000..18750c01c --- /dev/null +++ b/nodedb-cluster/src/calvin/sequencer/state_machine/history.rs @@ -0,0 +1,65 @@ +// SPDX-License-Identifier: BUSL-1.1 + +//! Where the history the sequencer state machine holds begins. +//! +//! A leader seeds its first epoch from the state machine. The seed is exact +//! only when the state holds the effect of every committed entry. That holds +//! for a state built from the log's first entry, and for one restored from a +//! snapshot that itself held complete history. It does not hold for a log +//! whose start was discarded with no snapshot to restore the state from. + +use super::core::SequencerStateMachine; + +/// Where the state this replica holds begins. +#[derive(Debug, Clone, Copy, PartialEq, Eq, Default)] +pub enum HistoryOrigin { + /// Built by applying the sequencer log from its first entry. + #[default] + LogStart, + /// Restored from a snapshot that holds the state through log index + /// `through`. Entries after it apply on top. + Snapshot { through: u64 }, + /// The log starts above its first entry, and no snapshot restored the + /// state the discarded entries built. + Unknown, +} + +impl HistoryOrigin { + /// Whether the state holds the effect of every committed entry. + pub fn is_known(self) -> bool { + !matches!(self, Self::Unknown) + } +} + +impl SequencerStateMachine { + /// Where the history this state machine holds begins. + pub fn history_origin(&self) -> HistoryOrigin { + self.history + } + + /// Record that the sequencer log starts above its first entry and no + /// snapshot restored the discarded state. A later snapshot restore + /// replaces it. + pub fn mark_history_unknown(&mut self) { + self.history = HistoryOrigin::Unknown; + } +} + +#[cfg(test)] +mod tests { + use std::collections::HashMap; + + use super::*; + use crate::calvin::CalvinCompletionRegistry; + + #[test] + fn a_fresh_state_machine_holds_history_from_the_log_start() { + let mut sm = + SequencerStateMachine::new(HashMap::new(), CalvinCompletionRegistry::new_detached()); + assert_eq!(sm.history_origin(), HistoryOrigin::LogStart); + assert!(sm.history_origin().is_known()); + sm.mark_history_unknown(); + assert_eq!(sm.history_origin(), HistoryOrigin::Unknown); + assert!(!sm.history_origin().is_known()); + } +} diff --git a/nodedb-cluster/src/calvin/sequencer/state_machine/mod.rs b/nodedb-cluster/src/calvin/sequencer/state_machine/mod.rs index d6cf02875..47ff562ba 100644 --- a/nodedb-cluster/src/calvin/sequencer/state_machine/mod.rs +++ b/nodedb-cluster/src/calvin/sequencer/state_machine/mod.rs @@ -3,6 +3,7 @@ pub mod apply; pub mod core; pub mod counters; +pub mod history; pub mod parts; pub mod restore; pub mod snapshot; @@ -14,4 +15,5 @@ pub use self::core::{ CutInstantHook, RestorePointHook, SequencerRestorePoint, SequencerStateMachine, }; pub use counters::StateMachineMetrics; +pub use history::HistoryOrigin; pub use snapshot::SequencerSnapshot; diff --git a/nodedb-cluster/src/calvin/sequencer/state_machine/restore.rs b/nodedb-cluster/src/calvin/sequencer/state_machine/restore.rs index f66828787..40591c3e5 100644 --- a/nodedb-cluster/src/calvin/sequencer/state_machine/restore.rs +++ b/nodedb-cluster/src/calvin/sequencer/state_machine/restore.rs @@ -4,9 +4,11 @@ //! backup or restore point takes, and the epoch floor a restored log opens //! with. +use std::sync::Arc; + use tracing::warn; -use crate::calvin::types::SchedulerInput; +use crate::calvin::types::{CutBarrierWire, SchedulerInput}; use super::core::{Delivery, NOT_YET_APPLIED, SequencerRestorePoint, SequencerStateMachine}; @@ -29,7 +31,14 @@ impl SequencerStateMachine { /// out to every vShard scheduler this node hosts. Same delivery as /// `ReserveRead`: a marker for an armed vShard, or one dropped at a full /// channel, is replayed by the scheduler's catch-up drain in log order. - pub(super) fn apply_cut_marker(&mut self, index: u64, hlc: u64, restore_point: u64) { + /// Every scheduler shares one copy of the marker's `barrier`. + pub(super) fn apply_cut_marker( + &mut self, + index: u64, + hlc: u64, + restore_point: u64, + barrier: Option, + ) { if restore_point != 0 && let Some(hook) = &self.restore_point_hook { @@ -46,9 +55,15 @@ impl SequencerStateMachine { if let Some(hook) = &self.cut_instant_hook { hook(hlc, index, self.last_epoch_system_ms); } + let barrier = barrier.map(Arc::new); let vshards: Vec = self.vshard_senders.keys().copied().collect(); for vshard in vshards { - match self.deliver(index, vshard, SchedulerInput::CutMarker { hlc }) { + let marker = SchedulerInput::CutMarker { + hlc, + restore_point, + barrier: barrier.clone(), + }; + match self.deliver(index, vshard, marker) { Delivery::NotHosted | Delivery::Sent | Delivery::Deferred => {} Delivery::DroppedFull => warn!( vshard, @@ -128,6 +143,7 @@ mod tests { &encode(&SequencerEntry::CutMarker { hlc: 900, restore_point: 0, + barrier: None, }), ); sm.apply( @@ -135,6 +151,7 @@ mod tests { &encode(&SequencerEntry::CutMarker { hlc: 1_000, restore_point: 77, + barrier: Some(CutBarrierWire { capture: None }), }), ); @@ -149,10 +166,11 @@ mod tests { }], "only a restore point's marker is reported" ); - for hlc in [900, 1_000] { + for (hlc, point, ordered) in [(900, 0, false), (1_000, 77, true)] { assert!(matches!( rx.try_recv(), - Ok(SchedulerInput::CutMarker { hlc: got }) if got == hlc + Ok(SchedulerInput::CutMarker { hlc: got, restore_point, barrier }) + if got == hlc && restore_point == point && barrier.is_some() == ordered )); } } @@ -172,6 +190,7 @@ mod tests { encode(&SequencerEntry::CutMarker { hlc, restore_point: 0, + barrier: None, }) }; sm.apply(1, &marker(10)); diff --git a/nodedb-cluster/src/calvin/sequencer/state_machine/snapshot.rs b/nodedb-cluster/src/calvin/sequencer/state_machine/snapshot.rs index 8b63c1fd3..21ea5fb62 100644 --- a/nodedb-cluster/src/calvin/sequencer/state_machine/snapshot.rs +++ b/nodedb-cluster/src/calvin/sequencer/state_machine/snapshot.rs @@ -10,7 +10,9 @@ //! mints an epoch or an instant at or below a committed one; //! - every open multi-part transaction, with the parts already applied, so //! the follower closes it on the same entry as every other replica, and -//! never abandons one the others committed. +//! never abandons one the others committed; +//! - whether the captured state holds every committed entry's effect, so a +//! follower seeds its epoch from it only when it does. //! //! The leader captures the snapshot at its applied index, on the Raft tick //! thread between apply batches, so it holds exactly the entries through @@ -20,6 +22,7 @@ use crate::calvin::TxnId; use crate::error::ClusterError; use super::core::{NOT_YET_APPLIED, SequencerStateMachine}; +use super::history::HistoryOrigin; /// The state the sequencer log built through `applied_index`. #[derive(Debug, Clone, PartialEq, Eq, zerompk::ToMessagePack, zerompk::FromMessagePack)] @@ -29,6 +32,7 @@ pub struct SequencerSnapshot { last_applied_epoch: Option, last_epoch_system_ms: Option, open_txns: Vec, + history_known: bool, } /// One open multi-part transaction in a [`SequencerSnapshot`]. @@ -83,16 +87,27 @@ impl SequencerStateMachine { last_applied_epoch: self.last_applied_epoch(), last_epoch_system_ms: self.last_epoch_system_ms, open_txns: self.open_parts.images(), + history_known: self.history.is_known(), } } /// Replace the state the log built with `snapshot`, installed as the /// group's log boundary. Entries after the snapshot apply on top. + /// + /// The history origin becomes the snapshot, or stays unknown when the + /// capturing replica's history was unknown. pub fn restore_snapshot(&mut self, snapshot: SequencerSnapshot) { self.last_applied_epoch = snapshot.last_applied_epoch.unwrap_or(NOT_YET_APPLIED); self.last_epoch_system_ms = snapshot.last_epoch_system_ms; self.last_committed_index = snapshot.applied_index; self.open_parts = super::parts::OpenParts::from_images(snapshot.open_txns); + self.history = if snapshot.history_known { + HistoryOrigin::Snapshot { + through: snapshot.applied_index, + } + } else { + HistoryOrigin::Unknown + }; } } @@ -211,6 +226,10 @@ mod tests { assert_eq!(follower.min_open_parts_index(), Some(1)); assert_eq!(follower.last_applied_epoch(), Some(0)); assert_eq!(follower.current_committed_index(), Some(3)); + assert_eq!( + follower.history_origin(), + HistoryOrigin::Snapshot { through: 3 } + ); follower.apply(4, &encode(&part(1, vshard))); assert!( @@ -222,4 +241,19 @@ mod tests { Ok(SchedulerInput::TxnPart { index: 1, .. }) )); } + + /// A snapshot captured with unknown history restores as unknown, so the + /// follower never treats it as complete. + #[test] + fn a_snapshot_of_unknown_history_restores_as_unknown() { + let mut leader = + SequencerStateMachine::new(HashMap::new(), CalvinCompletionRegistry::new_detached()); + leader.mark_history_unknown(); + let snapshot = leader.capture_snapshot(7); + + let mut follower = + SequencerStateMachine::new(HashMap::new(), CalvinCompletionRegistry::new_detached()); + follower.restore_snapshot(snapshot); + assert_eq!(follower.history_origin(), HistoryOrigin::Unknown); + } } diff --git a/nodedb-cluster/src/calvin/sequencer/validator.rs b/nodedb-cluster/src/calvin/sequencer/validator.rs index b1c9166c7..d2e45f60f 100644 --- a/nodedb-cluster/src/calvin/sequencer/validator.rs +++ b/nodedb-cluster/src/calvin/sequencer/validator.rs @@ -19,8 +19,8 @@ //! //! # Conflict handling (read-aware) //! -//! The commit lockset is all-exclusive over each txn's read ∪ write keys, so -//! same-key transactions serialize by `position`. Per-key last-write-LSN +//! The scheduler locks a txn's write keys exclusive and its read keys shared, +//! so a reader and a writer of one key serialize by `position`. Per-key last-write-LSN //! validation means a reader `R` of key `K` aborts at commit only if a //! *lower-position* writer `W` of `K` committed first (its write LSN exceeds //! `R`'s read version). Giving `R` a lower position than `W` converts that @@ -63,7 +63,8 @@ type KeyId = (u8, String, Vec); /// A single flattened key drawn from one transaction's read or write set. #[derive(Debug)] struct FlatKey { - /// Discriminant tag for the engine variant (Document=0, Vector=1, Kv=2, Edge=3). + /// Discriminant tag for the key-set variant (Document=0, Vector=1, Kv=2, + /// Edge=3, Array=4, Collection=5, Unique=6). discriminant: u8, /// Static engine name used when building conflict-context keys. engine_name: &'static str, @@ -153,6 +154,34 @@ fn flatten_set(set: &ReadWriteSet) -> Vec { }); } } + // The whole collection: one key with no row bytes. + EngineKeySet::Collection { collection, .. } => { + out.push(FlatKey { + discriminant: 5, + engine_name: "collection", + collection: collection.clone(), + key_bytes: Vec::new(), + }); + } + // One key per claimed value: the index name, a zero byte, the value. + EngineKeySet::Unique { + collection, + index, + values, + } => { + for value in values.as_slice() { + let mut key_bytes = Vec::with_capacity(index.len() + 1 + value.len()); + key_bytes.extend_from_slice(index.as_bytes()); + key_bytes.push(0); + key_bytes.extend_from_slice(value); + out.push(FlatKey { + discriminant: 6, + engine_name: "unique", + collection: collection.clone(), + key_bytes, + }); + } + } } } out diff --git a/nodedb-cluster/src/calvin/types/cut_barrier.rs b/nodedb-cluster/src/calvin/types/cut_barrier.rs new file mode 100644 index 000000000..50f208e31 --- /dev/null +++ b/nodedb-cluster/src/calvin/types/cut_barrier.rs @@ -0,0 +1,51 @@ +// SPDX-License-Identifier: BUSL-1.1 + +//! Plain-data wire twins of the data-group barrier a backup's cut orders. +//! +//! A cut marker that carries a [`CutBarrierWire`] asks every data group for +//! one barrier at the marker's watermark. The leader of each group proposes +//! the barrier once every scheduler of the group's vShards on that node +//! passed the marker. Until it applied the barrier, the leader holds the +//! redo of every transaction sequenced after the marker. So every slice of a +//! transaction before the marker sits before the barrier in its group, and +//! every slice of a later one sits after it. +//! +//! The sequencer crate cannot depend on the host's capture request, so +//! [`CutCaptureWire`] mirrors its fields. + +use serde::{Deserialize, Serialize}; + +/// Wire twin of a database backup's capture request. +#[derive( + Debug, + Clone, + PartialEq, + Eq, + Serialize, + Deserialize, + zerompk::ToMessagePack, + zerompk::FromMessagePack, +)] +pub struct CutCaptureWire { + /// Unique per backup. It keys every parked capture. + pub request_id: u64, + pub database_id: u64, + pub tenants: Vec, +} + +/// The barrier a cut marker orders into every data group. +#[derive( + Debug, + Clone, + PartialEq, + Eq, + Serialize, + Deserialize, + zerompk::ToMessagePack, + zerompk::FromMessagePack, +)] +pub struct CutBarrierWire { + /// The capture each group's leader takes when it applies the barrier, + /// `None` for a cut that captures nothing. + pub capture: Option, +} diff --git a/nodedb-cluster/src/calvin/types/lock_wire.rs b/nodedb-cluster/src/calvin/types/lock_wire.rs index ab5191ab1..ca16cd323 100644 --- a/nodedb-cluster/src/calvin/types/lock_wire.rs +++ b/nodedb-cluster/src/calvin/types/lock_wire.rs @@ -57,6 +57,14 @@ pub enum LockKeyWire { src: u32, dst: u32, }, + /// A whole collection. + Collection { collection: String }, + /// One claimed value of one UNIQUE index. + Unique { + collection: String, + index: String, + value: Vec, + }, } // ── ReleaseReason ───────────────────────────────────────────────────────────── @@ -116,6 +124,14 @@ mod tests { src: 1, dst: 2, }, + LockKeyWire::Collection { + collection: "orders".to_owned(), + }, + LockKeyWire::Unique { + collection: "users".to_owned(), + index: "users_email".to_owned(), + value: b"a@x".to_vec(), + }, ]; for v in variants { let bytes = zerompk::to_msgpack_vec(&v).expect("encode"); diff --git a/nodedb-cluster/src/calvin/types/mod.rs b/nodedb-cluster/src/calvin/types/mod.rs index 0828bd18d..c4bd86d01 100644 --- a/nodedb-cluster/src/calvin/types/mod.rs +++ b/nodedb-cluster/src/calvin/types/mod.rs @@ -1,5 +1,6 @@ // SPDX-License-Identifier: BUSL-1.1 +pub mod cut_barrier; pub mod lock_wire; pub mod multi_part; pub mod primitives; @@ -8,13 +9,14 @@ pub mod scheduler_input; pub mod sequencer; pub mod transaction; +pub use cut_barrier::{CutBarrierWire, CutCaptureWire}; pub use lock_wire::{LockKeyWire, ReleaseReason, TxnIdWire}; pub use multi_part::{ MultiPartPlans, PartStreamId, PlanPart, StreamedPart, TaskChunk, VShardParts, }; pub use primitives::{ - DependentReadSpec, EngineKeySet, EngineTag, PassiveReadKey, ReadKeyIdent, SortedVec, - VersionedReadEntry, VersionedReadSet, + DependentReadSpec, EngineKeySet, EngineTag, PassiveKey, PassiveReadKey, PassiveReadKeyId, + ReadKeyIdent, SortedVec, VersionedReadEntry, VersionedReadSet, }; pub use read_write_set::ReadWriteSet; pub use scheduler_input::SchedulerInput; diff --git a/nodedb-cluster/src/calvin/types/primitives.rs b/nodedb-cluster/src/calvin/types/primitives.rs index 5954a6ea7..8f1103ee9 100644 --- a/nodedb-cluster/src/calvin/types/primitives.rs +++ b/nodedb-cluster/src/calvin/types/primitives.rs @@ -12,14 +12,19 @@ use std::collections::BTreeMap; use serde::{Deserialize, Serialize}; pub use nodedb_types::calvin::{ - EngineKeySet, EngineTag, PassiveReadKey, ReadKeyIdent, SortedVec, VersionedReadEntry, - VersionedReadSet, + EngineKeySet, EngineTag, PassiveKey, PassiveReadKey, PassiveReadKeyId, ReadKeyIdent, SortedVec, + VersionedReadEntry, VersionedReadSet, }; /// Describes the passive-read participants for a dependent-read Calvin txn. /// -/// Each entry maps a vshard id to the keys that vshard must read and broadcast -/// to all active participants before any writes can proceed. +/// `passive_reads` maps each passive vShard to the keys it reads at its lock +/// grant and broadcasts to every active participant before any write stages. +/// +/// `expected` holds the value the coordinator read for each passive row +/// before it planned the writes: the stored bytes, `None` for an absent row. +/// An active participant whose broadcast values differ votes +/// `PredictionDrift`, and the coordinator reads again. /// /// `BTreeMap` is mandatory here: the sequencer and scheduler must iterate /// vshards in a deterministic order (determinism contract). @@ -36,20 +41,35 @@ pub use nodedb_types::calvin::{ pub struct DependentReadSpec { /// Passive participants: vshard → keys to read. pub passive_reads: BTreeMap>, + /// The value the coordinator read for each passive row. + pub expected: BTreeMap>>, } impl DependentReadSpec { - /// Total estimated serialized bytes across all passive read keys. + /// Total estimated serialized bytes across all passive read keys and + /// the expected values the coordinator read. /// /// Used by the sequencer admission check to enforce /// `max_dependent_read_bytes_per_txn`. This is an O(1)-per-key /// estimate, not an exact serialized size. pub fn total_bytes(&self) -> usize { - self.passive_reads + let keys: usize = self + .passive_reads .values() .flat_map(|ks| ks.iter()) .map(|k| k.engine_key.serialized_size_hint()) - .sum() + .sum(); + let expected: usize = self + .expected + .iter() + .map(|(id, value)| id.serialized_size_hint() + value.as_ref().map_or(0, Vec::len)) + .sum(); + keys + expected + } + + /// The passive vShards, in id order. + pub fn passive_vshards(&self) -> impl Iterator + '_ { + self.passive_reads.keys().copied() } } @@ -160,10 +180,39 @@ mod tests { ); m }, + expected: BTreeMap::from([( + PassiveReadKeyId::kv( + nodedb_types::QualifiedCollection::from_stored("sessions".to_owned()), + b"abc".to_vec(), + ), + Some(b"stored".to_vec()), + )]), }; let bytes = zerompk::to_msgpack_vec(&spec).unwrap(); let decoded: DependentReadSpec = zerompk::from_msgpack(&bytes).unwrap(); - assert_eq!(spec.passive_reads.len(), decoded.passive_reads.len()); - assert_eq!(spec.passive_reads.get(&1), decoded.passive_reads.get(&1)); + assert_eq!(decoded, spec); + } + + /// The byte estimate counts the expected values the coordinator read. + #[test] + fn total_bytes_counts_expected_values() { + let mut spec = DependentReadSpec { + passive_reads: BTreeMap::from([( + 1u32, + vec![PassiveReadKey { + engine_key: kv_set("s", vec![b"k".to_vec()]), + }], + )]), + expected: BTreeMap::new(), + }; + let keys_only = spec.total_bytes(); + spec.expected.insert( + PassiveReadKeyId::kv( + nodedb_types::QualifiedCollection::from_stored("s".to_owned()), + b"k".to_vec(), + ), + Some(vec![0u8; 100]), + ); + assert!(spec.total_bytes() >= keys_only + 100); } } diff --git a/nodedb-cluster/src/calvin/types/read_write_set.rs b/nodedb-cluster/src/calvin/types/read_write_set.rs index ea51c97a1..55265e90d 100644 --- a/nodedb-cluster/src/calvin/types/read_write_set.rs +++ b/nodedb-cluster/src/calvin/types/read_write_set.rs @@ -38,10 +38,12 @@ impl ReadWriteSet { /// Derive the set of vShards participating in this read/write set. /// - /// For Document/Vector/KV entries the vshard is derived from the - /// collection name (collection-level routing, consistent with the - /// per-vshard Raft groups that own each collection). KV collections - /// are also assigned a single vshard at creation time. + /// For Document/Vector/KV/Unique entries, and a Collection entry that + /// names no vShards, the vshard is derived from the collection name + /// (collection-level routing, consistent with the per-vshard Raft groups + /// that own each collection). KV collections are also assigned a single + /// vshard at creation time. A Collection entry that names vShards + /// participates on exactly those. /// /// For Edge entries the participating vShards are the edge's /// `home_vshards` (the `from_key(src)` / `from_key(dst)` key-hashed @@ -81,9 +83,19 @@ impl ReadWriteSet { } } } + EngineKeySet::Collection { vshards: homes, .. } if !homes.is_empty() => { + for &home in homes.as_slice() { + let vshard = VShardId::new(home); + if seen.insert(vshard.as_u32()) { + result.push(vshard); + } + } + } EngineKeySet::Document { .. } | EngineKeySet::Vector { .. } - | EngineKeySet::Kv { .. } => { + | EngineKeySet::Kv { .. } + | EngineKeySet::Collection { .. } + | EngineKeySet::Unique { .. } => { let vshard = CollectionKey::from_qualified_str(database_id, engine_set.collection())? .vshard(); diff --git a/nodedb-cluster/src/calvin/types/scheduler_input.rs b/nodedb-cluster/src/calvin/types/scheduler_input.rs index 23724d495..2704a5318 100644 --- a/nodedb-cluster/src/calvin/types/scheduler_input.rs +++ b/nodedb-cluster/src/calvin/types/scheduler_input.rs @@ -36,7 +36,15 @@ pub enum SchedulerInput { /// A backup's consistent-cut marker carrying its watermark `hlc`. Every /// transaction delivered before it must finish before the scheduler /// reports it; every transaction delivered after it commits above `hlc`. - CutMarker { hlc: u64 }, + /// `restore_point` names the cluster restore point the cut takes, `0` + /// for none. `barrier` is set when the cut places a barrier in every data + /// group: the leader holds the redo of every later transaction until it + /// applied its group's barrier. Shared by every vShard of this node. + CutMarker { + hlc: u64, + restore_point: u64, + barrier: Option>, + }, /// Part `index` of the plans of the multi-part transaction `txn`, whose /// first task is the transaction's task `first_task`. The bytes are /// shared by every scheduler of this node the part targets. `chunk` is diff --git a/nodedb-cluster/src/calvin/types/transaction.rs b/nodedb-cluster/src/calvin/types/transaction.rs index b53080eb5..4585a1d38 100644 --- a/nodedb-cluster/src/calvin/types/transaction.rs +++ b/nodedb-cluster/src/calvin/types/transaction.rs @@ -75,7 +75,7 @@ pub struct TxClass { /// Each entry carries the responding shard's write-LSN watermark at read /// time plus the point/predicate identity, so a participant can validate /// the read at the commit serialization point (the local commit vote on - /// `read_set_valid`). Empty for pure-write and autocommit transactions. + /// a stage response). Empty for pure-write and autocommit transactions. #[serde(default)] #[msgpack(default)] pub versioned_reads: VersionedReadSet, @@ -347,28 +347,39 @@ impl TxClass { Ok(participants) } - /// Ergonomic constructor for dependent-read Calvin transactions. + /// Constructor for a dependent-read Calvin transaction of `database_id`. /// - /// Equivalent to `TxClass::new(read_set, write_set, plans, tenant_id, - /// Some(dependent_reads), versioned_reads)`. + /// The write set spans two or more vShards, as [`Self::new_in_database`] + /// requires. Every passive vShard of `dependent_reads` participates. pub fn new_dependent( read_set: ReadWriteSet, write_set: ReadWriteSet, plans: Vec, tenant_id: TenantId, + database_id: DatabaseId, dependent_reads: DependentReadSpec, versioned_reads: VersionedReadSet, ) -> Result { - Self::new( + Self::new_in_database( read_set, write_set, plans, tenant_id, + database_id, Some(dependent_reads), versioned_reads, ) } + /// The vShards the write set names: the active participants of a + /// dependent-read transaction. + /// + /// Fails when a write-set collection name lacks the qualifier of the + /// class's database. Construction rejects such a class. + pub fn active_vshards(&self) -> Result, CalvinError> { + self.write_vshards() + } + /// The vShards that must receive this transaction's slice. /// /// Derived from the write set's collection names. Re-derived after @@ -494,7 +505,7 @@ mod tests { }; use super::*; use nodedb_types::id::CollectionKey; - use nodedb_types::{KeyRepr, Lsn}; + use nodedb_types::{KeyRepr, WriteVersion}; fn doc_set(collection: &str, surrogates: Vec) -> EngineKeySet { EngineKeySet::Document { @@ -734,6 +745,7 @@ mod tests { ); m }, + expected: std::collections::BTreeMap::new(), }; let tc = TxClass::new( @@ -764,17 +776,15 @@ mod tests { engine: EngineTag::Kv, collection: "kv_col".to_owned(), key: ReadKeyIdent::Point(KeyRepr::KvKey(Box::from(&b"k1"[..]))), - read_lsn: Lsn::new(7), + read_version: WriteVersion::logged(0, 7), home_vshard: None, - served_by: 0, }, VersionedReadEntry { engine: EngineTag::Document, collection: "doc_col".to_owned(), key: ReadKeyIdent::Predicate, - read_lsn: Lsn::new(11), + read_version: WriteVersion::logged(2, 11), home_vshard: None, - served_by: 0, }, ]) } @@ -805,7 +815,7 @@ mod tests { let mut decoded: TxClass = zerompk::from_msgpack(&bytes).expect("decode TxClass"); decoded.restore_derived().expect("restore derived"); - // Every read_lsn and the Point/Predicate distinction survive exactly. + // Every read version and the Point/Predicate distinction survive exactly. assert_eq!(decoded.versioned_reads, reads); assert_eq!(decoded.versioned_reads.len(), 2); let point = decoded @@ -813,7 +823,7 @@ mod tests { .iter() .find(|e| matches!(e.key, ReadKeyIdent::Point(_))) .expect("point entry"); - assert_eq!(point.read_lsn, Lsn::new(7)); + assert_eq!(point.read_version, WriteVersion::logged(0, 7)); assert_eq!( point.key, ReadKeyIdent::Point(KeyRepr::KvKey(Box::from(&b"k1"[..]))) @@ -823,7 +833,7 @@ mod tests { .iter() .find(|e| matches!(e.key, ReadKeyIdent::Predicate)) .expect("predicate entry"); - assert_eq!(predicate.read_lsn, Lsn::new(11)); + assert_eq!(predicate.read_version, WriteVersion::logged(2, 11)); } /// Mirror of `TxClass`'s wire shape from BEFORE `versioned_reads` existed: diff --git a/nodedb-cluster/src/circuit_breaker.rs b/nodedb-cluster/src/circuit_breaker.rs index c74ccfe39..a47380f2a 100644 --- a/nodedb-cluster/src/circuit_breaker.rs +++ b/nodedb-cluster/src/circuit_breaker.rs @@ -349,6 +349,8 @@ impl RetryPolicy { // A timed-out send may still have reached the peer, so resending // it can apply the request twice. ClusterError::ShardTimeout { .. } => false, + // The request was written, so the peer can have run it. + ClusterError::Unanswered { .. } => false, ClusterError::Raft(_) | ClusterError::VShardNotMapped { .. } | ClusterError::GroupNotFound { .. } diff --git a/nodedb-cluster/src/distributed_array/coordinator/read.rs b/nodedb-cluster/src/distributed_array/coordinator/read.rs index fa9ba1597..760c0b184 100644 --- a/nodedb-cluster/src/distributed_array/coordinator/read.rs +++ b/nodedb-cluster/src/distributed_array/coordinator/read.rs @@ -12,8 +12,8 @@ use crate::circuit_breaker::CircuitBreaker; use crate::error::{ClusterError, Result}; use super::super::merge::{ - ArrayAggPartial, any_truncated_before_horizon_agg, merge_slice_rows, merge_slice_rows_sorted, - reduce_agg_partials, + ArrayAggPartial, any_truncated_before_horizon_agg, merge_read_versions, merge_slice_rows, + merge_slice_rows_sorted, reduce_agg_partials, }; use super::super::rpc::ShardRpcDispatch; use super::super::scatter::{FanOutParams, FanOutPartitionedParams, fan_out, fan_out_partitioned}; @@ -50,6 +50,8 @@ pub struct ArrayCoordParams { pub struct CoordSliceResult { pub rows: Vec>, pub truncated_before_horizon: bool, + /// The write versions every shard leg observed, one per vShard. + pub read_versions: Vec, } /// Result of a coordinated aggregate fan-out. @@ -63,6 +65,8 @@ pub struct CoordSliceResult { pub struct CoordAggResult { pub partials: Vec, pub truncated_before_horizon: bool, + /// The write versions every shard leg observed, one per vShard. + pub read_versions: Vec, } /// Compute the inclusive Hilbert-prefix range `[lo, hi]` that vShard `shard_id` @@ -214,6 +218,7 @@ impl ArrayCoordinator { Ok(CoordSliceResult { rows, truncated_before_horizon, + read_versions: merge_read_versions(resps.iter().map(|r| r.read_versions.as_slice())), }) } @@ -263,6 +268,7 @@ impl ArrayCoordinator { Ok(CoordAggResult { partials: reduce_agg_partials(&resps), truncated_before_horizon: any_truncated_before_horizon_agg(&resps), + read_versions: merge_read_versions(resps.iter().map(|r| r.read_versions.as_slice())), }) } @@ -339,6 +345,14 @@ mod tests { use super::*; + /// The version a mock shard leg reports for its vShard. + fn leg_versions(vshard: u32) -> Vec { + vec![nodedb_types::ShardVersion { + vshard, + version: nodedb_types::WriteVersion::logged(1, u64::from(vshard) + 100), + }] + } + /// Mock dispatch that returns a pre-serialised `ArrayShardSliceResp`. struct SliceEchoDispatch { /// Rows to return from each shard. @@ -353,6 +367,7 @@ mod tests { rows_msgpack: self.rows.clone(), truncated: false, truncated_before_horizon: false, + read_versions: leg_versions(req.vshard_id), }; let payload = zerompk::to_msgpack_vec(&resp).unwrap(); Ok(VShardEnvelope::new( @@ -377,6 +392,7 @@ mod tests { shard_id: req.vshard_id, partials: self.partials.clone(), truncated_before_horizon: false, + read_versions: leg_versions(req.vshard_id), }; let payload = zerompk::to_msgpack_vec(&resp).unwrap(); Ok(VShardEnvelope::new( @@ -438,6 +454,16 @@ mod tests { .expect("coord_slice should succeed"); assert_eq!(result.rows.len(), 6); assert!(!result.truncated_before_horizon); + let mut versions = result.read_versions; + versions.sort(); + assert_eq!( + versions, + [0, 1, 2] + .into_iter() + .flat_map(leg_versions) + .collect::>(), + "the merge keeps every shard leg's version" + ); } #[tokio::test] @@ -533,6 +559,7 @@ mod tests { shard_id: req.vshard_id, partials, truncated_before_horizon: false, + read_versions: Vec::new(), }; let payload = zerompk::to_msgpack_vec(&resp).unwrap(); Ok(VShardEnvelope::new( @@ -596,6 +623,7 @@ mod tests { shard_id: req.vshard_id, partials, truncated_before_horizon: below, + read_versions: Vec::new(), }; let payload = zerompk::to_msgpack_vec(&resp).unwrap(); Ok(VShardEnvelope::new( diff --git a/nodedb-cluster/src/distributed_array/handler.rs b/nodedb-cluster/src/distributed_array/handler.rs index 2f8ae0acc..01cafb3e5 100644 --- a/nodedb-cluster/src/distributed_array/handler.rs +++ b/nodedb-cluster/src/distributed_array/handler.rs @@ -78,6 +78,7 @@ async fn handle_slice( rows_msgpack: exec.rows, truncated, truncated_before_horizon: exec.truncated_before_horizon, + read_versions: exec.read_versions, }; serialise(resp) } @@ -98,6 +99,7 @@ async fn handle_agg( shard_id: local_vshard_id, partials: exec.partials, truncated_before_horizon: exec.truncated_before_horizon, + read_versions: exec.read_versions, }; serialise(resp) } @@ -263,6 +265,14 @@ mod tests { }; use super::handle_array_shard_rpc; + /// The version a stub shard read reports for its vShard. + fn stub_versions(vshard: u32) -> Vec { + vec![nodedb_types::ShardVersion { + vshard, + version: nodedb_types::WriteVersion::logged(1, 7), + }] + } + /// Mock executor that returns a fixed set of row bytes for slice, /// a fixed bitmap for surrogate scan, fixed partials for agg, and /// echoes back the `wal_lsn` for put/delete. @@ -278,12 +288,13 @@ mod tests { impl ArrayLocalExecutor for StubExecutor { async fn exec_slice( &self, - _local_vshard_id: u32, + local_vshard_id: u32, _req: &super::super::wire::ArrayShardSliceReq, ) -> Result { Ok(ArraySliceExec { rows: self.rows.clone(), truncated_before_horizon: self.truncated_before_horizon, + read_versions: stub_versions(local_vshard_id), }) } @@ -298,12 +309,13 @@ mod tests { async fn exec_agg( &self, - _local_vshard_id: u32, + local_vshard_id: u32, _req: &ArrayShardAggReq, ) -> Result { Ok(ArrayAggExec { partials: self.partials.clone(), truncated_before_horizon: self.truncated_before_horizon, + read_versions: stub_versions(local_vshard_id), }) } @@ -422,6 +434,34 @@ mod tests { ); } + /// A shard leg that matched no row still reports its vShard's version, + /// and the version survives the response's wire encoding. + #[tokio::test] + async fn an_empty_shard_read_reports_its_version_over_the_wire() { + let executor: Arc = Arc::new(StubExecutor { + rows: vec![], + bitmap: vec![], + partials: vec![], + truncated_before_horizon: false, + affected: 0, + }); + let slice = + handle_array_shard_rpc(ARRAY_SHARD_SLICE_REQ, 5, &make_slice_req_bytes(), &executor) + .await + .expect("slice handler should succeed"); + let slice: ArrayShardSliceResp = + zerompk::from_msgpack(&slice).expect("response should deserialise"); + assert!(slice.rows_msgpack.is_empty()); + assert_eq!(slice.read_versions, stub_versions(5)); + + let agg = handle_array_shard_rpc(ARRAY_SHARD_AGG_REQ, 5, &make_agg_req_bytes(), &executor) + .await + .expect("agg handler should succeed"); + let agg: ArrayShardAggResp = + zerompk::from_msgpack(&agg).expect("response should deserialise"); + assert_eq!(agg.read_versions, stub_versions(5)); + } + #[tokio::test] async fn handle_agg_propagates_truncated_before_horizon() { let executor: Arc = Arc::new(StubExecutor { diff --git a/nodedb-cluster/src/distributed_array/local_executor.rs b/nodedb-cluster/src/distributed_array/local_executor.rs index 13dcf319b..68fcf31ff 100644 --- a/nodedb-cluster/src/distributed_array/local_executor.rs +++ b/nodedb-cluster/src/distributed_array/local_executor.rs @@ -29,6 +29,9 @@ use crate::error::Result; pub struct ArraySliceExec { pub rows: Vec>, pub truncated_before_horizon: bool, + /// The write versions the shard read observed, one per vShard it + /// covers. A transaction validates the read at these versions. + pub read_versions: Vec, } /// Result of a local shard partial aggregate: the per-group partial states @@ -36,6 +39,8 @@ pub struct ArraySliceExec { pub struct ArrayAggExec { pub partials: Vec, pub truncated_before_horizon: bool, + /// The write versions the shard read observed (see [`ArraySliceExec`]). + pub read_versions: Vec, } /// Result of a local shard write (put or delete): the LSN the coordinator diff --git a/nodedb-cluster/src/distributed_array/merge.rs b/nodedb-cluster/src/distributed_array/merge.rs index be00c0cfa..9d11396e7 100644 --- a/nodedb-cluster/src/distributed_array/merge.rs +++ b/nodedb-cluster/src/distributed_array/merge.rs @@ -101,6 +101,22 @@ pub fn any_truncated_before_horizon_agg(shard_resps: &[ArrayShardAggResp]) -> bo shard_resps.iter().any(|r| r.truncated_before_horizon) } +/// The write versions every shard leg observed, one per vShard. Each leg +/// reads its own vShard, so every leg's version is kept. Two legs that report +/// one vShard keep the higher version. +pub fn merge_read_versions<'a>( + legs: impl IntoIterator, +) -> Vec { + let mut merged: Vec = Vec::new(); + for shard in legs.into_iter().flatten() { + match merged.iter_mut().find(|seen| seen.vshard == shard.vshard) { + Some(seen) => seen.version = seen.version.max(shard.version), + None => merged.push(*shard), + } + } + merged +} + /// Merge row batches from multiple shards into one result set. /// /// Rows are concatenated in shard-arrival order (order-independent for @@ -232,11 +248,13 @@ mod tests { shard_id: 0, partials: vec![ArrayAggPartial::from_single(0, 10.0)], truncated_before_horizon: false, + read_versions: Vec::new(), }; let resp_b = ArrayShardAggResp { shard_id: 1, partials: vec![ArrayAggPartial::from_single(0, 20.0)], truncated_before_horizon: false, + read_versions: Vec::new(), }; let merged = reduce_agg_partials(&[resp_a, resp_b]); assert_eq!(merged.len(), 1); @@ -253,6 +271,7 @@ mod tests { ArrayAggPartial::from_single(1, 15.0), ], truncated_before_horizon: false, + read_versions: Vec::new(), }; let merged = reduce_agg_partials(&[resp]); assert_eq!(merged.len(), 2); @@ -277,12 +296,14 @@ mod tests { rows_msgpack: vec![vec![1u8], vec![2u8]], truncated: false, truncated_before_horizon: false, + read_versions: Vec::new(), }; let r1 = ArrayShardSliceResp { shard_id: 1, rows_msgpack: vec![vec![3u8]], truncated: false, truncated_before_horizon: false, + read_versions: Vec::new(), }; let rows = merge_slice_rows(&[r0, r1], 0); assert_eq!(rows.len(), 3); @@ -295,6 +316,7 @@ mod tests { rows_msgpack: vec![vec![1u8], vec![2u8], vec![3u8], vec![4u8], vec![5u8]], truncated: false, truncated_before_horizon: false, + read_versions: Vec::new(), }; let rows = merge_slice_rows(&[resp], 3); assert_eq!(rows.len(), 3); @@ -308,11 +330,13 @@ mod tests { shard_id: 0, partials: vec![ArrayAggPartial::from_single(0, 5.0)], truncated_before_horizon: false, + read_versions: Vec::new(), }; let resp_b = ArrayShardAggResp { shard_id: 1, partials: vec![ArrayAggPartial::from_single(0, 3.0)], truncated_before_horizon: false, + read_versions: Vec::new(), }; let merged = reduce_agg_partials(&[resp_a, resp_b]); assert_eq!(merged.len(), 1); @@ -325,11 +349,13 @@ mod tests { shard_id: 0, partials: vec![ArrayAggPartial::from_single(0, 5.0)], truncated_before_horizon: false, + read_versions: Vec::new(), }; let resp_b = ArrayShardAggResp { shard_id: 1, partials: vec![ArrayAggPartial::from_single(0, 99.0)], truncated_before_horizon: false, + read_versions: Vec::new(), }; let merged = reduce_agg_partials(&[resp_a, resp_b]); assert_eq!(merged.len(), 1); @@ -359,6 +385,7 @@ mod tests { ArrayAggPartial::from_single(1, 10.0), ], truncated_before_horizon: false, + read_versions: Vec::new(), }; let resp_b = ArrayShardAggResp { shard_id: 1, @@ -367,6 +394,7 @@ mod tests { ArrayAggPartial::from_single(2, 30.0), ], truncated_before_horizon: false, + read_versions: Vec::new(), }; let merged = reduce_agg_partials(&[resp_a, resp_b]); assert_eq!(merged.len(), 3); @@ -385,12 +413,14 @@ mod tests { rows_msgpack: vec![], truncated: false, truncated_before_horizon: true, + read_versions: Vec::new(), }; let r1 = ArrayShardSliceResp { shard_id: 1, rows_msgpack: vec![vec![1u8]], truncated: false, truncated_before_horizon: false, + read_versions: Vec::new(), }; assert!(any_truncated_before_horizon_slice(&[r0, r1])); @@ -398,11 +428,13 @@ mod tests { shard_id: 0, partials: vec![], truncated_before_horizon: false, + read_versions: Vec::new(), }; let a1 = ArrayShardAggResp { shard_id: 1, partials: vec![], truncated_before_horizon: true, + read_versions: Vec::new(), }; assert!(any_truncated_before_horizon_agg(&[a0, a1])); @@ -410,6 +442,7 @@ mod tests { shard_id: 2, partials: vec![], truncated_before_horizon: false, + read_versions: Vec::new(), }; assert!(!any_truncated_before_horizon_agg(&[a_none])); } @@ -421,11 +454,13 @@ mod tests { shard_id: 0, partials: vec![ArrayAggPartial::from_single(0, 7.0)], truncated_before_horizon: false, + read_versions: Vec::new(), }; let resp_b = ArrayShardAggResp { shard_id: 1, partials: vec![ArrayAggPartial::from_single(1, 13.0)], truncated_before_horizon: false, + read_versions: Vec::new(), }; let merged = reduce_agg_partials(&[resp_a, resp_b]); assert_eq!(merged.len(), 2); @@ -442,12 +477,14 @@ mod tests { rows_msgpack: vec![vec![1u8], vec![2u8]], truncated: false, truncated_before_horizon: false, + read_versions: Vec::new(), }; let r1 = ArrayShardSliceResp { shard_id: 1, rows_msgpack: vec![vec![3u8], vec![4u8]], truncated: false, truncated_before_horizon: false, + read_versions: Vec::new(), }; // Total 4 rows, limit 3 → first 3. let rows = merge_slice_rows(&[r0, r1], 3); @@ -476,12 +513,14 @@ mod tests { rows_msgpack: vec![audit_row(30, 3), audit_row(10, 1)], truncated: false, truncated_before_horizon: false, + read_versions: Vec::new(), }; let r1 = ArrayShardSliceResp { shard_id: 1, rows_msgpack: vec![audit_row(20, 2)], truncated: false, truncated_before_horizon: false, + read_versions: Vec::new(), }; let rows = merge_slice_rows_sorted(&[r0, r1], 0).expect("sorted merge"); let times: Vec = rows @@ -499,12 +538,14 @@ mod tests { rows_msgpack: vec![audit_row(50, 5), audit_row(10, 1)], truncated: false, truncated_before_horizon: false, + read_versions: Vec::new(), }; let r1 = ArrayShardSliceResp { shard_id: 1, rows_msgpack: vec![audit_row(30, 3), audit_row(20, 2)], truncated: false, truncated_before_horizon: false, + read_versions: Vec::new(), }; let rows = merge_slice_rows_sorted(&[r0, r1], 2).expect("sorted merge"); let times: Vec = rows @@ -523,6 +564,7 @@ mod tests { rows_msgpack: vec![audit_row(10, 1), vec![0xFF, 0xFF, 0xFF]], truncated: false, truncated_before_horizon: false, + read_versions: Vec::new(), }; assert!(merge_slice_rows_sorted(&[resp], 0).is_err()); } @@ -564,8 +606,30 @@ mod tests { rows_msgpack: (0u8..20).map(|i| vec![i]).collect(), truncated: false, truncated_before_horizon: false, + read_versions: Vec::new(), }; let rows = merge_slice_rows(&[resp], 0); assert_eq!(rows.len(), 20); } + + fn shard_version(vshard: u32, index: u64) -> nodedb_types::ShardVersion { + nodedb_types::ShardVersion { + vshard, + version: nodedb_types::WriteVersion::logged(1, index), + } + } + + /// Every leg's version survives the merge, a leg that matched no row + /// included. A vShard two legs report keeps the higher version. + #[test] + fn the_merge_keeps_every_legs_version() { + let legs = [ + vec![shard_version(3, 10)], + Vec::new(), + vec![shard_version(7, 4)], + vec![shard_version(3, 12)], + ]; + let merged = merge_read_versions(legs.iter().map(Vec::as_slice)); + assert_eq!(merged, vec![shard_version(3, 12), shard_version(7, 4)]); + } } diff --git a/nodedb-cluster/src/distributed_array/scatter.rs b/nodedb-cluster/src/distributed_array/scatter.rs index a697059f0..4c58f172d 100644 --- a/nodedb-cluster/src/distributed_array/scatter.rs +++ b/nodedb-cluster/src/distributed_array/scatter.rs @@ -94,7 +94,7 @@ fn counts_against_breaker(err: &ClusterError) -> bool { | ClusterError::StreamTerminal { .. } => false, // An unresponsive peer is exactly what the breaker exists to shed // load from, so a shard timeout counts like any other liveness failure. - ClusterError::ShardTimeout { .. } => true, + ClusterError::ShardTimeout { .. } | ClusterError::Unanswered { .. } => true, ClusterError::Raft(_) | ClusterError::VShardNotMapped { .. } | ClusterError::GroupNotFound { .. } diff --git a/nodedb-cluster/src/distributed_array/wire.rs b/nodedb-cluster/src/distributed_array/wire.rs index 0a3611830..2183236bb 100644 --- a/nodedb-cluster/src/distributed_array/wire.rs +++ b/nodedb-cluster/src/distributed_array/wire.rs @@ -72,6 +72,10 @@ pub struct ArrayShardSliceResp { /// True when `system_as_of` is below the oldest tile version on this shard /// and the shard produced zero rows as a result of that horizon. pub truncated_before_horizon: bool, + /// The write versions the shard read observed, one per vShard it covers, + /// even when it matched no row. A transaction validates the read at + /// these versions. + pub read_versions: Vec, } /// Scatter request: coordinator asks a shard to compute a partial aggregate. @@ -118,6 +122,9 @@ pub struct ArrayShardAggResp { /// True when `system_as_of` is below the oldest tile version on this shard /// and the shard produced zero rows as a result of that horizon. pub truncated_before_horizon: bool, + /// The write versions the shard read observed (see + /// [`ArrayShardSliceResp::read_versions`]). + pub read_versions: Vec, } /// Scatter request: coordinator forwards a cell write to the owning shard. diff --git a/nodedb-cluster/src/error.rs b/nodedb-cluster/src/error.rs index 5ee76b7b1..26e716a99 100644 --- a/nodedb-cluster/src/error.rs +++ b/nodedb-cluster/src/error.rs @@ -25,6 +25,20 @@ pub enum CalvinError { /// for the full variant set. #[error("sequencer error: {0}")] Sequencer(#[from] crate::calvin::sequencer::error::SequencerError), + + /// The data-group leader's write gate found a key of the proposed write + /// held, and a Calvin transaction can sequence the write. The proposer + /// submits the write to the Calvin sequencer instead. + #[error( + "a lock key of the proposed write is held on the data-group leader; \ + submit the write through the Calvin sequencer" + )] + RouteToSequencer, + + /// The data-group leader's write gate waited for the proposed write's + /// lock keys until its admission deadline. + #[error("the lock keys of the proposed write stayed held past the leader's admission deadline")] + AdmissionTimedOut, } /// Error emitted when applying or validating a `MigrationCheckpoint` entry. @@ -103,6 +117,14 @@ pub enum ClusterError { #[error("shard {vshard_id} RPC timed out after {elapsed_ms}ms")] ShardTimeout { vshard_id: u32, elapsed_ms: u64 }, + /// A request reached the wire to `node_id`, and no answer came back. + /// + /// The stream or connection failed, or the read timed out, after the + /// request was written. The peer can have run it, so its outcome is + /// unknown. A resend can run it twice. + #[error("request to node {node_id} was sent and got no answer: {detail}")] + Unanswered { node_id: u64, detail: String }, + /// Terminal error carried by a streaming `ExecuteStreamEnd` frame. /// /// Preserves the typed shape end-to-end so the coordinator can map a @@ -258,7 +280,10 @@ impl ClusterError { pub fn is_link_failure(&self) -> bool { matches!( self, - Self::Transport { .. } | Self::CircuitOpen { .. } | Self::NodeUnreachable { .. } + Self::Transport { .. } + | Self::Unanswered { .. } + | Self::CircuitOpen { .. } + | Self::NodeUnreachable { .. } ) } } diff --git a/nodedb-cluster/src/forward.rs b/nodedb-cluster/src/forward.rs index d6feb1cd2..272b63ffe 100644 --- a/nodedb-cluster/src/forward.rs +++ b/nodedb-cluster/src/forward.rs @@ -20,18 +20,17 @@ use crate::rpc_codec::{ExecuteRequest, ExecuteResponse, TypedClusterError}; /// stream was reset or finished early): the producer MUST stop pulling and /// return without writing a terminal frame. /// -/// `read_version_lsn` is that frame's per-collection read version (its scanned -/// collection's `coll_write_lsn` at read time), distinct from the core-global -/// `watermark_lsn`. The shuffle fan-out sink max-folds it so the producer can -/// report the observed read version for cross-shard OCC validation; the plain -/// `ExecuteStream` sink (which only carries `watermark_lsn` on the wire) ignores -/// it. +/// `read_versions` are that frame's read versions, one per vShard it read, +/// distinct from the core-global `watermark_lsn`. The shuffle fan-out sink +/// folds them so the producer can report what its scan observed for +/// cross-shard OCC validation. The plain `ExecuteStream` sink carries only +/// `watermark_lsn` on the wire and ignores them. pub trait ChunkSink: Send { fn send_chunk( &mut self, payload: Vec, watermark_lsn: u64, - read_version_lsn: u64, + read_versions: Vec, ) -> impl std::future::Future> + Send; } diff --git a/nodedb-cluster/src/lib.rs b/nodedb-cluster/src/lib.rs index c6e62ea1b..5ed99f51c 100644 --- a/nodedb-cluster/src/lib.rs +++ b/nodedb-cluster/src/lib.rs @@ -118,10 +118,10 @@ pub use migration_executor::{ pub use multi_raft::{GroupStatus, MultiRaft}; pub use raft_loop::{ ApplyPermit, AssignRemoteSurrogate, AuthLeaseService, BuiltGroupSnapshot, CalvinSubmit, - CalvinSubmitInbox, CommitApplier, GroupApplyGates, InstallPermit, MetadataSnapshotCapture, - RaftLoop, ReleaseReservation, ReserveRead, ShuffleAggregator, ShuffleConsumer, ShuffleProducer, - ShuffleReceiver, SnapshotApplier, SnapshotBuilder, SnapshotQuarantineHook, - VShardEnvelopeHandler, + CalvinSubmitInbox, CommitApplier, DataProposeGate, GroupApplyGates, InstallPermit, + MetadataSnapshotCapture, ProposeHold, RaftLoop, ReleaseReservation, ReserveRead, + ShuffleAggregator, ShuffleConsumer, ShuffleProducer, ShuffleReceiver, SnapshotApplier, + SnapshotBuilder, SnapshotQuarantineHook, VShardEnvelopeHandler, }; pub use reachability::{ NoopProber, ReachabilityDriver, ReachabilityDriverConfig, ReachabilityProber, TransportProber, diff --git a/nodedb-cluster/src/multi_raft/peer_contact.rs b/nodedb-cluster/src/multi_raft/peer_contact.rs index d858ebca4..6adfbae23 100644 --- a/nodedb-cluster/src/multi_raft/peer_contact.rs +++ b/nodedb-cluster/src/multi_raft/peer_contact.rs @@ -27,6 +27,15 @@ impl MultiRaft { .map(|node| node.current_term()) } + /// Index of the no-op this node appended when it won its current term in + /// `group_id`, while it leads the group, else `None`. Every entry of an + /// earlier term in its log sits below it and commits with it. + pub fn term_start_index(&self, group_id: u64) -> Option { + self.groups + .get(&group_id) + .and_then(|node| node.term_start_index()) + } + /// Whether this node heard from the leader of `group_id` within /// `window`. The leader itself always has. Unlike a staleness bound, a /// replica that is merely behind on apply still counts as in contact. diff --git a/nodedb-cluster/src/multi_raft/proposals.rs b/nodedb-cluster/src/multi_raft/proposals.rs index 6c5e4ef0e..5d8033ee7 100644 --- a/nodedb-cluster/src/multi_raft/proposals.rs +++ b/nodedb-cluster/src/multi_raft/proposals.rs @@ -30,10 +30,8 @@ impl MultiRaft { /// Maps the vshard to its Raft group via the routing table and reuses the /// existing local leader-role check — no new election. Returns `false` when /// the vshard has no group mapping or this node is a follower/learner for - /// the owning group. Used by the Calvin scheduler to stamp the per-node, - /// non-replicated `is_group_leader` dispatch flag so the OLLP optimistic-lock - /// verification runs only on the leader while every replica applies the same - /// predicted write-set (determinism). + /// the owning group. The Calvin scheduler proposes its owed sequencer + /// entries only while this holds. pub fn vshard_role_is_leader(&self, vshard_id: u32) -> bool { match self .routing @@ -207,10 +205,10 @@ impl MultiRaft { return Ok(false); }; // A sequencer replica that installs a snapshot never receives the - // Calvin inputs the snapshot covers, and its schedulers lose them. - // When every voter does, the inputs are gone from the cluster. So no - // replica compacts past an entry a voter's log may lack: no voter - // ever takes a sequencer snapshot, under this leader or a later one. + // Calvin inputs the snapshot covers. So no replica compacts past an + // entry a live voter's log may lack. A voter silent past the + // check-quorum window does not hold the floor. It recovers through a + // sequencer snapshot install once it answers again. let applied_index = if group_id == crate::calvin::SEQUENCER_GROUP_ID { applied_index.min(node.replicated_floor()) } else { diff --git a/nodedb-cluster/src/raft_loop/builder.rs b/nodedb-cluster/src/raft_loop/builder.rs index c04245500..f0b31ab4b 100644 --- a/nodedb-cluster/src/raft_loop/builder.rs +++ b/nodedb-cluster/src/raft_loop/builder.rs @@ -55,6 +55,7 @@ impl RaftLoop { calvin_submit_inbox: self.calvin_submit_inbox, reserve_read: self.reserve_read, release_reservation: self.release_reservation, + data_propose_gate: self.data_propose_gate, auth_lease: self.auth_lease, snapshot_builder: self.snapshot_builder, snapshot_applier: self.snapshot_applier, @@ -228,6 +229,19 @@ impl RaftLoop { self } + /// Attach the write gate for data-group proposals (builder chain). + /// + /// The supplied implementation (backed by `nodedb`'s write-admission + /// gate) admits every proposal to a data group this node leads, local + /// or forwarded, before the entry enters the log. + pub fn with_data_propose_gate( + mut self, + gate: Arc, + ) -> Self { + self.data_propose_gate = Some(gate); + self + } + /// Attach the per-group snapshot builder for the SEND path (builder chain). /// /// The supplied implementation (backed by `nodedb`'s Data Plane snapshot diff --git a/nodedb-cluster/src/raft_loop/handle_rpc/consensus.rs b/nodedb-cluster/src/raft_loop/handle_rpc/consensus.rs index b427e057a..f17d8c40a 100644 --- a/nodedb-cluster/src/raft_loop/handle_rpc/consensus.rs +++ b/nodedb-cluster/src/raft_loop/handle_rpc/consensus.rs @@ -12,7 +12,6 @@ use nodedb_raft::message::{ }; use super::super::loop_core::{CommitApplier, RaftLoop}; -use super::membership::TOPOLOGY_GROUP_ID; impl RaftLoop { /// The leader counts a successful answer as this node holding the @@ -165,9 +164,9 @@ impl RaftLoop { { Ok(crate::install_snapshot::ChunkOutcome::Committed(committed)) => { // The watcher means "state visible through N". It moves - // only when the host state machine holds the snapshot. - // Data-group watchers are bumped by the host apply loop. - if group_id == TOPOLOGY_GROUP_ID && committed.state_installed { + // only when the host state machine holds the snapshot, + // for every group kind. Later entries move it on apply. + if committed.state_installed { self.group_watchers.bump(group_id, last_included_index); } // The leader resumes replication after the snapshot index diff --git a/nodedb-cluster/src/raft_loop/handle_rpc/dispatch.rs b/nodedb-cluster/src/raft_loop/handle_rpc/dispatch.rs index 8439b9128..0e072827a 100644 --- a/nodedb-cluster/src/raft_loop/handle_rpc/dispatch.rs +++ b/nodedb-cluster/src/raft_loop/handle_rpc/dispatch.rs @@ -40,7 +40,7 @@ impl RaftRpcHandler for RaftLoop { // Metadata-group proposal forwarding. RaftRpc::MetadataProposeRequest(req) => self.handle_metadata_propose_rpc(req), // Data-group proposal forwarding. - RaftRpc::DataProposeRequest(req) => self.handle_data_propose_rpc(req), + RaftRpc::DataProposeRequest(req) => self.handle_data_propose_rpc(req).await, // Read index for a node that does not lead the group. RaftRpc::ReadIndexRequest(req) => self.handle_read_index_rpc(req).await, // The leader this node knows for a group, from its own Raft state. diff --git a/nodedb-cluster/src/raft_loop/handle_rpc/plan_dispatch.rs b/nodedb-cluster/src/raft_loop/handle_rpc/plan_dispatch.rs index af6cb2e2a..d684ae418 100644 --- a/nodedb-cluster/src/raft_loop/handle_rpc/plan_dispatch.rs +++ b/nodedb-cluster/src/raft_loop/handle_rpc/plan_dispatch.rs @@ -4,12 +4,12 @@ //! VShardEnvelope routing RPC bodies. use crate::calvin::SEQUENCER_GROUP_ID; -use crate::error::{ClusterError, Result}; +use crate::error::{CalvinError, ClusterError, Result}; use crate::forward::{ChunkSink, PlanExecutor}; use crate::multi_raft::MultiRaft; use crate::rpc_codec::{ DataProposeRequest, DataProposeResponse, ExecuteRequest, MetadataProposeRequest, ProposeTarget, - RaftRpc, TypedClusterError, VShardRefusal, + RaftRpc, TypedClusterError, VShardRefusal, forwarded_deadline, }; use super::super::loop_core::{CommitApplier, RaftLoop}; @@ -48,11 +48,29 @@ impl RaftLoop { // Data-group and sequencer-group proposal forwarding — apply locally if // we lead the target group, otherwise return NotLeader with a hint so the - // forwarder can chase the redirect. - pub(super) fn handle_data_propose_rpc(&self, req: DataProposeRequest) -> Result { - let resp = { - let mut mr = self.multi_raft.lock().unwrap_or_else(|p| p.into_inner()); - propose_forwarded(&mut mr, req) + // forwarder can chase the redirect. A data-group entry passes this + // leader's write gate first, exactly as a local proposal does. + pub(super) async fn handle_data_propose_rpc(&self, req: DataProposeRequest) -> Result { + let resp = match req.target { + ProposeTarget::VShard { + vshard_id, + deadline_remaining_ms, + } => match forwarded_deadline(deadline_remaining_ms) { + // The proposer stopped waiting: the entry is refused unproposed. + None => DataProposeResponse::refused(&ClusterError::Calvin( + CalvinError::AdmissionTimedOut, + )), + Some(deadline) => { + match self.propose_admitted(vshard_id, &req.bytes, deadline).await { + Ok((group_id, log_index)) => DataProposeResponse::ok(group_id, log_index), + Err(error) => DataProposeResponse::refused(&error), + } + } + }, + ProposeTarget::Sequencer => { + let mut mr = self.multi_raft.lock().unwrap_or_else(|p| p.into_inner()); + propose_to_sequencer(&mut mr, req.bytes) + } }; Ok(RaftRpc::DataProposeResponse(resp)) } @@ -92,19 +110,13 @@ fn vshard_answer(result: Result>) -> RaftRpc { } } -/// Propose a forwarded entry to its target group on this node. +/// Propose a forwarded sequencer entry to the sequencer group on this node. /// /// Answers `not leader` with the known leader as a hint when this node does -/// not lead the target group. -fn propose_forwarded(mr: &mut MultiRaft, req: DataProposeRequest) -> DataProposeResponse { - let proposed = match req.target { - ProposeTarget::VShard(vshard_id) => mr.propose(vshard_id, req.bytes), - ProposeTarget::Sequencer => mr - .propose_to_group(SEQUENCER_GROUP_ID, req.bytes) - .map(|log_index| (SEQUENCER_GROUP_ID, log_index)), - }; - match proposed { - Ok((group_id, log_index)) => DataProposeResponse::ok(group_id, log_index), +/// not lead the sequencer group. +fn propose_to_sequencer(mr: &mut MultiRaft, bytes: Vec) -> DataProposeResponse { + match mr.propose_to_group(SEQUENCER_GROUP_ID, bytes) { + Ok(log_index) => DataProposeResponse::ok(SEQUENCER_GROUP_ID, log_index), Err(error) => DataProposeResponse::refused(&error), } } @@ -178,7 +190,7 @@ mod tests { elect_sequencer_leader(&mut mr); let before = mr.last_log_index(SEQUENCER_GROUP_ID).unwrap_or(0); - let resp = propose_forwarded(&mut mr, sequencer_request()); + let resp = propose_to_sequencer(&mut mr, sequencer_request().bytes); assert!(resp.success, "{}", resp.error_message); assert_eq!(resp.group_id, SEQUENCER_GROUP_ID); @@ -191,7 +203,7 @@ mod tests { let dir = tempfile::tempdir().expect("tempdir"); let mut mr = multi_raft_with_sequencer(dir.path()); - let resp = propose_forwarded(&mut mr, sequencer_request()); + let resp = propose_to_sequencer(&mut mr, sequencer_request().bytes); assert!(!resp.success); assert_eq!( diff --git a/nodedb-cluster/src/raft_loop/handle_rpc/shuffle_calvin.rs b/nodedb-cluster/src/raft_loop/handle_rpc/shuffle_calvin.rs index fee16a61b..ab92b60e3 100644 --- a/nodedb-cluster/src/raft_loop/handle_rpc/shuffle_calvin.rs +++ b/nodedb-cluster/src/raft_loop/handle_rpc/shuffle_calvin.rs @@ -71,7 +71,7 @@ impl RaftLoop { message: "shuffle producer not configured (no ShuffleProducer installed)" .into(), }), - read_version_lsn: 0, + read_versions: Vec::new(), }, } } diff --git a/nodedb-cluster/src/raft_loop/hooks.rs b/nodedb-cluster/src/raft_loop/hooks.rs index ce03e3ca4..0b3c181b3 100644 --- a/nodedb-cluster/src/raft_loop/hooks.rs +++ b/nodedb-cluster/src/raft_loop/hooks.rs @@ -240,9 +240,9 @@ pub trait ShuffleProducer: Send + Sync + 'static { /// Run the local scan fragment, hash-partition its rows, and fan them out to /// the part-owners. Returns a [`ShuffleProduceResponse`] whose `error` is /// `None` on a clean produce or `Some(err)` on a terminal scan failure (after - /// every part has been `End`ed with the error), and whose `read_version_lsn` - /// carries the max per-collection read version the local scan observed (`0` on - /// failure) for the coordinator's cross-shard OCC read validation. + /// every part has been `End`ed with the error), and whose `read_versions` + /// carry the read versions the local scan observed (empty on failure) for + /// the coordinator's cross-shard OCC read validation. async fn on_shuffle_produce( &self, req: crate::rpc_codec::ShuffleProduceRequest, diff --git a/nodedb-cluster/src/raft_loop/loop_core.rs b/nodedb-cluster/src/raft_loop/loop_core.rs index 65061f2c1..20d6c1ca9 100644 --- a/nodedb-cluster/src/raft_loop/loop_core.rs +++ b/nodedb-cluster/src/raft_loop/loop_core.rs @@ -232,6 +232,13 @@ pub struct RaftLoop { /// configured" error. pub(super) release_reservation: Option>, + /// Optional write gate for data-group proposals on their leader. When + /// set (by the `nodedb` binary via `with_data_propose_gate`), a proposal + /// to a group this node leads takes its lock keys through it before the + /// entry enters the log. Cluster-only tests leave this `None`, which + /// proposes every entry ungated. + pub(super) data_propose_gate: Option>, + /// Optional authorization lease service. When set (by the `nodedb` /// binary via `with_auth_lease`), lease renewals and authorization /// barriers are answered through it. @@ -344,6 +351,7 @@ impl RaftLoop { calvin_submit_inbox: None, reserve_read: None, release_reservation: None, + data_propose_gate: None, auth_lease: None, snapshot_builder: None, snapshot_applier: None, diff --git a/nodedb-cluster/src/raft_loop/mod.rs b/nodedb-cluster/src/raft_loop/mod.rs index ebfc4b702..039f16c38 100644 --- a/nodedb-cluster/src/raft_loop/mod.rs +++ b/nodedb-cluster/src/raft_loop/mod.rs @@ -32,6 +32,7 @@ pub mod loop_core; mod membership_convergence; mod placement_reconcile; pub mod proposals; +pub mod propose_gate; mod read_index; mod routing_persist; mod run; @@ -48,3 +49,4 @@ pub use hooks::{ }; pub use in_flight_snapshots::{InFlightSnapshotGuard, InFlightSnapshots}; pub use loop_core::{CommitApplier, RaftLoop, VShardEnvelopeHandler}; +pub use propose_gate::{DataProposeGate, ProposeHold}; diff --git a/nodedb-cluster/src/raft_loop/proposals.rs b/nodedb-cluster/src/raft_loop/proposals.rs index 88ef3d8e7..d2076c66c 100644 --- a/nodedb-cluster/src/raft_loop/proposals.rs +++ b/nodedb-cluster/src/raft_loop/proposals.rs @@ -223,15 +223,30 @@ impl RaftLoop { /// a `DataProposeRequest` over QUIC. The receiving leader applies the /// proposal locally and returns `(group_id, log_index)`. /// + /// The leader admits the entry through its write gate before the entry + /// enters its log (see [`super::propose_gate`]). A gate refusal returns + /// as `ClusterError::Calvin`. The gate waits no later than `deadline`, + /// the caller's deadline. A forward carries what remains of it. + /// + /// The call ends no later than `deadline` plus [`crate::rpc_codec::REPLY_MARGIN`]. + /// `CalvinError::AdmissionTimedOut` means the entry was not proposed. + /// `ClusterError::ShardTimeout` means the leader's reply never came, so + /// the leader can have proposed the entry. `ClusterError::Unanswered` + /// means the request was written and its stream failed, with the same + /// unknown outcome. The transport never resends a written forward. A + /// `ClusterError::Transport` from the send means the request never went + /// out whole, so nothing was proposed. + /// /// On `NotLeader { leader_hint: None, .. }` (election in progress) the /// call returns the original `NotLeader` error so the caller can retry. pub async fn propose_via_data_leader( &self, vshard_id: u32, - data: Vec, + data: &[u8], + deadline: tokio::time::Instant, ) -> Result<(u64, u64)> { // First, try a local propose. - match self.propose(vshard_id, data.clone()) { + match self.propose_admitted(vshard_id, data, deadline).await { Ok(pair) => Ok(pair), Err(crate::error::ClusterError::Raft(nodedb_raft::RaftError::NotLeader { leader_hint, @@ -254,19 +269,30 @@ impl RaftLoop { )); } // Otherwise forward to the hinted leader. - self.forward_data_propose(leader_id, vshard_id, data).await + self.forward_data_propose(leader_id, vshard_id, data.to_vec(), deadline) + .await } Err(other) => Err(other), } } - /// Send a `DataProposeRequest` to `leader_id`. + /// Send a `DataProposeRequest` carrying the budget left to `deadline` to + /// `leader_id`, and wait for its verdict until `deadline` plus + /// [`crate::rpc_codec::REPLY_MARGIN`]. async fn forward_data_propose( &self, leader_id: u64, vshard_id: u32, data: Vec, + deadline: tokio::time::Instant, ) -> Result<(u64, u64)> { + let deadline_remaining_ms = crate::rpc_codec::remaining_budget_ms(deadline); + // The deadline passed before the forward left: nothing is proposed. + if deadline_remaining_ms == 0 { + return Err(crate::error::ClusterError::Calvin( + crate::error::CalvinError::AdmissionTimedOut, + )); + } { let topo = self.topology.read().unwrap_or_else(|p| p.into_inner()); let Some(node) = topo.get_node(leader_id) else { @@ -289,10 +315,31 @@ impl RaftLoop { let req = crate::rpc_codec::RaftRpc::DataProposeRequest(crate::rpc_codec::DataProposeRequest { - target: crate::rpc_codec::ProposeTarget::VShard(vshard_id), + target: crate::rpc_codec::ProposeTarget::VShard { + vshard_id, + deadline_remaining_ms, + }, bytes: data, }); - let resp = self.transport.send_rpc(leader_id, req).await?; + // The generic short RPC timeout ends the call before a contended + // write's gate wait does. The reply wait follows the forwarded budget. + let reply_budget = crate::rpc_codec::reply_wait(deadline_remaining_ms); + let sent = tokio::time::timeout( + reply_budget, + self.transport + .send_rpc_with_read_timeout(leader_id, req, reply_budget), + ) + .await; + let resp = match sent { + Ok(answer) => answer?, + // The leader can have proposed the entry: its outcome is unknown. + Err(_) => { + return Err(crate::error::ClusterError::ShardTimeout { + vshard_id, + elapsed_ms: u64::try_from(reply_budget.as_millis()).unwrap_or(u64::MAX), + }); + } + }; match resp { crate::rpc_codec::RaftRpc::DataProposeResponse(r) => { if r.success { @@ -312,7 +359,7 @@ impl RaftLoop { Err(r.refusal_error()) } } - other => Err(crate::error::ClusterError::Transport { + other => Err(crate::error::ClusterError::Codec { detail: format!("data propose forward: unexpected response variant {other:?}"), }), } diff --git a/nodedb-cluster/src/raft_loop/propose_gate.rs b/nodedb-cluster/src/raft_loop/propose_gate.rs new file mode 100644 index 000000000..198a854d5 --- /dev/null +++ b/nodedb-cluster/src/raft_loop/propose_gate.rs @@ -0,0 +1,89 @@ +// SPDX-License-Identifier: BUSL-1.1 + +//! The data-group leader's write gate. +//! +//! The leader of a vShard's data group admits every proposal to that group +//! through the host's write gate before the entry enters its log. The gate +//! takes the write's lock keys on the vShard's Calvin lock table. A Calvin +//! transaction that needs one of those keys then waits until this leader +//! started the entry's apply, so its own effects order after the write. +//! +//! A local proposal and a forwarded one pass the same gate: both reach the +//! leader through [`RaftLoop::propose_admitted`]. `nodedb-cluster` cannot +//! depend on `nodedb`, so the gate lives in the host crate behind +//! [`DataProposeGate`]. + +use crate::error::Result; +use crate::forward::PlanExecutor; + +use super::loop_core::{CommitApplier, RaftLoop}; + +/// The lock keys one admitted proposal holds. Dropping it releases them. +pub trait ProposeHold: Send { + /// The proposal landed at `log_index` of `group_id`. The host keeps the + /// keys until this node starts the apply of that entry. + fn landed(self: Box, group_id: u64, log_index: u64); +} + +/// The host's write gate for data-group proposals on their leader. +#[async_trait::async_trait] +pub trait DataProposeGate: Send + Sync + 'static { + /// Admit the encoded entry `entry`, proposed to the group of + /// `vshard_id`, on this leader. A contended entry waits for its keys + /// until `deadline`, the proposer's own deadline. + /// + /// - `Ok(None)`: the entry takes no lock key. + /// - `Ok(Some(hold))`: the entry holds its keys until `hold` releases + /// them. + /// - `Err(ClusterError::Calvin(CalvinError::RouteToSequencer))`: a held + /// key blocks the write, and the proposer submits it to the Calvin + /// sequencer. + /// - `Err(ClusterError::Calvin(CalvinError::AdmissionTimedOut))`: the + /// keys stayed held until `deadline`. Nothing is proposed. + async fn admit( + &self, + vshard_id: u32, + entry: &[u8], + deadline: tokio::time::Instant, + ) -> Result>>; +} + +impl RaftLoop { + /// Propose `data` to the group of `vshard_id` on this node. When this + /// node leads the group, the write gate admits the entry first, waiting + /// no later than `deadline`, the proposer's deadline. + /// + /// A node that does not lead the group proposes with no gate. The + /// propose refuses with `NotLeader`, and the caller forwards the entry to + /// the leader, which gates it there. A propose that fails after the gate + /// admitted the entry drops its hold, which releases the keys. + pub(super) async fn propose_admitted( + &self, + vshard_id: u32, + data: &[u8], + deadline: tokio::time::Instant, + ) -> Result<(u64, u64)> { + let hold = match &self.data_propose_gate { + Some(gate) if self.leads_group_of(vshard_id) => { + gate.admit(vshard_id, data, deadline).await? + } + Some(_) | None => None, + }; + let (group_id, log_index) = self.propose(vshard_id, data.to_vec())?; + if let Some(hold) = hold { + hold.landed(group_id, log_index); + } + Ok((group_id, log_index)) + } + + /// Whether this node leads the data group that owns `vshard_id`. + fn leads_group_of(&self, vshard_id: u32) -> bool { + let mr = self.multi_raft.lock().unwrap_or_else(|p| p.into_inner()); + let group_id = mr + .routing() + .read() + .unwrap_or_else(|p| p.into_inner()) + .group_for_vshard(vshard_id); + group_id.is_ok_and(|group_id| mr.is_group_leader(group_id)) + } +} diff --git a/nodedb-cluster/src/raft_loop/tick/apply_committed.rs b/nodedb-cluster/src/raft_loop/tick/apply_committed.rs index 5a7121b78..1cef50fac 100644 --- a/nodedb-cluster/src/raft_loop/tick/apply_committed.rs +++ b/nodedb-cluster/src/raft_loop/tick/apply_committed.rs @@ -202,29 +202,17 @@ impl RaftLoop { if let Err(e) = mr.advance_applied(group_id, last_applied) { warn!(group_id, error = %e, "failed to advance applied index"); } else if group_id == crate::calvin::SEQUENCER_GROUP_ID { - // Sequencer group: the host applies each - // entry to the sequencer state machine - // inline, before returning. The watcher - // is the sequencer's applied index that + // Sequencer group: the host applies each entry to the + // sequencer state machine inline, before returning. The + // watcher is the sequencer's applied index that // authorization lease coverage reports. // - // Data groups are NOT bumped here — for - // them `applier.apply_committed` only - // enqueues entries onto the - // `DistributedApplier` channel; the - // actual data lands in storage when - // `run_apply_loop` finishes the - // SPSC round-trip to the Data Plane. - // The host crate bumps the watcher - // there, so the watermark always means - // "data visible on this node up to - // index N" regardless of which group. - // - // Snapshot-install path also bumps - // (in `super::handle_rpc`) — covers - // jump-on-snapshot for both group - // kinds. The metadata lane bumps the - // metadata group's watcher. + // Data groups are not bumped here. `apply_committed` only + // hands their entries to the host apply loop, which bumps + // the watcher once the data is visible on this node. The + // metadata lane bumps the metadata group's watcher. A + // snapshot install bumps every group whose state it + // restored (see `super::super::handle_rpc`). self.group_watchers.bump(group_id, last_applied); } } diff --git a/nodedb-cluster/src/rpc_codec/data_plane_error.rs b/nodedb-cluster/src/rpc_codec/data_plane_error.rs index f67852305..e1157d806 100644 --- a/nodedb-cluster/src/rpc_codec/data_plane_error.rs +++ b/nodedb-cluster/src/rpc_codec/data_plane_error.rs @@ -237,6 +237,12 @@ pub enum DataPlaneErrorCode { object: String, detail: String, }, + /// The serving core fail-stopped and applied nothing. Another replica, or + /// the same one after a restart, serves the request. + CoreFailStopped { + core_id: u64, + detail: String, + }, } /// Wire mirror of `nodedb_types::text_search::TextColumnFault`. diff --git a/nodedb-cluster/src/rpc_codec/data_propose.rs b/nodedb-cluster/src/rpc_codec/data_propose.rs index dd2244da1..6bd70dae7 100644 --- a/nodedb-cluster/src/rpc_codec/data_propose.rs +++ b/nodedb-cluster/src/rpc_codec/data_propose.rs @@ -15,14 +15,57 @@ use crate::error::{ClusterError, Result}; /// The Raft group a forwarded proposal is for. #[derive(Debug, Clone, Copy, PartialEq, Eq, rkyv::Archive, rkyv::Serialize, rkyv::Deserialize)] pub enum ProposeTarget { - /// The data group that owns this vShard. The bytes are a serialized + /// The data group that owns `vshard_id`. The bytes are a serialized /// `ReplicatedEntry`. - VShard(u32), + /// + /// `deadline_remaining_ms` is what remains of the proposer's deadline + /// when it sends the request. The leader's write gate stops waiting for + /// the entry's keys once this budget runs out. `0` means the deadline + /// passed, and the leader refuses the entry unproposed. + VShard { + vshard_id: u32, + deadline_remaining_ms: u64, + }, /// The Calvin sequencer group. The bytes are a msgpack-encoded /// `SequencerEntry`. Sequencer, } +/// The budget a forward carries for `deadline`: the milliseconds left now. +/// `0` when the deadline passed. +pub fn remaining_budget_ms(deadline: tokio::time::Instant) -> u64 { + let remaining = deadline.saturating_duration_since(tokio::time::Instant::now()); + u64::try_from(remaining.as_millis()).unwrap_or(u64::MAX) +} + +/// How long a caller waits for a remote handler's reply past the budget the +/// request carries. The handler stops at that budget, so the margin covers +/// the round trip and its verdict arrives. +pub const REPLY_MARGIN: std::time::Duration = std::time::Duration::from_secs(2); + +/// How long a caller waits for the reply to a request that carries +/// `deadline_remaining_ms`: the request's own budget plus [`REPLY_MARGIN`]. +/// +/// A handler that can wait (a write gate, a Calvin queue) answers within the +/// budget it received. The transport's fixed RPC timeout never bounds such a +/// reply: a statement deadline longer than it would end early. +pub fn reply_wait(deadline_remaining_ms: u64) -> std::time::Duration { + std::time::Duration::from_millis(deadline_remaining_ms).saturating_add(REPLY_MARGIN) +} + +/// The leader's deadline for a forward that carries `deadline_remaining_ms`. +/// `None` when the budget is spent. +/// +/// The budget is used as it arrived. The proposer resolved it once, from the +/// statement's own timeout or its node default. Re-deciding it against the +/// leader's default gives one statement two budgets. +pub fn forwarded_deadline(deadline_remaining_ms: u64) -> Option { + match deadline_remaining_ms { + 0 => None, + ms => Some(tokio::time::Instant::now() + std::time::Duration::from_millis(ms)), + } +} + /// Forward an opaque proposal payload to the leader of its target group. #[derive(Debug, Clone, rkyv::Archive, rkyv::Serialize, rkyv::Deserialize)] pub struct DataProposeRequest { @@ -39,6 +82,12 @@ pub enum ForwardedProposeRefusal { NotLeader, /// A leadership transfer of the target group is in flight. LeadershipTransferInProgress, + /// The leader's write gate routes the write through the Calvin + /// sequencer. + RouteToSequencer, + /// The proposer's deadline passed before the leader's write gate + /// admitted the write. The leader did not propose it. + AdmissionTimedOut, /// Any other failure; `error_message` describes it. Failed, } @@ -82,6 +131,12 @@ impl DataProposeResponse { None, 0, ), + ClusterError::Calvin(crate::error::CalvinError::RouteToSequencer) => { + (ForwardedProposeRefusal::RouteToSequencer, None, 0) + } + ClusterError::Calvin(crate::error::CalvinError::AdmissionTimedOut) => { + (ForwardedProposeRefusal::AdmissionTimedOut, None, 0) + } // A refusal with no retry contract. The forwarding node reads it // as a transport error carrying the leader's message. ClusterError::Raft( @@ -105,6 +160,7 @@ impl DataProposeResponse { | ClusterError::GhostNotFound { .. } | ClusterError::Transport { .. } | ClusterError::ShardTimeout { .. } + | ClusterError::Unanswered { .. } | ClusterError::StreamTerminal { .. } | ClusterError::Storage { .. } | ClusterError::DataPlane { .. } @@ -147,8 +203,9 @@ impl DataProposeResponse { } /// The typed error a refused response stands for on the forwarding node. - /// A refusal the leader typed keeps its Raft error; any other stays a - /// transport error carrying the leader's message. + /// A refusal the leader typed keeps its Raft error. Any other becomes + /// `ClusterError::RemoteUntyped` with the leader's message. The leader + /// answered, so it is never a link failure. pub fn refusal_error(&self) -> ClusterError { match self.refusal { Some(ForwardedProposeRefusal::NotLeader) => { @@ -160,8 +217,14 @@ impl DataProposeResponse { Some(ForwardedProposeRefusal::LeadershipTransferInProgress) => { ClusterError::Raft(nodedb_raft::RaftError::LeadershipTransferInProgress) } - Some(ForwardedProposeRefusal::Failed) | None => ClusterError::Transport { - detail: format!("data propose forward failed: {}", self.error_message), + Some(ForwardedProposeRefusal::RouteToSequencer) => { + ClusterError::Calvin(crate::error::CalvinError::RouteToSequencer) + } + Some(ForwardedProposeRefusal::AdmissionTimedOut) => { + ClusterError::Calvin(crate::error::CalvinError::AdmissionTimedOut) + } + Some(ForwardedProposeRefusal::Failed) | None => ClusterError::RemoteUntyped { + detail: format!("data propose forward refused: {}", self.error_message), }, } } @@ -213,6 +276,21 @@ pub(super) fn decode_data_propose_resp(payload: &[u8]) -> Result { mod tests { use super::*; use crate::cluster_epoch::ClusterEpochState; + + /// A reply wait follows the request's own budget, past any fixed RPC + /// timeout, plus the margin. + #[test] + fn a_reply_wait_follows_the_request_budget() { + assert_eq!( + reply_wait(30_000), + std::time::Duration::from_millis(30_000) + REPLY_MARGIN + ); + assert_eq!(reply_wait(0), REPLY_MARGIN); + assert_eq!( + reply_wait(u64::MAX), + std::time::Duration::from_millis(u64::MAX) + REPLY_MARGIN + ); + } use crate::rpc_codec::{decode, encode}; fn roundtrip(target: ProposeTarget) -> DataProposeRequest { @@ -235,10 +313,41 @@ mod tests { assert_eq!(req.bytes, vec![1, 2, 3]); } + /// The vShard target keeps the proposer's remaining budget. + #[test] + fn vshard_target_keeps_its_deadline_across_the_wire() { + let target = ProposeTarget::VShard { + vshard_id: 42, + deadline_remaining_ms: 1_500, + }; + let req = roundtrip(target); + assert_eq!(req.target, target); + } + + /// A spent budget gives the leader no deadline, so it proposes nothing. + #[test] + fn a_spent_budget_has_no_leader_deadline() { + assert_eq!(forwarded_deadline(0), None); + } + + /// The leader's deadline ends no later than the budget the forward + /// carries. #[test] - fn vshard_target_survives_the_wire() { - let req = roundtrip(ProposeTarget::VShard(42)); - assert_eq!(req.target, ProposeTarget::VShard(42)); + fn the_leader_deadline_ends_within_the_forwarded_budget() { + let budget = std::time::Duration::from_millis(50); + let latest = tokio::time::Instant::now() + budget; + let deadline = forwarded_deadline(50).expect("a live budget"); + let received = tokio::time::Instant::now() + budget; + assert!(deadline <= received); + assert!(deadline >= latest); + } + + /// A passed deadline leaves no budget to forward. + #[test] + fn a_passed_deadline_leaves_no_budget() { + assert_eq!(remaining_budget_ms(tokio::time::Instant::now()), 0); + let live = tokio::time::Instant::now() + std::time::Duration::from_secs(10); + assert!(remaining_budget_ms(live) > 9_000); } fn refusal_across_the_wire(error: ClusterError) -> ClusterError { @@ -281,9 +390,30 @@ mod tests { )); } + /// A write-gate refusal keeps its Calvin error across the wire, so the + /// forwarding node routes the write or reports the deadline. + #[test] + fn a_write_gate_refusal_keeps_its_calvin_error_across_the_wire() { + let routed = refusal_across_the_wire(ClusterError::Calvin( + crate::error::CalvinError::RouteToSequencer, + )); + assert!(matches!( + routed, + ClusterError::Calvin(crate::error::CalvinError::RouteToSequencer) + )); + let timed_out = refusal_across_the_wire(ClusterError::Calvin( + crate::error::CalvinError::AdmissionTimedOut, + )); + assert!(matches!( + timed_out, + ClusterError::Calvin(crate::error::CalvinError::AdmissionTimedOut) + )); + } + #[test] - fn any_other_refusal_stays_a_transport_error() { + fn any_other_refusal_is_an_untyped_answer() { let error = refusal_across_the_wire(ClusterError::VShardNotMapped { vshard_id: 7 }); - assert!(matches!(error, ClusterError::Transport { .. })); + assert!(matches!(error, ClusterError::RemoteUntyped { .. })); + assert!(!error.is_link_failure()); } } diff --git a/nodedb-cluster/src/rpc_codec/execute/codec.rs b/nodedb-cluster/src/rpc_codec/execute/codec.rs index c156002f1..4ea76e09d 100644 --- a/nodedb-cluster/src/rpc_codec/execute/codec.rs +++ b/nodedb-cluster/src/rpc_codec/execute/codec.rs @@ -222,10 +222,14 @@ mod tests { #[test] fn roundtrip_execute_response_success() { + let read_versions = vec![nodedb_types::ShardVersion { + vshard: 513, + version: nodedb_types::WriteVersion::logged(2, 0xBEEF_5678), + }]; let resp = ExecuteResponse::ok( vec![b"row1".to_vec(), b"row2".to_vec()], 0xCAFE_1234, - 0xBEEF_5678, + read_versions.clone(), ); let decoded = roundtrip_resp(resp); assert!(decoded.success); @@ -237,11 +241,32 @@ mod tests { "read watermark roundtrips on the response body" ); assert_eq!( - decoded.read_version_lsn, 0xBEEF_5678, - "per-collection read-version LSN roundtrips distinct from the watermark" + decoded.read_versions, read_versions, + "read versions roundtrip distinct from the watermark" ); } + #[test] + fn a_refusal_roundtrips_the_versions_it_observed() { + let read_versions = vec![nodedb_types::ShardVersion { + vshard: 7, + version: nodedb_types::WriteVersion::logged(1, 40), + }]; + let resp = ExecuteResponse::refused_with_versions( + TypedClusterError::Internal { + code: 0, + message: "no row".into(), + }, + 12, + read_versions.clone(), + ); + let decoded = roundtrip_resp(resp); + assert!(!decoded.success); + assert!(decoded.error.is_some()); + assert_eq!(decoded.watermark_lsn, 12); + assert_eq!(decoded.read_versions, read_versions); + } + #[test] fn roundtrip_execute_response_not_leader() { let resp = ExecuteResponse::err(TypedClusterError::NotLeader { @@ -256,9 +281,9 @@ mod tests { decoded.watermark_lsn, 0, "error responses carry no watermark" ); - assert_eq!( - decoded.read_version_lsn, 0, - "error responses carry no read-version LSN" + assert!( + decoded.read_versions.is_empty(), + "error responses carry no read versions" ); match decoded.error { Some(TypedClusterError::NotLeader { diff --git a/nodedb-cluster/src/rpc_codec/execute/types.rs b/nodedb-cluster/src/rpc_codec/execute/types.rs index ce85b3aaa..c52225b3f 100644 --- a/nodedb-cluster/src/rpc_codec/execute/types.rs +++ b/nodedb-cluster/src/rpc_codec/execute/types.rs @@ -63,12 +63,12 @@ pub struct ExecuteResponse { /// writes/errors. Mirrors [`ExecuteStreamChunk::watermark_lsn`]: raw `u64` /// on the wire, converted to `Lsn` at the coordinator via `Lsn::new`. pub watermark_lsn: u64, - /// Per-collection read-version LSN for the scanned collection (its - /// `coll_write_lsn` at read time, a WAL LSN); 0 for - /// writes/errors. The sound comparand for cross-shard OCC read validation, - /// distinct from the core-global `watermark_lsn`. Raw `u64` on the wire, - /// converted to `Lsn` at the coordinator via `Lsn::new`. - pub read_version_lsn: u64, + /// The read versions the executing node's cores observed, one per vShard + /// read. A write reports the versions it recorded. Empty for errors other + /// than a read that found no row. The + /// comparand for cross-shard OCC read validation, distinct from the + /// core-global `watermark_lsn`. + pub read_versions: Vec, } /// Typed error returned by the remote executor. @@ -142,22 +142,42 @@ pub struct ExecuteStreamEnd { } impl ExecuteResponse { - pub fn ok(payloads: Vec>, watermark_lsn: u64, read_version_lsn: u64) -> Self { + pub fn ok( + payloads: Vec>, + watermark_lsn: u64, + read_versions: Vec, + ) -> Self { Self { success: true, payloads, error: None, watermark_lsn, - read_version_lsn, + read_versions, } } + /// A refusal that still observed the versions a read validates: a read + /// that found no row. + pub fn refused_with_versions( + error: TypedClusterError, + watermark_lsn: u64, + read_versions: Vec, + ) -> Self { + Self { + success: false, + payloads: vec![], + error: Some(error), + watermark_lsn, + read_versions, + } + } + pub fn err(error: TypedClusterError) -> Self { Self { success: false, payloads: vec![], error: Some(error), watermark_lsn: 0, - read_version_lsn: 0, + read_versions: Vec::new(), } } } diff --git a/nodedb-cluster/src/rpc_codec/mod.rs b/nodedb-cluster/src/rpc_codec/mod.rs index 83ade303b..e75b05d7c 100644 --- a/nodedb-cluster/src/rpc_codec/mod.rs +++ b/nodedb-cluster/src/rpc_codec/mod.rs @@ -27,6 +27,7 @@ pub mod raft_msgs; pub mod raft_rpc; pub mod read_index; pub mod request_refusal; +pub mod resend; pub mod reservation; pub mod shard_error; pub mod shuffle; @@ -53,7 +54,8 @@ pub use data_plane_error::{ DataPlaneCounterFault, DataPlaneErrorCode, DataPlaneSyncHold, DataPlaneTextColumnFault, }; pub use data_propose::{ - DataProposeRequest, DataProposeResponse, ForwardedProposeRefusal, ProposeTarget, + DataProposeRequest, DataProposeResponse, ForwardedProposeRefusal, ProposeTarget, REPLY_MARGIN, + forwarded_deadline, remaining_budget_ms, reply_wait, }; pub use execute::{ DescriptorVersionEntry, ExecuteRequest, ExecuteResponse, ExecuteStreamChunk, ExecuteStreamEnd, diff --git a/nodedb-cluster/src/rpc_codec/resend.rs b/nodedb-cluster/src/rpc_codec/resend.rs new file mode 100644 index 000000000..c600e4283 --- /dev/null +++ b/nodedb-cluster/src/rpc_codec/resend.rs @@ -0,0 +1,116 @@ +// SPDX-License-Identifier: BUSL-1.1 + +//! Which requests the transport can send a second time after a written +//! attempt got no answer. + +use super::raft_rpc::RaftRpc; + +impl RaftRpc { + /// Whether a second run of this request on the peer leaves the same + /// state and answer as one run. + /// + /// The transport resends a written request with no answer only when + /// this holds. The peer can have run the first copy. Every request that + /// proposes, submits, executes, assigns or moves state answers `false`, + /// and a lost answer ends as an unknown outcome. A reply variant never + /// goes out as a request, so it answers `false`. + pub fn resend_safe(&self) -> bool { + match self { + // Raft log matching makes a repeated append a no-op. + RaftRpc::AppendEntriesRequest(_) + // A voter grants at most one candidate per term, and a repeated + // request from that candidate gets the same grant. + | RaftRpc::RequestVoteRequest(_) + // A pre-vote changes no Raft state. + | RaftRpc::PreVoteRequest(_) + // Reads of the peer's state. + | RaftRpc::Ping(_) + | RaftRpc::ReadIndexRequest(_) + | RaftRpc::LeaderStatusRequest(_) => true, + // A second copy starts a second election. + RaftRpc::TimeoutNowRequest(_) + // The receiver takes a chunk only at its next expected offset, + // so it refuses a repeat of a chunk that landed. + | RaftRpc::InstallSnapshotRequest(_) + // Membership, topology and lease changes. + | RaftRpc::JoinRequest(_) + | RaftRpc::TopologyUpdate(_) + | RaftRpc::AuthLeaseRenewRequest(_) + | RaftRpc::AuthBarrierRequest(_) + // Proposals, submits and executions. + | RaftRpc::VShardEnvelope(_) + | RaftRpc::MetadataProposeRequest(_) + | RaftRpc::ExecuteRequest(_) + | RaftRpc::ExecuteStreamRequest(_) + | RaftRpc::ShufflePushRequest(_) + | RaftRpc::ShufflePushChunk(_) + | RaftRpc::ShufflePushEnd(_) + | RaftRpc::ShuffleProduceRequest(_) + | RaftRpc::ShuffleConsumeRequest(_) + | RaftRpc::ShuffleAggregateConsumeRequest(_) + | RaftRpc::AssignSurrogateRequest(_) + | RaftRpc::SubmitCalvinTxnRequest(_) + | RaftRpc::SubmitCalvinInboxRequest(_) + | RaftRpc::CalvinPartsRequest(_) + | RaftRpc::ReserveReadRequest(_) + | RaftRpc::ReleaseReservationRequest(_) + | RaftRpc::DataProposeRequest(_) + // Replies. + | RaftRpc::AppendEntriesResponse(_) + | RaftRpc::RequestVoteResponse(_) + | RaftRpc::PreVoteResponse(_) + | RaftRpc::InstallSnapshotResponse(_) + | RaftRpc::JoinResponse(_) + | RaftRpc::Pong(_) + | RaftRpc::TopologyAck(_) + | RaftRpc::MetadataProposeResponse(_) + | RaftRpc::ExecuteResponse(_) + | RaftRpc::ExecuteStreamChunk(_) + | RaftRpc::ExecuteStreamEnd(_) + | RaftRpc::ShuffleProduceResponse(_) + | RaftRpc::ShuffleConsumeResponse(_) + | RaftRpc::ShuffleAggregateConsumeResponse(_) + | RaftRpc::AssignSurrogateResponse(_) + | RaftRpc::SubmitCalvinTxnResponse(_) + | RaftRpc::SubmitCalvinInboxResponse(_) + | RaftRpc::CalvinPartsResponse(_) + | RaftRpc::ReserveReadResponse(_) + | RaftRpc::ReleaseReservationResponse(_) + | RaftRpc::DataProposeResponse(_) + | RaftRpc::ReadIndexResponse(_) + | RaftRpc::LeaderStatusResponse(_) + | RaftRpc::AuthLeaseRenewResponse(_) + | RaftRpc::AuthBarrierResponse(_) + | RaftRpc::VShardRefusal(_) + | RaftRpc::FrameRefused(_) + | RaftRpc::RequestRefused(_) => false, + } + } +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::rpc_codec::{DataProposeRequest, PingRequest, ProposeTarget}; + + #[test] + fn a_data_propose_is_never_resent() { + let propose = RaftRpc::DataProposeRequest(DataProposeRequest { + target: ProposeTarget::VShard { + vshard_id: 3, + deadline_remaining_ms: 100, + }, + bytes: vec![1, 2, 3], + }); + assert!(!propose.resend_safe()); + } + + #[test] + fn a_ping_is_resent() { + let ping = RaftRpc::Ping(PingRequest { + sender_id: 1, + topology_version: 0, + }); + assert!(ping.resend_safe()); + } +} diff --git a/nodedb-cluster/src/rpc_codec/shard_error/convert.rs b/nodedb-cluster/src/rpc_codec/shard_error/convert.rs index a0b1f4c9f..71cccdbaf 100644 --- a/nodedb-cluster/src/rpc_codec/shard_error/convert.rs +++ b/nodedb-cluster/src/rpc_codec/shard_error/convert.rs @@ -42,6 +42,7 @@ impl From for ShardErrorWire { Self::GhostNotFound { node_id, shard_id } } ClusterError::Transport { detail } => Self::Transport { detail }, + ClusterError::Unanswered { node_id, detail } => Self::Unanswered { node_id, detail }, ClusterError::ShardTimeout { vshard_id, elapsed_ms, @@ -175,6 +176,7 @@ impl From for ClusterError { Self::GhostNotFound { node_id, shard_id } } ShardErrorWire::Transport { detail } => Self::Transport { detail }, + ShardErrorWire::Unanswered { node_id, detail } => Self::Unanswered { node_id, detail }, ShardErrorWire::ShardTimeout { vshard_id, elapsed_ms, diff --git a/nodedb-cluster/src/rpc_codec/shard_error/wire.rs b/nodedb-cluster/src/rpc_codec/shard_error/wire.rs index 6c585d80a..d85cfb6ce 100644 --- a/nodedb-cluster/src/rpc_codec/shard_error/wire.rs +++ b/nodedb-cluster/src/rpc_codec/shard_error/wire.rs @@ -123,4 +123,8 @@ pub enum ShardErrorWire { error: TypedClusterError, detail: String, }, + Unanswered { + node_id: u64, + detail: String, + }, } diff --git a/nodedb-cluster/src/rpc_codec/shuffle.rs b/nodedb-cluster/src/rpc_codec/shuffle.rs index 379046d5d..76dfa23b6 100644 --- a/nodedb-cluster/src/rpc_codec/shuffle.rs +++ b/nodedb-cluster/src/rpc_codec/shuffle.rs @@ -125,14 +125,12 @@ pub struct ShuffleProduceRequest { #[derive(Debug, Clone, rkyv::Archive, rkyv::Serialize, rkyv::Deserialize)] pub struct ShuffleProduceResponse { pub error: Option, - /// Max per-collection read-version LSN observed by the producer's local scan - /// (its scanned collection's `coll_write_lsn` at read time, a WAL LSN); 0 for - /// a failed produce. The sound comparand the coordinator max-folds across + /// The read versions the producer's local scan observed, one per vShard + /// read. Empty for a failed produce. The coordinator folds them across /// producers for cross-shard OCC read validation of an in-transaction - /// distributed aggregate — distinct from the core-global watermark. Raw `u64` - /// on the wire, converted to `Lsn` at the coordinator via `Lsn::new`. Mirrors - /// [`ExecuteResponse::read_version_lsn`](super::execute::ExecuteResponse::read_version_lsn). - pub read_version_lsn: u64, + /// distributed aggregate. Mirrors + /// [`ExecuteResponse::read_versions`](super::execute::ExecuteResponse::read_versions). + pub read_versions: Vec, } /// One `(left_key, right_key)` equi-join pair of a [`ShuffleConsumeRequest`]'s @@ -639,14 +637,18 @@ mod tests { #[test] fn roundtrip_shuffle_produce_response_clean() { + let read_versions = vec![nodedb_types::ShardVersion { + vshard: 9, + version: nodedb_types::WriteVersion::logged(1, 0xABCD_1234), + }]; let decoded = roundtrip_produce_resp(ShuffleProduceResponse { error: None, - read_version_lsn: 0xABCD_1234, + read_versions: read_versions.clone(), }); assert!(decoded.error.is_none()); assert_eq!( - decoded.read_version_lsn, 0xABCD_1234, - "producer read-version LSN roundtrips on the produce reply" + decoded.read_versions, read_versions, + "producer read versions roundtrip on the produce reply" ); } @@ -657,11 +659,11 @@ mod tests { code: 0x55, message: "produce scan failed".into(), }), - read_version_lsn: 0, + read_versions: Vec::new(), }); - assert_eq!( - decoded.read_version_lsn, 0, - "a failed produce carries no read-version LSN" + assert!( + decoded.read_versions.is_empty(), + "a failed produce carries no read versions" ); match decoded.error { Some(TypedClusterError::Internal { code, message }) => { diff --git a/nodedb-cluster/src/transport/client/attempt.rs b/nodedb-cluster/src/transport/client/attempt.rs index 888e3eadb..3ad90fd5c 100644 --- a/nodedb-cluster/src/transport/client/attempt.rs +++ b/nodedb-cluster/src/transport/client/attempt.rs @@ -9,6 +9,11 @@ //! typed refusal included. A refusal goes back to the caller as its typed //! error and is not resent. //! +//! The peer can have run a request written before a link failure. The +//! transport resends such a request only when [`RaftRpc::resend_safe`] +//! holds. Any other request ends as `ClusterError::Unanswered`, an unknown +//! outcome. +//! //! A link failure drops the pooled connection only when the connection //! itself failed. Other sends share that connection, so one stream's //! failure must not cut them off. The connection fails when: @@ -47,22 +52,50 @@ enum Attempt { /// A link failure, and the pooled connection it condemns. #[derive(Debug)] struct LinkFailure { - error: ClusterError, + error: Box, /// Stable id of the connection to drop from the pool. `None` when no /// connection was obtained or the connection still works. condemned: Option, + /// Whether the whole request was written before the failure. The peer + /// can have run a written request. + written: bool, } impl LinkFailure { - /// A failure that leaves the pool as it is. + /// A failure before the request went out that leaves the pool as it is. fn keep_connection(error: ClusterError) -> Self { Self { - error, + error: Box::new(error), condemned: None, + written: false, + } + } + + /// The error of a written request that got no answer from `target`. + fn into_unanswered(self, target: u64) -> ClusterError { + ClusterError::Unanswered { + node_id: target, + detail: self.error.to_string(), } } } +/// Whether a failed attempt is sent again. +/// +/// `Ok` holds the error kept while the next attempt goes out. `Err` ends the +/// call. A written request that is not [`RaftRpc::resend_safe`] can have run +/// on the peer. It is never resent and ends as `ClusterError::Unanswered`. +fn resend_verdict(failure: LinkFailure, target: u64, resend_safe: bool) -> Result { + if failure.written && !resend_safe { + return Err(failure.into_unanswered(target)); + } + if RetryPolicy::is_retryable(&failure.error) { + Ok(*failure.error) + } else { + Err(*failure.error) + } +} + /// Sort the outcome of one send into an [`Attempt`]. /// /// The outer error is a link failure. The inner result is what the peer @@ -110,6 +143,7 @@ impl NexarTransport { // Each retry wraps it in a fresh envelope so the seq advances // per attempt — a retry is a new frame, not a replayed frame. let inner = rpc_codec::encode(&rpc, &self.auth.epoch)?; + let resend_safe = rpc.resend_safe(); let mut last_err = None; for attempt in 0..self.retry_policy.max_attempts { @@ -124,10 +158,9 @@ impl NexarTransport { match self.attempt(target, &inner, read_timeout, admission).await { Attempt::Answered(answer) => return answer, Attempt::FrameRefused(e) => last_err = Some(e), - Attempt::LinkFailed(failure) if RetryPolicy::is_retryable(&failure.error) => { - last_err = Some(failure.error) + Attempt::LinkFailed(failure) => { + last_err = Some(resend_verdict(failure, target, resend_safe)?) } - Attempt::LinkFailed(failure) => return Err(failure.error), } } @@ -152,7 +185,7 @@ impl NexarTransport { { Attempt::Answered(answer) => answer, Attempt::FrameRefused(e) => Err(e), - Attempt::LinkFailed(failure) => Err(failure.error), + Attempt::LinkFailed(failure) => Err(*failure.error), } } @@ -192,7 +225,8 @@ impl NexarTransport { /// The outer error is a link failure. The inner result is the peer's /// reply. A stream the peer finished without a reply is an inner error: /// the peer refused the request before its handler, and a resend gets - /// the same answer. + /// the same answer. The peer resets a stream whose handler started and + /// sent no answer, and that reset is a link failure of a written request. async fn try_send_once( &self, target: u64, @@ -206,23 +240,26 @@ impl NexarTransport { let stable_id = conn.stable_id(); if let Err(error) = self.verify_connection_target(&conn, target) { return Err(LinkFailure { - error, + error: Box::new(error), condemned: Some(stable_id), + written: false, }); } let received_before = received_datagrams(&conn); self.exchange(&conn, target, inner, read_timeout) .await .map_err(|failure| { - let (error, timed_out) = match failure { - StreamFailure::ReadTimeout(error) => (error, true), - StreamFailure::Other(error) => (error, false), + let (error, timed_out, written) = match failure { + StreamFailure::Unsent(error) => (error, false, false), + StreamFailure::ReadTimeout(error) => (error, true, true), + StreamFailure::Unanswered(error) => (error, false, true), }; let heard = received_datagrams(&conn) > received_before; let broken = condemns_connection(timed_out, conn.close_reason().is_some(), heard); LinkFailure { - error, + error: Box::new(error), condemned: broken.then_some(stable_id), + written, } }) } @@ -236,19 +273,21 @@ impl NexarTransport { read_timeout: Duration, ) -> std::result::Result, StreamFailure> { let (mut send, mut recv) = conn.open_bi().await.map_err(|e| { - StreamFailure::Other(ClusterError::Transport { + StreamFailure::Unsent(ClusterError::Transport { detail: format!("open_bi to node {target}: {e}"), }) })?; - let envelope = self.wrap_inner(inner).map_err(StreamFailure::Other)?; + let envelope = self.wrap_inner(inner).map_err(StreamFailure::Unsent)?; + // A failed `write_all` left part of the frame unbuffered, so the peer + // cannot hold the whole request. Once it returns, the peer can. send.write_all(&envelope).await.map_err(|e| { - StreamFailure::Other(ClusterError::Transport { + StreamFailure::Unsent(ClusterError::Transport { detail: format!("write to node {target}: {e}"), }) })?; send.finish().map_err(|e| { - StreamFailure::Other(ClusterError::Transport { + StreamFailure::Unanswered(ClusterError::Transport { detail: format!("finish send to node {target}: {e}"), }) })?; @@ -264,7 +303,7 @@ impl NexarTransport { ), }) })? - .map_err(StreamFailure::Other)?; + .map_err(StreamFailure::Unanswered)?; let Some(response_envelope) = response_envelope else { return Ok(Err(ClusterError::RemoteUntyped { detail: format!("node {target} finished the stream without a reply"), @@ -281,10 +320,15 @@ impl NexarTransport { /// How one stream on a pooled connection failed. #[derive(Debug)] enum StreamFailure { - /// The reply did not arrive within the read timeout. + /// The request did not go out whole: the stream did not open, the + /// envelope did not build, or the write failed. + Unsent(ClusterError), + /// The request was written, and the reply did not arrive within the + /// read timeout. ReadTimeout(ClusterError), - /// Opening, writing or reading the stream failed. - Other(ClusterError), + /// The request was written, and finishing the stream or reading the + /// reply failed. + Unanswered(ClusterError), } /// Whether a stream failure condemns its connection. @@ -355,6 +399,46 @@ mod tests { assert_eq!(client.circuit_breaker().failure_count(1), 1); } + fn lost(written: bool) -> LinkFailure { + LinkFailure { + error: Box::new(ClusterError::Transport { + detail: "connection lost".into(), + }), + condemned: None, + written, + } + } + + /// A propose that went out can have run on the peer. It is never + /// resent, and it ends as an unknown outcome. + #[test] + fn a_written_request_that_is_not_resend_safe_ends_unanswered() { + match resend_verdict(lost(true), 2, false) { + Err(ClusterError::Unanswered { node_id: 2, .. }) => {} + other => panic!("expected Unanswered, got {other:?}"), + } + } + + /// A request that never went out ran nowhere, so it is resent. + #[test] + fn an_unsent_request_is_resent() { + for resend_safe in [false, true] { + assert!(matches!( + resend_verdict(lost(false), 2, resend_safe), + Ok(ClusterError::Transport { .. }) + )); + } + } + + /// A request whose second run is harmless is resent after a write. + #[test] + fn a_written_resend_safe_request_is_resent() { + assert!(matches!( + resend_verdict(lost(true), 2, true), + Ok(ClusterError::Transport { .. }) + )); + } + #[test] fn a_typed_refusal_is_a_final_answer() { let refusal = RaftRpc::RequestRefused(RequestRefusal::from(ClusterError::GroupNotFound { diff --git a/nodedb-cluster/src/transport/client/transport.rs b/nodedb-cluster/src/transport/client/transport.rs index 742e31357..74c569bf7 100644 --- a/nodedb-cluster/src/transport/client/transport.rs +++ b/nodedb-cluster/src/transport/client/transport.rs @@ -442,7 +442,8 @@ pub(super) mod tests { /// The group `EchoHandler` does not host. const UNHOSTED_GROUP: u64 = 404; - /// The group whose vote requests `EchoHandler` answers after [`STALL`]. + /// The group whose vote and snapshot requests `EchoHandler` answers + /// after [`STALL`]. pub(crate) const STALLED_GROUP: u64 = 503; /// How long `EchoHandler` holds a vote request of [`STALLED_GROUP`]. @@ -486,6 +487,12 @@ pub(super) mod tests { vote_granted: true, })) } + RaftRpc::InstallSnapshotRequest(req) if req.group_id == STALLED_GROUP => { + tokio::time::sleep(STALL).await; + Ok(RaftRpc::InstallSnapshotResponse(InstallSnapshotResponse { + term: req.term, + })) + } RaftRpc::InstallSnapshotRequest(req) => { Ok(RaftRpc::InstallSnapshotResponse(InstallSnapshotResponse { term: req.term, @@ -505,7 +512,7 @@ pub(super) mod tests { // Emit three deterministic chunks then clean EOF. Exercises the // multi-frame chunk/end envelope path end-to-end over loopback QUIC. for i in 0u64..3 { - if let Err(_e) = sink.send_chunk(vec![i as u8; 4], i + 100, 0).await { + if let Err(_e) = sink.send_chunk(vec![i as u8; 4], i + 100, Vec::new()).await { // Coordinator gone — stop producing, no terminal frame. return None; } @@ -540,7 +547,7 @@ pub(super) mod tests { ) -> crate::rpc_codec::ShuffleProduceResponse { crate::rpc_codec::ShuffleProduceResponse { error: None, - read_version_lsn: 0, + read_versions: Vec::new(), } } @@ -1034,4 +1041,36 @@ pub(super) mod tests { assert_eq!(breaker.state(1), CircuitState::Closed); assert_eq!(breaker.failure_count(1), 0); } + + /// A node that shuts down while its handler runs a request leaves the + /// request unanswered. The handler can have run it, so the sender never + /// reads it as a refusal and never resends it. + #[tokio::test] + async fn a_request_whose_node_shuts_down_mid_handler_ends_unanswered() { + let (_server, client, shutdown) = serve_echo().await; + let req = RaftRpc::InstallSnapshotRequest(InstallSnapshotRequest { + term: 7, + leader_id: 2, + last_included_index: 500, + last_included_term: 6, + offset: 0, + data: Vec::new(), + done: true, + group_id: STALLED_GROUP, + total_size: 0, + voters: Vec::new(), + learners: Vec::new(), + }); + assert!(!req.resend_safe()); + let send = tokio::spawn(async move { client.send_rpc(1, req).await }); + tokio::time::sleep(STALL / 4).await; + shutdown.send(true).expect("signal shutdown"); + + let err = send.await.expect("send task").unwrap_err(); + + assert!( + matches!(err, ClusterError::Unanswered { node_id: 1, .. }), + "{err:?}" + ); + } } diff --git a/nodedb-cluster/src/transport/mod.rs b/nodedb-cluster/src/transport/mod.rs index 65a244415..5e0cecad5 100644 --- a/nodedb-cluster/src/transport/mod.rs +++ b/nodedb-cluster/src/transport/mod.rs @@ -9,6 +9,7 @@ mod identity_admission; pub mod peer_identity_store; pub mod peer_identity_verifier; pub mod pinned_verifier; +mod reply_stream; pub mod rpc_handler; pub mod server; mod shuffle_drain; diff --git a/nodedb-cluster/src/transport/reply_stream.rs b/nodedb-cluster/src/transport/reply_stream.rs new file mode 100644 index 000000000..6d9cfe8c3 --- /dev/null +++ b/nodedb-cluster/src/transport/reply_stream.rs @@ -0,0 +1,70 @@ +// SPDX-License-Identifier: BUSL-1.1 + +//! The send half of an inbound RPC stream, reset when its request went +//! unanswered. +//! +//! A sender reads a stream finished without a reply as a refusal before the +//! handler: the request never ran, and a resend is safe. quinn finishes a +//! dropped `SendStream`, so a stream dropped once its handler started would +//! read the same way. A node that shuts down mid-request, a reply that fails +//! to write, and a handler panic all drop it there. +//! +//! [`ReplyStream`] closes that gap. Once armed, it resets the stream on drop +//! unless the request was answered. The sender reads a reset as a written +//! request with no answer, an unknown outcome it never resends. + +use std::ops::{Deref, DerefMut}; + +/// The QUIC stream error code of a request whose handler started and that +/// got no answer. +const UNANSWERED_RESET_CODE: u32 = 1; + +/// The send half of one inbound RPC stream. +pub(super) struct ReplyStream { + send: quinn::SendStream, + /// Whether the handler started and no answer went out yet. + armed: bool, +} + +impl ReplyStream { + pub(super) fn new(send: quinn::SendStream) -> Self { + Self { send, armed: false } + } + + /// Mark the request as handed to its handler. A drop after this point + /// resets the stream. + pub(super) fn arm(&mut self) { + self.armed = true; + } + + /// Mark the request as answered. The drop then leaves the stream to + /// quinn's finish. + pub(super) fn answered(&mut self) { + self.armed = false; + } +} + +impl Deref for ReplyStream { + type Target = quinn::SendStream; + + fn deref(&self) -> &Self::Target { + &self.send + } +} + +impl DerefMut for ReplyStream { + fn deref_mut(&mut self) -> &mut Self::Target { + &mut self.send + } +} + +impl Drop for ReplyStream { + fn drop(&mut self) { + if self.armed { + // A stream already closed by the connection needs no reset. + let _ = self + .send + .reset(quinn::VarInt::from_u32(UNANSWERED_RESET_CODE)); + } + } +} diff --git a/nodedb-cluster/src/transport/rpc_handler.rs b/nodedb-cluster/src/transport/rpc_handler.rs index 385fee368..13d2e4c9d 100644 --- a/nodedb-cluster/src/transport/rpc_handler.rs +++ b/nodedb-cluster/src/transport/rpc_handler.rs @@ -71,8 +71,8 @@ pub trait RaftRpcHandler: Send + Sync + 'static { /// (looping back for parts this node owns). Returns a /// [`ShuffleProduceResponse`] whose `error` is `None` on a clean produce or /// `Some(err)` if the scan failed (every part has already been `End`ed with - /// the same error), and whose `read_version_lsn` carries the max - /// per-collection read version the scan observed. The transport writes + /// the same error), and whose `read_versions` carry the read versions the + /// scan observed, one per vShard. The transport writes /// exactly one `ShuffleProduceResponse` carrying this outcome back to the /// coordinator. fn on_shuffle_produce( diff --git a/nodedb-cluster/src/transport/server.rs b/nodedb-cluster/src/transport/server.rs index ec2cd16d6..a52417851 100644 --- a/nodedb-cluster/src/transport/server.rs +++ b/nodedb-cluster/src/transport/server.rs @@ -50,6 +50,7 @@ use crate::transport::rpc_handler::RaftRpcHandler; use crate::wire_version::handshake_io::{local_version_range, perform_version_handshake_server}; use super::frame_io::{finish_stream, read_envelope, reply_and_finish, write_rpc_frame}; +use super::reply_stream::ReplyStream; use super::shuffle_drain::drain_shuffle_push; use super::stream_dispatch; use super::stream_identity::{reject_peer_identity, verify_stream_identity}; @@ -71,10 +72,10 @@ impl ChunkSink for QuicChunkSink<'_> { &mut self, payload: Vec, watermark_lsn: u64, - // The `ExecuteStream` wire chunk only carries `watermark_lsn`; the - // per-collection read version is surfaced separately on the shuffle - // produce reply, not on this streaming path. - _read_version_lsn: u64, + // The `ExecuteStream` wire chunk only carries `watermark_lsn`. The + // read versions reach the coordinator on the shuffle produce reply, + // not on this streaming path. + _read_versions: Vec, ) -> Result<()> { let rpc = RaftRpc::ExecuteStreamChunk(ExecuteStreamChunk { payload, @@ -220,9 +221,10 @@ struct StreamContext { /// module docstring for the rationale. async fn handle_stream( ctx: StreamContext, - mut send: quinn::SendStream, + send: quinn::SendStream, mut recv: quinn::RecvStream, ) -> Result<()> { + let mut send = ReplyStream::new(send); let StreamContext { handler, auth, @@ -279,6 +281,9 @@ async fn handle_stream( fields.from_node_id, &request, )?; + // The handler can run the request from here on. A stream that ends + // without an answer is reset, so the sender never resends it. + send.arm(); // 4b. Streaming path: an `ExecuteStreamRequest` produces a multi-frame // response — N `RPC_EXECUTE_STREAM_CHUNK` envelopes (each written @@ -338,11 +343,15 @@ async fn handle_stream( reply_and_finish(&mut send, &auth, &response, "response").await }; - tokio::select! { + let outcome = tokio::select! { biased; - _ = shutdown.changed() => Ok(()), - result = work => result, + _ = shutdown.changed() => Ok(false), + result = work => result.map(|()| true), + }; + if let Ok(true) = outcome { + send.answered(); } + outcome.map(|_| ()) } /// The frame that answers a one-shot request: the handler's response, or a diff --git a/nodedb-physical/src/physical_plan/calvin_reply.rs b/nodedb-physical/src/physical_plan/calvin_reply.rs new file mode 100644 index 000000000..335f0baf4 --- /dev/null +++ b/nodedb-physical/src/physical_plan/calvin_reply.rs @@ -0,0 +1,176 @@ +// SPDX-License-Identifier: Apache-2.0 + +//! The reply a committed Calvin slice answers with, in the form a redo +//! entry carries to every replica. +//! +//! The leader's stage decides the reply. Every replica renders it after its +//! own install of the slice's redo, so every replica answers the same bytes. + +use nodedb_types::Surrogate; + +use super::ReturningSpec; + +/// What a Calvin slice answers once its redo installs. +#[derive( + Debug, + Clone, + PartialEq, + serde::Serialize, + serde::Deserialize, + zerompk::ToMessagePack, + zerompk::FromMessagePack, +)] +pub enum CalvinReplySpec { + /// An affected count, decided when the plan staged. + Count(Vec), + /// `RETURNING` rows, decided when the plan staged. + Rows(Vec), + /// `RETURNING` rows read from base after the install. + PostImages(CalvinPostImagesSpec), + /// `RETURNING` rows of a resolved timeseries ingest, rendered from what + /// its install stored. + InstalledTimeseries(CalvinInstalledTimeseriesSpec), +} + +/// The engine that stores a row a `RETURNING` reply reads after the install. +#[derive( + Debug, + Clone, + Copy, + PartialEq, + Eq, + serde::Serialize, + serde::Deserialize, + zerompk::ToMessagePack, + zerompk::FromMessagePack, +)] +pub enum CalvinRowEngine { + Document, + Crdt, + Kv, + Vector, + Columnar, + Timeseries, +} + +/// One row a `RETURNING` plan wrote: its client identity and its surrogate. +#[derive( + Debug, + Clone, + PartialEq, + serde::Serialize, + serde::Deserialize, + zerompk::ToMessagePack, + zerompk::FromMessagePack, +)] +pub struct CalvinReplyRow { + pub identity: String, + pub surrogate: Surrogate, +} + +/// The rows of a `RETURNING` plan read from base after the install. +#[derive( + Debug, + Clone, + PartialEq, + serde::Serialize, + serde::Deserialize, + zerompk::ToMessagePack, + zerompk::FromMessagePack, +)] +pub struct CalvinPostImagesSpec { + pub spec: ReturningSpec, + pub rls_filters: Vec, + pub collection: String, + pub engine: CalvinRowEngine, + /// Each row the plan wrote, in the order the plan wrote them. + pub rows: Vec, +} + +/// The `RETURNING` projection of a resolved timeseries ingest. +#[derive( + Debug, + Clone, + PartialEq, + serde::Serialize, + serde::Deserialize, + zerompk::ToMessagePack, + zerompk::FromMessagePack, +)] +pub struct CalvinInstalledTimeseriesSpec { + pub spec: ReturningSpec, + pub rls_filters: Vec, + pub collection: String, + /// The ingest's position among the slice's timeseries ingests. Its + /// install is the install at this position. + pub ordinal: u64, +} + +/// What a stamped redo install of a Calvin slice needs beyond the redo +/// record. +#[derive( + Debug, + Clone, + PartialEq, + serde::Serialize, + serde::Deserialize, + zerompk::ToMessagePack, + zerompk::FromMessagePack, +)] +pub struct CalvinInstall { + pub epoch: u64, + pub position: u32, + /// The epoch's deterministic instant. The install stamps graph versions + /// at the transaction's ordinal and renders the reply at this instant. + pub epoch_system_ms: i64, + /// The reply the install answers with. + pub reply: CalvinReplySpec, + /// Whether the install raises the tenant's write mark. A slice that only + /// installs schema raises none. + pub user_write: bool, +} + +/// The answer of `MetaOp::CalvinResolve`: the slice's resolved redo record +/// and the reply its stage decided. +#[derive( + Debug, + Clone, + PartialEq, + serde::Serialize, + serde::Deserialize, + zerompk::ToMessagePack, + zerompk::FromMessagePack, +)] +pub struct CalvinResolved { + /// The zerompk-encoded redo record. + pub redo: Vec, + pub reply: CalvinReplySpec, +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::physical_plan::ReturningColumns; + + #[test] + fn a_resolved_answer_round_trips() { + let resolved = CalvinResolved { + redo: vec![1, 2, 3], + reply: CalvinReplySpec::PostImages(CalvinPostImagesSpec { + spec: ReturningSpec { + columns: ReturningColumns::Star, + }, + rls_filters: vec![9], + collection: "orders".into(), + engine: CalvinRowEngine::Kv, + rows: vec![CalvinReplyRow { + identity: "k1".into(), + surrogate: Surrogate::new(4), + }], + }), + }; + let bytes = zerompk::to_msgpack_vec(&resolved).expect("encode"); + let decoded: CalvinResolved = zerompk::from_msgpack(&bytes).expect("decode"); + assert_eq!(decoded, resolved); + } +} diff --git a/nodedb-physical/src/physical_plan/collection.rs b/nodedb-physical/src/physical_plan/collection.rs index 4bd095a99..02a6c8598 100644 --- a/nodedb-physical/src/physical_plan/collection.rs +++ b/nodedb-physical/src/physical_plan/collection.rs @@ -158,15 +158,14 @@ impl PhysicalPlan { /// Every user collection this plan names: each collection a committed /// redo install writes, or else the one [`Self::collection`] reports. /// - /// A committed-redo apply and a Calvin flush install one record that can - /// write several collections, so [`Self::collection`] reports none for - /// them. A caller that keys on a collection name uses this instead. + /// A committed-redo apply installs one record that can write several + /// collections, so [`Self::collection`] reports none for it. A caller that + /// keys on a collection name uses this instead. pub fn named_collections(&self) -> Vec<&str> { match self { - PhysicalPlan::Meta( - MetaOp::ApplyTransactionRedo { collections, .. } - | MetaOp::CalvinFlush { collections, .. }, - ) => collections.iter().map(String::as_str).collect(), + PhysicalPlan::Meta(MetaOp::ApplyTransactionRedo { collections, .. }) => { + collections.iter().map(String::as_str).collect() + } PhysicalPlan::Meta(MetaOp::RestoreRedo(batch)) => { batch.collections.iter().map(String::as_str).collect() } @@ -186,22 +185,14 @@ mod tests { fn a_redo_install_names_every_collection_it_writes() { let collections = vec!["a".to_string(), "b".to_string()]; let redo = PhysicalPlan::Meta(MetaOp::ApplyTransactionRedo { - redo: Vec::new(), - collections: collections.clone(), - sum_targets: Vec::new(), - origin: crate::physical_plan::RedoOrigin::Commit, - }); - let flush = PhysicalPlan::Meta(MetaOp::CalvinFlush { - epoch: 1, - position: 0, redo: Vec::new(), collections, sum_targets: Vec::new(), + origin: crate::physical_plan::RedoOrigin::Commit, + calvin: None, }); - for plan in [redo, flush] { - assert_eq!(plan.collection(), None); - assert_eq!(plan.named_collections(), vec!["a", "b"]); - } + assert_eq!(redo.collection(), None); + assert_eq!(redo.named_collections(), vec!["a", "b"]); } #[test] diff --git a/nodedb-physical/src/physical_plan/document/sum_target.rs b/nodedb-physical/src/physical_plan/document/sum_target.rs index bb619a42d..1824eecd7 100644 --- a/nodedb-physical/src/physical_plan/document/sum_target.rs +++ b/nodedb-physical/src/physical_plan/document/sum_target.rs @@ -129,6 +129,11 @@ impl ResolvedSumTarget { /// A transaction redo record carries source post-images only. Every replica /// applying it folds the source rows into their target rows, so the target /// identities travel with the redo, keyed by source collection. +/// +/// The table holds only the targets the source rows fold on their own core. +/// A target whose balance travels on an `ApplyBalanceDelta` task has no entry +/// here. The install then skips it. The table is merged over every write of +/// the collection, so an entry here applies to every row of the collection. #[derive( Debug, Clone, @@ -142,11 +147,8 @@ impl ResolvedSumTarget { pub struct RedoSumTargets { /// SOURCE collection the transaction wrote. pub collection: String, - /// Every target row the transaction's writes to `collection` resolved. + /// Every target row the transaction's writes to `collection` fold into. pub resolved: Vec, - /// TARGET collections whose delta travels on its own `ApplyBalanceDelta` - /// task, so the fold skips them. - pub deferred: Vec, } /// The surrogate `resolved` binds `target_collection`'s `join_value` to. diff --git a/nodedb-physical/src/physical_plan/meta.rs b/nodedb-physical/src/physical_plan/meta.rs index 52b842968..75853a614 100644 --- a/nodedb-physical/src/physical_plan/meta.rs +++ b/nodedb-physical/src/physical_plan/meta.rs @@ -8,7 +8,7 @@ use nodedb_types::calvin::{PassiveReadKey, VersionedReadEntry}; use nodedb_types::timeseries::continuous_agg::ContinuousAggregateDef; use nodedb_types::{QualifiedCollection, TenantId, Value}; -pub use super::meta_calvin::PassiveReadKeyId; +pub use super::meta_calvin::{PassiveKey, PassiveReadKeyId}; /// Meta / maintenance physical operations. #[derive( @@ -32,7 +32,7 @@ pub enum MetaOp { /// A transaction's plans as one batch. An embedded (Lite) engine executes /// the sub-plans atomically. An Origin core refuses it: a committed /// transaction installs there only through its redo record - /// (`ApplyTransactionRedo`, `CalvinFlush`). + /// (`ApplyTransactionRedo`). /// /// `txn_id` identifies the committing session transaction whose staging /// overlay holds the resolve-time bitemporal stamps an install must reuse. @@ -132,6 +132,12 @@ pub enum MetaOp { /// their edges too. #[serde(default)] group_vshards: Vec, + /// The version each of the group's vShards holds once the install + /// lands: the log position of the snapshot's cut. The installed rows + /// carry no per-row versions, so a read older than the cut no longer + /// validates on this node. Empty = no group install. + #[serde(default)] + version_floor: Vec, }, /// Purge ALL data for a tenant across every engine and cache. @@ -314,10 +320,12 @@ pub enum MetaOp { /// /// The Calvin scheduler dispatches this variant after lock acquisition for /// transactions whose read/write set is fully known at submission time (the - /// common case). The Data Plane handler validates the read-set and stages - /// `plans` without mutating base. Once the global verdict is commit, the - /// scheduler resolves the staged plans into a redo record, appends it, and - /// flushes it (`CalvinResolve`, `CalvinFlush`). + /// common case). Only the vShard's data-group leader dispatches it. The + /// Data Plane handler validates the read-set and stages `plans` without + /// mutating base. Once the global verdict is commit, the scheduler + /// resolves the staged plans into a redo record (`CalvinResolve`) and + /// proposes it to the data group. Every replica installs it from the log + /// (`ApplyTransactionRedo`). /// /// NOTE: This variant occupies the same msgpack positional tag as the /// original `CalvinExecute` variant it replaces, preserving wire @@ -337,13 +345,6 @@ pub enum MetaOp { /// the wall clock independently. Wire-additive: defaults to 0 on decode /// of older entries. epoch_system_ms: i64, - /// Whether THIS node is the leader of the data-group owning this vshard, - /// stamped by the dispatching scheduler. OLLP verification (`actual != - /// predicted` → `OllpRetryRequired`) is leader-only; followers apply the - /// carried `ollp_predicted_surrogates` verbatim so every replica mutates - /// the identical set (Calvin determinism). Per-node, never replicated. - /// Wire-additive: defaults to `false` on decode of older entries. - is_group_leader: bool, /// The transaction's LSN-versioned read-set from the replicated `TxClass`. /// Each participant checks its own vShard's reads at apply and records /// whether they were still current, without gating apply. Wire-additive: @@ -357,15 +358,16 @@ pub enum MetaOp { body_plans: Vec, }, - /// Calvin dependent-read executor: passive participant reads keys and - /// returns values for broadcast. + /// Calvin dependent-read executor: a passive participant reads its keys + /// and returns their values for broadcast. /// - /// Dispatched by the scheduler to passive vshards (those holding only - /// read keys, not write keys) for a dependent-read Calvin transaction. - /// The Data Plane handler reads each key from the local engine and - /// returns a msgpack-encoded `Vec<(PassiveReadKeyId, Value)>` payload. - /// The scheduler then proposes a `ReplicatedWrite::CalvinReadResult` - /// to the per-vshard Raft group so all replicas see the same values. + /// The data-group leader of each passive vShard dispatches it once the + /// transaction holds its locks there. The Data Plane handler reads each + /// key from base storage and returns a zerompk-encoded + /// `Vec<(PassiveReadKeyId, Value)>`: the stored bytes as `Value::Bytes`, + /// `Value::Null` for an absent row. A successful read votes commit. The + /// scheduler proposes the payload as a `ReplicatedWrite::CalvinReadResult` + /// to the data group of every active vShard. CalvinExecutePassive { /// Sequencer epoch this transaction belongs to. epoch: u64, @@ -405,13 +407,6 @@ pub enum MetaOp { /// Wall-clock ms read once on the sequencer leader at epoch creation. /// Same semantics as `CalvinExecuteStatic::epoch_system_ms`. epoch_system_ms: i64, - /// Whether THIS node is the leader of the data-group owning this - /// vshard. Same semantics as `CalvinExecuteStatic::is_group_leader`: - /// gates the LEADER-ONLY OLLP verification + `OllpRetryRequired`; every - /// replica applies the carried `ollp_predicted_surrogates` set verbatim - /// for determinism. Per-node, non-replicated; wire-additive (defaults to - /// `false`). - is_group_leader: bool, }, /// Rebuild a collection's indexes (HNSW, full-text, graph CSR) on @@ -505,49 +500,13 @@ pub enum MetaOp { savepoint: u64, }, - /// Record the per-key write versions of a committed Calvin transaction's - /// locally-applied write plans. - /// - /// The scheduler stamps the transaction's committed LSN onto this op's - /// `wal_lsn` and dispatches it back to the same core once the flush - /// completes. The core funnels `plans` through the shared write-version - /// recorder at that LSN — the same shard-local WAL-LSN space the - /// single-shard fast path and read watermarks use. Records only: no base - /// mutation, no WAL append, no event emission. - RecordCalvinWriteVersions { - /// Tenant scope for all plans. - tenant_id: TenantId, - /// The locally-applied write plans whose keys' versions are recorded. - plans: Vec, - }, - - /// Install a committed Calvin transaction's redo record on base storage. - /// - /// `CalvinExecuteStatic` STAGES the transaction's plans without mutating - /// base, and `CalvinResolve` resolves them into one redo record, which the - /// scheduler appends to the WAL as a `TransactionRedo` record. This op - /// carries that record's bytes, and the request carries its LSN. The core - /// installs it through the same passes restart replay drives: validate, - /// install with undo, then settle and cover. It then drops the staged - /// state keyed by `(epoch, position)`. - /// - /// `redo` is empty when the transaction wrote nothing. `collections` names - /// every collection the transaction wrote. `sum_targets` is the - /// materialized-sum resolution its document writes fold into. - CalvinFlush { - epoch: u64, - position: u32, - redo: Vec, - collections: Vec, - sum_targets: Vec, - }, - /// Discard the staged writes of a Calvin transaction. /// - /// Dispatched by the scheduler when the local commit vote resolves to abort. - /// The handler removes the staged plans keyed by `(epoch, position)` and - /// fires nothing — no base mutation, no side effects. Absent key is an - /// idempotent no-op. + /// Dispatched by the leader's scheduler when the slice ends with no log + /// entry (an abort verdict, or a COMMIT for a slice with no write), and + /// by a demoted leader for state it staged. The handler removes the + /// staged plans keyed by `(epoch, position)` and fires nothing — no base + /// mutation, no side effects. Absent key is an idempotent no-op. CalvinDrop { epoch: u64, position: u32 }, /// Resolve a committing transaction's staged writes into ONE replayable @@ -581,8 +540,13 @@ pub enum MetaOp { /// `commit_pending` under `(epoch, position, vshard)` and the per-core /// staging overlay written under the corresponding synthetic `TxnId` /// (see `calvin_synthetic_txn_id`). Dispatched by the scheduler once the - /// global verdict is commit, ahead of `CalvinFlush`, which installs the - /// record this op returns. No base engine is touched during resolve. + /// global verdict is commit. The scheduler proposes the record this op + /// returns as a stamped `TransactionRedo` entry, and + /// `ApplyTransactionRedo` installs it. No base engine is touched during + /// resolve. + /// + /// The response payload is a zerompk-encoded [`super::CalvinResolved`]: + /// the redo record and the reply the staged plans decided. CalvinResolve { epoch: u64, position: u32 }, /// Apply one committed transaction's resolved redo record on the core that @@ -598,11 +562,17 @@ pub enum MetaOp { /// version at the record's LSN. `sum_targets` is the materialized-sum /// resolution the transaction's document writes fold into their targets. /// `origin` decides which commit-boundary checks the apply runs. + /// + /// `calvin` is set when the record is a committed Calvin slice. The + /// install then stamps the slice's ordinal on the core clock, consumes + /// the slice's staged state when this core staged it, and answers the + /// reply `calvin.reply` describes. ApplyTransactionRedo { redo: Vec, collections: Vec, sum_targets: Vec, origin: super::RedoOrigin, + calvin: Option, }, /// Install one batch of a RESTORE as part of a Calvin transaction, on diff --git a/nodedb-physical/src/physical_plan/meta_calvin.rs b/nodedb-physical/src/physical_plan/meta_calvin.rs index 8fed613b7..2e53bcbcb 100644 --- a/nodedb-physical/src/physical_plan/meta_calvin.rs +++ b/nodedb-physical/src/physical_plan/meta_calvin.rs @@ -1,34 +1,9 @@ // SPDX-License-Identifier: Apache-2.0 //! Calvin-specific identity types carried by `MetaOp` Calvin variants. +//! +//! [`PassiveReadKeyId`] keys `MetaOp::CalvinExecuteActive::injected_reads`. +//! It lives in `nodedb-types`, so the replicated transaction class in +//! `nodedb-cluster` names the same identity. -/// Identity of a single key read by a passive Calvin participant. -/// -/// Used as the map key in `MetaOp::CalvinExecuteActive::injected_reads` so -/// active participants can look up which value belongs to which key. -/// -/// `collection` is database-qualified (`QualifiedCollection`) so a key from -/// one database never collides with the same bare name in another. -/// -/// `BTreeMap` key: `Ord` is derived lexicographically — collection first, -/// surrogate second. This is the determinism contract: all replicas must -/// iterate `injected_reads` in the same order. -#[derive( - Debug, - Clone, - PartialEq, - Eq, - PartialOrd, - Ord, - Hash, - serde::Serialize, - serde::Deserialize, - zerompk::ToMessagePack, - zerompk::FromMessagePack, -)] -pub struct PassiveReadKeyId { - /// Collection the key belongs to. - pub collection: nodedb_types::QualifiedCollection, - /// Global surrogate for the row. - pub surrogate: u32, -} +pub use nodedb_types::calvin::{PassiveKey, PassiveReadKeyId}; diff --git a/nodedb-physical/src/physical_plan/meta_home.rs b/nodedb-physical/src/physical_plan/meta_home.rs index cec6fb3d0..ff9c52950 100644 --- a/nodedb-physical/src/physical_plan/meta_home.rs +++ b/nodedb-physical/src/physical_plan/meta_home.rs @@ -53,9 +53,9 @@ pub struct HomeVersion { zerompk::FromMessagePack, )] pub enum HomeAnswer { - /// The collection's write floor on the probe's core, or that core's - /// watermark for a probe with no collection. - Version(u64), + /// The collection's write floor on the probe's vShard, or that vShard's + /// latest version for a probe with no collection. + Version(nodedb_types::WriteVersion), /// The node does not hold the leader lease of the probe's group, so it /// cannot answer. `leader_node` is the leader its routing table names at /// `leader_term`, `0` when it names none. @@ -74,7 +74,7 @@ mod tests { vshard: 7, collection: Some("db1.edges".into()), }, - answer: HomeAnswer::Version(42), + answer: HomeAnswer::Version(nodedb_types::WriteVersion::logged(3, 42)), }, HomeVersion { probe: HomeVersionProbe { diff --git a/nodedb-physical/src/physical_plan/meta_snapshot.rs b/nodedb-physical/src/physical_plan/meta_snapshot.rs index 0af8302af..d83aedf0f 100644 --- a/nodedb-physical/src/physical_plan/meta_snapshot.rs +++ b/nodedb-physical/src/physical_plan/meta_snapshot.rs @@ -29,7 +29,7 @@ pub struct SnapshotClearTarget { /// A database backup's request to capture its tenants at its cut barrier. /// -/// The barrier the backup proposes into each data group carries it. The +/// The barrier each data group's leader places for the backup carries it. The /// group's leader, when it applies the first barrier of `request_id`, /// snapshots every tenant of `tenants` in `database_id` before it applies the /// next entry, and parks the capture for the backup to collect. diff --git a/nodedb-physical/src/physical_plan/mod.rs b/nodedb-physical/src/physical_plan/mod.rs index 5e1919506..f403546ae 100644 --- a/nodedb-physical/src/physical_plan/mod.rs +++ b/nodedb-physical/src/physical_plan/mod.rs @@ -7,6 +7,7 @@ //! isolated. pub mod array; +pub mod calvin_reply; pub mod cluster_array; pub mod cluster_event; pub mod collection; @@ -37,6 +38,10 @@ pub mod vector; pub mod wire; pub use array::{ArrayBinaryOp, ArrayOp, ArrayReducer}; +pub use calvin_reply::{ + CalvinInstall, CalvinInstalledTimeseriesSpec, CalvinPostImagesSpec, CalvinReplyRow, + CalvinReplySpec, CalvinResolved, CalvinRowEngine, +}; pub use cluster_array::ClusterArrayOp; pub use cluster_event::{ClusterEventOp, MAX_REMOTE_CDC_COMMITTED_OFFSETS}; pub use columnar::{ColumnarInsertIntent, ColumnarOp}; diff --git a/nodedb-raft/Cargo.toml b/nodedb-raft/Cargo.toml index 8f43ac9e2..50ac35058 100644 --- a/nodedb-raft/Cargo.toml +++ b/nodedb-raft/Cargo.toml @@ -12,8 +12,15 @@ documentation = "https://docs.rs/nodedb-raft" keywords = ["raft", "consensus", "replication", "distributed", "leader-election"] categories = ["algorithms", "network-programming"] +[features] +default = [] +# Arms the node-scoped commit gate (`raft::hold_commit::group`) cluster +# tests use to keep an entry uncommitted across a leadership change. +failpoints = ["dep:nodedb-types", "nodedb-types/failpoints"] + [dependencies] # Workspace +nodedb-types = { workspace = true, optional = true } serde = { workspace = true } serde_json = { workspace = true } sonic-rs = { workspace = true } diff --git a/nodedb-raft/src/fail_gate.rs b/nodedb-raft/src/fail_gate.rs new file mode 100644 index 000000000..a79be1ce9 --- /dev/null +++ b/nodedb-raft/src/fail_gate.rs @@ -0,0 +1,36 @@ +// SPDX-License-Identifier: BUSL-1.1 + +//! Fail gates for Raft tests. +//! +//! Compiled only with the `failpoints` feature. A gate answers without +//! waiting: Raft steps are synchronous, so a gate holds a step back and the +//! driver asks again on its next step. + +use nodedb_types::fail_point::{FailAction, FailScope, lookup}; + +/// Whether the commit gate of `group_id` on `node_id` holds now. Named +/// `raft::hold_commit::group` and scoped to the node, armed with +/// `wait_file()`. It holds while the file is absent. +/// +/// A leader under the gate counts no quorum, so it commits no new entry. +/// The entries it appends still replicate. A test keeps an entry +/// uncommitted this way, then moves the leadership: the entry commits only +/// in the next term. On its first hold the gate creates `.parked`. +pub(crate) fn holds_commit(node_id: u64, group_id: u64) -> bool { + let name = format!("raft::hold_commit::group{group_id}"); + let Some(FailAction::WaitForFile(path)) = lookup(FailScope::Node(node_id), &name) else { + return false; + }; + if path.exists() { + return false; + } + let mut parked = path.into_os_string(); + parked.push(".parked"); + let parked = std::path::PathBuf::from(parked); + if !parked.exists() + && let Err(e) = std::fs::write(&parked, &name) + { + tracing::warn!(gate = %name, node_id, error = %e, "fail gate parked marker not created"); + } + true +} diff --git a/nodedb-raft/src/lib.rs b/nodedb-raft/src/lib.rs index a7e7f0e2a..203c93d80 100644 --- a/nodedb-raft/src/lib.rs +++ b/nodedb-raft/src/lib.rs @@ -10,6 +10,8 @@ //! catalog and the cross-engine surrogate counter). pub mod error; +#[cfg(feature = "failpoints")] +mod fail_gate; pub mod log; pub mod message; pub mod node; diff --git a/nodedb-raft/src/log.rs b/nodedb-raft/src/log.rs index a9ebb232c..a2ecd0279 100644 --- a/nodedb-raft/src/log.rs +++ b/nodedb-raft/src/log.rs @@ -4,6 +4,11 @@ use crate::error::{RaftError, Result}; use crate::message::LogEntry; use crate::storage::LogStorage; +/// Most entry data one AppendEntries request carries. Half the 64 MiB RPC +/// frame limit, so a batch and its envelope fit one frame. A single entry +/// larger than this ships alone, and the host bounds an entry's size. +pub const MAX_APPEND_BATCH_BYTES: usize = 32 * 1024 * 1024; + /// In-memory Raft log backed by a pluggable `LogStorage`. /// /// The log is 1-indexed. Index 0 is a sentinel (term 0, empty data). @@ -78,6 +83,23 @@ impl RaftLog { self.entries.get(offset) } + /// Entries from `lo` up to `hi` inclusive whose data sums to at most + /// `max_bytes`. The first entry is always included, so an entry larger + /// than `max_bytes` still ships, alone. + pub fn entries_within(&self, lo: u64, hi: u64, max_bytes: usize) -> Result<&[LogEntry]> { + let range = self.entries_range(lo, hi)?; + let mut total = 0usize; + let mut end = 0usize; + for entry in range { + total = total.saturating_add(entry.data.len()); + if end > 0 && total > max_bytes { + break; + } + end += 1; + } + Ok(&range[..end]) + } + /// Get entries in range [lo, hi] inclusive. pub fn entries_range(&self, lo: u64, hi: u64) -> Result<&[LogEntry]> { if lo <= self.snapshot_index { @@ -265,6 +287,24 @@ mod tests { assert_eq!(range[2].index, 4); } + #[test] + fn entries_within_stops_at_the_byte_budget_and_keeps_one_entry() { + let mut log = RaftLog::new(MemStorage::new()); + for i in 1..=4 { + log.append(LogEntry { + term: 1, + index: i, + data: vec![0; 10], + }) + .unwrap(); + } + assert_eq!(log.entries_within(1, 4, 25).unwrap().len(), 2); + assert_eq!(log.entries_within(1, 4, 40).unwrap().len(), 4); + // An entry past the budget ships alone. + assert_eq!(log.entries_within(2, 4, 5).unwrap().len(), 1); + assert!(log.entries_within(5, 4, 5).unwrap().is_empty()); + } + #[test] fn conflict_detection() { let mut log = RaftLog::new(MemStorage::new()); diff --git a/nodedb-raft/src/node/core.rs b/nodedb-raft/src/node/core.rs index c31da6293..d895ec21a 100644 --- a/nodedb-raft/src/node/core.rs +++ b/nodedb-raft/src/node/core.rs @@ -116,6 +116,12 @@ pub struct RaftNode { /// Per-voter highest lease round acknowledged in the current term, and /// when the latest answer arrived. Cleared on step-down. pub(super) quorum_window: Vec, + /// When this node won its current leadership term. `None` off the + /// leader path. A voter not yet heard this term counts as last seen here. + pub(super) leader_since: Option, + /// Index of the no-op this node appended when it won its current + /// leadership term. `None` off the leader path. + pub(super) term_start_index: Option, /// Leader-lease rounds and anchor (see [`super::leader_lease`]). pub(super) lease: LeaseState, /// Until this instant the node refuses every vote but a transfer vote. @@ -178,6 +184,8 @@ impl RaftNode { leader_contact: None, last_quorum_contact: None, quorum_window: Vec::new(), + leader_since: None, + term_start_index: None, lease: LeaseState::new(), boot_vote_fence: now + config.election_timeout_max, compaction_ceiling: None, @@ -250,6 +258,16 @@ impl RaftNode { self.log.last_index() } + /// Index of the no-op this leader appended when it won its current + /// term. Every entry of an earlier term in its log sits below it and + /// commits with it. `None` when this node does not lead. + pub fn term_start_index(&self) -> Option { + if self.role != NodeRole::Leader { + return None; + } + self.term_start_index + } + /// Override election deadline (for testing). pub fn election_deadline_override(&mut self, deadline: Instant) { self.election_deadline = deadline; diff --git a/nodedb-raft/src/node/internal.rs b/nodedb-raft/src/node/internal.rs index 978c54400..cc9eb796a 100644 --- a/nodedb-raft/src/node/internal.rs +++ b/nodedb-raft/src/node/internal.rs @@ -147,6 +147,8 @@ impl RaftNode { // The contact window belongs to a leader term; it means nothing here. self.last_quorum_contact = None; self.quorum_window.clear(); + self.leader_since = None; + self.term_start_index = None; self.persist_hard_state(); self.reset_election_timeout(); @@ -197,7 +199,9 @@ impl RaftNode { index: self.log.last_index() + 1, data: Vec::new(), }; + let noop_index = noop.index; let _ = self.log.append(noop); + self.term_start_index = Some(noop_index); // Single-voter cluster: the no-op commits once it is durable here. if self.config.cluster_size() == 1 { @@ -255,7 +259,11 @@ impl RaftNode { }; let entries = if next_index <= self.log.last_index() { - match self.log.entries_range(next_index, self.log.last_index()) { + match self.log.entries_within( + next_index, + self.log.last_index(), + crate::log::MAX_APPEND_BATCH_BYTES, + ) { Ok(slice) => slice.to_vec(), Err(RaftError::LogCompacted { .. }) => { debug!( @@ -345,7 +353,11 @@ impl RaftNode { }; let entries = if next_index <= self.log.last_index() { - match self.log.entries_range(next_index, self.log.last_index()) { + match self.log.entries_within( + next_index, + self.log.last_index(), + crate::log::MAX_APPEND_BATCH_BYTES, + ) { Ok(slice) => slice.to_vec(), Err(crate::error::RaftError::LogCompacted { .. }) => { debug!( @@ -395,6 +407,10 @@ impl RaftNode { /// up for promotion) but intentionally excluded from this calculation /// so adding a learner never weakens the commit quorum. pub(super) fn try_advance_commit_index(&mut self) { + #[cfg(feature = "failpoints")] + if crate::fail_gate::holds_commit(self.config.node_id, self.config.group_id) { + return; + } let leader = match &self.leader_state { Some(ls) => ls, None => return, @@ -516,6 +532,25 @@ mod tests { ); } + /// A leader names the index of the no-op it appended when it won its + /// term. Later proposals leave it unchanged, and a step-down clears it. + #[test] + fn a_leader_names_its_term_start_index() { + let mut node = RaftNode::new(test_config(1, vec![]), MemStorage::new()); + assert_eq!(node.term_start_index(), None, "a follower names none"); + force_election(&mut node); + let noop = node.last_log_index(); + assert_eq!(node.term_start_index(), Some(noop)); + + node.propose(b"write".to_vec()) + .expect("single voter commits immediately"); + assert_eq!(node.term_start_index(), Some(noop)); + + let term = node.current_term(); + node.become_follower(term + 1); + assert_eq!(node.term_start_index(), None); + } + /// A committed range below the log's first available index is a hard /// error in `Ready`, never a silently empty delivery. #[test] diff --git a/nodedb-raft/src/node/peer_contact.rs b/nodedb-raft/src/node/peer_contact.rs index 61d986374..c8dcfe5bf 100644 --- a/nodedb-raft/src/node/peer_contact.rs +++ b/nodedb-raft/src/node/peer_contact.rs @@ -18,29 +18,43 @@ impl RaftNode { .map(|leader| leader.ack_count_for(peer)) } - /// The highest log index every voter's log is known to hold. A log - /// compacted at or below it never sends a voter a snapshot, under this - /// leader or a later one: every voter holds the compacted entries. + /// The highest log index every live voter's log is known to hold. See + /// [`Self::replicated_floor_at`]. + pub fn replicated_floor(&self) -> u64 { + self.replicated_floor_at(Instant::now()) + } + + /// The highest log index every live voter's log is known to hold at + /// `now`. A log compacted at or below it never sends a live voter a + /// snapshot. /// /// The leader takes the lowest `match_index` over the other voters, /// capped at its commit index: a committed entry stays in every log that /// holds it. It sends the value with each `AppendEntries`. Every replica - /// keeps the highest value it saw, so the floor never moves back. A voter - /// that stops answering holds it where it is. Learners and observers do - /// not count: a new replica takes a snapshot. - pub fn replicated_floor(&self) -> u64 { + /// keeps the highest value it saw, so the floor never moves back. + /// + /// Only voters that answered within the check-quorum window count. A + /// voter not yet heard this term counts until the window passes from the + /// election win. A silent voter never holds compaction on any replica. + /// This assumes a voter that falls behind the compacted boundary recovers + /// through `InstallSnapshot` once it answers again. + /// + /// Learners and observers never count: a new replica takes a snapshot. + /// Off the leader path this is the highest floor a leader sent. + pub fn replicated_floor_at(&self, now: Instant) -> u64 { let Some(leader) = self.leader_state.as_ref() else { return self.replicated_floor; }; - let lowest_voter = self + let lowest_live_voter = self .config .peers .iter() + .filter(|&&peer| self.voter_heard_within_window(peer, now)) .map(|&peer| leader.match_index_for(peer)) .min() .unwrap_or(u64::MAX); self.replicated_floor - .max(lowest_voter.min(self.volatile.commit_index)) + .max(lowest_live_voter.min(self.volatile.commit_index)) } /// Whether `peer`'s latest response to this leader asked for a snapshot. @@ -73,7 +87,7 @@ impl RaftNode { #[cfg(test)] mod tests { - use std::time::Duration; + use std::time::{Duration, Instant}; use crate::message::{AppendEntriesRequest, RequestVoteResponse}; use crate::node::core::RaftNode; @@ -143,8 +157,9 @@ mod tests { assert_eq!(node.live_leader(std::time::Instant::now()), 0); } - /// The leader's floor is the lowest voter's committed match. A voter - /// that has not answered holds it at 0. + /// The leader's floor is the lowest voter's committed match. Within the + /// window from the election win, a voter that has not answered holds it + /// at 0. #[test] fn the_replicated_floor_waits_for_every_voter() { let mut node = RaftNode::new(test_config(1, vec![2, 3]), MemStorage::new()); @@ -172,6 +187,106 @@ mod tests { assert_eq!(node.replicated_floor(), 1); } + /// Elect node 1 over `cfg`, with every voter granting its vote. + fn elect(cfg: crate::node::config::RaftConfig) -> RaftNode { + let peers = cfg.peers.clone(); + let mut node = RaftNode::new(cfg, MemStorage::new()); + force_election(&mut node); + for peer in peers { + if node.role() == NodeRole::Leader { + break; + } + node.handle_request_vote_response( + peer, + &RequestVoteResponse { + term: node.current_term(), + vote_granted: true, + }, + ); + } + assert_eq!(node.role(), NodeRole::Leader); + let _ = node.take_ready(); + node + } + + /// Peer `peer` answers the latest round, acking `last_log_index`. + fn answer(node: &mut RaftNode, peer: u64, success: bool, last_log_index: u64) { + let resp = crate::message::AppendEntriesResponse { + term: node.current_term(), + success, + last_log_index, + round: node.lease.next_round - 1, + needs_snapshot: false, + }; + node.handle_append_entries_response(peer, &resp); + } + + /// Move the election win back past the check-quorum window. + fn age_term_start(node: &mut RaftNode) { + let window = node.config.election_timeout_max; + node.leader_since = Some(Instant::now() - window - Duration::from_millis(1)); + } + + /// A voter silent for the whole check-quorum window stops holding the + /// floor. The voter that answered sets it. + #[test] + fn a_silent_voter_stops_holding_the_floor_after_the_window() { + let mut node = elect(test_config(1, vec![2, 3])); + answer(&mut node, 2, true, 1); + assert_eq!(node.commit_index(), 1); + assert_eq!( + node.replicated_floor(), + 0, + "voter 3 counts within the window from the election win" + ); + age_term_start(&mut node); + answer(&mut node, 2, true, 1); + assert_eq!( + node.replicated_floor(), + 1, + "silent voter 3 no longer counts" + ); + } + + /// A voter that answered within the window holds the floor at its + /// `match_index`, even when that answer was a rejection. Voter 3 answers + /// before the commit moves: the floor never moves back. + #[test] + fn a_voter_heard_within_the_window_still_holds_the_floor() { + let mut node = elect(test_config(1, vec![2, 3])); + age_term_start(&mut node); + answer(&mut node, 3, false, 0); + assert!(node.voter_heard_within_window(3, Instant::now())); + answer(&mut node, 2, true, 1); + assert_eq!(node.commit_index(), 1); + assert_eq!( + node.replicated_floor(), + 0, + "voter 3 answered and holds nothing" + ); + + let stale = Instant::now() + node.config.election_timeout_max; + assert!( + !node.voter_heard_within_window(3, stale), + "the answer ages out of the window" + ); + } + + /// A learner never holds the floor, answered or silent. + #[test] + fn learners_never_count_toward_the_floor() { + let mut cfg = test_config(1, vec![2, 3]); + cfg.learners = vec![4]; + let mut node = elect(cfg); + answer(&mut node, 4, false, 0); + answer(&mut node, 2, true, 1); + answer(&mut node, 3, true, 1); + assert_eq!(node.commit_index(), 1); + assert_eq!(node.replicated_floor(), 1, "learner 4 holds nothing"); + age_term_start(&mut node); + assert_eq!(node.replicated_floor(), 1); + } + /// A follower keeps the highest floor a leader sent it. #[test] fn a_follower_keeps_the_highest_replicated_floor() { diff --git a/nodedb-raft/src/node/quorum_contact.rs b/nodedb-raft/src/node/quorum_contact.rs index 2b44ac90d..74f32f7a2 100644 --- a/nodedb-raft/src/node/quorum_contact.rs +++ b/nodedb-raft/src/node/quorum_contact.rs @@ -66,10 +66,24 @@ impl RaftNode { /// grant does not make a voter refuse other candidates. pub(super) fn arm_quorum_window(&mut self, now: Instant) { self.last_quorum_contact = Some(now); + self.leader_since = Some(now); self.quorum_window.clear(); self.lease.begin_term(); } + /// Whether voter `peer` answered this leader within the check-quorum + /// window at `now`. A voter not yet heard this term counts from the + /// election win. Always false off the leader path. + pub(super) fn voter_heard_within_window(&self, peer: u64, now: Instant) -> bool { + let last_seen = match self.voter_ack(peer) { + Some(ack) => Some(ack.arrived), + None => self.leader_since, + }; + last_seen.is_some_and(|seen| { + now.saturating_duration_since(seen) < self.config.election_timeout_max + }) + } + /// Move contact up to the latest instant by which a quorum of voters /// answered this term. No-op for any role but leader. /// diff --git a/nodedb-test-support/Cargo.toml b/nodedb-test-support/Cargo.toml index c93ed50f0..385462da9 100644 --- a/nodedb-test-support/Cargo.toml +++ b/nodedb-test-support/Cargo.toml @@ -10,6 +10,11 @@ description = "Shared test harness for NodeDB integration tests (not for end-use [lib] path = "src/lib.rs" +[features] +# Re-exports the fail-point test API (`fail_point::FailGuard`) and arms the +# fail points of the nodes this harness spawns. +failpoints = ["nodedb/failpoints", "nodedb-types/failpoints"] + [dependencies] # Production crates this harness drives. nodedb = { workspace = true } diff --git a/nodedb-test-support/src/booted_state.rs b/nodedb-test-support/src/booted_state.rs index 68984ba5d..6cbb67fa8 100644 --- a/nodedb-test-support/src/booted_state.rs +++ b/nodedb-test-support/src/booted_state.rs @@ -186,6 +186,7 @@ async fn boot_node(options: BootOptions) -> Result { let (core_stop_tx, core_stop_rx) = std::sync::mpsc::channel::<()>(); let core = crate::core_loop_runner::spawn_core_loop(crate::core_loop_runner::CoreLoopSpawn { idx: 0, + node_id: state.node_id, num_cores: 1, data_side, core_dir: dir.path().to_path_buf(), diff --git a/nodedb-test-support/src/cluster_harness/cluster/leadership.rs b/nodedb-test-support/src/cluster_harness/cluster/leadership.rs new file mode 100644 index 000000000..4c5e5c562 --- /dev/null +++ b/nodedb-test-support/src/cluster_harness/cluster/leadership.rs @@ -0,0 +1,191 @@ +// SPDX-License-Identifier: BUSL-1.1 + +//! Data-group leadership moves and waits on a [`TestCluster`]. +//! +//! A move asks the group's leader to hand its leadership to a target, the +//! path the leader balance takes. The leader balance keeps running, so it +//! can move a group back to its preferred leader a few seconds later. + +use std::time::{Duration, Instant}; + +use super::TestCluster; +use crate::cluster_harness::wait::wait_for_report; + +/// Which node a wait expects to lead a group. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum GroupLeader { + /// Any live node. + Any, + /// The node with this id. + Node(u64), +} + +/// How long one transfer request gets before the leader is asked again. +const TRANSFER_ROUND: Duration = Duration::from_secs(3); + +/// How long a leadership move gets in all. +const TRANSFER_DEADLINE: Duration = Duration::from_secs(30); + +/// Pause between two leadership reads. +const LEADER_POLL: Duration = Duration::from_millis(20); + +impl TestCluster { + /// The node that leads `group_id`: it leads in its own Raft state, and + /// every live node hosting the group names it leader. `None` while no + /// node leads or the nodes disagree. + pub fn group_leader(&self, group_id: u64) -> Option { + self.leader_report(group_id).ok() + } + + /// Wait until `leader` leads `group_id` as [`Self::group_leader`] reads + /// it, and return the leader's id. Panics at `timeout` with each node's + /// view of the group. + pub async fn wait_for_group_leader( + &self, + group_id: u64, + leader: GroupLeader, + timeout: Duration, + ) -> u64 { + let expected = match leader { + GroupLeader::Any => "any node".to_owned(), + GroupLeader::Node(node_id) => format!("node {node_id}"), + }; + let mut found = 0; + wait_for_report( + &format!("{expected} leads group {group_id}"), + timeout, + LEADER_POLL, + || { + let current = self.leader_report(group_id)?; + match leader { + GroupLeader::Node(node_id) if node_id != current => { + Err(format!("node {current} leads")) + } + GroupLeader::Any | GroupLeader::Node(_) => { + found = current; + Ok(()) + } + } + }, + ) + .await; + found + } + + /// Move the leadership of `group_id` to node `target`, and wait until + /// [`Self::group_leader`] names it. Each round asks the group's current + /// leader again: a transfer the leader refused, or one that lapsed, + /// starts nothing. Panics when the move does not finish within 30 s, + /// with the last refusal and each node's view. + pub async fn transfer_leadership(&self, group_id: u64, target: u64) { + let deadline = Instant::now() + TRANSFER_DEADLINE; + let mut last_refusal = String::from("no transfer was refused"); + loop { + if self.group_leader(group_id) == Some(target) { + return; + } + if Instant::now() >= deadline { + panic!( + "group {group_id} did not move to node {target} within \ + {TRANSFER_DEADLINE:?}; last refusal: {last_refusal}; views: {}", + self.leader_views(group_id) + ); + } + let leader = self.nodes.iter().find(|node| node.leads_group(group_id)); + if let Some(leader) = leader + && leader.node_id != target + && let Err(refusal) = leader.transfer_group_leadership(group_id, target) + { + last_refusal = refusal; + } + let round_end = Instant::now() + TRANSFER_ROUND; + while Instant::now() < round_end { + if self.group_leader(group_id) == Some(target) { + return; + } + tokio::time::sleep(LEADER_POLL).await; + } + } + } + + /// Wait until each data group is led by its preferred leader. The + /// leader balance then moves no group by itself, so the leaders stay + /// put until a test moves them. Panics at `timeout`. + pub async fn wait_for_preferred_leaders(&self, timeout: Duration) { + wait_for_report( + "every data group is led by its preferred leader", + timeout, + LEADER_POLL, + || { + let preferred = { + let routing = self.nodes[0] + .shared + .cluster_routing + .as_ref() + .ok_or_else(|| "node has no cluster routing".to_owned())? + .read() + .unwrap_or_else(|p| p.into_inner()); + nodedb_cluster::rebalancer::preferred_leaders(&routing) + }; + for (group_id, node_id) in preferred { + let current = self.leader_report(group_id)?; + if current != node_id { + return Err(format!( + "node {current} leads group {group_id}, node {node_id} is preferred" + )); + } + } + Ok(()) + }, + ) + .await; + } + + /// The agreed leader of `group_id`, or why there is none. + fn leader_report(&self, group_id: u64) -> Result { + let mut named = None; + for node in &self.nodes { + let Some(leader) = node + .all_group_leaders() + .into_iter() + .find(|(group, _)| *group == group_id) + .map(|(_, leader)| leader) + else { + continue; + }; + match named { + None => named = Some(leader), + Some(first) if first != leader => { + return Err(format!( + "the nodes disagree on group {group_id}'s leader: {}", + self.leader_views(group_id) + )); + } + Some(_) => {} + } + } + let leader = named + .filter(|leader| *leader != 0) + .ok_or_else(|| format!("no node leads group {group_id}"))?; + let leads = self + .nodes + .iter() + .any(|node| node.node_id == leader && node.leads_group(group_id)); + if leads { + Ok(leader) + } else { + Err(format!( + "node {leader} is named leader of group {group_id} but does not lead it" + )) + } + } + + /// Each live node's view of `group_id`'s leader, for a failure report. + fn leader_views(&self, group_id: u64) -> String { + self.nodes + .iter() + .map(|node| node.group_status_line(group_id)) + .collect::>() + .join("; ") + } +} diff --git a/nodedb-test-support/src/cluster_harness/cluster/mod.rs b/nodedb-test-support/src/cluster_harness/cluster/mod.rs index 63fd6d1e4..d0ed6e4ba 100644 --- a/nodedb-test-support/src/cluster_harness/cluster/mod.rs +++ b/nodedb-test-support/src/cluster_harness/cluster/mod.rs @@ -6,15 +6,18 @@ //! [`types`]; spawn-variant convenience wrappers in [`spawn_variants`]; //! the bringup body in [`bringup`]; the convergence barriers in [`ready`]; //! the in-place restart in [`restart`]; post-spawn membership + DDL helpers -//! in [`membership`]. +//! in [`membership`]; data-group leadership moves and waits in +//! [`leadership`]. mod bringup; +mod leadership; mod membership; mod ready; mod restart; mod spawn_variants; mod types; +pub use leadership::GroupLeader; pub use restart::{StoppedCluster, StoppedMember, StoppedNodeInfo}; pub use types::TestCluster; pub(crate) use types::{ClusterSpawnConfig, DEFAULT_NUM_GROUPS}; diff --git a/nodedb-test-support/src/cluster_harness/mod.rs b/nodedb-test-support/src/cluster_harness/mod.rs index ebabcb540..eb4f9f0e3 100644 --- a/nodedb-test-support/src/cluster_harness/mod.rs +++ b/nodedb-test-support/src/cluster_harness/mod.rs @@ -22,7 +22,7 @@ pub mod retriable; pub mod shared_steps; pub mod wait; -pub use cluster::{StoppedCluster, StoppedNodeInfo, TestCluster}; +pub use cluster::{GroupLeader, StoppedCluster, StoppedNodeInfo, TestCluster}; pub use node::TestClusterNode; pub use pitr::PitrStorage; pub use retriable::{is_no_serving_leader, read_once_a_leader_exists}; diff --git a/nodedb-test-support/src/cluster_harness/node/inspect/core_placement.rs b/nodedb-test-support/src/cluster_harness/node/inspect/core_placement.rs index cdaa555ad..24cd79998 100644 --- a/nodedb-test-support/src/cluster_harness/node/inspect/core_placement.rs +++ b/nodedb-test-support/src/cluster_harness/node/inspect/core_placement.rs @@ -42,6 +42,23 @@ impl TestClusterNode { /// The document keys of `tenant` in the default database that core /// `core_id` stores, from that core's own tenant snapshot. pub async fn document_keys_on_core(&self, core_id: usize, tenant: TenantId) -> Vec { + self.tenant_snapshot_on_core(core_id, tenant) + .await + .documents + .into_iter() + .map(|(k, _)| k) + .collect() + } + + /// The tenant snapshot of `tenant` in the default database that core + /// `core_id` of this node holds. The read never leaves this node, so it + /// shows this replica's own stored rows. Panics when the core answers + /// with an error. + pub async fn tenant_snapshot_on_core( + &self, + core_id: usize, + tenant: TenantId, + ) -> TenantDataSnapshot { let request_id = RequestId::new(PLACEMENT_REQUEST_ID.fetch_add(1, Ordering::Relaxed)); let request = Request { request_id, @@ -67,6 +84,7 @@ impl TestClusterNode { wal_lsn: None, resolved_now_ms: None, commit_hlc: None, + entry_version: None, admission: nodedb::bridge::envelope::Admission::Exempt( nodedb::bridge::envelope::ExemptReason::Read, ), @@ -87,8 +105,6 @@ impl TestClusterNode { Status::Ok, "core {core_id} snapshot failed" ); - let snap: TenantDataSnapshot = - zerompk::from_msgpack(response.payload.as_bytes()).expect("core snapshot decodes"); - snap.documents.into_iter().map(|(k, _)| k).collect() + zerompk::from_msgpack(response.payload.as_bytes()).expect("core snapshot decodes") } } diff --git a/nodedb-test-support/src/cluster_harness/node/inspect/crdt.rs b/nodedb-test-support/src/cluster_harness/node/inspect/crdt.rs index 24df83b47..d55e8aba2 100644 --- a/nodedb-test-support/src/cluster_harness/node/inspect/crdt.rs +++ b/nodedb-test-support/src/cluster_harness/node/inspect/crdt.rs @@ -57,6 +57,7 @@ impl TestClusterNode { wal_lsn: None, resolved_now_ms: None, commit_hlc: None, + entry_version: None, admission: nodedb::bridge::envelope::Admission::Exempt( nodedb::bridge::envelope::ExemptReason::Read, ), @@ -131,6 +132,7 @@ impl TestClusterNode { wal_lsn: None, resolved_now_ms: None, commit_hlc: None, + entry_version: None, admission: nodedb::bridge::envelope::Admission::Admitted, }; diff --git a/nodedb-test-support/src/cluster_harness/node/inspect/snapshot.rs b/nodedb-test-support/src/cluster_harness/node/inspect/snapshot.rs index 1a25df725..17d79e170 100644 --- a/nodedb-test-support/src/cluster_harness/node/inspect/snapshot.rs +++ b/nodedb-test-support/src/cluster_harness/node/inspect/snapshot.rs @@ -80,6 +80,7 @@ impl TestClusterNode { wal_lsn: None, resolved_now_ms: None, commit_hlc: None, + entry_version: None, admission: nodedb::bridge::envelope::Admission::Exempt( nodedb::bridge::envelope::ExemptReason::Read, ), @@ -133,6 +134,7 @@ impl TestClusterNode { replace_mode: true, collections_to_clear: Vec::new(), group_vshards: Vec::new(), + version_floor: Vec::new(), }), deadline: std::time::Instant::now() + std::time::Duration::from_secs(5), priority: Priority::Normal, @@ -147,6 +149,7 @@ impl TestClusterNode { wal_lsn: None, resolved_now_ms: None, commit_hlc: None, + entry_version: None, admission: nodedb::bridge::envelope::Admission::Exempt( nodedb::bridge::envelope::ExemptReason::Read, ), diff --git a/nodedb-test-support/src/cluster_harness/node/inspect/timeseries.rs b/nodedb-test-support/src/cluster_harness/node/inspect/timeseries.rs index e16519549..0b922c5ca 100644 --- a/nodedb-test-support/src/cluster_harness/node/inspect/timeseries.rs +++ b/nodedb-test-support/src/cluster_harness/node/inspect/timeseries.rs @@ -68,6 +68,7 @@ impl TestClusterNode { wal_lsn: None, resolved_now_ms: None, commit_hlc: None, + entry_version: None, admission: nodedb::bridge::envelope::Admission::Exempt( nodedb::bridge::envelope::ExemptReason::Read, ), diff --git a/nodedb-test-support/src/cluster_harness/node/leadership.rs b/nodedb-test-support/src/cluster_harness/node/leadership.rs new file mode 100644 index 000000000..99b2a9c22 --- /dev/null +++ b/nodedb-test-support/src/cluster_harness/node/leadership.rs @@ -0,0 +1,39 @@ +// SPDX-License-Identifier: BUSL-1.1 + +//! Raft leadership reads and moves on one [`TestClusterNode`], through the +//! node's own Raft groups. + +use crate::cluster_harness::node::lifecycle::TestClusterNode; + +impl TestClusterNode { + /// Whether this node leads `group_id` in its own Raft state now. + pub fn leads_group(&self, group_id: u64) -> bool { + self.shared.multi_raft.get().is_some_and(|multi_raft| { + multi_raft + .lock() + .unwrap_or_else(|p| p.into_inner()) + .group_role_is_leader(group_id) + }) + } + + /// Ask this node, the leader of `group_id`, to hand the leadership to + /// node `target`. The move finishes once `target` wins the next term. + /// The error names the node, the group and the target. + pub fn transfer_group_leadership(&self, group_id: u64, target: u64) -> Result<(), String> { + let multi_raft = self + .shared + .multi_raft + .get() + .ok_or_else(|| format!("node {}: its Raft groups are not started", self.node_id))?; + multi_raft + .lock() + .unwrap_or_else(|p| p.into_inner()) + .transfer_leadership(group_id, target) + .map_err(|e| { + format!( + "node {}: transfer of group {group_id} to node {target}: {e}", + self.node_id + ) + }) + } +} diff --git a/nodedb-test-support/src/cluster_harness/node/lifecycle/spawn_full.rs b/nodedb-test-support/src/cluster_harness/node/lifecycle/spawn_full.rs index 566288684..004496689 100644 --- a/nodedb-test-support/src/cluster_harness/node/lifecycle/spawn_full.rs +++ b/nodedb-test-support/src/cluster_harness/node/lifecycle/spawn_full.rs @@ -245,6 +245,7 @@ impl TestClusterNode { let core_handle = crate::core_loop_runner::spawn_core_loop(crate::core_loop_runner::CoreLoopSpawn { idx, + node_id, num_cores, data_side, core_dir: data_dir_path.clone(), diff --git a/nodedb-test-support/src/cluster_harness/node/mod.rs b/nodedb-test-support/src/cluster_harness/node/mod.rs index f2bbfa68b..19ac65c78 100644 --- a/nodedb-test-support/src/cluster_harness/node/mod.rs +++ b/nodedb-test-support/src/cluster_harness/node/mod.rs @@ -5,6 +5,7 @@ pub mod graph; pub mod inspect; +pub mod leadership; pub mod lifecycle; pub mod native_client; pub mod raw_pgwire; diff --git a/nodedb-test-support/src/core_loop_runner.rs b/nodedb-test-support/src/core_loop_runner.rs index ef19a6922..1d20df3a5 100644 --- a/nodedb-test-support/src/core_loop_runner.rs +++ b/nodedb-test-support/src/core_loop_runner.rs @@ -63,6 +63,9 @@ pub struct WalReplay { pub struct CoreLoopSpawn { /// Core index within the data plane (0-based). pub idx: usize, + /// The node this core runs on, the scope of its fail points. An + /// in-process cluster runs several nodes, so a test arms one node's core. + pub node_id: u64, /// Total number of Data-Plane cores in this node. The committed-redo apply /// routes each record to `vshard_id % num_cores`, the same rule the /// dispatcher routes requests by. @@ -131,6 +134,7 @@ pub fn event_interest_for(shared: &SharedState) -> Arc { pub fn spawn_core_loop(spawn: CoreLoopSpawn) -> tokio::task::JoinHandle<()> { let CoreLoopSpawn { idx, + node_id, num_cores, data_side, core_dir, @@ -162,6 +166,7 @@ pub fn spawn_core_loop(spawn: CoreLoopSpawn) -> tokio::task::JoinHandle<()> { core_array_catalog, ) .expect("CoreLoop::open_with_array_catalog"); + core.set_fail_scope(node_id); core.set_event_producer(event_producer); core.set_event_interest(event_interest); core.set_num_cores(num_cores); diff --git a/nodedb-test-support/src/fail_point.rs b/nodedb-test-support/src/fail_point.rs new file mode 100644 index 000000000..9ae616d2c --- /dev/null +++ b/nodedb-test-support/src/fail_point.rs @@ -0,0 +1,9 @@ +// SPDX-License-Identifier: BUSL-1.1 + +//! The fail-point test API. +//! +//! An in-process cluster shares one fail-point registry across its nodes. +//! [`FailGuard::for_node`] arms a point on one node, and +//! [`FailGuard::install`] arms it on every node. + +pub use nodedb_types::fail_point::{FailAction, FailGuard, FailScope}; diff --git a/nodedb-test-support/src/lib.rs b/nodedb-test-support/src/lib.rs index f2c785607..4c2aef699 100644 --- a/nodedb-test-support/src/lib.rs +++ b/nodedb-test-support/src/lib.rs @@ -7,6 +7,8 @@ pub mod booted_state; pub mod catalog_fixtures; pub mod cluster_harness; pub mod core_loop_runner; +#[cfg(feature = "failpoints")] +pub mod fail_point; pub mod ilp_client; pub mod insert_returning_engines; pub mod jwks_fixture; diff --git a/nodedb-test-support/src/pgwire_harness/multicore.rs b/nodedb-test-support/src/pgwire_harness/multicore.rs index 24b04b9bb..85e1add27 100644 --- a/nodedb-test-support/src/pgwire_harness/multicore.rs +++ b/nodedb-test-support/src/pgwire_harness/multicore.rs @@ -68,6 +68,7 @@ impl TestServer { let core_handle = crate::core_loop_runner::spawn_core_loop(crate::core_loop_runner::CoreLoopSpawn { idx, + node_id: shared.node_id, num_cores, data_side, core_dir: dir.path().to_path_buf(), diff --git a/nodedb-test-support/src/pgwire_harness/restart.rs b/nodedb-test-support/src/pgwire_harness/restart.rs index 860b419a6..9d270e9fa 100644 --- a/nodedb-test-support/src/pgwire_harness/restart.rs +++ b/nodedb-test-support/src/pgwire_harness/restart.rs @@ -247,6 +247,7 @@ impl TestServer { let core_handle = crate::core_loop_runner::spawn_core_loop(crate::core_loop_runner::CoreLoopSpawn { idx, + node_id: shared.node_id, // Single-core harness (`Dispatcher::new(1, ..)`). num_cores: 1, data_side, diff --git a/nodedb-test-support/src/pgwire_harness/start.rs b/nodedb-test-support/src/pgwire_harness/start.rs index 86c27cdcd..c3ce2e977 100644 --- a/nodedb-test-support/src/pgwire_harness/start.rs +++ b/nodedb-test-support/src/pgwire_harness/start.rs @@ -268,6 +268,7 @@ impl TestServer { let core_handle = crate::core_loop_runner::spawn_core_loop(crate::core_loop_runner::CoreLoopSpawn { idx, + node_id: shared.node_id, // Single-core harness (`Dispatcher::new(1, ..)`). num_cores: 1, data_side, diff --git a/nodedb-test-support/src/tx_batch_helpers.rs b/nodedb-test-support/src/tx_batch_helpers.rs index ca8b2fa46..d99205fda 100644 --- a/nodedb-test-support/src/tx_batch_helpers.rs +++ b/nodedb-test-support/src/tx_batch_helpers.rs @@ -62,6 +62,7 @@ pub fn make_request(plan: PhysicalPlan) -> Request { wal_lsn: None, resolved_now_ms: None, commit_hlc: None, + entry_version: None, admission: nodedb::bridge::envelope::Admission::Admitted, } } diff --git a/nodedb-test-support/src/tx_commit.rs b/nodedb-test-support/src/tx_commit.rs index 62bc8fade..31cbc7987 100644 --- a/nodedb-test-support/src/tx_commit.rs +++ b/nodedb-test-support/src/tx_commit.rs @@ -84,6 +84,7 @@ pub fn commit_plans( collections: written_collections(&plans), sum_targets: redo_sum_targets(&plans), origin: nodedb_physical::physical_plan::RedoOrigin::Commit, + calvin: None, })); install.wal_lsn = Some(Lsn::new(lsn)); send_request(core, tx, rx, install) diff --git a/nodedb-types/src/calvin.rs b/nodedb-types/src/calvin.rs index 003662c56..e8d44d6c7 100644 --- a/nodedb-types/src/calvin.rs +++ b/nodedb-types/src/calvin.rs @@ -10,7 +10,9 @@ use serde::{Deserialize, Serialize}; -use crate::{KeyRepr, Lsn}; +use crate::{KeyRepr, WriteVersion}; + +pub use crate::calvin_passive::{PassiveKey, PassiveReadKeyId}; /// A newtype over `Vec` that guarantees sorted, deduplicated contents. /// @@ -121,6 +123,27 @@ pub enum EngineKeySet { collection: String, vshards: SortedVec, }, + /// A whole collection on each vShard in `vshards`, or on the + /// collection's own vShard when `vshards` is empty. + /// + /// In a write set the transaction locks the collection exclusively: a + /// truncate, bulk, predicate, or identity-less write. In a read set it + /// locks the collection shared: a predicate, scan, or index read. A read + /// that observed the collection on named vShards, such as a cross-shard + /// graph read, names them in `vshards`. + Collection { + collection: String, + vshards: SortedVec, + }, + /// UNIQUE values a write claims in one index of `collection`. + /// + /// Each value is the canonical index value the Data Plane judges. Two + /// transactions that claim one value lock one key, so they serialize. + Unique { + collection: String, + index: String, + values: SortedVec>, + }, } impl EngineKeySet { @@ -142,6 +165,15 @@ impl EngineKeySet { Self::Edge { edges, .. } => edges.len() * 8, // Array: one u32 vShard each. Self::Array { vshards, .. } => vshards.len() * 4, + // Collection: the name plus one u32 vShard each. + Self::Collection { + collection, + vshards, + } => collection.len() + vshards.len() * 4, + // Unique: the index name plus each value's bytes. + Self::Unique { index, values, .. } => { + index.len() + values.iter().map(|v| v.len()).sum::() + } } } @@ -152,7 +184,9 @@ impl EngineKeySet { | Self::Vector { collection, .. } | Self::Kv { collection, .. } | Self::Edge { collection, .. } - | Self::Array { collection, .. } => collection, + | Self::Array { collection, .. } + | Self::Collection { collection, .. } + | Self::Unique { collection, .. } => collection, } } @@ -164,17 +198,18 @@ impl EngineKeySet { Self::Kv { keys, .. } => keys.is_empty(), Self::Edge { edges, .. } => edges.is_empty(), Self::Array { vshards, .. } => vshards.is_empty(), + Self::Collection { .. } => false, + Self::Unique { values, .. } => values.is_empty(), } } } -/// A single key that a passive participant must read and broadcast. +/// A key set a passive participant reads and broadcasts. /// -/// Wraps an [`EngineKeySet`]; per the dependent-read protocol each -/// `PassiveReadKey` contains a single-element (or small) key set. The -/// sequencer does not enforce single-element sets; the scheduler enforces the -/// total byte budget via `DependentReadSpec::total_bytes()` (which lives in -/// `nodedb-cluster`). +/// Wraps an [`EngineKeySet`]. A passive vShard reads document rows by +/// surrogate and key-value rows by key. It refuses every other key set. The +/// sequencer bounds the bytes a transaction reads passively through +/// `DependentReadSpec::total_bytes()` in `nodedb-cluster`. #[derive( Debug, Clone, @@ -265,13 +300,13 @@ pub enum ReadKeyIdent { }, } -/// One LSN-versioned, predicate-aware read observed by a transaction, carried -/// on the replicated Calvin `TxClass` so participants can validate it at the +/// One versioned, predicate-aware read observed by a transaction, carried on +/// the replicated Calvin `TxClass` so participants can validate it at the /// commit serialization point. /// -/// `read_lsn` is the responding shard's write-LSN watermark at read time. The -/// enclosing `TxClass` scopes the tenant; per-database scoping is carried by -/// the transaction as a whole. +/// `read_version` is a data-group log position, so any replica of the read's +/// vShard validates it. The enclosing `TxClass` scopes the tenant; +/// per-database scoping is carried by the transaction as a whole. #[derive( Debug, Clone, @@ -289,27 +324,23 @@ pub struct VersionedReadEntry { pub collection: String, /// Point-key or collection-scoped-predicate identity of the observation. pub key: ReadKeyIdent, - /// The responding shard's write-LSN watermark at read time. - pub read_lsn: Lsn, + /// The version of what the read observed on its home vShard. + pub read_version: WriteVersion, /// The vShard whose write versions validate this read. `None` homes the /// read to its collection's vShard. A graph read names the key vShard it /// read edges on, because edges live on their endpoints' vShards. A homed /// read with an empty `collection` observed every collection there, so it - /// validates against the shard's core watermark. + /// validates against the vShard's latest version. pub home_vshard: Option, - /// The node that served the read. `read_lsn` is a position in that node's - /// WAL, so a participant on any other node treats the read as changed. - /// `0` when no one node is known to have served it. - pub served_by: u64, } -/// The LSN-versioned read-set of a Calvin transaction. +/// The versioned read-set of a Calvin transaction. /// /// Empty for pure-write transactions and for autocommit statements (which /// accumulate no session read-set). Populated at commit time from the neutral /// session read-set, and validated per-participant against the local /// write-version index at Calvin stage time (the commit vote on -/// `read_set_valid`). +/// a stage response). #[derive( Debug, Clone, diff --git a/nodedb-types/src/calvin_passive.rs b/nodedb-types/src/calvin_passive.rs new file mode 100644 index 000000000..a777120ca --- /dev/null +++ b/nodedb-types/src/calvin_passive.rs @@ -0,0 +1,110 @@ +// SPDX-License-Identifier: Apache-2.0 + +//! Identity of one row a passive Calvin participant reads. +//! +//! A dependent-read transaction names the rows its passive vShards read. Each +//! passive vShard reads them at its lock grant and broadcasts one value per +//! row through the data-group log of every active vShard. The active vShard +//! matches each value to its row by [`PassiveReadKeyId`]. +//! +//! The identity is exact: a key-value row keeps its key bytes, so two keys +//! never share an identity. + +use serde::{Deserialize, Serialize}; + +use crate::QualifiedCollection; + +/// The row within a collection a passive read names. +#[derive( + Debug, + Clone, + PartialEq, + Eq, + PartialOrd, + Ord, + Hash, + Serialize, + Deserialize, + zerompk::ToMessagePack, + zerompk::FromMessagePack, +)] +pub enum PassiveKey { + /// A document row, by its surrogate. + Surrogate { surrogate: u32 }, + /// A key-value row, by its key bytes. + Kv { key: Vec }, +} + +/// Identity of one row a passive participant reads. +/// +/// The map key of every passive read value. `Ord` is derived, collection +/// first, so every replica iterates a map of them in one order. +#[derive( + Debug, + Clone, + PartialEq, + Eq, + PartialOrd, + Ord, + Hash, + Serialize, + Deserialize, + zerompk::ToMessagePack, + zerompk::FromMessagePack, +)] +pub struct PassiveReadKeyId { + /// The database-qualified collection of the row. + pub collection: QualifiedCollection, + /// The row within the collection. + pub key: PassiveKey, +} + +impl PassiveReadKeyId { + /// The identity of document row `surrogate` of `collection`. + pub fn surrogate(collection: QualifiedCollection, surrogate: u32) -> Self { + Self { + collection, + key: PassiveKey::Surrogate { surrogate }, + } + } + + /// The identity of key-value row `key` of `collection`. + pub fn kv(collection: QualifiedCollection, key: Vec) -> Self { + Self { + collection, + key: PassiveKey::Kv { key }, + } + } + + /// Bytes this identity adds to a replicated entry, estimated. + pub fn serialized_size_hint(&self) -> usize { + let key = match &self.key { + PassiveKey::Surrogate { .. } => 4, + PassiveKey::Kv { key } => key.len(), + }; + self.collection.as_str().len() + key + } +} + +#[cfg(test)] +mod tests { + use super::*; + + /// Two key-value rows keep two identities, whatever their keys hash to. + #[test] + fn kv_rows_keep_distinct_identities() { + let collection = QualifiedCollection::from_stored("items".to_owned()); + let a = PassiveReadKeyId::kv(collection.clone(), b"alice:sword".to_vec()); + let b = PassiveReadKeyId::kv(collection, b"alice:shield".to_vec()); + assert_ne!(a, b); + } + + /// An identity survives a msgpack round trip. + #[test] + fn identity_round_trips_through_msgpack() { + let id = PassiveReadKeyId::surrogate(QualifiedCollection::from_stored("o".to_owned()), 9); + let bytes = zerompk::to_msgpack_vec(&id).expect("encode"); + let back: PassiveReadKeyId = zerompk::from_msgpack(&bytes).expect("decode"); + assert_eq!(back, id); + } +} diff --git a/nodedb-types/src/config/tuning/calvin.rs b/nodedb-types/src/config/tuning/calvin.rs new file mode 100644 index 000000000..03c5e1faf --- /dev/null +++ b/nodedb-types/src/config/tuning/calvin.rs @@ -0,0 +1,179 @@ +// SPDX-License-Identifier: Apache-2.0 + +//! Calvin scheduler tuning — channel sizes, deadlines, and the in-flight +//! bounds of each per-vShard scheduler. +//! +//! The epoch duration belongs to the sequencer, so it is not set here. The +//! scheduler reads it from the sequencer config it runs beside. + +use serde::{Deserialize, Serialize}; + +fn default_channel_capacity() -> usize { + 512 +} + +fn default_txn_deadline_multiplier() -> u32 { + 3 +} + +fn default_dependent_read_passive_timeout_ms() -> u64 { + // A passive read crosses two data groups: its read result is proposed to + // each active vShard's group, often led by another node. The timeout only + // bounds how long a lost result holds the active vShard's locks, so it + // leaves room for a leader change on the way. 250 sequencer epochs. + 5_000 +} + +fn default_verdict_stall_warn_ms() -> u64 { + // 250 sequencer epochs of 20 ms. + 5_000 +} + +fn default_max_inflight_backlog() -> usize { + // The per-core request queue capacity of the bridge dispatcher. Every + // dispatch of one vShard goes to one Data Plane core. + 1_024 +} + +fn default_catch_up_window() -> u64 { + // One channel of missed entries per drain. + 512 +} + +fn default_max_redo_entry_bytes() -> usize { + // Well below the 64 MiB RPC frame limit, so one entry and the frame + // around it always fit. + 8 * 1024 * 1024 +} + +fn default_max_open_redo_bytes() -> u64 { + // Sixty-four streams of the default entry size. + 512 * 1024 * 1024 +} + +fn default_restage_attempts() -> u32 { + 5 +} + +fn default_restage_backoff_ms() -> u64 { + // Five sequencer epochs of 20 ms. Five doubling waits span about 3 s. + 100 +} + +/// Tuning knobs for the Calvin scheduler that runs per hosted vShard. +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] +pub struct CalvinTuning { + /// Capacity of each scheduler's bounded input, completion, and verdict + /// channels. + #[serde(default = "default_channel_capacity")] + pub channel_capacity: usize, + + /// Transaction deadline, in sequencer epochs. + #[serde(default = "default_txn_deadline_multiplier")] + pub txn_deadline_multiplier: u32, + + /// Wait in milliseconds of an active vShard's dependent-read barrier for + /// the read results of its passive vShards. Past it, the vShard's + /// data-group leader proposes the barrier's timeout entry, and every + /// replica aborts a barrier that entry finds incomplete. + #[serde(default = "default_dependent_read_passive_timeout_ms")] + pub dependent_read_passive_timeout_ms: u64, + + /// Interval in milliseconds between stall warnings for a transaction + /// parked on its global verdict. The transaction keeps waiting and never + /// aborts. The idle sweep runs at a quarter of this interval. + #[serde(default = "default_verdict_stall_warn_ms")] + pub verdict_stall_warn_ms: u64, + + /// In-flight backlog at which a scheduler stops taking new sequenced + /// input. The backlog counts pending, blocked, and dependent-barrier + /// transactions. It also bounds the dependent-read barrier entries a + /// vShard holds in memory for transactions its scheduler has not taken + /// yet. Entries past it wait in the system catalog alone. + #[serde(default = "default_max_inflight_backlog")] + pub max_inflight_backlog: usize, + + /// Most sequencer log entries one catch-up drain reads and replays. + #[serde(default = "default_catch_up_window")] + pub catch_up_window: u64, + + /// Largest encoded redo entry a proposer puts in a data-group log. A + /// committed redo past it travels as chunk entries and a final entry. + #[serde(default = "default_max_redo_entry_bytes")] + pub max_redo_entry_bytes: usize, + + /// Most bytes of chunked redo streams a node holds open. A group leader + /// refuses a new stream that would pass it on the leader's node. A node + /// holds at most this much per data group it hosts. + #[serde(default = "default_max_open_redo_bytes")] + pub max_open_redo_bytes: u64, + + /// Most times a leader stages a committed transaction again after its + /// own stage failed. An earlier leader's COMMIT vote decided the + /// transaction, so it cannot drop. The scheduler halts once the stages + /// run out. + #[serde(default = "default_restage_attempts")] + pub restage_attempts: u32, + + /// Wait in milliseconds before the first restage of a committed + /// transaction. Each later restage waits twice as long as the one + /// before, at most 60 seconds. + #[serde(default = "default_restage_backoff_ms")] + pub restage_backoff_ms: u64, +} + +impl Default for CalvinTuning { + fn default() -> Self { + Self { + channel_capacity: default_channel_capacity(), + txn_deadline_multiplier: default_txn_deadline_multiplier(), + dependent_read_passive_timeout_ms: default_dependent_read_passive_timeout_ms(), + verdict_stall_warn_ms: default_verdict_stall_warn_ms(), + max_inflight_backlog: default_max_inflight_backlog(), + catch_up_window: default_catch_up_window(), + max_redo_entry_bytes: default_max_redo_entry_bytes(), + max_open_redo_bytes: default_max_open_redo_bytes(), + restage_attempts: default_restage_attempts(), + restage_backoff_ms: default_restage_backoff_ms(), + } + } +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn defaults() { + let t = CalvinTuning::default(); + assert_eq!(t.channel_capacity, 512); + assert_eq!(t.txn_deadline_multiplier, 3); + assert_eq!(t.dependent_read_passive_timeout_ms, 5_000); + assert_eq!(t.verdict_stall_warn_ms, 5_000); + assert_eq!(t.max_inflight_backlog, 1_024); + assert_eq!(t.catch_up_window, 512); + assert_eq!(t.max_redo_entry_bytes, 8 * 1024 * 1024); + assert_eq!(t.max_open_redo_bytes, 512 * 1024 * 1024); + assert_eq!(t.restage_attempts, 5); + assert_eq!(t.restage_backoff_ms, 100); + } + + #[test] + fn partial_override() { + let toml_str = r#" +max_inflight_backlog = 256 +verdict_stall_warn_ms = 10000 +"#; + let t: CalvinTuning = toml::from_str(toml_str).expect("deserialize"); + assert_eq!(t.max_inflight_backlog, 256); + assert_eq!(t.verdict_stall_warn_ms, 10_000); + assert_eq!(t.channel_capacity, 512); + assert_eq!(t.catch_up_window, 512); + } + + #[test] + fn empty_table_keeps_the_defaults() { + let t: CalvinTuning = toml::from_str("").expect("deserialize"); + assert_eq!(t, CalvinTuning::default()); + } +} diff --git a/nodedb-types/src/config/tuning/config.rs b/nodedb-types/src/config/tuning/config.rs index aac570e7f..b7b690db9 100644 --- a/nodedb-types/src/config/tuning/config.rs +++ b/nodedb-types/src/config/tuning/config.rs @@ -5,6 +5,7 @@ use serde::{Deserialize, Serialize}; use super::bitemporal::BitemporalTuning; +use super::calvin::CalvinTuning; use super::data_plane::{DataPlaneTuning, QueryTuning}; use super::engines::{GraphTuning, KvTuning, SparseTuning, TimeseriesToning, VectorTuning}; use super::maintenance::MaintenanceTuning; @@ -51,6 +52,8 @@ pub struct TuningConfig { pub bitemporal: BitemporalTuning, #[serde(default)] pub maintenance: MaintenanceTuning, + #[serde(default)] + pub calvin: CalvinTuning, } impl TuningConfig { @@ -112,6 +115,18 @@ mod tests { assert_eq!(parsed.memory.doc_cache_entries, 4096); assert_eq!(parsed.shutdown.deadline_ms, 900); assert_eq!(parsed.maintenance.auto_analyze_min_mutations, 1000); + assert_eq!(parsed.calvin, CalvinTuning::default()); + } + + #[test] + fn calvin_section_overrides_the_scheduler_backlog() { + let toml_str = r#" +[calvin] +max_inflight_backlog = 256 +"#; + let cfg: TuningConfig = toml::from_str(toml_str).expect("deserialize"); + assert_eq!(cfg.calvin.max_inflight_backlog, 256); + assert_eq!(cfg.calvin.channel_capacity, 512); } #[test] diff --git a/nodedb-types/src/config/tuning/mod.rs b/nodedb-types/src/config/tuning/mod.rs index d521e851f..445ce2e86 100644 --- a/nodedb-types/src/config/tuning/mod.rs +++ b/nodedb-types/src/config/tuning/mod.rs @@ -1,6 +1,7 @@ // SPDX-License-Identifier: Apache-2.0 mod bitemporal; +mod calvin; mod config; mod data_plane; mod engines; @@ -11,6 +12,7 @@ mod scheduler; mod shutdown; pub use bitemporal::BitemporalTuning; +pub use calvin::CalvinTuning; pub use config::TuningConfig; pub use data_plane::{DataPlaneTuning, QueryTuning}; pub use engines::{ diff --git a/nodedb-types/src/fail_point.rs b/nodedb-types/src/fail_point.rs deleted file mode 100644 index 88948da0e..000000000 --- a/nodedb-types/src/fail_point.rs +++ /dev/null @@ -1,347 +0,0 @@ -// SPDX-License-Identifier: Apache-2.0 - -//! Minimal feature-gated fail-point framework for crash-injection tests. -//! -//! When the `failpoints` Cargo feature is OFF (the default and what release -//! builds compile with), `fail_point!(name)` expands to nothing — zero -//! runtime cost. -//! -//! When the feature is ON, each invocation looks up `name` in a process-wide -//! registry. If a [`FailAction`] has been installed for that name, the action -//! fires: -//! - `Panic`: panic immediately with a message naming the fail point. -//! - `Sleep(d)`: sleep for `d`, useful for race-condition tests. -//! - `Fail(detail)`: make the injected call site return an error. Only -//! [`fail_point_err!`] can honour this — the call site supplies the -//! mapping into its own error type, so no crate has to know about -//! anyone else's. -//! - `WaitForFile(path)`: park the call site until `path` exists, so a test -//! releases it at the moment it chooses. Only an async call site can -//! honour this: it awaits the file with the crate's own timer and parks -//! only its own task. A synchronous call site refuses it. -//! -//! The framework is deliberately tiny — no fail-rs dep, no parsing of env -//! vars, no list of probabilities. Tests install actions explicitly via -//! `set` / `clear` and must clean up in their own teardown (or use -//! `FailGuard` for RAII cleanup). -//! -//! Feature gating is per-crate: the macros expand under the *calling* crate's -//! `failpoints` feature, so every crate that injects a fail point declares its -//! own `failpoints = ["nodedb-types/failpoints"]`. - -#[cfg(feature = "failpoints")] -mod imp { - use std::collections::HashMap; - use std::sync::{LazyLock, Mutex}; - use std::time::Duration; - - /// Action a fail point performs when triggered. - #[derive(Debug, Clone)] - pub enum FailAction { - /// Panic with a message that names the fail-point. - Panic, - /// Kill the whole process immediately via `abort()`. - /// - /// `Panic` only unwinds the current task, which a supervising runtime - /// absorbs — no use for simulating a crash in a spawned server. This - /// is the action a process-kill harness arms. - Abort, - /// Sleep for the given duration; execution then continues normally. - Sleep(Duration), - /// Return an error from the injected call site, carrying this detail. - /// Ignored by bare `fail_point!` — use `fail_point_err!`. - Fail(String), - /// Park the call site until this file exists. Only an async call site - /// that looks the action up with [`lookup`] can honour it. - WaitForFile(std::path::PathBuf), - } - - /// Environment variable read once, the first time any fail point is - /// evaluated. Lets a test arm injections in a *spawned* process — the - /// in-process `set` API cannot reach a server the test only supervises. - /// - /// Format: comma-separated `name=action`, where action is `panic`, - /// `abort`, `sleep()`, `fail()`, or `wait_file()`. - /// For example: - /// `NODEDB_FAILPOINTS='checkpoint::after_marker_before_truncate=panic'` - pub const FAILPOINTS_ENV: &str = "NODEDB_FAILPOINTS"; - - static REGISTRY: LazyLock>> = - LazyLock::new(|| Mutex::new(parse_env(std::env::var(FAILPOINTS_ENV).ok().as_deref()))); - - pub(super) fn parse_env(spec: Option<&str>) -> HashMap { - let mut actions = HashMap::new(); - let Some(spec) = spec else { - return actions; - }; - for entry in spec.split(',').map(str::trim).filter(|e| !e.is_empty()) { - let Some((name, action)) = entry.split_once('=') else { - panic!("{FAILPOINTS_ENV} entry {entry:?} is not `name=action`"); - }; - let action = match action.trim() { - "panic" => FailAction::Panic, - "abort" => FailAction::Abort, - rest if rest.starts_with("sleep(") && rest.ends_with(')') => { - let millis = - rest["sleep(".len()..rest.len() - 1] - .parse() - .unwrap_or_else(|_| { - panic!("{FAILPOINTS_ENV} entry {entry:?} has a non-numeric sleep") - }); - FailAction::Sleep(Duration::from_millis(millis)) - } - rest if rest.starts_with("fail(") && rest.ends_with(')') => { - FailAction::Fail(rest["fail(".len()..rest.len() - 1].to_string()) - } - rest if rest.starts_with("wait_file(") && rest.ends_with(')') => { - FailAction::WaitForFile(std::path::PathBuf::from( - &rest["wait_file(".len()..rest.len() - 1], - )) - } - other => panic!("{FAILPOINTS_ENV} entry {entry:?} has unknown action {other:?}"), - }; - actions.insert(name.trim().to_string(), action); - } - actions - } - - /// Install an action for a named fail point. Subsequent - /// `fail_point!(name)` invocations will fire the action. - pub fn set(name: &str, action: FailAction) { - REGISTRY - .lock() - .unwrap_or_else(|p| p.into_inner()) - .insert(name.to_string(), action); - } - - /// Remove any installed action for the named fail point. - pub fn clear(name: &str) { - REGISTRY - .lock() - .unwrap_or_else(|p| p.into_inner()) - .remove(name); - } - - /// Look up the installed action for a fail point (None if not set). - pub fn lookup(name: &str) -> Option { - REGISTRY - .lock() - .unwrap_or_else(|p| p.into_inner()) - .get(name) - .cloned() - } - - /// Evaluate the installed action for a fail point. Used by the - /// `fail_point!` macro; not intended to be called directly. - pub fn eval(name: &str) { - if let Some(action) = lookup(name) { - match action { - FailAction::Panic => panic!("fail_point fired: {name}"), - FailAction::Abort => abort(name), - FailAction::Sleep(d) => std::thread::sleep(d), - // A caller that cannot return an error must not silently - // swallow the injection: an armed failpoint that does nothing - // makes a test pass for the wrong reason. - FailAction::Fail(detail) => { - panic!( - "fail_point {name} installed Fail({detail}) but the call site cannot return an error — use fail_point_err!" - ) - } - // Blocking the thread would stall every task that shares it. - FailAction::WaitForFile(path) => panic!( - "fail_point {name} installed WaitForFile({}) but the call site is \ - synchronous — only an async call site can park", - path.display() - ), - } - } - } - - /// Kill the process at a fail point. Flushes the reason first — an - /// unexplained SIGABRT in a test log is indistinguishable from a real bug. - fn abort(name: &str) -> ! { - eprintln!("fail_point aborting process: {name}"); - std::process::abort() - } - - /// Evaluate a fail point that can return an error, yielding the detail - /// string when the installed action is [`FailAction::Fail`]. Used by the - /// `fail_point_err!` macro; not intended to be called directly. - pub fn eval_fail(name: &str) -> Option { - match lookup(name) { - Some(FailAction::Fail(detail)) => Some(detail), - Some(FailAction::Panic) => panic!("fail_point fired: {name}"), - Some(FailAction::Abort) => abort(name), - Some(FailAction::Sleep(d)) => { - std::thread::sleep(d); - None - } - Some(FailAction::WaitForFile(path)) => panic!( - "fail_point {name} installed WaitForFile({}) but the call site is synchronous \ - — only an async call site can park", - path.display() - ), - None => None, - } - } - - /// RAII guard that clears a fail point on drop. Use to keep tests - /// from leaking installed actions across cases. - pub struct FailGuard { - name: String, - } - - impl FailGuard { - pub fn install(name: &str, action: FailAction) -> Self { - set(name, action); - Self { - name: name.to_string(), - } - } - - /// Arm a fail point to return an error carrying `detail`. - pub fn fail(name: &str, detail: &str) -> Self { - Self::install(name, FailAction::Fail(detail.to_string())) - } - } - - impl Drop for FailGuard { - fn drop(&mut self) { - clear(&self.name); - } - } -} - -#[cfg(feature = "failpoints")] -pub use imp::{FAILPOINTS_ENV, FailAction, FailGuard, clear, eval, eval_fail, lookup, set}; - -/// Inject a fail point. Expands to nothing without the `failpoints` feature. -/// -/// Usage in production code: -/// `nodedb_types::fail_point!("calvin_static::during_overlay_stage");` -/// -/// Tests opt in by enabling the feature and installing actions: -/// `fail_point::set("calvin_static::during_overlay_stage", -/// fail_point::FailAction::Panic);` -#[macro_export] -macro_rules! fail_point { - ($name:expr) => { - #[cfg(feature = "failpoints")] - $crate::fail_point::eval($name); - }; -} - -/// Inject a fail point that can abort the enclosing function with an error. -/// -/// `$map` receives the detail string installed with [`FailAction::Fail`] and -/// returns the error value to propagate, so each crate injects its own error -/// type without the framework knowing about it: -/// -/// ```ignore -/// fail_point_err!("wal::flush_out_of_space", |_| WalError::OutOfSpace { -/// context: "WAL segment append (failpoint)", -/// }); -/// ``` -/// -/// Expands to nothing without the `failpoints` feature. -#[macro_export] -macro_rules! fail_point_err { - ($name:expr, $map:expr) => { - #[cfg(feature = "failpoints")] - if let Some(detail) = $crate::fail_point::eval_fail($name) { - return Err(($map)(detail)); - } - }; -} - -#[cfg(all(test, feature = "failpoints"))] -mod tests { - use super::*; - - #[test] - fn unset_fail_point_is_noop() { - eval("nodedb::test::unset"); - assert_eq!(eval_fail("nodedb::test::unset"), None); - } - - #[test] - #[should_panic(expected = "fail_point fired: nodedb::test::panic_target")] - fn set_panic_fires() { - let _g = FailGuard::install("nodedb::test::panic_target", FailAction::Panic); - eval("nodedb::test::panic_target"); - } - - #[test] - fn fail_action_yields_its_detail() { - let _g = FailGuard::fail("nodedb::test::fail_target", "disk full"); - assert_eq!( - eval_fail("nodedb::test::fail_target"), - Some("disk full".to_string()) - ); - } - - #[test] - #[should_panic(expected = "cannot return an error")] - fn fail_action_at_an_infallible_call_site_is_loud() { - let _g = FailGuard::fail("nodedb::test::fail_at_infallible", "nope"); - eval("nodedb::test::fail_at_infallible"); - } - - #[test] - fn env_spec_parses_every_action() { - let actions = super::imp::parse_env(Some( - "a::panic=panic, b::sleep=sleep(25), c::fail=fail(disk full)", - )); - assert!(matches!(actions.get("a::panic"), Some(FailAction::Panic))); - assert!(matches!( - actions.get("b::sleep"), - Some(FailAction::Sleep(d)) if *d == std::time::Duration::from_millis(25) - )); - assert!(matches!( - actions.get("c::fail"), - Some(FailAction::Fail(detail)) if detail == "disk full" - )); - } - - #[test] - fn env_spec_parses_a_file_gate() { - let actions = super::imp::parse_env(Some("d::gate=wait_file(/tmp/release-d)")); - assert!(matches!( - actions.get("d::gate"), - Some(FailAction::WaitForFile(path)) if path == std::path::Path::new("/tmp/release-d") - )); - } - - #[test] - #[should_panic(expected = "only an async call site can park")] - fn a_file_gate_at_a_synchronous_call_site_is_loud() { - let _g = FailGuard::install( - "nodedb::test::gate_at_sync", - FailAction::WaitForFile(std::path::PathBuf::from("/nonexistent")), - ); - eval("nodedb::test::gate_at_sync"); - } - - #[test] - fn empty_env_spec_arms_nothing() { - assert!(super::imp::parse_env(None).is_empty()); - assert!(super::imp::parse_env(Some("")).is_empty()); - } - - #[test] - #[should_panic(expected = "unknown action")] - fn malformed_env_spec_is_loud() { - // A typo that silently armed nothing would make a crash test pass - // without ever injecting the crash. - super::imp::parse_env(Some("a::b=explode")); - } - - #[test] - fn fail_guard_clears_on_drop() { - { - let _g = FailGuard::install("nodedb::test::guard_clear", FailAction::Panic); - assert!(lookup("nodedb::test::guard_clear").is_some()); - } - assert!(lookup("nodedb::test::guard_clear").is_none()); - } -} diff --git a/nodedb-types/src/fail_point/action.rs b/nodedb-types/src/fail_point/action.rs new file mode 100644 index 000000000..08c9d9069 --- /dev/null +++ b/nodedb-types/src/fail_point/action.rs @@ -0,0 +1,26 @@ +// SPDX-License-Identifier: Apache-2.0 + +//! What an armed fail point does when it fires. + +use std::time::Duration; + +/// Action a fail point performs when triggered. +#[derive(Debug, Clone)] +pub enum FailAction { + /// Panic with a message that names the fail-point. + Panic, + /// Kill the whole process immediately via `abort()`. + /// + /// `Panic` only unwinds the current task, which a supervising runtime + /// absorbs — no use for simulating a crash in a spawned server. This + /// is the action a process-kill harness arms. + Abort, + /// Sleep for the given duration; execution then continues normally. + Sleep(Duration), + /// Return an error from the injected call site, carrying this detail. + /// Ignored by bare `fail_point!` — use `fail_point_err!`. + Fail(String), + /// Park the call site until this file exists. Only an async call site + /// that looks the action up with [`super::lookup`] can honour it. + WaitForFile(std::path::PathBuf), +} diff --git a/nodedb-types/src/fail_point/env.rs b/nodedb-types/src/fail_point/env.rs new file mode 100644 index 000000000..67de29130 --- /dev/null +++ b/nodedb-types/src/fail_point/env.rs @@ -0,0 +1,181 @@ +// SPDX-License-Identifier: Apache-2.0 + +//! The environment spec that arms fail points in a spawned process. + +use std::time::Duration; + +use super::action::FailAction; +use super::registry::ArmedActions; +use super::scope::FailScope; + +/// Environment variable read once, the first time any fail point is +/// evaluated. Lets a test arm injections in a *spawned* process — the +/// in-process `set` API cannot reach a server the test only supervises. +/// +/// Format: comma-separated `name=action` or `name@node=action`. The +/// first arms every node, the second only node `N`. The action is `panic`, +/// `abort`, `sleep()`, `fail()`, or `wait_file()`. +/// For example: +/// `NODEDB_FAILPOINTS='checkpoint::after_marker_before_truncate=panic'` +pub const FAILPOINTS_ENV: &str = "NODEDB_FAILPOINTS"; + +/// Parse a [`FAILPOINTS_ENV`] spec. Panics on a malformed entry: a typo that +/// armed nothing would let a crash test pass without injecting the crash. +pub(crate) fn parse_env(spec: Option<&str>) -> ArmedActions { + let mut actions = ArmedActions::new(); + let Some(spec) = spec else { + return actions; + }; + for entry in spec.split(',').map(str::trim).filter(|e| !e.is_empty()) { + let Some((target, action)) = entry.split_once('=') else { + panic!( + "{FAILPOINTS_ENV} entry {entry:?} is not `name=action` or `name@node=action`" + ); + }; + let (name, scope) = parse_target(entry, target.trim()); + let action = parse_action(entry, action.trim()); + actions + .entry(name.to_string()) + .or_default() + .insert(scope, action); + } + actions +} + +/// Split `name@node` into the name and `Node(N)`. A target with no `@` +/// is scoped to every node. +fn parse_target<'a>(entry: &str, target: &'a str) -> (&'a str, FailScope) { + let (name, scope) = match target.rsplit_once('@') { + None => (target, FailScope::Any), + Some((name, scope)) => { + let node = scope + .trim() + .strip_prefix("node") + .and_then(|id| id.parse().ok()) + .unwrap_or_else(|| { + panic!("{FAILPOINTS_ENV} entry {entry:?} has scope {scope:?}, not `node`") + }); + (name.trim(), FailScope::Node(node)) + } + }; + if name.is_empty() { + panic!("{FAILPOINTS_ENV} entry {entry:?} names no fail point"); + } + (name, scope) +} + +fn parse_action(entry: &str, action: &str) -> FailAction { + match action { + "panic" => FailAction::Panic, + "abort" => FailAction::Abort, + rest if rest.starts_with("sleep(") && rest.ends_with(')') => { + let millis = rest["sleep(".len()..rest.len() - 1] + .parse() + .unwrap_or_else(|_| { + panic!("{FAILPOINTS_ENV} entry {entry:?} has a non-numeric sleep") + }); + FailAction::Sleep(Duration::from_millis(millis)) + } + rest if rest.starts_with("fail(") && rest.ends_with(')') => { + FailAction::Fail(rest["fail(".len()..rest.len() - 1].to_string()) + } + rest if rest.starts_with("wait_file(") && rest.ends_with(')') => FailAction::WaitForFile( + std::path::PathBuf::from(&rest["wait_file(".len()..rest.len() - 1]), + ), + other => panic!("{FAILPOINTS_ENV} entry {entry:?} has unknown action {other:?}"), + } +} + +#[cfg(test)] +mod tests { + use super::*; + + fn armed<'a>( + actions: &'a ArmedActions, + scope: FailScope, + name: &str, + ) -> Option<&'a FailAction> { + actions.get(name).and_then(|scopes| scopes.get(&scope)) + } + + #[test] + fn env_spec_parses_every_action() { + let actions = parse_env(Some( + "a::panic=panic, b::sleep=sleep(25), c::fail=fail(disk full)", + )); + assert!(matches!( + armed(&actions, FailScope::Any, "a::panic"), + Some(FailAction::Panic) + )); + assert!(matches!( + armed(&actions, FailScope::Any, "b::sleep"), + Some(FailAction::Sleep(d)) if *d == Duration::from_millis(25) + )); + assert!(matches!( + armed(&actions, FailScope::Any, "c::fail"), + Some(FailAction::Fail(detail)) if detail == "disk full" + )); + } + + #[test] + fn env_spec_parses_a_file_gate() { + let actions = parse_env(Some("d::gate=wait_file(/tmp/release-d)")); + assert!(matches!( + armed(&actions, FailScope::Any, "d::gate"), + Some(FailAction::WaitForFile(path)) if path == std::path::Path::new("/tmp/release-d") + )); + } + + #[test] + fn env_spec_scopes_an_entry_to_one_node() { + let actions = parse_env(Some("e::hold@node3=abort, e::hold=panic")); + assert!(matches!( + armed(&actions, FailScope::Node(3), "e::hold"), + Some(FailAction::Abort) + )); + assert!(matches!( + armed(&actions, FailScope::Any, "e::hold"), + Some(FailAction::Panic) + )); + assert!(armed(&actions, FailScope::Node(2), "e::hold").is_none()); + } + + #[test] + fn a_node_scope_leaves_the_action_path_alone() { + let actions = parse_env(Some("f::gate@node1=wait_file(/tmp/a@b=c)")); + assert!(matches!( + armed(&actions, FailScope::Node(1), "f::gate"), + Some(FailAction::WaitForFile(path)) if path == std::path::Path::new("/tmp/a@b=c") + )); + } + + #[test] + fn empty_env_spec_arms_nothing() { + assert!(parse_env(None).is_empty()); + assert!(parse_env(Some("")).is_empty()); + } + + #[test] + #[should_panic(expected = "unknown action")] + fn malformed_env_spec_is_loud() { + parse_env(Some("a::b=explode")); + } + + #[test] + #[should_panic(expected = "not `node`")] + fn a_scope_that_is_not_a_node_is_loud() { + parse_env(Some("a::b@core1=panic")); + } + + #[test] + #[should_panic(expected = "not `node`")] + fn a_node_scope_without_an_id_is_loud() { + parse_env(Some("a::b@node=panic")); + } + + #[test] + #[should_panic(expected = "names no fail point")] + fn a_scope_with_no_name_is_loud() { + parse_env(Some("@node1=panic")); + } +} diff --git a/nodedb-types/src/fail_point/guard.rs b/nodedb-types/src/fail_point/guard.rs new file mode 100644 index 000000000..56daf936f --- /dev/null +++ b/nodedb-types/src/fail_point/guard.rs @@ -0,0 +1,45 @@ +// SPDX-License-Identifier: Apache-2.0 + +//! RAII arming of a fail point for one test. + +use super::action::FailAction; +use super::registry::{clear, set}; +use super::scope::FailScope; + +/// Arms a fail point and disarms it on drop. Use to keep tests from +/// leaking armed actions across cases. +pub struct FailGuard { + scope: FailScope, + name: String, +} + +impl FailGuard { + /// Arm `name` on every node. + pub fn install(name: &str, action: FailAction) -> Self { + Self::arm(FailScope::Any, name, action) + } + + /// Arm `name` on node `node_id` only. Every other node runs past it. + pub fn for_node(node_id: u64, name: &str, action: FailAction) -> Self { + Self::arm(FailScope::Node(node_id), name, action) + } + + /// Arm `name` on every node to return an error carrying `detail`. + pub fn fail(name: &str, detail: &str) -> Self { + Self::install(name, FailAction::Fail(detail.to_string())) + } + + fn arm(scope: FailScope, name: &str, action: FailAction) -> Self { + set(scope, name, action); + Self { + scope, + name: name.to_string(), + } + } +} + +impl Drop for FailGuard { + fn drop(&mut self) { + clear(self.scope, &self.name); + } +} diff --git a/nodedb-types/src/fail_point/macros.rs b/nodedb-types/src/fail_point/macros.rs new file mode 100644 index 000000000..cd3be8550 --- /dev/null +++ b/nodedb-types/src/fail_point/macros.rs @@ -0,0 +1,69 @@ +// SPDX-License-Identifier: Apache-2.0 + +//! The `fail_point!` and `fail_point_err!` injection macros. +//! +//! Each takes the scope the point is evaluated in as its first argument. A +//! call site with a node identity passes `FailScope::Node(id)`. The form +//! without a scope evaluates in `FailScope::Any`, for code with no node +//! identity. Without the calling crate's `failpoints` feature, a scoped form +//! only type-checks its scope and evaluates no name. + +/// Inject a fail point. Does nothing at runtime without the `failpoints` +/// feature. +/// +/// Usage in production code: +/// `nodedb_types::fail_point!(scope, "calvin_static::during_overlay_stage");` +/// `nodedb_types::fail_point!("wal::roll_before_seal");` (no node identity) +/// +/// Tests opt in by enabling the feature and arming actions: +/// `FailGuard::for_node(node_id, "calvin_static::during_overlay_stage", +/// FailAction::Panic);` +#[macro_export] +macro_rules! fail_point { + ($name:expr) => { + #[cfg(feature = "failpoints")] + $crate::fail_point::eval($crate::fail_point::FailScope::Any, $name); + }; + ($scope:expr, $name:expr) => { + #[cfg(feature = "failpoints")] + $crate::fail_point::eval($scope, $name); + #[cfg(not(feature = "failpoints"))] + let _: $crate::fail_point::FailScope = $scope; + }; +} + +/// Inject a fail point that can abort the enclosing function with an error. +/// +/// `$map` receives the detail string armed with `FailAction::Fail` and +/// returns the error value to propagate, so each crate injects its own error +/// type without the framework knowing about it: +/// +/// ```ignore +/// fail_point_err!("wal::flush_out_of_space", |_| WalError::OutOfSpace { +/// context: "WAL segment append (failpoint)", +/// }); +/// fail_point_err!(FailScope::Node(node_id), "cut_floor::before_persist", |detail| { +/// Error::Internal { detail } +/// }); +/// ``` +/// +/// Does nothing at runtime without the `failpoints` feature. +#[macro_export] +macro_rules! fail_point_err { + ($name:expr, $map:expr) => { + #[cfg(feature = "failpoints")] + if let Some(detail) = + $crate::fail_point::eval_fail($crate::fail_point::FailScope::Any, $name) + { + return Err(($map)(detail)); + } + }; + ($scope:expr, $name:expr, $map:expr) => { + #[cfg(feature = "failpoints")] + if let Some(detail) = $crate::fail_point::eval_fail($scope, $name) { + return Err(($map)(detail)); + } + #[cfg(not(feature = "failpoints"))] + let _: $crate::fail_point::FailScope = $scope; + }; +} diff --git a/nodedb-types/src/fail_point/mod.rs b/nodedb-types/src/fail_point/mod.rs new file mode 100644 index 000000000..84db4f922 --- /dev/null +++ b/nodedb-types/src/fail_point/mod.rs @@ -0,0 +1,44 @@ +// SPDX-License-Identifier: Apache-2.0 + +//! Minimal feature-gated fail-point framework for crash-injection tests. +//! +//! When the `failpoints` Cargo feature is OFF (the default and what release +//! builds compile with), `fail_point!` expands to nothing at runtime. +//! +//! When the feature is ON, each invocation looks up its scope and name in a +//! process-wide registry. A point evaluated with scope `Node(n)` fires the +//! action armed for `(Node(n), name)`, else the one armed for `(Any, name)`. +//! A point evaluated with scope `Any` fires only the `(Any, name)` action. +//! The actions are listed on [`FailAction`]. +//! +//! An in-process cluster test runs several nodes in one process, so they +//! share the registry. The scope arms one node's point and leaves the +//! others running. +//! +//! Tests arm actions with `set` / `clear` or the RAII [`FailGuard`]. A +//! spawned server reads its actions once from [`FAILPOINTS_ENV`]. +//! +//! Feature gating is per-crate: the macros expand under the *calling* crate's +//! `failpoints` feature, so every crate that injects a fail point declares its +//! own `failpoints = ["nodedb-types/failpoints"]`. + +#[cfg(feature = "failpoints")] +pub mod action; +#[cfg(feature = "failpoints")] +pub mod env; +#[cfg(feature = "failpoints")] +pub mod guard; +pub mod macros; +#[cfg(feature = "failpoints")] +pub mod registry; +pub mod scope; + +#[cfg(feature = "failpoints")] +pub use action::FailAction; +#[cfg(feature = "failpoints")] +pub use env::FAILPOINTS_ENV; +#[cfg(feature = "failpoints")] +pub use guard::FailGuard; +#[cfg(feature = "failpoints")] +pub use registry::{clear, eval, eval_fail, lookup, set}; +pub use scope::FailScope; diff --git a/nodedb-types/src/fail_point/registry.rs b/nodedb-types/src/fail_point/registry.rs new file mode 100644 index 000000000..37e38c0c4 --- /dev/null +++ b/nodedb-types/src/fail_point/registry.rs @@ -0,0 +1,224 @@ +// SPDX-License-Identifier: Apache-2.0 + +//! The process-wide registry of armed fail points, and their evaluation. + +use std::collections::HashMap; +use std::sync::{LazyLock, Mutex, MutexGuard}; + +use super::action::FailAction; +use super::env::{FAILPOINTS_ENV, parse_env}; +use super::scope::FailScope; + +/// Armed actions by fail point name, then by scope. +pub(crate) type ArmedActions = HashMap>; + +static REGISTRY: LazyLock> = + LazyLock::new(|| Mutex::new(parse_env(std::env::var(FAILPOINTS_ENV).ok().as_deref()))); + +fn registry() -> MutexGuard<'static, ArmedActions> { + REGISTRY.lock().unwrap_or_else(|p| p.into_inner()) +} + +/// Arm `name` for `scope`. A later point evaluated in a matching scope fires +/// `action`. +pub fn set(scope: FailScope, name: &str, action: FailAction) { + registry() + .entry(name.to_string()) + .or_default() + .insert(scope, action); +} + +/// Disarm `name` for `scope`. Actions armed for other scopes stay. +pub fn clear(scope: FailScope, name: &str) { + let mut armed = registry(); + if let Some(scopes) = armed.get_mut(name) { + scopes.remove(&scope); + if scopes.is_empty() { + armed.remove(name); + } + } +} + +/// The action a point `name` evaluated in `scope` fires, if any. +/// +/// `Node(n)` takes the action armed for `Node(n)`, else the one armed for +/// `Any`. `Any` takes only the `Any` action. +pub fn lookup(scope: FailScope, name: &str) -> Option { + let armed = registry(); + let scopes = armed.get(name)?; + scopes + .get(&scope) + .or_else(|| scopes.get(&FailScope::Any)) + .cloned() +} + +/// Evaluate point `name` in `scope`. Used by the `fail_point!` macro; not +/// intended to be called directly. +pub fn eval(scope: FailScope, name: &str) { + if let Some(action) = lookup(scope, name) { + match action { + FailAction::Panic => panic!("fail_point fired: {name} ({scope})"), + FailAction::Abort => abort(scope, name), + FailAction::Sleep(d) => std::thread::sleep(d), + // A caller that cannot return an error must not silently + // swallow the injection: an armed failpoint that does nothing + // makes a test pass for the wrong reason. + FailAction::Fail(detail) => panic!( + "fail_point {name} ({scope}) installed Fail({detail}) but the call site cannot \ + return an error — use fail_point_err!" + ), + // Blocking the thread would stall every task that shares it. + FailAction::WaitForFile(path) => panic!( + "fail_point {name} ({scope}) installed WaitForFile({}) but the call site is \ + synchronous — only an async call site can park", + path.display() + ), + } + } +} + +/// Evaluate point `name` in `scope` at a call site that can return an error. +/// Yields the detail when the action is [`FailAction::Fail`]. Used by the +/// `fail_point_err!` macro; not intended to be called directly. +pub fn eval_fail(scope: FailScope, name: &str) -> Option { + match lookup(scope, name) { + Some(FailAction::Fail(detail)) => Some(detail), + Some(FailAction::Panic) => panic!("fail_point fired: {name} ({scope})"), + Some(FailAction::Abort) => abort(scope, name), + Some(FailAction::Sleep(d)) => { + std::thread::sleep(d); + None + } + Some(FailAction::WaitForFile(path)) => panic!( + "fail_point {name} ({scope}) installed WaitForFile({}) but the call site is \ + synchronous — only an async call site can park", + path.display() + ), + None => None, + } +} + +/// Kill the process at a fail point. Flushes the reason first — an +/// unexplained SIGABRT in a test log is indistinguishable from a real bug. +fn abort(scope: FailScope, name: &str) -> ! { + eprintln!("fail_point aborting process: {name} ({scope})"); + std::process::abort() +} + +#[cfg(test)] +mod tests { + use super::super::guard::FailGuard; + use super::*; + + #[test] + fn unset_fail_point_is_noop() { + eval(FailScope::Any, "nodedb::test::unset"); + assert_eq!(eval_fail(FailScope::Node(1), "nodedb::test::unset"), None); + } + + #[test] + #[should_panic(expected = "fail_point fired: nodedb::test::panic_target")] + fn set_panic_fires() { + let _g = FailGuard::install("nodedb::test::panic_target", FailAction::Panic); + eval(FailScope::Any, "nodedb::test::panic_target"); + } + + #[test] + fn fail_action_yields_its_detail() { + let _g = FailGuard::fail("nodedb::test::fail_target", "disk full"); + assert_eq!( + eval_fail(FailScope::Any, "nodedb::test::fail_target"), + Some("disk full".to_string()) + ); + } + + #[test] + #[should_panic(expected = "cannot return an error")] + fn fail_action_at_an_infallible_call_site_is_loud() { + let _g = FailGuard::fail("nodedb::test::fail_at_infallible", "nope"); + eval(FailScope::Any, "nodedb::test::fail_at_infallible"); + } + + #[test] + #[should_panic(expected = "only an async call site can park")] + fn a_file_gate_at_a_synchronous_call_site_is_loud() { + let _g = FailGuard::install( + "nodedb::test::gate_at_sync", + FailAction::WaitForFile(std::path::PathBuf::from("/nonexistent")), + ); + eval(FailScope::Any, "nodedb::test::gate_at_sync"); + } + + #[test] + fn an_any_action_fires_on_every_node() { + let _g = FailGuard::fail("nodedb::test::any_scope", "every node"); + for scope in [FailScope::Any, FailScope::Node(1), FailScope::Node(2)] { + assert_eq!( + eval_fail(scope, "nodedb::test::any_scope"), + Some("every node".to_string()), + "{scope}" + ); + } + } + + #[test] + fn a_node_action_fires_only_on_its_node() { + let _g = FailGuard::for_node( + 2, + "nodedb::test::node_scope", + FailAction::Fail("node two".to_string()), + ); + assert_eq!( + eval_fail(FailScope::Node(2), "nodedb::test::node_scope"), + Some("node two".to_string()) + ); + assert_eq!( + eval_fail(FailScope::Node(1), "nodedb::test::node_scope"), + None + ); + assert_eq!(eval_fail(FailScope::Any, "nodedb::test::node_scope"), None); + } + + #[test] + fn a_node_action_wins_over_an_any_action() { + let _any = FailGuard::fail("nodedb::test::both_scopes", "any"); + let _node = FailGuard::for_node( + 3, + "nodedb::test::both_scopes", + FailAction::Fail("node three".to_string()), + ); + assert_eq!( + eval_fail(FailScope::Node(3), "nodedb::test::both_scopes"), + Some("node three".to_string()) + ); + assert_eq!( + eval_fail(FailScope::Node(4), "nodedb::test::both_scopes"), + Some("any".to_string()) + ); + } + + #[test] + fn clearing_one_scope_keeps_the_other() { + let _any = FailGuard::fail("nodedb::test::clear_one", "any"); + { + let _node = FailGuard::for_node( + 5, + "nodedb::test::clear_one", + FailAction::Fail("node five".to_string()), + ); + } + assert_eq!( + eval_fail(FailScope::Node(5), "nodedb::test::clear_one"), + Some("any".to_string()) + ); + } + + #[test] + fn fail_guard_clears_on_drop() { + { + let _g = FailGuard::install("nodedb::test::guard_clear", FailAction::Panic); + assert!(lookup(FailScope::Any, "nodedb::test::guard_clear").is_some()); + } + assert!(lookup(FailScope::Any, "nodedb::test::guard_clear").is_none()); + } +} diff --git a/nodedb-types/src/fail_point/scope.rs b/nodedb-types/src/fail_point/scope.rs new file mode 100644 index 000000000..78c8e5aed --- /dev/null +++ b/nodedb-types/src/fail_point/scope.rs @@ -0,0 +1,27 @@ +// SPDX-License-Identifier: Apache-2.0 + +//! The node scope of a fail point. + +use std::fmt; + +/// The nodes an armed fail point applies to, and the node a point is +/// evaluated on. +/// +/// Compiled without the `failpoints` feature too, so a call site can name +/// its scope in every build. +#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)] +pub enum FailScope { + /// Every node. Code with no node identity evaluates its points here. + Any, + /// The node with this id. + Node(u64), +} + +impl fmt::Display for FailScope { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + match self { + Self::Any => f.write_str("any node"), + Self::Node(id) => write!(f, "node {id}"), + } + } +} diff --git a/nodedb-types/src/lib.rs b/nodedb-types/src/lib.rs index a1a207475..13b965607 100644 --- a/nodedb-types/src/lib.rs +++ b/nodedb-types/src/lib.rs @@ -19,6 +19,7 @@ pub mod ascii; pub mod backup_envelope; pub mod bbox; pub mod calvin; +pub mod calvin_passive; pub mod clone; pub mod collection; pub mod collection_config; @@ -73,6 +74,7 @@ pub mod vector_index_stats; pub mod vector_model; pub mod volatility; pub mod wire_version; +pub mod write_version; pub use approx::{CountMinSketch, HyperLogLog, SpaceSaving, TDigest}; pub use array_cell::ArrayCell; @@ -150,3 +152,4 @@ pub use vector_index_stats::{ }; pub use vector_model::{VectorModelEntry, VectorModelMetadata}; pub use volatility::Volatility; +pub use write_version::{ShardVersion, WriteVersion}; diff --git a/nodedb-types/src/write_version.rs b/nodedb-types/src/write_version.rs new file mode 100644 index 000000000..0dc0d05d4 --- /dev/null +++ b/nodedb-types/src/write_version.rs @@ -0,0 +1,164 @@ +// SPDX-License-Identifier: Apache-2.0 + +//! The version a committed write gives the rows it writes. +//! +//! A write that applies a data-group Raft entry takes the entry's log +//! position. Every replica applies the entry at the same position, so a +//! version read on one replica compares with the versions every other +//! replica holds. + +use serde::{Deserialize, Serialize}; + +/// The position of the write that produced a row version, within one vShard. +/// +/// A write that applies a data-group entry records `epoch`, the epoch at +/// which its vShard moved to the entry's group, and `index`, the entry's log +/// index. `local` is `0`. +/// +/// A write that applies no entry records its WAL LSN in `local`, on top of +/// the latest version its vShard holds on the core. Such a write exists on +/// one node only: a single-node write, or a local-only write on a cluster +/// node. It sorts after every version its vShard already holds and before +/// the next entry the vShard applies. +/// +/// Versions compare only within one vShard. The writes of one vShard are +/// totally ordered by `(epoch, index, local)`. +#[derive( + Debug, + Clone, + Copy, + Default, + PartialEq, + Eq, + PartialOrd, + Ord, + Hash, + Serialize, + Deserialize, + rkyv::Archive, + rkyv::Serialize, + rkyv::Deserialize, + zerompk::ToMessagePack, + zerompk::FromMessagePack, +)] +pub struct WriteVersion { + /// The epoch at which the vShard moved to the group that applied the + /// write. `0` while the vShard stays in its initial group. + pub epoch: u64, + /// The log index of the data-group entry that applied the write. + pub index: u64, + /// The WAL LSN of a write that applies no entry, `0` otherwise. + pub local: u64, +} + +impl WriteVersion { + /// The version below every write. + pub const ZERO: Self = Self { + epoch: 0, + index: 0, + local: 0, + }; + + /// The version of a write that applies the entry at `index`, in `epoch`. + pub const fn logged(epoch: u64, index: u64) -> Self { + Self { + epoch, + index, + local: 0, + } + } + + /// The version of a write that applies no entry, at WAL LSN `lsn`, on a + /// vShard whose latest version is `latest`. + pub const fn local_after(latest: Self, lsn: u64) -> Self { + Self { + epoch: latest.epoch, + index: latest.index, + local: lsn, + } + } + + /// How far `self` sits above `older` in one epoch: the log entries + /// between them, or the WAL LSNs between two writes of one entry + /// position. `None` across epochs, or when `older` is not below `self`. + pub fn distance_above(self, older: Self) -> Option { + if self.epoch != older.epoch || self <= older { + return None; + } + if self.index > older.index { + return Some(self.index - older.index); + } + Some(self.local.saturating_sub(older.local)) + } +} + +/// One vShard's write version. +#[derive( + Debug, + Clone, + Copy, + PartialEq, + Eq, + PartialOrd, + Ord, + Hash, + Serialize, + Deserialize, + rkyv::Archive, + rkyv::Serialize, + rkyv::Deserialize, + zerompk::ToMessagePack, + zerompk::FromMessagePack, +)] +pub struct ShardVersion { + /// The vShard whose history positions `version`. + pub vshard: u32, + pub version: WriteVersion, +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn a_later_entry_sorts_after_a_local_write_of_the_earlier_one() { + let entry = WriteVersion::logged(0, 10); + let local = WriteVersion::local_after(entry, 900); + let next = WriteVersion::logged(0, 11); + assert!(entry < local); + assert!(local < next); + } + + #[test] + fn a_later_epoch_sorts_after_every_index_of_an_earlier_one() { + assert!(WriteVersion::logged(0, 1_000) < WriteVersion::logged(5, 1)); + } + + #[test] + fn distance_counts_entries_then_wal_lsns_within_one_epoch() { + let base = WriteVersion::logged(2, 10); + assert_eq!(WriteVersion::logged(2, 15).distance_above(base), Some(5)); + let local = WriteVersion::local_after(base, 40); + assert_eq!( + WriteVersion::local_after(base, 90).distance_above(local), + Some(50) + ); + assert_eq!(WriteVersion::logged(3, 1).distance_above(base), None); + assert_eq!(base.distance_above(WriteVersion::logged(2, 12)), None); + } + + #[test] + fn versions_round_trip_through_msgpack() { + let version = ShardVersion { + vshard: 7, + version: WriteVersion { + epoch: 3, + index: 41, + local: 9, + }, + }; + let bytes = zerompk::to_msgpack_vec(&version).expect("encode"); + let decoded: ShardVersion = zerompk::from_msgpack(&bytes).expect("decode"); + assert_eq!(decoded, version); + } +} diff --git a/nodedb-wal/src/lib.rs b/nodedb-wal/src/lib.rs index 8fb9fc1d9..40c321eba 100644 --- a/nodedb-wal/src/lib.rs +++ b/nodedb-wal/src/lib.rs @@ -87,8 +87,8 @@ pub use preamble::{ pub use reader::{StopReason, WalReader}; #[cfg(not(target_arch = "wasm32"))] pub use record::{ - CalvinAppliedPayload, FtsDeletePayload, FtsIndexPayload, RecordStamp, RecordTarget, RecordType, - SpatialDeletePayload, SpatialPutPayload, WalRecord, WalRecordArgs, WriteAbortedPayload, + FtsDeletePayload, FtsIndexPayload, RecordStamp, RecordTarget, RecordType, SpatialDeletePayload, + SpatialPutPayload, WalRecord, WalRecordArgs, WriteAbortedPayload, }; pub use record::{NO_EVENT_SOURCE, RecordHeader}; #[cfg(not(target_arch = "wasm32"))] diff --git a/nodedb-wal/src/record/calvin.rs b/nodedb-wal/src/record/calvin.rs deleted file mode 100644 index 542126688..000000000 --- a/nodedb-wal/src/record/calvin.rs +++ /dev/null @@ -1,107 +0,0 @@ -// SPDX-License-Identifier: Apache-2.0 - -//! Calvin scheduler WAL payload types. -//! -//! Uses fixed little-endian encoding (no msgpack framing) for zero-allocation -//! replay, consistent with other WAL payload types in this crate. - -use crate::error::{Result, WalError}; - -/// Fixed size of a `CalvinApplied` payload on disk. -/// -/// Layout: `epoch (u64 LE) | position (u32 LE) | vshard_id (u32 LE)` = 16 bytes. -pub const CALVIN_APPLIED_PAYLOAD_SIZE: usize = 16; - -/// Payload for [`super::types::RecordType::CalvinApplied`]. -/// -/// Written by the Calvin executor after a `MetaOp::CalvinExecute` batch -/// commits successfully. Encodes `{ epoch, position, vshard_id }` so the -/// scheduler's restart path can scan the WAL and find `last_applied_epoch` -/// for a given vshard without reading the full Raft log. -/// -/// Fixed little-endian encoding — no msgpack framing. -#[derive(Debug, Clone, Copy, PartialEq, Eq)] -pub struct CalvinAppliedPayload { - /// Sequencer epoch of the applied transaction. - pub epoch: u64, - /// Zero-based position within the epoch batch. - pub position: u32, - /// The vshard that applied this transaction. - pub vshard_id: u32, -} - -impl CalvinAppliedPayload { - /// Construct a new payload. - pub fn new(epoch: u64, position: u32, vshard_id: u32) -> Self { - Self { - epoch, - position, - vshard_id, - } - } - - /// Serialize to fixed-size bytes for WAL append. - /// - /// Layout: `epoch (8 LE) | position (4 LE) | vshard_id (4 LE)`. - pub fn to_bytes(&self) -> [u8; CALVIN_APPLIED_PAYLOAD_SIZE] { - let mut buf = [0u8; CALVIN_APPLIED_PAYLOAD_SIZE]; - buf[..8].copy_from_slice(&self.epoch.to_le_bytes()); - buf[8..12].copy_from_slice(&self.position.to_le_bytes()); - buf[12..16].copy_from_slice(&self.vshard_id.to_le_bytes()); - buf - } - - /// Deserialize from bytes read from the WAL. - pub fn from_bytes(bytes: &[u8]) -> Result { - if bytes.len() < CALVIN_APPLIED_PAYLOAD_SIZE { - return Err(WalError::InvalidPayload { - detail: format!( - "CalvinApplied payload must be {CALVIN_APPLIED_PAYLOAD_SIZE} bytes, got {}", - bytes.len() - ), - }); - } - let epoch = u64::from_le_bytes(bytes[..8].try_into().expect( - "invariant: bounds-checked above (bytes.len() >= CALVIN_APPLIED_PAYLOAD_SIZE >= 8)", - )); - let position = u32::from_le_bytes(bytes[8..12].try_into().expect( - "invariant: bounds-checked above (bytes.len() >= CALVIN_APPLIED_PAYLOAD_SIZE >= 12)", - )); - let vshard_id = u32::from_le_bytes(bytes[12..16].try_into().expect( - "invariant: bounds-checked above (bytes.len() >= CALVIN_APPLIED_PAYLOAD_SIZE >= 16)", - )); - Ok(Self { - epoch, - position, - vshard_id, - }) - } -} - -#[cfg(test)] -mod tests { - use super::*; - - #[test] - fn roundtrip() { - let p = CalvinAppliedPayload::new(42, 7, 3); - let bytes = p.to_bytes(); - let decoded = CalvinAppliedPayload::from_bytes(&bytes).unwrap(); - assert_eq!(decoded.epoch, 42); - assert_eq!(decoded.position, 7); - assert_eq!(decoded.vshard_id, 3); - } - - #[test] - fn too_short_payload_returns_error() { - let result = CalvinAppliedPayload::from_bytes(&[0u8; 5]); - assert!(result.is_err()); - } - - #[test] - fn payload_size_is_correct() { - let p = CalvinAppliedPayload::new(u64::MAX, u32::MAX, u32::MAX); - let bytes = p.to_bytes(); - assert_eq!(bytes.len(), CALVIN_APPLIED_PAYLOAD_SIZE); - } -} diff --git a/nodedb-wal/src/record/mod.rs b/nodedb-wal/src/record/mod.rs index 280e177ea..ce0bebbfc 100644 --- a/nodedb-wal/src/record/mod.rs +++ b/nodedb-wal/src/record/mod.rs @@ -5,8 +5,6 @@ pub mod aborted; #[cfg(not(target_arch = "wasm32"))] pub mod anchor; #[cfg(not(target_arch = "wasm32"))] -pub mod calvin; -#[cfg(not(target_arch = "wasm32"))] pub mod fts_spatial; pub mod header; #[cfg(not(target_arch = "wasm32"))] @@ -27,8 +25,6 @@ pub use aborted::{WRITE_ABORTED_PAYLOAD_SIZE, WriteAbortedPayload}; #[cfg(not(target_arch = "wasm32"))] pub use anchor::{TIME_ANCHOR_PAYLOAD_SIZE, TimeAnchorPayload}; #[cfg(not(target_arch = "wasm32"))] -pub use calvin::CalvinAppliedPayload; -#[cfg(not(target_arch = "wasm32"))] pub use fts_spatial::{FtsDeletePayload, FtsIndexPayload, SpatialDeletePayload, SpatialPutPayload}; pub use header::{ ENCRYPTED_FLAG, HEADER_SIZE, MAX_WAL_PAYLOAD_SIZE, NO_EVENT_SOURCE, RecordHeader, diff --git a/nodedb-wal/src/record/types.rs b/nodedb-wal/src/record/types.rs index 19a16c25f..48334772f 100644 --- a/nodedb-wal/src/record/types.rs +++ b/nodedb-wal/src/record/types.rs @@ -195,8 +195,9 @@ pub enum RecordType { /// uses, so replay reconstitutes a `WalRecord` per sub-op and feeds it to /// that engine's existing replay path — no tag loss, no re-encoding. /// - /// May also carry a Calvin stamp so a cross-shard transaction's durable - /// record doubles as its sequencer applied-marker. + /// May also carry a Calvin stamp. A stamped record marks its Calvin slice + /// applied on its vShard. + /// /// Payload: zerompk-encoded `RedoRecord` (see the `wal::redo` module). /// /// Required: skipping this record on replay would drop a committed @@ -258,21 +259,6 @@ pub enum RecordType { /// versions resurrected and diverge from the leader's state. TemporalPurge = 103 | 0x8000, - /// Calvin scheduler: marks a sequenced transaction as applied on this - /// vshard. - /// - /// Written by the Calvin executor after a `MetaOp::CalvinExecute` batch - /// commits successfully. The scheduler's rebuild path scans the WAL for - /// these records to determine `last_applied_epoch` on restart. - /// - /// Payload: zerompk-encoded `CalvinAppliedPayload { epoch: u64, - /// position: u32, vshard_id: u32 }`. - /// - /// Required: a replay that skipped this record would leave the scheduler - /// believing the transaction was not applied and re-dispatch it after - /// restart, causing double-application. - CalvinApplied = 110 | 0x8000, - /// Sync idempotency watermark — advances the durable per-stream /// high-watermark for a given producer so the receiver can safely /// deduplicate re-delivered ingest frames on reconnect. @@ -423,6 +409,16 @@ pub enum RecordType { /// /// Required: skipping it leaves every truncated edge live after replay. GraphEdgeCut = 68 | 0x8000, + + /// One piece of a chunk of a chunked transaction redo stream, or the + /// close of such a stream. Payload: a `RedoChunkRecord`. A chunk past the + /// WAL record limit spans several pieces, and only its last piece carries + /// the chunk entry's apply key. Boot rebuilds every open stream from these + /// records. Never replayed into any engine. + /// + /// Required: skipping it loses the bytes an open stream holds, and the + /// stream's final entry then refuses on this replica alone. + RedoChunk = 69 | 0x8000, } impl RecordType { @@ -468,7 +464,6 @@ impl RecordType { x if x == 101 | 0x8000 => Some(Self::CollectionTombstoned), 102 => Some(Self::TimeAnchor), x if x == 103 | 0x8000 => Some(Self::TemporalPurge), - x if x == 110 | 0x8000 => Some(Self::CalvinApplied), x if x == 53 | 0x8000 => Some(Self::SyncSeqAdvance), x if x == 54 | 0x8000 => Some(Self::FtsIndex), x if x == 55 | 0x8000 => Some(Self::FtsDelete), @@ -484,6 +479,7 @@ impl RecordType { x if x == 66 | 0x8000 => Some(Self::SnapshotInstalled), x if x == 67 | 0x8000 => Some(Self::WriteGroup), x if x == 68 | 0x8000 => Some(Self::GraphEdgeCut), + x if x == 69 | 0x8000 => Some(Self::RedoChunk), _ => None, } } @@ -521,6 +517,7 @@ mod tests { // Without the flag an older reader skips the abort marker and replays // the refused write it names — the exact bug it exists to prevent. assert!(RecordType::is_required(RecordType::WriteAborted as u32)); + assert!(RecordType::is_required(RecordType::RedoChunk as u32)); } #[test] @@ -560,7 +557,6 @@ mod tests { RecordType::CollectionTombstoned, RecordType::TimeAnchor, RecordType::TemporalPurge, - RecordType::CalvinApplied, RecordType::SyncSeqAdvance, RecordType::FtsIndex, RecordType::FtsDelete, @@ -576,6 +572,7 @@ mod tests { RecordType::SnapshotInstalled, RecordType::WriteGroup, RecordType::GraphEdgeCut, + RecordType::RedoChunk, ] { assert_eq!(RecordType::from_raw(ty as u32), Some(ty)); } diff --git a/nodedb-wal/src/uring_writer.rs b/nodedb-wal/src/uring_writer.rs index 6affceb8a..a3ce1827a 100644 --- a/nodedb-wal/src/uring_writer.rs +++ b/nodedb-wal/src/uring_writer.rs @@ -320,7 +320,10 @@ impl UringWriter { fn submit_and_wait_fsync(&mut self) -> Result<()> { // Crash injection: the kernel reports a writeback error at the fsync. #[cfg(feature = "failpoints")] - if let Some(detail) = nodedb_types::fail_point::eval_fail("wal::fsync_failure") { + if let Some(detail) = nodedb_types::fail_point::eval_fail( + nodedb_types::fail_point::FailScope::Any, + "wal::fsync_failure", + ) { return Err(self.durability.poison(format!( "io_uring fsync failed: failpoint wal::fsync_failure: {detail}" ))); diff --git a/nodedb-wal/src/writer/durability.rs b/nodedb-wal/src/writer/durability.rs index 72ac9f8a3..2a74541f2 100644 --- a/nodedb-wal/src/writer/durability.rs +++ b/nodedb-wal/src/writer/durability.rs @@ -98,7 +98,10 @@ pub(crate) fn fsync_and_track(file: &File, state: &mut DurabilityState) -> Resul // Crash injection: the kernel reports a writeback error at the fsync. // Everything already handed to the page cache is unrecoverable from here. #[cfg(feature = "failpoints")] - if let Some(detail) = nodedb_types::fail_point::eval_fail("wal::fsync_failure") { + if let Some(detail) = nodedb_types::fail_point::eval_fail( + nodedb_types::fail_point::FailScope::Any, + "wal::fsync_failure", + ) { return Err(state.poison(format!( "fsync failed: failpoint wal::fsync_failure: {detail}" ))); diff --git a/nodedb/Cargo.toml b/nodedb/Cargo.toml index c9cf82312..a45320c78 100644 --- a/nodedb/Cargo.toml +++ b/nodedb/Cargo.toml @@ -188,4 +188,4 @@ io-uring = { workspace = true } default = [] # Enables crash-injection points used by recovery tests. Off by default; tests # that exercise the framework opt in via `--features failpoints`. -failpoints = ["nodedb-types/failpoints", "nodedb-wal/failpoints"] +failpoints = ["nodedb-types/failpoints", "nodedb-wal/failpoints", "nodedb-cluster/failpoints"] diff --git a/nodedb/src/bootstrap/calvin_catch_up.rs b/nodedb/src/bootstrap/calvin_catch_up.rs new file mode 100644 index 000000000..26b853a13 --- /dev/null +++ b/nodedb/src/bootstrap/calvin_catch_up.rs @@ -0,0 +1,181 @@ +// SPDX-License-Identifier: BUSL-1.1 + +//! Calvin scheduler catch-up gate. +//! +//! Each Calvin scheduler starts with a rebuild target: the highest epoch its +//! recovery found applied for the vShard. Until its fully-applied watermark +//! reaches that target, the vShard re-applies epochs from the sequencer log. +//! A gateway opened meanwhile serves a vShard whose Calvin state is behind +//! what this node acknowledged before the restart. +//! +//! This gate holds startup until every scheduler running on this node reports +//! caught up. A scheduler with nothing to rebuild reports at once, and a node +//! that runs no scheduler passes at once. A vShard whose scheduler halted can +//! never catch up, so its halt fails the wait at once. The wait shares the +//! data-group recovery bound and fails closed when the bound expires. + +use std::sync::Arc; +use std::time::{Duration, Instant}; + +use tracing::{debug, info}; + +use crate::bootstrap::data_group_recovery::DATA_GROUP_RECOVERY_TIMEOUT; +use crate::control::cluster::CalvinApplyHalt; +use crate::control::state::SharedState; + +/// How often the caught-up registry is re-read while waiting. Catch-up drains +/// run on a scheduler tick of about a second, so a coarse poll costs nothing. +const POLL_INTERVAL: Duration = Duration::from_millis(50); + +/// Most lagging vShards one error message names. The count names the rest. +const MAX_NAMED_VSHARDS: usize = 16; + +/// Why the scheduler catch-up wait cannot pass yet. +#[derive(Debug, Clone, PartialEq, Eq)] +enum CatchUpWait { + /// These vShards' schedulers are behind their rebuild target. + Lagging(Vec), + /// A lagging vShard's scheduler halted, so it will never catch up. + Halted(CalvinApplyHalt), +} + +/// The wait state for `lagging` vShards, given the node's first apply halt. +fn catch_up_wait(lagging: Vec, halt: Option<&CalvinApplyHalt>) -> Option { + if lagging.is_empty() { + return None; + } + if let Some(halt) = halt + && lagging.binary_search(&halt.vshard_id).is_ok() + { + return Some(CatchUpWait::Halted(halt.clone())); + } + Some(CatchUpWait::Lagging(lagging)) +} + +/// Name at most [`MAX_NAMED_VSHARDS`] of `lagging`, then count the rest. +fn describe_lagging(lagging: &[u32]) -> String { + let named = lagging + .iter() + .take(MAX_NAMED_VSHARDS) + .map(u32::to_string) + .collect::>() + .join(", "); + match lagging.len().checked_sub(MAX_NAMED_VSHARDS) { + Some(rest) if rest > 0 => format!("vShards {named} and {rest} more"), + _ => format!("vShards {named}"), + } +} + +/// Hold startup until every Calvin scheduler on this node reaches its rebuild +/// target. +pub async fn await_calvin_catch_up(shared: &Arc) -> anyhow::Result<()> { + let registry = &shared.calvin.caught_up; + let deadline = Instant::now() + DATA_GROUP_RECOVERY_TIMEOUT; + + loop { + let halt = shared.sequencer_halt.apply_halt().report(); + match catch_up_wait(registry.lagging(), halt) { + None => { + info!("every calvin scheduler reached its rebuild target"); + return Ok(()); + } + Some(CatchUpWait::Halted(halt)) => { + return Err(anyhow::anyhow!( + "calvin scheduler for vShard {} halted at epoch {} position {} \ + ({}, step {}): {}; it cannot reach its rebuild target", + halt.vshard_id, + halt.epoch, + halt.position, + halt.reason, + halt.step, + halt.error + )); + } + Some(CatchUpWait::Lagging(lagging)) => { + if Instant::now() >= deadline { + return Err(anyhow::anyhow!( + "calvin scheduler catch-up timeout after \ + {DATA_GROUP_RECOVERY_TIMEOUT:?}: {} behind their rebuild target", + describe_lagging(&lagging) + )); + } + debug!( + lagging = lagging.len(), + "waiting for calvin schedulers to reach their rebuild target" + ); + } + } + tokio::time::sleep(POLL_INTERVAL).await; + } +} + +#[cfg(test)] +mod tests { + use super::*; + + fn halt(vshard_id: u32) -> CalvinApplyHalt { + CalvinApplyHalt { + vshard_id, + epoch: 9, + position: 1, + reason: "redo_apply_failed", + step: "redo_apply", + error: "redo install did not become durable".to_string(), + } + } + + #[test] + fn no_lagging_scheduler_passes() { + assert_eq!(catch_up_wait(Vec::new(), None), None); + assert_eq!(catch_up_wait(Vec::new(), Some(&halt(3))), None); + } + + #[test] + fn a_lagging_scheduler_waits() { + assert_eq!( + catch_up_wait(vec![2, 5], None), + Some(CatchUpWait::Lagging(vec![2, 5])) + ); + } + + #[test] + fn a_halt_on_another_vshard_keeps_waiting() { + assert_eq!( + catch_up_wait(vec![2, 5], Some(&halt(3))), + Some(CatchUpWait::Lagging(vec![2, 5])) + ); + } + + #[test] + fn a_halted_lagging_scheduler_fails_at_once() { + assert_eq!( + catch_up_wait(vec![2, 5], Some(&halt(5))), + Some(CatchUpWait::Halted(halt(5))) + ); + } + + #[test] + fn a_long_lagging_list_names_the_first_vshards_and_counts_the_rest() { + let lagging: Vec = (0..20).collect(); + let text = describe_lagging(&lagging); + assert!(text.starts_with("vShards 0, 1, 2"), "{text}"); + assert!(text.ends_with("15 and 4 more"), "{text}"); + assert_eq!(describe_lagging(&[7]), "vShards 7"); + } + + /// A node that runs no scheduler, as a fresh single node does before its + /// first sequenced write, passes at once. + #[tokio::test] + async fn a_node_with_no_scheduler_passes_at_once() { + let dir = tempfile::tempdir().expect("tmpdir"); + let wal = Arc::new( + crate::wal::WalManager::open_for_testing(&dir.path().join("catch-up.wal")) + .expect("open wal"), + ); + let (dispatcher, _data_sides) = crate::bridge::Dispatcher::new(1, 64); + let shared = SharedState::new(dispatcher, wal).expect("build shared state"); + await_calvin_catch_up(&shared) + .await + .expect("no scheduler holds the gate"); + } +} diff --git a/nodedb/src/bootstrap/cluster_ready.rs b/nodedb/src/bootstrap/cluster_ready.rs index f35bfabf0..f6b24d6e0 100644 --- a/nodedb/src/bootstrap/cluster_ready.rs +++ b/nodedb/src/bootstrap/cluster_ready.rs @@ -161,6 +161,15 @@ pub async fn await_cluster_ready( return Err(e); } + // A replayed data group says nothing about Calvin: each scheduler still + // re-applies epochs from the sequencer log up to the highest epoch it had + // applied before the restart. Hold the gateway until every scheduler on + // this node reaches that target. Fail closed, like the wait above. + if let Err(e) = crate::bootstrap::calvin_catch_up::await_calvin_catch_up(shared).await { + data_groups_gate.fail(format!("calvin scheduler catch-up failed: {e}")); + return Err(e); + } + // Retry every owed reclaim before the gateway opens. A row that does not // reclaim keeps the drain hold its retry took, so a same-name CREATE waits // for the worker's retry and never opens over storage the retry erases. diff --git a/nodedb/src/bootstrap/constraint_reconcile.rs b/nodedb/src/bootstrap/constraint_reconcile.rs index 7dff168fd..90e2b28b1 100644 --- a/nodedb/src/bootstrap/constraint_reconcile.rs +++ b/nodedb/src/bootstrap/constraint_reconcile.rs @@ -169,7 +169,8 @@ pub async fn reconcile_once( }, ); - match propose_replicated_entry(shared, &proposer, entry).await { + let deadline = crate::control::wal_replication::statement_propose_deadline(shared); + match propose_replicated_entry(shared, &proposer, entry, deadline).await { Ok(_) => { // Record only on commit. A transient / NotLeader error // leaves the map untouched so the next tick retries. diff --git a/nodedb/src/bootstrap/data_plane.rs b/nodedb/src/bootstrap/data_plane.rs index ed877955f..3b29aabba 100644 --- a/nodedb/src/bootstrap/data_plane.rs +++ b/nodedb/src/bootstrap/data_plane.rs @@ -327,6 +327,7 @@ pub fn spawn_data_plane_cores( columnar_schema_seed, } = resources; let num_cores = config.server.data_plane_cores; + let node_id = crate::control::cluster::configured_node_id(config); let compaction_cfg = CoreCompactionConfig { interval: config.checkpoint.compaction_interval(), tombstone_threshold: config.checkpoint.compaction_tombstone_threshold, @@ -347,6 +348,7 @@ pub fn spawn_data_plane_cores( let (replay_done_tx, replay_done_rx) = tokio::sync::oneshot::channel(); let (handle, notifier) = spawn_core(SpawnCoreParams { core_id, + node_id, request_rx: data_side.request_rx, response_tx: data_side.response_tx, data_dir: &config.server.data_dir, diff --git a/nodedb/src/bootstrap/mod.rs b/nodedb/src/bootstrap/mod.rs index 005a9adb9..d57dfdff0 100644 --- a/nodedb/src/bootstrap/mod.rs +++ b/nodedb/src/bootstrap/mod.rs @@ -1,6 +1,7 @@ // SPDX-License-Identifier: BUSL-1.1 pub mod background_loops; +pub mod calvin_catch_up; pub mod catalog_open; pub mod cluster_ready; pub mod constraint_reconcile; diff --git a/nodedb/src/bootstrap/write_group_settle/settle.rs b/nodedb/src/bootstrap/write_group_settle/settle.rs index 87420f5da..bb06bed83 100644 --- a/nodedb/src/bootstrap/write_group_settle/settle.rs +++ b/nodedb/src/bootstrap/write_group_settle/settle.rs @@ -18,8 +18,6 @@ //! transaction record is the exception: it always applies, so it stays, //! its group closes with one part with no rows, and replay runs its //! install. -//! - A committed Calvin record is whole at append and takes no part. Its -//! stored write set holds no row, so boot journals nothing for it. //! //! Restart replay, a point-in-time restore of the archived WAL, and the event //! stream rebuilt from it then agree with the stores. Boot makes the appended @@ -157,11 +155,6 @@ fn complete_group( scanned: &WalGroups, capture: &WriteSetCapture, ) -> crate::Result { - // A Calvin flush's origin is a committed record whole at append: no part - // follows it. - if is_calvin_flush(&stored_plan(&capture.origin_append)?) { - return Ok(false); - } let appender = part_appender(wal, &capture.origin_append)?; let target = target(capture, Lsn::new(capture.origin)); let write_set = capture.write_set(); @@ -188,16 +181,6 @@ fn journal_cut_write(wal: &WalManager, capture: &WriteSetCapture) -> crate::Resu let inputs = &capture.origin_append; let appender = part_appender(wal, inputs)?; let plan = stored_plan(inputs)?; - // A Calvin flush's origin is the transaction's committed record. The - // sequencer holds the transaction, and its recovery runs it again. - if is_calvin_flush(&plan) { - tracing::warn!( - origin = capture.origin, - "boot found a Calvin flush whose committed record the crash cut; the \ - sequencer runs the transaction again" - ); - return Ok(false); - } let home = target(capture, Lsn::ZERO); if let Some((epoch, group_id, log_index)) = inputs.change_position { let marker = ChangePositionMarker { @@ -244,14 +227,6 @@ fn stored_plan(inputs: &OriginAppend) -> crate::Result { }) } -/// Whether `plan` is a Calvin flush, whose origin is whole at append. -fn is_calvin_flush(plan: &PhysicalPlan) -> bool { - matches!( - plan, - PhysicalPlan::Meta(nodedb_physical::physical_plan::MetaOp::CalvinFlush { .. }) - ) -} - /// The appender every record of the write carried. fn part_appender<'a>(wal: &'a WalManager, inputs: &OriginAppend) -> crate::Result> { let appender = wal diff --git a/nodedb/src/bridge/admission_chokepoint.rs b/nodedb/src/bridge/admission_chokepoint.rs index e1d775ef3..fde8582da 100644 --- a/nodedb/src/bridge/admission_chokepoint.rs +++ b/nodedb/src/bridge/admission_chokepoint.rs @@ -145,6 +145,7 @@ mod tests { wal_lsn: None, resolved_now_ms: None, commit_hlc: None, + entry_version: None, admission, } } diff --git a/nodedb/src/bridge/dispatch/core_channel.rs b/nodedb/src/bridge/dispatch/core_channel.rs index 4f2175c31..9ae4ea45a 100644 --- a/nodedb/src/bridge/dispatch/core_channel.rs +++ b/nodedb/src/bridge/dispatch/core_channel.rs @@ -44,7 +44,7 @@ pub struct CoreChannel { /// **Lifetime**: entries are written in place and never reach a "remove" /// path on their own. Stale databases that no longer enqueue requests /// retain a `Normal` (or last-observed) entry until the surrounding - /// dispatcher is dropped or `recalculate_tenant_limits` rotates state. + /// dispatcher is dropped. /// The map is bounded by the universe of `database_id`s that have ever /// been dispatched against this core, so unbounded growth is not a /// concern in practice. diff --git a/nodedb/src/bridge/dispatch/dispatcher.rs b/nodedb/src/bridge/dispatch/dispatcher.rs index d69296ce3..8847beeb8 100644 --- a/nodedb/src/bridge/dispatch/dispatcher.rs +++ b/nodedb/src/bridge/dispatch/dispatcher.rs @@ -24,6 +24,7 @@ use super::core_channel::{CoreChannel, CoreChannelDataSide}; use super::dispatched_lsns::DispatchedLsns; use super::journal::WriteSetJournal; use super::outcome_floor::OutcomeFloor; +use super::tenant_inflight::TenantInflight; /// Per-core request queue capacity of the server's bridge dispatcher. /// @@ -104,16 +105,11 @@ pub struct Dispatcher { /// Routes vShards to core IDs. pub(super) router: VShardRouter, - /// Per-tenant in-flight request count across all cores. - pub(super) tenant_inflight: HashMap, + /// Per-tenant in-flight counts across all cores and the fair-share cap + /// they derive. + pub(super) tenants: TenantInflight, - /// Maps request_id → tenant_id for in-flight requests. - pub(super) request_tenant: HashMap, - - /// Maximum in-flight requests per tenant (0 = unlimited). - pub(super) max_per_tenant_inflight: u32, - - /// Per-core queue capacity (used in tenant fairness recalculation). + /// Per-core queue capacity. pub(super) per_core_capacity: u32, /// Resolves priority class for a database_id (consulted on enqueue). @@ -184,9 +180,7 @@ impl Dispatcher { Self { cores, router, - tenant_inflight: HashMap::new(), - request_tenant: HashMap::new(), - max_per_tenant_inflight: total_capacity as u32, + tenants: TenantInflight::new(u32::try_from(total_capacity).unwrap_or(u32::MAX)), per_core_capacity: queue_capacity as u32, priority_resolver, data_plane_draining: false, diff --git a/nodedb/src/bridge/dispatch/drain.rs b/nodedb/src/bridge/dispatch/drain.rs index 064ba8155..de930e493 100644 --- a/nodedb/src/bridge/dispatch/drain.rs +++ b/nodedb/src/bridge/dispatch/drain.rs @@ -27,7 +27,6 @@ use crate::bridge::envelope::{ErrorCode, Payload, Status}; use crate::types::{Lsn, RequestId}; use super::dispatcher::Dispatcher; -use super::enqueue::release_inflight_slot; /// Work one core still owes the Control Plane. #[derive(Debug, Clone, Copy, PartialEq, Eq)] @@ -155,8 +154,7 @@ impl Dispatcher { for rid in ids { // The node is shutting down, and this core publishes nothing more. self.dispatched_lsns.settle(rid); - freed |= - release_inflight_slot(&mut self.request_tenant, &mut self.tenant_inflight, rid); + freed |= self.tenants.release(rid); abandoned.push(envelope::Response { request_id: RequestId::new(rid), status: Status::Error, @@ -170,8 +168,8 @@ impl Dispatcher { shutdown drain deadline" ), })), - read_set_valid: None, - read_version_lsn: Lsn::ZERO, + stage_vote: None, + read_versions: crate::types::ReadVersions::new(), write_set: Vec::new(), }); } diff --git a/nodedb/src/bridge/dispatch/enqueue.rs b/nodedb/src/bridge/dispatch/enqueue.rs index ade60a51f..5716f6aaf 100644 --- a/nodedb/src/bridge/dispatch/enqueue.rs +++ b/nodedb/src/bridge/dispatch/enqueue.rs @@ -6,8 +6,6 @@ //! A capacity limit refuses with [`crate::Error::DispatchCapacity`] and hands //! the request back. Every other refusal is terminal. -use std::collections::HashMap; - use tracing::warn; use crate::DispatchCapacityScope; @@ -18,6 +16,7 @@ use crate::types::{Lsn, VShardId}; use super::dispatcher::Dispatcher; use super::journal::JournalGroup; use super::refusal::DispatchRefusal; +use super::tenant_inflight::TenantAtCap; impl Dispatcher { /// Dispatch a request to the correct Data Plane core. @@ -58,19 +57,16 @@ impl Dispatcher { let wal_lsn = request.wal_lsn; // Per-tenant fairness: refuse while the tenant holds its in-flight cap. - if self.max_per_tenant_inflight > 0 { - let inflight = self.tenant_inflight.get(&tenant_id).copied().unwrap_or(0); - if inflight >= self.max_per_tenant_inflight { - let scope = DispatchCapacityScope::TenantInflight { - tenant_id: request.tenant_id, - inflight, - cap: self.max_per_tenant_inflight, - }; - return Err(DispatchRefusal::boxed( - crate::Error::DispatchCapacity { scope }, - request, - )); - } + if let Some(TenantAtCap { inflight, cap }) = self.tenants.at_cap(tenant_id) { + let scope = DispatchCapacityScope::TenantInflight { + tenant_id: request.tenant_id, + inflight, + cap, + }; + return Err(DispatchRefusal::boxed( + crate::Error::DispatchCapacity { scope }, + request, + )); } let Some(core_id) = self.router.resolve(request.vshard_id) else { @@ -167,14 +163,6 @@ impl Dispatcher { Ok(()) } - /// Recalculate the per-tenant in-flight limit based on active tenants. - pub fn recalculate_tenant_limits(&mut self) { - let active = self.tenant_inflight.len().max(1) as u32; - let total_capacity: u32 = self.cores.len() as u32 * self.per_core_capacity; - self.max_per_tenant_inflight = (total_capacity / active).max(2); - self.tenant_inflight.retain(|_, count| *count > 0); - } - /// Bookkeeping once a request sits in `core_id`'s weighted-fair queue: /// hold the outcome floor below its WAL LSN, flush it toward the ring, /// record pressure, track it as outstanding and in flight for its tenant, @@ -217,8 +205,7 @@ impl Dispatcher { channel.outstanding.insert(req_id); // Track per-tenant in-flight + request→tenant mapping for response routing. - *self.tenant_inflight.entry(tenant_id).or_insert(0) += 1; - self.request_tenant.insert(req_id, tenant_id); + self.tenants.admit(tenant_id, req_id); // Wake the Data Plane core via eventfd. if let Some(ref notifier) = self.cores[core_id].wake_notifier { @@ -227,27 +214,6 @@ impl Dispatcher { } } -/// Release the in-flight slot `request_id` holds for its tenant. -/// -/// Returns `true` when a slot was freed. A free function over the two maps, -/// so a caller can release while it holds a borrow of one core's channel. -pub(super) fn release_inflight_slot( - request_tenant: &mut HashMap, - tenant_inflight: &mut HashMap, - request_id: u64, -) -> bool { - let Some(tenant_id) = request_tenant.remove(&request_id) else { - return false; - }; - match tenant_inflight.get_mut(&tenant_id) { - Some(count) if *count > 0 => { - *count -= 1; - true - } - _ => false, - } -} - #[cfg(test)] mod tests { use super::*; @@ -306,6 +272,72 @@ mod tests { ); } + /// Dispatch a request of `tenant` with id and database `id`, then pop it + /// off the ring so the ring and the weighted-fair queue stay empty. + fn dispatch_popped( + dispatcher: &mut Dispatcher, + data_side: &mut crate::bridge::dispatch::CoreChannelDataSide, + tenant: u64, + id: u64, + ) -> crate::Result<()> { + let mut request = make_request_for_db(0, id, id); + request.tenant_id = TenantId::new(tenant); + dispatcher.dispatch(request)?; + while data_side.request_rx.try_pop().is_ok() {} + Ok(()) + } + + fn tenant_cap_of(error: crate::Error) -> u32 { + match error { + crate::Error::DispatchCapacity { + scope: DispatchCapacityScope::TenantInflight { cap, .. }, + } => cap, + other => panic!("expected a tenant-cap refusal, got: {other}"), + } + } + + #[test] + fn two_active_tenants_split_the_capacity_and_an_idle_one_returns_its_share() { + let (mut dispatcher, mut data_sides) = Dispatcher::new(1, 8); + let data_side = &mut data_sides[0]; + + // Tenant 2 holds one request, so tenant 1 gets half of the eight slots. + dispatch_popped(&mut dispatcher, data_side, 2, 100).expect("tenant 2 admitted"); + for id in 1..=4u64 { + dispatch_popped(&mut dispatcher, data_side, 1, id).expect("within half the capacity"); + } + let refused = dispatch_popped(&mut dispatcher, data_side, 1, 5) + .expect_err("tenant 1 is at half the capacity"); + assert_eq!(tenant_cap_of(refused), 4); + + // Tenant 2's only request completes, so tenant 1 is the sole tenant. + data_side + .response_tx + .try_push(BridgeResponse { + inner: envelope::Response { + request_id: RequestId::new(100), + status: Status::Ok, + attempt: 1, + partial: false, + payload: Payload::empty(), + watermark_lsn: Lsn::ZERO, + error_code: None, + stage_vote: None, + read_versions: crate::types::ReadVersions::new(), + write_set: Vec::new(), + }, + }) + .expect("response ring has room"); + assert_eq!(dispatcher.poll_responses().len(), 1); + + for id in 5..=8u64 { + dispatch_popped(&mut dispatcher, data_side, 1, id).expect("the full capacity is back"); + } + let refused = dispatch_popped(&mut dispatcher, data_side, 1, 9) + .expect_err("tenant 1 is at the full capacity"); + assert_eq!(tenant_cap_of(refused), 8); + } + #[test] fn full_weighted_fair_queue_is_refused_with_queue_full_scope() { // Distinct tenants and databases keep the tenant cap and the per-DB @@ -339,8 +371,8 @@ mod tests { dispatcher.dispatch_to_core(1, request).unwrap(); - assert_eq!(dispatcher.tenant_inflight.get(&tenant_id), Some(&1)); - assert_eq!(dispatcher.request_tenant.get(&request_id), Some(&tenant_id)); + assert_eq!(dispatcher.tenants.inflight(tenant_id), 1); + assert_eq!(dispatcher.tenants.tenant_of(request_id), Some(tenant_id)); assert_eq!(data_sides[1].request_rx.len(), 1); let _req = data_sides[1].request_rx.try_pop().unwrap(); @@ -355,8 +387,8 @@ mod tests { payload: Payload::empty(), watermark_lsn: Lsn::ZERO, error_code: None, - read_set_valid: None, - read_version_lsn: crate::types::Lsn::ZERO, + stage_vote: None, + read_versions: crate::types::ReadVersions::new(), write_set: Vec::new(), }, }) @@ -364,8 +396,8 @@ mod tests { let responses = dispatcher.poll_responses(); assert_eq!(responses.len(), 1); - assert_eq!(dispatcher.tenant_inflight.get(&tenant_id), Some(&0)); - assert!(!dispatcher.request_tenant.contains_key(&request_id)); + assert_eq!(dispatcher.tenants.inflight(tenant_id), 0); + assert_eq!(dispatcher.tenants.tenant_of(request_id), None); } #[test] diff --git a/nodedb/src/bridge/dispatch/mod.rs b/nodedb/src/bridge/dispatch/mod.rs index dbfa33695..ce643e371 100644 --- a/nodedb/src/bridge/dispatch/mod.rs +++ b/nodedb/src/bridge/dispatch/mod.rs @@ -10,6 +10,7 @@ mod journal; mod outcome_floor; mod refusal; mod response_poll; +mod tenant_inflight; #[cfg(test)] mod test_requests; diff --git a/nodedb/src/bridge/dispatch/response_poll.rs b/nodedb/src/bridge/dispatch/response_poll.rs index a999c5007..29e2738a9 100644 --- a/nodedb/src/bridge/dispatch/response_poll.rs +++ b/nodedb/src/bridge/dispatch/response_poll.rs @@ -10,7 +10,6 @@ use crate::bridge::envelope::{ErrorCode, Payload, Status}; use crate::types::{Lsn, RequestId}; use super::dispatcher::Dispatcher; -use super::enqueue::release_inflight_slot; impl Dispatcher { /// Poll responses from all Data Plane cores. @@ -21,7 +20,7 @@ impl Dispatcher { /// everything dispatched earlier that it never answered. Those travel back /// with the real responses so the single completion loop in the caller /// finishes each waiter, and the loop below releases each request's - /// `tenant_inflight` slot exactly as a real response would — without which + /// tenant in-flight slot exactly as a real response would — without which /// one dead core ratchets the tenant's in-flight count until the tenant is /// rejected on healthy cores too. /// @@ -43,11 +42,7 @@ impl Dispatcher { if !br.inner.partial { channel.outstanding.remove(&rid); self.dispatched_lsns.settle(rid); - freed |= release_inflight_slot( - &mut self.request_tenant, - &mut self.tenant_inflight, - rid, - ); + freed |= self.tenants.release(rid); } responses.push(br.inner); } @@ -85,8 +80,7 @@ impl Dispatcher { for rid in lost { // A dead core never publishes a watermark again. self.dispatched_lsns.settle(rid); - freed |= - release_inflight_slot(&mut self.request_tenant, &mut self.tenant_inflight, rid); + freed |= self.tenants.release(rid); responses.push(envelope::Response { request_id: RequestId::new(rid), status: Status::Error, @@ -99,8 +93,8 @@ impl Dispatcher { "core-{core_id} is gone; the request can never be executed" ), })), - read_set_valid: None, - read_version_lsn: Lsn::ZERO, + stage_vote: None, + read_versions: crate::types::ReadVersions::new(), write_set: Vec::new(), }); } @@ -139,8 +133,8 @@ mod tests { payload: Payload::from_vec(b"result".to_vec()), watermark_lsn: Lsn::new(42), error_code: None, - read_set_valid: None, - read_version_lsn: crate::types::Lsn::ZERO, + stage_vote: None, + read_versions: crate::types::ReadVersions::new(), write_set: Vec::new(), }, }) @@ -170,8 +164,8 @@ mod tests { payload: Payload::empty(), watermark_lsn: Lsn::ZERO, error_code: None, - read_set_valid: None, - read_version_lsn: Lsn::ZERO, + stage_vote: None, + read_versions: crate::types::ReadVersions::new(), write_set: Vec::new(), }, }) @@ -196,7 +190,7 @@ mod tests { // When a Data Plane core's consumer/producer is dropped (the core thread // died), `Dispatcher` must synthesize an error `Response` for every // request it knows is outstanding on that core, rather than dropping the - // request silently and leaking the caller's waiter + `tenant_inflight` + // request silently and leaking the caller's waiter + tenant in-flight // slot forever. Dropping one element of the `data_sides` vector handed // back by `Dispatcher::new`/`with_resolver` simulates that core thread // dying, matching how `dispatch_routes_to_correct_core` and @@ -236,7 +230,7 @@ mod tests { #[test] fn dead_core_synthesized_response_resets_tenant_inflight() { - // The ratchet: `tenant_inflight` is incremented on dispatch and must + // The ratchet: the tenant in-flight count is incremented on dispatch and must // return to its pre-dispatch value once the synthesized response for // the lost request is drained through `poll_responses` — otherwise it // climbs forever and eventually starves the tenant on healthy cores. @@ -247,31 +241,23 @@ mod tests { let request = make_request_for_db(0, 1, 1); let tenant_id = request.tenant_id.as_u64(); - let before = dispatcher - .tenant_inflight - .get(&tenant_id) - .copied() - .unwrap_or(0); + let before = dispatcher.tenants.inflight(tenant_id); dispatcher.dispatch_to_core(dead_core, request).unwrap(); assert_eq!( - dispatcher.tenant_inflight.get(&tenant_id).copied(), - Some(before + 1), - "dispatch must still increment tenant_inflight even though the core is dead" + dispatcher.tenants.inflight(tenant_id), + before + 1, + "dispatch must still increment the tenant in-flight count even though the core is dead" ); let responses = dispatcher.poll_responses(); assert_eq!(responses.len(), 1); assert_eq!( - dispatcher - .tenant_inflight - .get(&tenant_id) - .copied() - .unwrap_or(0), + dispatcher.tenants.inflight(tenant_id), before, - "tenant_inflight must return to its pre-dispatch value, not ratchet upward" + "the tenant in-flight count must return to its pre-dispatch value, not ratchet upward" ); - assert!(!dispatcher.request_tenant.contains_key(&1)); + assert_eq!(dispatcher.tenants.tenant_of(1), None); } #[test] @@ -307,8 +293,8 @@ mod tests { payload: Payload::empty(), watermark_lsn: Lsn::ZERO, error_code: None, - read_set_valid: None, - read_version_lsn: crate::types::Lsn::ZERO, + stage_vote: None, + read_versions: crate::types::ReadVersions::new(), write_set: Vec::new(), }, }) @@ -402,8 +388,8 @@ mod tests { payload: Payload::empty(), watermark_lsn: Lsn::ZERO, error_code: None, - read_set_valid: None, - read_version_lsn: Lsn::ZERO, + stage_vote: None, + read_versions: crate::types::ReadVersions::new(), write_set: Vec::new(), }, } diff --git a/nodedb/src/bridge/dispatch/tenant_inflight.rs b/nodedb/src/bridge/dispatch/tenant_inflight.rs new file mode 100644 index 000000000..530318505 --- /dev/null +++ b/nodedb/src/bridge/dispatch/tenant_inflight.rs @@ -0,0 +1,189 @@ +// SPDX-License-Identifier: BUSL-1.1 + +//! Per-tenant in-flight accounting and the fair-share cap it derives. +//! +//! Each tenant with a request in flight is active. The cap is the dispatcher's +//! total capacity divided by the active tenant count, at least two. The cap +//! moves only when a tenant turns active or idle. A move gates new admissions +//! only: a request already admitted keeps its slot until it is answered. + +use std::collections::HashMap; + +/// Smallest per-tenant cap. One slot will serialize every request a tenant +/// sends, so the floor keeps one request in flight while the next one queues. +const MIN_TENANT_CAP: u32 = 2; + +/// A tenant held at its in-flight cap. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub(super) struct TenantAtCap { + pub(super) inflight: u32, + pub(super) cap: u32, +} + +/// In-flight counts per tenant, the request-to-tenant map, and the cap. +#[derive(Debug)] +pub(super) struct TenantInflight { + /// In-flight count of every active tenant. A tenant at zero has no entry. + by_tenant: HashMap, + /// The tenant of every admitted request not yet answered. + request_tenant: HashMap, + /// Requests the dispatcher holds across every core. + total_capacity: u32, + /// Most requests one tenant can hold in flight. Zero means no cap, which + /// holds only for a dispatcher with no capacity. + cap: u32, +} + +impl TenantInflight { + /// Accounting for a dispatcher that holds `total_capacity` requests. + pub(super) fn new(total_capacity: u32) -> Self { + Self { + by_tenant: HashMap::new(), + request_tenant: HashMap::new(), + total_capacity, + cap: total_capacity, + } + } + + /// The refusal for one more request of `tenant_id`, when it is at its cap. + pub(super) fn at_cap(&self, tenant_id: u64) -> Option { + if self.cap == 0 { + return None; + } + let inflight = self.inflight(tenant_id); + (inflight >= self.cap).then_some(TenantAtCap { + inflight, + cap: self.cap, + }) + } + + /// Count `request_id` in flight for `tenant_id`. A tenant that turns + /// active lowers the cap of every tenant. + pub(super) fn admit(&mut self, tenant_id: u64, request_id: u64) { + let count = self.by_tenant.entry(tenant_id).or_insert(0); + *count += 1; + let turned_active = *count == 1; + self.request_tenant.insert(request_id, tenant_id); + if turned_active { + self.recalculate_cap(); + } + } + + /// Release the slot `request_id` holds. Returns `true` when a slot was + /// freed. A tenant that turns idle raises the cap of every other tenant. + pub(super) fn release(&mut self, request_id: u64) -> bool { + let Some(tenant_id) = self.request_tenant.remove(&request_id) else { + return false; + }; + let Some(count) = self.by_tenant.get_mut(&tenant_id) else { + return false; + }; + *count = count.saturating_sub(1); + if *count == 0 { + self.by_tenant.remove(&tenant_id); + self.recalculate_cap(); + } + true + } + + /// Requests `tenant_id` holds in flight. + pub(super) fn inflight(&self, tenant_id: u64) -> u32 { + self.by_tenant.get(&tenant_id).copied().unwrap_or(0) + } + + /// The tenant of `request_id`, while it is in flight. + #[cfg(test)] + pub(super) fn tenant_of(&self, request_id: u64) -> Option { + self.request_tenant.get(&request_id).copied() + } + + /// The current per-tenant cap. + #[cfg(test)] + pub(super) fn cap(&self) -> u32 { + self.cap + } + + /// Set the cap to the total capacity split across the active tenants. + fn recalculate_cap(&mut self) { + if self.total_capacity == 0 { + return; + } + let active = u32::try_from(self.by_tenant.len()) + .unwrap_or(u32::MAX) + .max(1); + self.cap = (self.total_capacity / active).max(MIN_TENANT_CAP); + } +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn a_single_active_tenant_holds_the_whole_capacity() { + let mut tenants = TenantInflight::new(8); + tenants.admit(1, 10); + assert_eq!(tenants.cap(), 8); + assert_eq!(tenants.at_cap(1), None); + } + + #[test] + fn a_second_active_tenant_halves_the_cap_and_its_exit_restores_it() { + let mut tenants = TenantInflight::new(8); + tenants.admit(1, 10); + tenants.admit(2, 20); + assert_eq!(tenants.cap(), 4); + + assert!(tenants.release(20)); + assert_eq!(tenants.cap(), 8); + assert_eq!(tenants.inflight(2), 0); + } + + #[test] + fn a_lower_cap_keeps_every_admitted_slot() { + let mut tenants = TenantInflight::new(8); + for request_id in 0..6 { + tenants.admit(1, request_id); + } + tenants.admit(2, 100); + assert_eq!(tenants.cap(), 4); + assert_eq!(tenants.inflight(1), 6, "admitted requests stay in flight"); + assert_eq!( + tenants.at_cap(1), + Some(TenantAtCap { + inflight: 6, + cap: 4 + }) + ); + for request_id in 0..6 { + assert!(tenants.release(request_id)); + } + assert_eq!(tenants.inflight(1), 0); + } + + #[test] + fn the_cap_never_falls_below_the_floor() { + let mut tenants = TenantInflight::new(4); + for tenant_id in 0..10 { + tenants.admit(tenant_id, tenant_id); + } + assert_eq!(tenants.cap(), MIN_TENANT_CAP); + } + + #[test] + fn releasing_an_unknown_request_frees_nothing() { + let mut tenants = TenantInflight::new(8); + assert!(!tenants.release(42)); + tenants.admit(1, 10); + assert!(tenants.release(10)); + assert!(!tenants.release(10)); + assert_eq!(tenants.tenant_of(10), None); + } + + #[test] + fn a_dispatcher_with_no_capacity_has_no_cap() { + let mut tenants = TenantInflight::new(0); + tenants.admit(1, 10); + assert_eq!(tenants.at_cap(1), None); + } +} diff --git a/nodedb/src/bridge/dispatch/test_requests.rs b/nodedb/src/bridge/dispatch/test_requests.rs index efb02d80d..f2a8592db 100644 --- a/nodedb/src/bridge/dispatch/test_requests.rs +++ b/nodedb/src/bridge/dispatch/test_requests.rs @@ -39,6 +39,7 @@ pub(super) fn make_request(vshard: u32) -> envelope::Request { wal_lsn: None, resolved_now_ms: None, commit_hlc: None, + entry_version: None, admission: Admission::Exempt(ExemptReason::Read), } } @@ -72,6 +73,7 @@ pub(super) fn make_request_for_db(vshard: u32, db: u64, req_id: u64) -> envelope wal_lsn: None, resolved_now_ms: None, commit_hlc: None, + entry_version: None, admission: Admission::Exempt(ExemptReason::Read), } } diff --git a/nodedb/src/bridge/envelope/error_code.rs b/nodedb/src/bridge/envelope/error_code.rs index a63576fcb..d21d282b6 100644 --- a/nodedb/src/bridge/envelope/error_code.rs +++ b/nodedb/src/bridge/envelope/error_code.rs @@ -25,6 +25,10 @@ pub enum ErrorCode { /// a retry channel can tell a retry apart from a permanent refusal instead /// of collapsing both into a terminal rejection. RetryableRefusal { reason: String }, + /// The core fail-stopped: its state is unknown until a restart rebuilds + /// it from the WAL. It refused the request and applied nothing. Another + /// replica, or this one after its restart, serves the request. + CoreFailStopped { core_id: usize, detail: String }, /// A sync frame the validator refused for good. Nothing applied. The /// stream's high-water mark advanced to `provenance`, so the frame is /// never admitted again, and `applied_seq` is the mark after the refusal. diff --git a/nodedb/src/bridge/envelope/mod.rs b/nodedb/src/bridge/envelope/mod.rs index 371b131b2..337b116e2 100644 --- a/nodedb/src/bridge/envelope/mod.rs +++ b/nodedb/src/bridge/envelope/mod.rs @@ -8,6 +8,7 @@ pub mod error_code_from; pub mod payload; pub mod request; pub mod response; +pub mod stage_vote; pub mod status; pub mod sync_hold; @@ -17,5 +18,6 @@ pub use nodedb_physical::physical_plan::PhysicalPlan; pub use payload::Payload; pub use request::{Admission, ExemptReason, Request}; pub use response::{EdgeImage, Response, RowEffect, RowVersion, WriteSetEntry}; +pub use stage_vote::StageVote; pub use status::{Priority, Status}; pub use sync_hold::SyncHold; diff --git a/nodedb/src/bridge/envelope/request.rs b/nodedb/src/bridge/envelope/request.rs index 2c2c29e8d..b1e074483 100644 --- a/nodedb/src/bridge/envelope/request.rs +++ b/nodedb/src/bridge/envelope/request.rs @@ -110,6 +110,13 @@ pub struct Request { /// of the WAL record they reproduce. pub commit_hlc: Option, + /// The log position of the data-group Raft entry this write applies. The + /// Data Plane records it as the version of every row the write touches, + /// so every replica that applies the entry records the same version. + /// `None` for reads, control ops, and writes that apply no entry: those + /// version by their WAL LSN on the core. + pub entry_version: Option, + /// Write-admission decision for this request. /// /// Every write-class [`PhysicalPlan`] MUST pass the neutral write-admission @@ -232,6 +239,7 @@ mod tests { wal_lsn: None, resolved_now_ms: None, commit_hlc: None, + entry_version: None, admission: Admission::Exempt(ExemptReason::Read), } } @@ -287,6 +295,7 @@ mod tests { wal_lsn: None, resolved_now_ms: None, commit_hlc: None, + entry_version: None, admission: Admission::Exempt(ExemptReason::Read), }; match req.plan { diff --git a/nodedb/src/bridge/envelope/response.rs b/nodedb/src/bridge/envelope/response.rs index 53fff8144..ea1a53c39 100644 --- a/nodedb/src/bridge/envelope/response.rs +++ b/nodedb/src/bridge/envelope/response.rs @@ -4,6 +4,7 @@ use super::error_code::ErrorCode; use super::payload::Payload; +use super::stage_vote::StageVote; use super::status::Status; use crate::types::{Lsn, RequestId}; use nodedb_types::RowIdentity; @@ -192,32 +193,26 @@ pub struct Response { /// Watermark LSN at the time of read (for snapshot consistency tracking). pub watermark_lsn: Lsn, - /// Per-collection read-version LSN (the scanned collection's `coll_write_lsn` - /// at read time, a WAL LSN) — the sound comparand for cross-shard OCC read - /// validation. Distinct from `watermark_lsn` (core-global max, used for - /// snapshot/SI reporting). + /// The write versions the read observed, one per vShard: the read + /// collection's version on its home vShard, or every vShard's latest + /// version a graph read walked. The comparand for cross-shard OCC read + /// validation. Distinct from `watermark_lsn` (this core's WAL position). /// /// On a WRITE response it is the POST-write version of the written - /// collection (the handlers record before responding), which is how the Raft - /// apply path returns a committed write's own version to its proposer. - /// `Lsn::ZERO` when the plan names no single user collection. - pub read_version_lsn: Lsn, + /// collection on the write's vShard (the handlers record before + /// responding), which is how the Raft apply path returns a committed + /// write's own version to its proposer. Empty when the plan names no + /// user collection. + pub read_versions: crate::types::ReadVersions, /// Error code if status is not Ok. pub error_code: Option>, - /// Whether this response's originating transaction found its slice of the - /// versioned read-set still current against the local write versions. - /// `Some(true)` = still current (or no reads observed for this slice); - /// `Some(false)` = at least one read was superseded; `None` = the response - /// did not carry a read-set check (reads, control ops, and every - /// non-transaction response). + /// The local commit vote of a Calvin stage response. /// - /// For the direct-apply (dependent/active, fast-path) path this is reporting - /// only — the apply commits regardless. For a staged static Calvin - /// transaction it is the LOCAL COMMIT VOTE: the scheduler flushes the staged - /// buffer to base on `Some(true)` and drops it on `Some(false)`. - pub read_set_valid: Option, + /// Every Calvin stage path sets it. `None` means the response is not a + /// Calvin stage outcome. It never means an unknown vote. + pub stage_vote: Option, /// Row-level effects the Control Plane turns into durable redo records /// *after* the Data Plane applied them (see [`WriteSetEntry`]). Empty for @@ -239,8 +234,8 @@ mod tests { payload: Payload::from_vec(b"result".to_vec()), watermark_lsn: Lsn::new(42), error_code: None, - read_set_valid: None, - read_version_lsn: Lsn::ZERO, + stage_vote: None, + read_versions: crate::types::ReadVersions::new(), write_set: Vec::new(), }; assert_eq!(resp.status, Status::Ok); @@ -258,8 +253,8 @@ mod tests { payload: Payload::empty(), watermark_lsn: Lsn::ZERO, error_code: Some(Box::new(ErrorCode::DeadlineExceeded)), - read_set_valid: None, - read_version_lsn: Lsn::ZERO, + stage_vote: None, + read_versions: crate::types::ReadVersions::new(), write_set: Vec::new(), }; assert_eq!( diff --git a/nodedb/src/bridge/envelope/stage_vote.rs b/nodedb/src/bridge/envelope/stage_vote.rs new file mode 100644 index 000000000..c62144995 --- /dev/null +++ b/nodedb/src/bridge/envelope/stage_vote.rs @@ -0,0 +1,81 @@ +// SPDX-License-Identifier: BUSL-1.1 + +//! The local vote a Calvin stage response carries to its scheduler. + +use super::error_code::ErrorCode; + +/// A participant's local vote on its slice of a staged Calvin transaction. +/// +/// Every Calvin stage path sets one on its response. No other response +/// carries one. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum StageVote { + /// The slice staged, and its slice of the read-set is still current. + Commit, + /// The slice staged, but a read it validates was superseded. + SerializationConflict, + /// A state the coordinator predicted moved. The slice holds no staged + /// state, and the coordinator reads again and retries. + PredictionDrift, + /// The stage failed. The slice holds no staged state. + ParticipantError, +} + +impl StageVote { + /// The vote of a stage that refused with `error`. + /// + /// `OllpRetryRequired` reports a moved prediction. Every other error is a + /// participant error. + pub fn of_stage_error(error: &ErrorCode) -> Self { + if matches!(error, ErrorCode::OllpRetryRequired) { + Self::PredictionDrift + } else { + Self::ParticipantError + } + } + + /// The vote of a staged slice whose read-set check found it `current`. + pub fn of_read_set(current: bool) -> Self { + if current { + Self::Commit + } else { + Self::SerializationConflict + } + } +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn ollp_retry_refusal_votes_prediction_drift() { + assert_eq!( + StageVote::of_stage_error(&ErrorCode::OllpRetryRequired), + StageVote::PredictionDrift + ); + } + + #[test] + fn any_other_refusal_votes_participant_error() { + assert_eq!( + StageVote::of_stage_error(&ErrorCode::DeadlineExceeded), + StageVote::ParticipantError + ); + assert_eq!( + StageVote::of_stage_error(&ErrorCode::Internal { + detail: "stage".into() + }), + StageVote::ParticipantError + ); + } + + #[test] + fn read_set_check_maps_to_commit_or_serialization_conflict() { + assert_eq!(StageVote::of_read_set(true), StageVote::Commit); + assert_eq!( + StageVote::of_read_set(false), + StageVote::SerializationConflict + ); + } +} diff --git a/nodedb/src/config/server/config.rs b/nodedb/src/config/server/config.rs index 2656ba073..b83609e69 100644 --- a/nodedb/src/config/server/config.rs +++ b/nodedb/src/config/server/config.rs @@ -434,6 +434,107 @@ mod tests { assert!(msg.contains("tuning.wal.write_buffer_size"), "{msg}"); } + #[test] + fn from_file_rejects_a_zero_calvin_backlog() { + let path = write_temp_config( + "nodedb-domain-calvin-backlog.toml", + "[tuning.calvin]\nmax_inflight_backlog = 0\n", + ); + let err = ServerConfig::from_file(&path).unwrap_err(); + std::fs::remove_file(&path).ok(); + let msg = err.to_string(); + assert!(msg.contains("tuning.calvin.max_inflight_backlog"), "{msg}"); + assert!(msg.contains("positive integer"), "{msg}"); + } + + #[test] + fn from_file_rejects_a_zero_calvin_restage_bound() { + let path = write_temp_config( + "nodedb-domain-calvin-restage.toml", + "[tuning.calvin]\nrestage_attempts = 0\n", + ); + let err = ServerConfig::from_file(&path).unwrap_err(); + std::fs::remove_file(&path).ok(); + let msg = err.to_string(); + assert!(msg.contains("tuning.calvin.restage_attempts"), "{msg}"); + assert!(msg.contains("positive integer"), "{msg}"); + } + + #[test] + fn from_file_rejects_a_calvin_stall_warning_below_the_floor() { + let path = write_temp_config( + "nodedb-domain-calvin-stall.toml", + "[tuning.calvin]\nverdict_stall_warn_ms = 3\n", + ); + let err = ServerConfig::from_file(&path).unwrap_err(); + std::fs::remove_file(&path).ok(); + let msg = err.to_string(); + assert!(msg.contains("tuning.calvin.verdict_stall_warn_ms"), "{msg}"); + assert!(msg.contains("at least 4 milliseconds"), "{msg}"); + } + + #[test] + fn from_file_reads_the_calvin_section() { + let path = write_temp_config( + "nodedb-domain-calvin-ok.toml", + "[tuning.calvin]\nmax_inflight_backlog = 256\ncatch_up_window = 64\n", + ); + let cfg = ServerConfig::from_file(&path).expect("valid calvin section"); + std::fs::remove_file(&path).ok(); + assert_eq!(cfg.tuning.calvin.max_inflight_backlog, 256); + assert_eq!(cfg.tuning.calvin.catch_up_window, 64); + assert_eq!(cfg.tuning.calvin.channel_capacity, 512); + } + + #[test] + fn from_file_rejects_a_redo_entry_size_past_the_rpc_frame() { + let path = write_temp_config( + "nodedb-domain-redo-entry-large.toml", + "[tuning.calvin]\nmax_redo_entry_bytes = 67108864\n", + ); + let err = ServerConfig::from_file(&path).unwrap_err(); + std::fs::remove_file(&path).ok(); + let msg = err.to_string(); + assert!(msg.contains("tuning.calvin.max_redo_entry_bytes"), "{msg}"); + assert!(msg.contains("RPC frame limit"), "{msg}"); + } + + #[test] + fn from_file_rejects_a_zero_redo_entry_size() { + let path = write_temp_config( + "nodedb-domain-redo-entry-zero.toml", + "[tuning.calvin]\nmax_redo_entry_bytes = 0\n", + ); + let err = ServerConfig::from_file(&path).unwrap_err(); + std::fs::remove_file(&path).ok(); + let msg = err.to_string(); + assert!(msg.contains("tuning.calvin.max_redo_entry_bytes"), "{msg}"); + } + + #[test] + fn from_file_rejects_an_open_redo_cap_below_one_entry() { + let path = write_temp_config( + "nodedb-domain-redo-open-cap.toml", + "[tuning.calvin]\nmax_redo_entry_bytes = 1048576\nmax_open_redo_bytes = 65536\n", + ); + let err = ServerConfig::from_file(&path).unwrap_err(); + std::fs::remove_file(&path).ok(); + let msg = err.to_string(); + assert!(msg.contains("tuning.calvin.max_open_redo_bytes"), "{msg}"); + } + + #[test] + fn from_file_reads_the_redo_sizes() { + let path = write_temp_config( + "nodedb-domain-redo-ok.toml", + "[tuning.calvin]\nmax_redo_entry_bytes = 65536\nmax_open_redo_bytes = 1048576\n", + ); + let cfg = ServerConfig::from_file(&path).expect("valid redo sizes"); + std::fs::remove_file(&path).ok(); + assert_eq!(cfg.tuning.calvin.max_redo_entry_bytes, 65_536); + assert_eq!(cfg.tuning.calvin.max_open_redo_bytes, 1_048_576); + } + /// Every shipped default satisfies every bound the gate enforces. #[test] fn the_compiled_defaults_are_in_domain() { diff --git a/nodedb/src/config/server/domain.rs b/nodedb/src/config/server/domain.rs index bb2cd2950..b152a1300 100644 --- a/nodedb/src/config/server/domain.rs +++ b/nodedb/src/config/server/domain.rs @@ -18,6 +18,22 @@ pub(super) const MIN_WAL_WRITE_BUFFER_BYTES: usize = 64 * 1024; /// shorter sweep costs more than the resolution it buys. pub(super) const MIN_SCOPE_EXPIRY_SECS: u64 = 10; +/// Smallest Calvin verdict stall warning interval. The scheduler's idle sweep +/// ticks at a quarter of it, and a zero-length tick panics the timer. +pub(super) const MIN_CALVIN_VERDICT_STALL_WARN_MS: u64 = 4; + +/// Smallest redo entry size. A chunk entry carries its stream header beside +/// its bytes, so a smaller entry leaves too little room for the bytes. +pub(super) const MIN_REDO_ENTRY_BYTES: usize = 64 * 1024; + +/// Room a redo entry leaves in one RPC frame for the frame header, the +/// AppendEntries envelope, and the entry's own routing fields. +pub(super) const REDO_ENTRY_ENVELOPE_BYTES: usize = 1024 * 1024; + +/// Largest redo entry size: one entry and its envelope fit one RPC frame. +pub(super) const MAX_REDO_ENTRY_BYTES: usize = + nodedb_cluster::rpc_codec::MAX_RPC_PAYLOAD_SIZE as usize - REDO_ENTRY_ENVELOPE_BYTES; + /// Rejects an endpoint that carries no `http://` or `https://` host. pub(super) fn otlp_endpoint_has_host(raw: &str) -> bool { raw.strip_prefix("http://") @@ -103,6 +119,8 @@ pub(super) fn validate_domain(config: &ServerConfig) -> crate::Result<()> { )); } + validate_calvin(config)?; + if let Some(cluster) = config.cluster.as_ref() { positive_u64( u64::from(cluster.join_retry_max_attempts), @@ -129,3 +147,63 @@ pub(super) fn validate_domain(config: &ServerConfig) -> crate::Result<()> { Ok(()) } + +/// Checks `[tuning.calvin]`. A zero channel capacity panics the channel +/// constructor, and a zero bound stalls every scheduler. +fn validate_calvin(config: &ServerConfig) -> crate::Result<()> { + let calvin = &config.tuning.calvin; + positive_usize(calvin.channel_capacity, "tuning.calvin.channel_capacity")?; + positive_u64( + u64::from(calvin.txn_deadline_multiplier), + "tuning.calvin.txn_deadline_multiplier", + )?; + positive_u64( + calvin.dependent_read_passive_timeout_ms, + "tuning.calvin.dependent_read_passive_timeout_ms", + )?; + if calvin.verdict_stall_warn_ms < MIN_CALVIN_VERDICT_STALL_WARN_MS { + return Err(reject( + "tuning.calvin.verdict_stall_warn_ms", + calvin.verdict_stall_warn_ms, + "an interval of at least 4 milliseconds", + )); + } + positive_usize( + calvin.max_inflight_backlog, + "tuning.calvin.max_inflight_backlog", + )?; + positive_u64(calvin.catch_up_window, "tuning.calvin.catch_up_window")?; + positive_u64( + u64::from(calvin.restage_attempts), + "tuning.calvin.restage_attempts", + )?; + positive_u64( + calvin.restage_backoff_ms, + "tuning.calvin.restage_backoff_ms", + )?; + validate_redo_sizes(calvin) +} + +/// Checks the redo entry size against one RPC frame, and the open-stream +/// cap against one entry. +fn validate_redo_sizes(calvin: &nodedb_types::config::tuning::CalvinTuning) -> crate::Result<()> { + let entry = calvin.max_redo_entry_bytes; + if !(MIN_REDO_ENTRY_BYTES..=MAX_REDO_ENTRY_BYTES).contains(&entry) { + return Err(reject( + "tuning.calvin.max_redo_entry_bytes", + entry, + &format!( + "a size of at least {MIN_REDO_ENTRY_BYTES} bytes and at most \ + {MAX_REDO_ENTRY_BYTES} bytes, the RPC frame limit less its envelope" + ), + )); + } + if calvin.max_open_redo_bytes < entry as u64 { + return Err(reject( + "tuning.calvin.max_open_redo_bytes", + calvin.max_open_redo_bytes, + "a size of at least tuning.calvin.max_redo_entry_bytes", + )); + } + Ok(()) +} diff --git a/nodedb/src/control/array_catalog/ddl.rs b/nodedb/src/control/array_catalog/ddl.rs index fd3c8e08c..328b02c0d 100644 --- a/nodedb/src/control/array_catalog/ddl.rs +++ b/nodedb/src/control/array_catalog/ddl.rs @@ -99,8 +99,8 @@ pub(crate) async fn run_trusted_array_ddl( payload: Payload::from_vec(payload), watermark_lsn: Lsn::ZERO, error_code: None, - read_set_valid: None, - read_version_lsn: Lsn::ZERO, + stage_vote: None, + read_versions: crate::types::ReadVersions::new(), write_set: Vec::new(), }) } diff --git a/nodedb/src/control/array_sync/raft_apply/common.rs b/nodedb/src/control/array_sync/raft_apply/common.rs index 4e9635db1..140ec0c1a 100644 --- a/nodedb/src/control/array_sync/raft_apply/common.rs +++ b/nodedb/src/control/array_sync/raft_apply/common.rs @@ -126,12 +126,18 @@ pub(super) async fn submit_array_write( // committed plan: the proposer's LSN is deliberately not carried on // the wire, and the array engine's tile state has no other // durability path than this record's replay. - // An array write emits no Data-Plane change event to position. + // The entry's position versions the write's rows alike on every + // replica, and its marker gives restart replay the same version. durability: WalDurability::AppendHere { now_override: resolved_now_ms, apply_key, commit_hlc, - change_position: None, + change_position: Some(crate::event::cdc::position::entry_position( + state, + vshard.as_u32(), + group_id, + log_index, + )), }, // Raft committed this entry at a fixed log index and every replica // applies it in that order; re-entering the write-admission gate @@ -296,6 +302,7 @@ pub(super) fn build_array_request( wal_lsn: None, resolved_now_ms: None, commit_hlc: None, + entry_version: None, admission: crate::bridge::envelope::Admission::Exempt( crate::bridge::envelope::ExemptReason::AlreadyOrdered, ), @@ -351,8 +358,8 @@ mod tests { payload: Payload::empty(), watermark_lsn: Lsn::ZERO, error_code: code.map(Box::new), - read_set_valid: None, - read_version_lsn: Lsn::ZERO, + stage_vote: None, + read_versions: crate::types::ReadVersions::new(), write_set: Vec::new(), } } diff --git a/nodedb/src/control/backup/capture.rs b/nodedb/src/control/backup/capture.rs index d8d58ed4e..d2b0f26ed 100644 --- a/nodedb/src/control/backup/capture.rs +++ b/nodedb/src/control/backup/capture.rs @@ -39,7 +39,7 @@ pub async fn capture_collections( arrays: bool, ) -> Result { let assignment = source_assignment(state)?; - let watermark = super::cut::consistent_cut(state, tenant_id).await?; + let watermark = super::cut::consistent_cut(state).await?; let per_node = join_all( assignment diff --git a/nodedb/src/control/backup/cut.rs b/nodedb/src/control/backup/cut.rs index 02cb48bde..5ea8eed68 100644 --- a/nodedb/src/control/backup/cut.rs +++ b/nodedb/src/control/backup/cut.rs @@ -12,63 +12,68 @@ //! //! Three waits make the cut: //! -//! - **Raft data groups.** A replicated write carries its proposer's commit -//! stamp and applies in log order. The cut proposes a -//! [`ReplicatedWrite::CutBarrier`] carrying `W` into every data group this -//! node hosts and waits for this node's apply of it. Every entry before the -//! barrier applied first. Every entry after it records a commit HLC above -//! `W`, however early its proposer stamped it. A database backup's barrier -//! also captures its tenants at that log position (see -//! [`super::cut_capture`]). //! - **Calvin transactions.** A Calvin transaction commits at its place in //! the sequencer log. The cut proposes a `CutMarker` carrying `W` into the //! sequencer log and waits until every Calvin scheduler on this node passed //! it: every transaction delivered before the marker installed or dropped. //! Every transaction delivered after it records a commit HLC above `W`. +//! - **Raft data groups.** A replicated write carries its proposer's commit +//! stamp and applies in log order. The cut places a +//! [`ReplicatedWrite::CutBarrier`] carrying `W` in every data group and +//! waits for this node's apply of the barrier of each group it hosts. +//! Every entry before the barrier applied first. Every entry after it +//! records a commit HLC above `W`, however early its proposer stamped it. A +//! database backup's barrier also captures its tenants at that log +//! position (see [`super::cut_capture`]). //! - **Local write windows.** A write the funnel appends here stamps itself //! after its record is minted inside an outcome-floor window. The cut reads //! the highest LSN any window minted, after it picks `W`, and waits for the //! outcome floor to reach it. A record minted later stamps itself above `W`. //! +//! A Calvin slice installs from a stamped redo entry its group's leader +//! proposes into the data-group log. The barrier is ordered with the marker +//! (see [`super::cut_order`]): the marker carries the barrier, and each +//! group's leader proposes it once its schedulers passed the marker, while +//! it holds the redo of every later transaction until it applied the +//! barrier. In every group, each slice of a transaction sequenced before the +//! marker sits before the barrier, and each slice of a later one after it, +//! so no transaction is in a capture or a restore point on one group only. +//! //! Two stamps can share a wall time. Once it picks `W`, the cut moves this //! node's clock past `W`, so every later stamp here reads above it. //! //! The waits bind this node's replicas. A remote source node takes the same //! cut at `W` itself before it snapshots: the snapshot plan sent to it carries //! `W`, and its Control Plane runs [`cut_at`] first. +//! +//! [`ReplicatedWrite::CutBarrier`]: crate::control::wal_replication::ReplicatedWrite::CutBarrier -use std::collections::BTreeMap; use std::time::Duration; -use nodedb_cluster::METADATA_GROUP_ID; -use nodedb_cluster::calvin::SEQUENCER_GROUP_ID; -use nodedb_cluster::calvin::SequencerEntry; use nodedb_physical::physical_plan::CutCaptureRequest; use nodedb_types::Hlc; use crate::Error; -use crate::control::security::auth_fence::cluster::{group_of_vshard, hosts_group, routed_groups}; use crate::control::state::SharedState; -use crate::control::wal_replication::{ReplicatedEntry, ReplicatedWrite, propose_replicated_entry}; -use crate::types::{DatabaseId, VShardId}; + +use super::cut_order::OrderedCut; +use super::cut_order::driver::{ensure_driver, hosted_data_groups}; +use super::cut_order::marker::{Marker, place_marker}; +use super::cut_order::wait::await_group_barriers; /// Pick the envelope watermark and wait until every write committed below it /// has a final outcome on this node. Returns the watermark. -pub(super) async fn consistent_cut(state: &SharedState, tenant_id: u64) -> Result { +pub(super) async fn consistent_cut(state: &SharedState) -> Result { let watermark = state.hlc_clock.now().wall_ns; - cut_at(state, tenant_id, watermark).await?; + cut_at(state, watermark).await?; Ok(watermark) } /// Take the consistent cut at `watermark` on this node: wait until every /// write committed below it has a final outcome here. A remote source node /// runs it for the watermark the backup's coordinator picked. -pub(crate) async fn cut_at( - state: &SharedState, - tenant_id: u64, - watermark: u64, -) -> Result<(), Error> { - cut_at_point(state, tenant_id, watermark, 0).await +pub(crate) async fn cut_at(state: &SharedState, watermark: u64) -> Result<(), Error> { + cut_at_point(state, watermark, 0).await } /// [`cut_at`] for the cluster restore point `restore_point`, `0` for a @@ -76,11 +81,18 @@ pub(crate) async fn cut_at( /// every replica records its group's place at the point. pub(crate) async fn cut_at_point( state: &SharedState, - tenant_id: u64, watermark: u64, restore_point: u64, ) -> Result<(), Error> { - cut_barriers(state, tenant_id, watermark, restore_point, None).await + cut_barriers( + state, + OrderedCut { + hlc: watermark, + restore_point, + capture: None, + }, + ) + .await } /// [`cut_at`] whose barriers carry a database backup's capture `request`. @@ -88,35 +100,46 @@ pub(crate) async fn cut_at_point( /// when it applies the request's first barrier in that group. pub(crate) async fn cut_with_capture( state: &SharedState, - tenant_id: u64, watermark: u64, request: &CutCaptureRequest, ) -> Result<(), Error> { - cut_barriers(state, tenant_id, watermark, 0, Some(request)).await + cut_barriers( + state, + OrderedCut { + hlc: watermark, + restore_point: 0, + capture: Some(request.clone()), + }, + ) + .await } -async fn cut_barriers( - state: &SharedState, - tenant_id: u64, - watermark: u64, - restore_point: u64, - capture: Option<&CutCaptureRequest>, -) -> Result<(), Error> { +async fn cut_barriers(state: &SharedState, cut: OrderedCut) -> Result<(), Error> { state .hlc_clock - .update(Hlc::new(watermark.saturating_add(1), 0)); + .update(Hlc::new(cut.hlc.saturating_add(1), 0)); let timeout = Duration::from_secs(state.tuning.network.default_deadline_secs); let deadline = tokio::time::Instant::now() + timeout; // Read after the watermark: a record minted after this read stamps its // write above the watermark. let target = state.outcome_floor.max_noted(); - let (groups, calvin) = tokio::join!( - cut_data_groups(state, tenant_id, watermark, restore_point, capture), - cut_calvin_point(state, watermark, restore_point, deadline), + let groups = hosted_data_groups(state); + if !groups.is_empty() { + // This node places the barrier of every group it leads. + ensure_driver(&state.self_arc()?, &cut); + } + let marker = Marker { + hlc: cut.hlc, + restore_point: cut.restore_point, + barrier: Some(&cut), + }; + let (barriers, calvin) = tokio::join!( + await_group_barriers(state, &cut, &groups, deadline), + place_marker(state, marker, &groups, deadline), ); - groups?; calvin?; + barriers?; if !state.outcome_floor.await_floor(target, deadline).await { return Err(Error::Internal { @@ -134,155 +157,17 @@ async fn cut_barriers( Ok(()) } -/// Propose a cut barrier carrying `watermark` into every data group this node -/// hosts, and wait for this node's apply of each. -async fn cut_data_groups( - state: &SharedState, - tenant_id: u64, - watermark: u64, - restore_point: u64, - capture: Option<&CutCaptureRequest>, -) -> Result<(), Error> { - let proposer = state.async_raft_proposer()?; - let barriers = futures::future::join_all(barrier_vshards(state).into_iter().map( - |(group_id, vshard_id)| { - // The barrier orders every entry of its group, whatever database - // the entry writes, so one barrier per group cuts every database. - // The entry's database id only frames it. - let entry = ReplicatedEntry::new( - tenant_id, - DatabaseId::DEFAULT.as_u64(), - vshard_id, - ReplicatedWrite::CutBarrier { - hlc: watermark, - restore_point, - capture: capture.cloned(), - }, - ); - async move { - propose_replicated_entry(state, proposer, entry) - .await - .map_err(|error| (group_id, error)) - } - }, - )) - .await; - for barrier in barriers { - if let Err((group_id, error)) = barrier { - // A group this node left while the barrier waited holds no - // replica here to cut: the source node that snapshots it takes - // its own cut. - if !hosts_group(state, group_id) { - tracing::info!( - group_id, - %error, - "backup: this node left the group before its cut barrier applied here; \ - the group needs no cut on this node" - ); - continue; - } - return Err(Error::Internal { - detail: format!( - "backup: the consistent-cut barrier of raft group {group_id} did not \ - apply on this node: {error}. Retry the backup" - ), - }); - } - } - Ok(()) -} - -/// How long the cut waits for its Calvin marker before it proposes the -/// marker again. A leader change can drop a proposed marker; a second copy is -/// harmless, since a scheduler passes each marker once its earlier -/// transactions finished. -const CUT_MARKER_RETRY: Duration = Duration::from_secs(1); - -/// Propose a Calvin cut marker carrying `watermark`, and wait until every -/// Calvin scheduler on this node passed it, or `deadline`. +/// Propose a Calvin cut marker carrying `watermark` and no barrier, and wait +/// until every Calvin scheduler on this node passed it, or `deadline`. pub(crate) async fn cut_calvin( state: &SharedState, watermark: u64, deadline: tokio::time::Instant, ) -> Result<(), Error> { - cut_calvin_point(state, watermark, 0, deadline).await -} - -/// [`cut_calvin`] with a marker that carries the cluster restore point -/// `restore_point`, `0` for none. -async fn cut_calvin_point( - state: &SharedState, - watermark: u64, - restore_point: u64, - deadline: tokio::time::Instant, -) -> Result<(), Error> { - let cuts = &state.calvin.cuts; - if cuts.is_empty() { - // No Calvin scheduler runs here: this node applies no Calvin write. - return Ok(()); - } - let proposer = state - .calvin - .sequencer_proposer - .get() - .ok_or_else(|| Error::Internal { - detail: "backup: Calvin schedulers run on this node, but no sequencer proposer \ - is set, so the consistent cut cannot place its marker. Retry the backup \ - once the cluster finished starting" - .into(), - })?; - let marker = zerompk::to_msgpack_vec(&SequencerEntry::CutMarker { + let marker = Marker { hlc: watermark, - restore_point, - }) - .map_err(|error| Error::Internal { - detail: format!("backup: encode the Calvin cut marker: {error}"), - })?; - let mut last_refusal = None; - loop { - if let Err(error) = proposer.propose(marker.clone()) { - last_refusal = Some(error.to_string()); - } - let attempt_deadline = deadline.min(tokio::time::Instant::now() + CUT_MARKER_RETRY); - let lagging = cuts.await_passed(watermark, attempt_deadline).await; - if lagging.is_empty() { - return Ok(()); - } - if tokio::time::Instant::now() >= deadline { - return Err(Error::Internal { - detail: format!( - "backup: the Calvin schedulers of vShards {lagging:?} did not pass the \ - consistent-cut marker in time, so Calvin transactions sequenced before \ - the backup may not be installed (last marker refusal: {}). Retry the \ - backup", - last_refusal.as_deref().unwrap_or("none") - ), - }); - } - } -} - -/// One vShard per data group this node hosts: the barrier of a group is -/// proposed through any vShard the group homes. -fn barrier_vshards(state: &SharedState) -> BTreeMap { - let hosted: Vec = routed_groups(state) - .into_iter() - .filter(|group_id| { - *group_id != METADATA_GROUP_ID - && *group_id != SEQUENCER_GROUP_ID - && hosts_group(state, *group_id) - }) - .collect(); - let mut vshards = BTreeMap::new(); - for vshard_id in 0..VShardId::COUNT { - if vshards.len() == hosted.len() { - break; - } - if let Ok(group_id) = group_of_vshard(state, vshard_id) - && hosted.contains(&group_id) - { - vshards.entry(group_id).or_insert(vshard_id); - } - } - vshards + restore_point: 0, + barrier: None, + }; + place_marker(state, marker, &[], deadline).await } diff --git a/nodedb/src/control/backup/cut_capture/collect.rs b/nodedb/src/control/backup/cut_capture/collect.rs index f37668f4d..61f916827 100644 --- a/nodedb/src/control/backup/cut_capture/collect.rs +++ b/nodedb/src/control/backup/cut_capture/collect.rs @@ -4,7 +4,7 @@ //! //! The coordinator picks the cut watermark and a request id, then takes the //! cut with capturing barriers on its own Raft data groups, a one-node -//! cluster's included. Every other source node receives the request, takes +//! cluster's included. Each group's leader places the group's one barrier. Every other source node receives the request, takes //! the same cut on its groups, and answers with the captures it parked. Each //! group's capture comes from its leader at the apply of the request's first //! barrier. A group with no capture fails the backup with a retryable error @@ -66,10 +66,10 @@ pub(crate) async fn capture_database( database_id: database_id.as_u64(), tenants: tenants.iter().copied().collect(), }; - // The barrier names a tenant only to frame its Raft entry. + // The snapshot request names a tenant only to frame it. let framing = tenants.first().copied().unwrap_or(0); - super::super::cut::cut_with_capture(state, framing, cut, &request).await?; + super::super::cut::cut_with_capture(state, cut, &request).await?; let mut replies = take_replies(state, request.request_id); let remote: Vec = source_assignment(state)? .into_iter() @@ -101,7 +101,11 @@ pub(crate) async fn capture_database( // A test parks the backup here: every capture is taken, so a write from // now on lies above the cut. #[cfg(feature = "failpoints")] - crate::control::fail_gate::wait("backup::database::after_cut").await; + crate::control::fail_gate::wait( + crate::fail_point::FailScope::Node(state.node_id), + "backup::database::after_cut", + ) + .await; assemble(state, cut, replies) } @@ -130,11 +134,10 @@ pub(crate) fn take_replies(state: &SharedState, request: u64) -> Vec /// the encoded captures this node parked. pub(crate) async fn cut_and_reply( state: &SharedState, - framing: u64, watermark: u64, request: &CutCaptureRequest, ) -> Result, Error> { - super::super::cut::cut_with_capture(state, framing, watermark, request).await?; + super::super::cut::cut_with_capture(state, watermark, request).await?; zerompk::to_msgpack_vec(&take_replies(state, request.request_id)).map_err(|e| { Error::Serialization { format: "msgpack".into(), diff --git a/nodedb/src/control/backup/cut_capture/registry.rs b/nodedb/src/control/backup/cut_capture/registry.rs index 1d4e17055..9fd741ce8 100644 --- a/nodedb/src/control/backup/cut_capture/registry.rs +++ b/nodedb/src/control/backup/cut_capture/registry.rs @@ -5,8 +5,9 @@ //! //! Every replica of a group applies the same log, so every replica sees the //! same first barrier of a request in that group. Only that first barrier -//! captures. A later barrier of the same request, proposed by another node, -//! sits above entries the first barrier already cut away, and never captures. +//! captures. A later barrier of the same request, a leader's second proposal +//! of it, sits above entries the first barrier already cut away, and never +//! captures. use std::collections::HashMap; use std::sync::Mutex; diff --git a/nodedb/src/control/backup/cut_order/driver.rs b/nodedb/src/control/backup/cut_order/driver.rs new file mode 100644 index 000000000..18ebb9772 --- /dev/null +++ b/nodedb/src/control/backup/cut_order/driver.rs @@ -0,0 +1,227 @@ +// SPDX-License-Identifier: BUSL-1.1 + +//! The task that places one ordered cut's barrier in every data group this +//! node leads. +//! +//! A node starts the driver when it learns of the cut: its coordinator takes +//! the cut here, or a scheduler here receives the cut's marker. The driver +//! runs until every data group this node hosts applied the cut's barrier +//! here, or the cut's window closes. While this node leads a group, the +//! driver proposes the barrier once every scheduler of the group's vShards +//! on this node passed the cut's marker: every transaction sequenced before +//! the marker then installed here, so its redo is in the log before the +//! barrier. A proposal a later term overwrote is proposed again. + +use std::sync::{Arc, Mutex}; +use std::time::Duration; + +use nodedb_cluster::calvin::SEQUENCER_GROUP_ID; +use nodedb_cluster::{METADATA_GROUP_ID, MultiRaft}; + +use crate::control::security::auth_fence::cluster::{hosts_group, routed_groups}; +use crate::control::state::SharedState; +use crate::control::wal_replication::ReplicatedEntry; +use crate::types::DatabaseId; + +use super::ordered_cut::{CutKey, CutWindow, OrderedCut}; +use super::registry::Proposed; + +/// How often a driver looks again when no change woke it. A leadership move +/// and a group applying past a lost proposal wake nothing. +const DRIVE_POLL: Duration = Duration::from_millis(20); + +/// The tenant a barrier entry is framed under. A barrier orders every entry +/// of its group, whatever tenant writes it. No step reads the tenant of a +/// barrier: the apply's barrier arm ignores it, and a barrier raises no +/// tenant mark. +const BARRIER_FRAME_TENANT: u64 = 0; + +/// Every data group this node hosts. +pub(crate) fn hosted_data_groups(state: &SharedState) -> Vec { + routed_groups(state) + .into_iter() + .filter(|group_id| { + *group_id != METADATA_GROUP_ID + && *group_id != SEQUENCER_GROUP_ID + && hosts_group(state, *group_id) + }) + .collect() +} + +/// Every vShard `group_id` homes, from this node's routing table. +fn group_vshards(state: &SharedState, group_id: u64) -> Vec { + state + .cluster_routing + .as_ref() + .map_or_else(Vec::new, |routing| { + routing + .read() + .unwrap_or_else(|p| p.into_inner()) + .vshards_for_group(group_id) + }) +} + +/// Start the driver of `cut` on this node, unless one runs for it. +pub(crate) fn ensure_driver(state: &Arc, cut: &OrderedCut) { + if !state.calvin.cut_barriers.claim_driver(cut) { + return; + } + let key = cut.key(); + let state = Arc::clone(state); + tokio::spawn(async move { + drive(&state, key).await; + state.calvin.cut_barriers.driver_stopped(key); + }); +} + +/// Place `key`'s barrier in every group this node leads, until every group +/// this node hosts applied it here or the cut's window closes. +async fn drive(state: &Arc, key: CutKey) { + let Some(cut) = state.calvin.cut_barriers.cut(key) else { + return; + }; + let window = CutWindow::on(state, cut.hlc); + let mut shutdown = state.shutdown.subscribe(); + loop { + // Registered before the checks, so a change between a check and the + // wait still wakes it. + let changed = state.calvin.cut_barriers.notified(); + tokio::pin!(changed); + changed.as_mut().enable(); + let open: Vec = hosted_data_groups(state) + .into_iter() + .filter(|group_id| state.calvin.cut_barriers.applied(key, *group_id).is_none()) + .collect(); + if open.is_empty() { + return; + } + // no-determinism: the window bounds only when this leader proposes. + // The barrier's place in the log is the one every replica applies. + if state.hlc_clock.now().wall_ns > window.propose_until { + for group_id in open { + report_unplaced(state, &cut, group_id); + } + return; + } + for group_id in open { + if let Err(error) = propose_if_due(state, &cut, group_id) { + tracing::error!( + group_id, + hlc = cut.hlc, + %error, + "backup cut: the cut's barrier entry cannot be built; the cut fails" + ); + report_unplaced(state, &cut, group_id); + return; + } + } + tokio::select! { + () = shutdown.wait_cancelled() => return, + () = &mut changed => {} + () = tokio::time::sleep(DRIVE_POLL) => {} + } + } +} + +/// This node's leader term of `group_id`, while it leads the group. +fn leader_term(multi_raft: &Mutex, group_id: u64) -> Option { + multi_raft + .lock() + .unwrap_or_else(|p| p.into_inner()) + .leader_term(group_id) +} + +/// Propose `cut`'s barrier into `group_id` when this node leads the group, +/// every scheduler of the group's vShards here passed the cut's marker, and +/// no proposal of the current term is still in the log. Fails only when the +/// barrier entry cannot be built. +fn propose_if_due(state: &SharedState, cut: &OrderedCut, group_id: u64) -> crate::Result<()> { + let Some(multi_raft) = state.multi_raft.get() else { + return Ok(()); + }; + let key = cut.key(); + let Some(term) = leader_term(multi_raft, group_id) else { + return Ok(()); + }; + if let Some(proposed) = state.calvin.cut_barriers.proposal(key, group_id) { + // An entry of an earlier term that the group applied past with no + // barrier of the cut is gone: a later term overwrote it. + let lost = proposed.term != term + || state.applied_index_watcher(group_id).current() >= proposed.log_index; + if !lost { + return Ok(()); + } + state + .calvin + .cut_barriers + .forget_proposal(key, group_id, proposed); + if state.calvin.cut_barriers.applied(key, group_id).is_some() { + return Ok(()); + } + } + let vshards = group_vshards(state, group_id); + if !state + .calvin + .cuts + .lagging_among(cut.hlc, &vshards) + .is_empty() + { + return Ok(()); + } + let Some(&vshard_id) = vshards.first() else { + return Ok(()); + }; + let bytes = ReplicatedEntry::new( + BARRIER_FRAME_TENANT, + DatabaseId::DEFAULT.as_u64(), + vshard_id, + cut.barrier_write(), + ) + .encode()?; + let landed = { + let mut mr = multi_raft.lock().unwrap_or_else(|p| p.into_inner()); + // Leadership can move between the checks and the proposal. + if mr.leader_term(group_id) != Some(term) { + return Ok(()); + } + mr.propose(vshard_id, bytes) + }; + match landed { + Ok((landed_group, log_index)) => { + state + .calvin + .cut_barriers + .note_proposed(key, landed_group, Proposed { term, log_index }) + } + Err(error) => tracing::debug!( + group_id, + hlc = cut.hlc, + %error, + "backup cut: the group refused the cut's barrier; the driver proposes it again" + ), + } + Ok(()) +} + +/// Report that the cut's window closed before `group_id` applied its +/// barrier here. +fn report_unplaced(state: &SharedState, cut: &OrderedCut, group_id: u64) { + let led_here = state + .multi_raft + .get() + .is_some_and(|multi_raft| leader_term(multi_raft, group_id).is_some()); + let lagging = state + .calvin + .cuts + .lagging_among(cut.hlc, &group_vshards(state, group_id)); + tracing::warn!( + group_id, + hlc = cut.hlc, + restore_point = cut.restore_point, + led_here, + ?lagging, + "backup cut: the cut's window closed before its barrier applied in this group; \ + the backup or restore point that took the cut failed" + ); + crate::diag::cut_barrier_not_placed(group_id, cut.hlc, cut.restore_point, led_here, lagging); +} diff --git a/nodedb/src/control/backup/cut_order/marker.rs b/nodedb/src/control/backup/cut_order/marker.rs new file mode 100644 index 000000000..1600292c2 --- /dev/null +++ b/nodedb/src/control/backup/cut_order/marker.rs @@ -0,0 +1,106 @@ +// SPDX-License-Identifier: BUSL-1.1 + +//! A cut's Calvin marker: proposed into the sequencer log until it took. + +use std::time::Duration; + +use nodedb_cluster::calvin::SequencerEntry; + +use crate::Error; +use crate::control::state::SharedState; + +use super::ordered_cut::OrderedCut; +use super::wait::await_placed; + +/// How long the cut waits for its Calvin marker before it proposes the +/// marker again. A leader change can drop a proposed marker. A second copy +/// is harmless: it names the same cut, and every scheduler keeps the place +/// of the first copy it received. +const CUT_MARKER_RETRY: Duration = Duration::from_secs(1); + +/// What one cut marker carries. +#[derive(Clone, Copy)] +pub(crate) struct Marker<'a> { + pub hlc: u64, + pub restore_point: u64, + /// The cut whose barriers the marker orders, `None` for a marker with no + /// barrier. + pub barrier: Option<&'a OrderedCut>, +} + +/// Propose `marker` into the sequencer log, and wait until every Calvin +/// scheduler on this node passed it, or `deadline`. +/// +/// With no scheduler here, nothing here shows that the marker applied. An +/// ordered cut then proposes the marker again until the barrier of every +/// group of `groups` applied here: the groups' leaders place the barriers +/// only once their schedulers passed the marker. +pub(crate) async fn place_marker( + state: &SharedState, + marker: Marker<'_>, + groups: &[u64], + deadline: tokio::time::Instant, +) -> Result<(), Error> { + let cuts = &state.calvin.cuts; + let calvin_here = !cuts.is_empty(); + // With no scheduler here, the barriers of this node's groups show that + // the marker applied. + let watched = marker + .barrier + .filter(|_| !calvin_here && !groups.is_empty()); + if !calvin_here && watched.is_none() { + // No Calvin scheduler runs here and no group here waits on the + // marker: this node applies no Calvin write. + return Ok(()); + } + let proposer = state + .calvin + .sequencer_proposer + .get() + .ok_or_else(|| Error::Internal { + detail: "backup: this node takes a Calvin cut, but no sequencer proposer is set, so \ + the consistent cut cannot place its marker. Retry the backup once the \ + cluster finished starting" + .into(), + })?; + let entry = zerompk::to_msgpack_vec(&SequencerEntry::CutMarker { + hlc: marker.hlc, + restore_point: marker.restore_point, + barrier: marker.barrier.map(OrderedCut::to_wire), + }) + .map_err(|error| Error::Internal { + detail: format!("backup: encode the Calvin cut marker: {error}"), + })?; + let mut last_refusal = None; + loop { + if let Err(error) = proposer.propose(entry.clone()) { + last_refusal = Some(error.to_string()); + } + let attempt_deadline = deadline.min(tokio::time::Instant::now() + CUT_MARKER_RETRY); + let lagging = cuts.await_passed(marker.hlc, attempt_deadline).await; + if lagging.is_empty() { + let Some(cut) = watched else { + return Ok(()); + }; + if await_placed(state, cut.key(), groups, attempt_deadline).await + || tokio::time::Instant::now() >= deadline + { + // A group with no barrier by the deadline fails the cut's + // barrier wait, which names it. + return Ok(()); + } + continue; + } + if tokio::time::Instant::now() >= deadline { + return Err(Error::Internal { + detail: format!( + "backup: the Calvin schedulers of vShards {lagging:?} did not pass the \ + consistent-cut marker in time, so Calvin transactions sequenced before \ + the backup may not be installed (last marker refusal: {}). Retry the \ + backup", + last_refusal.as_deref().unwrap_or("none") + ), + }); + } + } +} diff --git a/nodedb/src/control/backup/cut_order/mod.rs b/nodedb/src/control/backup/cut_order/mod.rs new file mode 100644 index 000000000..73d10b7f7 --- /dev/null +++ b/nodedb/src/control/backup/cut_order/mod.rs @@ -0,0 +1,10 @@ +// SPDX-License-Identifier: BUSL-1.1 + +pub mod driver; +pub mod marker; +pub mod ordered_cut; +pub mod registry; +pub mod wait; + +pub use ordered_cut::{CutKey, CutWindow, OrderedCut}; +pub use registry::CutBarriers; diff --git a/nodedb/src/control/backup/cut_order/ordered_cut.rs b/nodedb/src/control/backup/cut_order/ordered_cut.rs new file mode 100644 index 000000000..29b6fa860 --- /dev/null +++ b/nodedb/src/control/backup/cut_order/ordered_cut.rs @@ -0,0 +1,177 @@ +// SPDX-License-Identifier: BUSL-1.1 + +//! One cut whose data-group barriers are ordered with its Calvin marker. +//! +//! The cut's marker carries the barrier. In every data group, the leader +//! proposes the barrier once every scheduler of the group's vShards on that +//! node passed the marker, and holds the redo of every later transaction +//! until it applied the barrier. A transaction sequenced before the marker +//! installed before its scheduler passed it, so its redo sits before the +//! barrier. A later one proposes its redo only after the barrier. +//! +//! The ordering has a window, measured from the cut's watermark on the HLC +//! wall clock. A leader proposes the barrier only inside +//! [`CutWindow::propose_until`]. A scheduler holds later redo only until +//! [`CutWindow::hold_until`], a clock-skew margin later. A leader that +//! released its holds at the end of the window therefore never sees a new +//! leader place the barrier after the released redo, while clock skew stays +//! below the margin. A cut whose window closed with no barrier in a group +//! failed: its coordinator's deadline ends inside the window. + +use std::time::Duration; + +use nodedb_cluster::calvin::{CutBarrierWire, CutCaptureWire}; +use nodedb_physical::physical_plan::CutCaptureRequest; + +use crate::control::state::SharedState; +use crate::control::wal_replication::ReplicatedWrite; + +/// The clock skew between two nodes a cut's window tolerates. +const CLOCK_SKEW_MARGIN: Duration = Duration::from_secs(30); + +/// What names one cut on every node: its watermark, its restore point, and +/// its capture request, `0` for none. +#[derive(Debug, Clone, Copy, PartialEq, Eq, PartialOrd, Ord, Hash)] +pub struct CutKey { + pub hlc: u64, + pub restore_point: u64, + pub capture: u64, +} + +/// A cut whose barriers the data-group leaders order with its marker. +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct OrderedCut { + /// The cut's watermark: HLC wall time in nanoseconds. + pub hlc: u64, + /// The cluster restore point the cut takes, `0` for a backup's cut. + pub restore_point: u64, + /// A database backup's capture request. + pub capture: Option, +} + +impl OrderedCut { + pub fn key(&self) -> CutKey { + CutKey { + hlc: self.hlc, + restore_point: self.restore_point, + capture: self + .capture + .as_ref() + .map_or(0, |request| request.request_id), + } + } + + /// The cut a marker carrying `hlc`, `restore_point` and `barrier` names. + pub fn from_marker(hlc: u64, restore_point: u64, barrier: &CutBarrierWire) -> Self { + Self { + hlc, + restore_point, + capture: barrier.capture.as_ref().map(|wire| CutCaptureRequest { + request_id: wire.request_id, + database_id: wire.database_id, + tenants: wire.tenants.clone(), + }), + } + } + + /// The barrier this cut's marker carries. + pub fn to_wire(&self) -> CutBarrierWire { + CutBarrierWire { + capture: self.capture.as_ref().map(|request| CutCaptureWire { + request_id: request.request_id, + database_id: request.database_id, + tenants: request.tenants.clone(), + }), + } + } + + /// The write of this cut's barrier entry. + pub fn barrier_write(&self) -> ReplicatedWrite { + ReplicatedWrite::CutBarrier { + hlc: self.hlc, + restore_point: self.restore_point, + capture: self.capture.clone(), + } + } +} + +/// The ordering window of one cut on this node. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub struct CutWindow { + /// The last HLC wall time a leader proposes the cut's barrier at. + pub propose_until: u64, + /// The last HLC wall time a scheduler holds later redo at. + pub hold_until: u64, +} + +impl CutWindow { + /// The window of the cut at `hlc`, given the statement deadline + /// `deadline`. A coordinator waits one deadline from its watermark, and + /// a remote source node one more from the request it receives. + pub fn of(hlc: u64, deadline: Duration) -> Self { + let propose = deadline.saturating_mul(2); + let propose_until = hlc.saturating_add(nanos(propose)); + Self { + propose_until, + hold_until: propose_until.saturating_add(nanos(CLOCK_SKEW_MARGIN)), + } + } + + /// The window of the cut at `hlc` under `state`'s statement deadline. + pub fn on(state: &SharedState, hlc: u64) -> Self { + Self::of( + hlc, + Duration::from_secs(state.tuning.network.default_deadline_secs), + ) + } +} + +/// `duration` in nanoseconds, saturating. +fn nanos(duration: Duration) -> u64 { + u64::try_from(duration.as_nanos()).unwrap_or(u64::MAX) +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn a_cut_round_trips_through_its_marker() { + let cut = OrderedCut { + hlc: 7, + restore_point: 3, + capture: Some(CutCaptureRequest { + request_id: 11, + database_id: 2, + tenants: vec![1, 4], + }), + }; + let back = OrderedCut::from_marker(7, 3, &cut.to_wire()); + assert_eq!(back, cut); + assert_eq!( + back.key(), + CutKey { + hlc: 7, + restore_point: 3, + capture: 11 + } + ); + let plain = OrderedCut { + hlc: 7, + restore_point: 0, + capture: None, + }; + assert_eq!(plain.key().capture, 0); + assert_ne!(plain.key(), cut.key(), "the capture keeps two cuts apart"); + } + + #[test] + fn the_hold_outlasts_the_proposal_window_by_the_skew_margin() { + let window = CutWindow::of(1_000, Duration::from_secs(30)); + assert_eq!(window.propose_until, 1_000 + 60_000_000_000); + assert_eq!( + window.hold_until - window.propose_until, + nanos(CLOCK_SKEW_MARGIN) + ); + } +} diff --git a/nodedb/src/control/backup/cut_order/registry.rs b/nodedb/src/control/backup/cut_order/registry.rs new file mode 100644 index 000000000..dcdb15e71 --- /dev/null +++ b/nodedb/src/control/backup/cut_order/registry.rs @@ -0,0 +1,271 @@ +// SPDX-License-Identifier: BUSL-1.1 + +//! Where a node tracks each ordered cut's barrier per data group. +//! +//! For every cut it knows, the node keeps, per group: +//! +//! - where it proposed the cut's barrier while it led the group, as the +//! leader term and log index; +//! - the log index of the first barrier of the cut it applied in the group. +//! +//! A leader's schedulers hold the redo of every transaction sequenced after +//! a cut's marker until the group's barrier applied here. An applied entry is +//! committed, so every redo proposed afterwards sits after it in the log, +//! whatever term the proposer leads. +//! +//! One driver task per cut runs on a node (see [`super::driver`]). It +//! proposes the barrier of every group this node leads, and proposes it +//! again when a later term overwrote the proposal. + +use std::collections::BTreeMap; +use std::sync::Mutex; +use std::time::{Duration, Instant}; + +use tokio::sync::Notify; +use tokio::sync::futures::Notified; + +use super::ordered_cut::{CutKey, OrderedCut}; + +/// How long a node remembers a cut no driver runs for. Every hold and every +/// wait on a cut ends within its window, far below this. +const CUT_TTL: Duration = Duration::from_secs(30 * 60); + +/// Where this node proposed a cut's barrier. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub struct Proposed { + /// The group's leader term the proposal was made in. + pub term: u64, + pub log_index: u64, +} + +#[derive(Debug, Default, Clone, Copy)] +struct GroupBarrier { + proposed: Option, + applied: Option, +} + +#[derive(Debug)] +struct TrackedCut { + cut: OrderedCut, + groups: BTreeMap, + driving: bool, + touched: Instant, +} + +impl TrackedCut { + fn new(cut: &OrderedCut, now: Instant) -> Self { + Self { + cut: cut.clone(), + groups: BTreeMap::new(), + driving: false, + touched: now, + } + } +} + +/// Every ordered cut this node knows, by key. +#[derive(Debug, Default)] +pub struct CutBarriers { + cuts: Mutex>, + changed: Notify, +} + +impl CutBarriers { + pub fn new() -> Self { + Self::default() + } + + /// Track `cut` and claim its driver. `true` when no driver ran for it: + /// the caller starts one. + pub(crate) fn claim_driver(&self, cut: &OrderedCut) -> bool { + self.with_cut(cut, |tracked| { + !std::mem::replace(&mut tracked.driving, true) + }) + } + + /// The driver of `key` stopped. A later claim starts a new one. + pub(crate) fn driver_stopped(&self, key: CutKey) { + let mut cuts = self.cuts.lock().unwrap_or_else(|p| p.into_inner()); + if let Some(tracked) = cuts.get_mut(&key) { + tracked.driving = false; + tracked.touched = Instant::now(); + } + } + + /// The cut `key` names, while this node tracks it. + pub(crate) fn cut(&self, key: CutKey) -> Option { + self.cuts + .lock() + .unwrap_or_else(|p| p.into_inner()) + .get(&key) + .map(|tracked| tracked.cut.clone()) + } + + /// Note a barrier of `cut` this node applies at `log_index` of `group`. + /// The first barrier of the cut in the group keeps its index. + pub(crate) fn note_applied(&self, cut: &OrderedCut, group: u64, log_index: u64) { + self.with_cut(cut, |tracked| { + let barrier = tracked.groups.entry(group).or_default(); + barrier.applied.get_or_insert(log_index); + }); + self.changed.notify_waiters(); + } + + /// The log index of the first barrier of `key` this node applied in + /// `group`. + pub(crate) fn applied(&self, key: CutKey, group: u64) -> Option { + self.group(key, group).and_then(|barrier| barrier.applied) + } + + /// Note that this node proposed `key`'s barrier into `group` while it + /// led the group. + pub(crate) fn note_proposed(&self, key: CutKey, group: u64, proposed: Proposed) { + { + let mut cuts = self.cuts.lock().unwrap_or_else(|p| p.into_inner()); + if let Some(tracked) = cuts.get_mut(&key) { + tracked.groups.entry(group).or_default().proposed = Some(proposed); + } + } + self.changed.notify_waiters(); + } + + /// Where this node proposed `key`'s barrier into `group`. + pub(crate) fn proposal(&self, key: CutKey, group: u64) -> Option { + self.group(key, group).and_then(|barrier| barrier.proposed) + } + + /// Forget the proposal `lost` of `key` in `group`: a later term + /// overwrote it. A newer proposal stays. + pub(crate) fn forget_proposal(&self, key: CutKey, group: u64, lost: Proposed) { + let mut cuts = self.cuts.lock().unwrap_or_else(|p| p.into_inner()); + if let Some(barrier) = cuts + .get_mut(&key) + .and_then(|tracked| tracked.groups.get_mut(&group)) + && barrier.proposed == Some(lost) + { + barrier.proposed = None; + } + } + + /// A wait that resolves at the next change of any cut. Enable it before + /// the check it guards. + pub(crate) fn notified(&self) -> Notified<'_> { + self.changed.notified() + } + + fn group(&self, key: CutKey, group: u64) -> Option { + self.cuts + .lock() + .unwrap_or_else(|p| p.into_inner()) + .get(&key) + .and_then(|tracked| tracked.groups.get(&group).copied()) + } + + /// Apply `update` to the tracked `cut`, tracking it when new. Drops every + /// cut past its TTL that no driver runs for. + fn with_cut(&self, cut: &OrderedCut, update: impl FnOnce(&mut TrackedCut) -> T) -> T { + let now = Instant::now(); + let mut cuts = self.cuts.lock().unwrap_or_else(|p| p.into_inner()); + cuts.retain(|_, tracked| { + tracked.driving || now.saturating_duration_since(tracked.touched) < CUT_TTL + }); + let tracked = cuts + .entry(cut.key()) + .or_insert_with(|| TrackedCut::new(cut, now)); + tracked.touched = now; + update(tracked) + } +} + +#[cfg(test)] +mod tests { + use super::*; + + fn cut(hlc: u64) -> OrderedCut { + OrderedCut { + hlc, + restore_point: 0, + capture: None, + } + } + + #[test] + fn one_driver_runs_per_cut() { + let barriers = CutBarriers::new(); + assert!(barriers.claim_driver(&cut(5))); + assert!(!barriers.claim_driver(&cut(5)), "a copy finds the driver"); + assert!(barriers.claim_driver(&cut(6)), "another cut has its own"); + barriers.driver_stopped(cut(5).key()); + assert!( + barriers.claim_driver(&cut(5)), + "a stopped driver starts again" + ); + } + + #[test] + fn the_first_applied_barrier_keeps_its_index() { + let barriers = CutBarriers::new(); + let key = cut(5).key(); + assert_eq!(barriers.applied(key, 1), None); + barriers.note_applied(&cut(5), 1, 40); + barriers.note_applied(&cut(5), 1, 52); + assert_eq!(barriers.applied(key, 1), Some(40)); + assert_eq!(barriers.applied(key, 2), None, "a barrier binds its group"); + assert_eq!(barriers.cut(key), Some(cut(5))); + } + + /// A lost proposal is forgotten only when it is the one recorded. + #[test] + fn a_lost_proposal_is_forgotten_once() { + let barriers = CutBarriers::new(); + let key = cut(5).key(); + barriers.claim_driver(&cut(5)); + assert_eq!(barriers.proposal(key, 1), None); + + let proposed = Proposed { + term: 3, + log_index: 17, + }; + barriers.note_proposed(key, 1, proposed); + assert_eq!(barriers.proposal(key, 1), Some(proposed)); + assert_eq!(barriers.proposal(key, 2), None, "another group"); + + barriers.forget_proposal( + key, + 1, + Proposed { + term: 2, + log_index: 9, + }, + ); + assert_eq!( + barriers.proposal(key, 1), + Some(proposed), + "an older loss keeps the newer proposal" + ); + barriers.forget_proposal(key, 1, proposed); + assert_eq!(barriers.proposal(key, 1), None); + } + + #[tokio::test] + async fn an_applied_barrier_wakes_a_waiter() { + let barriers = std::sync::Arc::new(CutBarriers::new()); + let waiter = std::sync::Arc::clone(&barriers); + let woken = tokio::spawn(async move { + let notified = waiter.notified(); + tokio::pin!(notified); + notified.as_mut().enable(); + if waiter.applied(cut(5).key(), 1).is_none() { + notified.await; + } + waiter.applied(cut(5).key(), 1) + }); + tokio::task::yield_now().await; + barriers.note_applied(&cut(5), 1, 8); + let applied = tokio::time::timeout(Duration::from_secs(5), woken) + .await + .expect("the waiter wakes") + .expect("the waiter runs"); + assert_eq!(applied, Some(8)); + } +} diff --git a/nodedb/src/control/backup/cut_order/wait.rs b/nodedb/src/control/backup/cut_order/wait.rs new file mode 100644 index 000000000..20959e8a9 --- /dev/null +++ b/nodedb/src/control/backup/cut_order/wait.rs @@ -0,0 +1,157 @@ +// SPDX-License-Identifier: BUSL-1.1 + +//! A cut's wait on this node for the barrier of each data group it hosts. + +use std::time::Duration; + +use crate::Error; +use crate::control::security::auth_fence::cluster::{hosts_group, wait_applied}; +use crate::control::state::SharedState; + +use super::ordered_cut::{CutKey, OrderedCut}; + +/// How often a wait looks again when no change woke it. A group this node +/// leaves wakes nothing. +const WAIT_POLL: Duration = Duration::from_millis(50); + +/// Wait until every group of `groups` applied `cut`'s barrier here, and +/// every entry before the barrier applied here, or `deadline`. A group this +/// node left needs no wait: the source node that snapshots it takes its own +/// cut. +pub(crate) async fn await_group_barriers( + state: &SharedState, + cut: &OrderedCut, + groups: &[u64], + deadline: tokio::time::Instant, +) -> Result<(), Error> { + let waits = futures::future::join_all( + groups + .iter() + .map(|group_id| await_group_barrier(state, cut, *group_id, deadline)), + ) + .await; + waits.into_iter().collect() +} + +/// Wait until every group of `groups` applied the barrier of `key` here, or +/// `until`. A group this node left counts as placed. Returns whether every +/// group's barrier applied. +pub(crate) async fn await_placed( + state: &SharedState, + key: CutKey, + groups: &[u64], + until: tokio::time::Instant, +) -> bool { + loop { + let changed = state.calvin.cut_barriers.notified(); + tokio::pin!(changed); + changed.as_mut().enable(); + let placed = groups.iter().all(|group_id| { + state.calvin.cut_barriers.applied(key, *group_id).is_some() + || !hosts_group(state, *group_id) + }); + if placed { + return true; + } + if tokio::time::Instant::now() >= until { + return false; + } + tokio::select! { + () = &mut changed => {} + () = tokio::time::sleep(WAIT_POLL) => {} + () = tokio::time::sleep_until(until) => {} + } + } +} + +async fn await_group_barrier( + state: &SharedState, + cut: &OrderedCut, + group_id: u64, + deadline: tokio::time::Instant, +) -> Result<(), Error> { + let key = cut.key(); + loop { + // Registered before the checks, so a change between a check and the + // wait still wakes it. + let changed = state.calvin.cut_barriers.notified(); + tokio::pin!(changed); + changed.as_mut().enable(); + if let Some(barrier_index) = state.calvin.cut_barriers.applied(key, group_id) { + let remaining = deadline.saturating_duration_since(tokio::time::Instant::now()); + return match wait_applied(state, group_id, barrier_index, remaining).await { + Ok(()) => Ok(()), + Err(error) if !hosts_group(state, group_id) => { + note_left(group_id, &error); + Ok(()) + } + Err(error) => Err(Error::Internal { + detail: format!( + "backup: the consistent-cut barrier of raft group {group_id} at log \ + index {barrier_index} did not apply on this node: {error}. Retry the \ + backup" + ), + }), + }; + } + if !hosts_group(state, group_id) { + note_left(group_id, &"no barrier applied yet"); + return Ok(()); + } + if tokio::time::Instant::now() >= deadline { + return Err(unplaced_error(state, cut, group_id)); + } + tokio::select! { + () = &mut changed => {} + () = tokio::time::sleep(WAIT_POLL) => {} + () = tokio::time::sleep_until(deadline) => {} + } + } +} + +fn note_left(group_id: u64, why: &dyn std::fmt::Display) { + tracing::info!( + group_id, + %why, + "backup: this node left the group before its cut barrier applied here; the group \ + needs no cut on this node" + ); +} + +/// The error of a group whose barrier did not apply here by the deadline. +fn unplaced_error(state: &SharedState, cut: &OrderedCut, group_id: u64) -> Error { + let leads = state.multi_raft.get().is_some_and(|multi_raft| { + multi_raft + .lock() + .unwrap_or_else(|p| p.into_inner()) + .leader_term(group_id) + .is_some() + }); + let place = if leads { + let vshards = state + .cluster_routing + .as_ref() + .map_or_else(Vec::new, |routing| { + routing + .read() + .unwrap_or_else(|p| p.into_inner()) + .vshards_for_group(group_id) + }); + let lagging = state.calvin.cuts.lagging_among(cut.hlc, &vshards); + format!( + "This node leads the group. The Calvin schedulers of its vShards {lagging:?} here \ + have not passed the cut's marker" + ) + } else { + "Another node leads the group and places the barrier once its Calvin schedulers \ + passed the cut's marker" + .to_owned() + }; + Error::Internal { + detail: format!( + "backup: raft group {group_id} applied no consistent-cut barrier at watermark {} on \ + this node in time. {place}. Retry the backup", + cut.hlc + ), + } +} diff --git a/nodedb/src/control/backup/mod.rs b/nodedb/src/control/backup/mod.rs index d266eaab2..245c80885 100644 --- a/nodedb/src/control/backup/mod.rs +++ b/nodedb/src/control/backup/mod.rs @@ -4,6 +4,7 @@ pub mod bind_capture; pub mod capture; pub mod cut; pub mod cut_capture; +pub mod cut_order; pub mod database; pub mod detect; pub mod metadata; diff --git a/nodedb/src/control/backup/orchestrator.rs b/nodedb/src/control/backup/orchestrator.rs index 1ebd1d5db..04d2b4c1d 100644 --- a/nodedb/src/control/backup/orchestrator.rs +++ b/nodedb/src/control/backup/orchestrator.rs @@ -52,7 +52,7 @@ pub async fn backup_tenant(state: &Arc, tenant_id: u64) -> Result { })? .with_event_source(EventSource::Restore) .with_restore_id(self.target.restore_id); - crate::control::wal_replication::propose_replicated_entry(state, proposer, entry).await?; + crate::control::wal_replication::propose_replicated_entry( + state, + proposer, + entry, + crate::control::wal_replication::statement_propose_deadline(state), + ) + .await?; Ok(()) } } diff --git a/nodedb/src/control/backup/restore/crdt_reissue.rs b/nodedb/src/control/backup/restore/crdt_reissue.rs index 70cff33a7..9016067ab 100644 --- a/nodedb/src/control/backup/restore/crdt_reissue.rs +++ b/nodedb/src/control/backup/restore/crdt_reissue.rs @@ -51,7 +51,9 @@ async fn reissue_crdt_collection( })? .with_event_source(EventSource::Restore) .with_restore_id(target.restore_id); - crate::control::wal_replication::propose_replicated_entry(state, proposer, entry).await?; + let deadline = crate::control::wal_replication::statement_propose_deadline(state); + crate::control::wal_replication::propose_replicated_entry(state, proposer, entry, deadline) + .await?; Ok(()) } diff --git a/nodedb/src/control/backup/restore/durable.rs b/nodedb/src/control/backup/restore/durable.rs index ea4d7fa95..3fc14e7d1 100644 --- a/nodedb/src/control/backup/restore/durable.rs +++ b/nodedb/src/control/backup/restore/durable.rs @@ -67,12 +67,14 @@ pub(crate) async fn reissue_plan_durably( // fire again for it. .with_event_source(crate::event::EventSource::Restore) .with_restore_id(target.restore_id); - let (_, write_version) = - crate::control::wal_replication::propose_replicated_entry(state, proposer, entry).await?; + let deadline = crate::control::wal_replication::statement_propose_deadline(state); + let (_, write_versions) = + crate::control::wal_replication::propose_replicated_entry(state, proposer, entry, deadline) + .await?; tracing::debug!( collection, vshard_id = vshard.as_u32(), - write_version = write_version.as_u64(), + write_version = ?write_versions.of(vshard), "restore: re-issued write applied on this node" ); Ok(()) diff --git a/nodedb/src/control/backup/restore/redo_reissue/reissue.rs b/nodedb/src/control/backup/restore/redo_reissue/reissue.rs index a896b27fe..eab133411 100644 --- a/nodedb/src/control/backup/restore/redo_reissue/reissue.rs +++ b/nodedb/src/control/backup/restore/redo_reissue/reissue.rs @@ -53,11 +53,15 @@ pub(in crate::control::backup::restore) async fn reissue_rows_and_edges( stats.records += commit_collection(state, tenant, target.restore_id, collection).await?; } // Fails the re-issue after the rows committed and before any edge. - crate::fail_point_err!("restore::reissue::before_edges", |detail: String| { - crate::Error::Internal { - detail: format!("fail point: {detail}"), + crate::fail_point_err!( + crate::fail_point::FailScope::Node(state.node_id), + "restore::reissue::before_edges", + |detail: String| { + crate::Error::Internal { + detail: format!("fail point: {detail}"), + } } - }); + ); let (versions, edges) = edge_units(state, tenant_id, target, rows.edges).await?; stats.edges = versions; // Both homes of an edge commit in one transaction. Each version is diff --git a/nodedb/src/control/backup/restore/sections.rs b/nodedb/src/control/backup/restore/sections.rs index fc83afab6..995c09e1c 100644 --- a/nodedb/src/control/backup/restore/sections.rs +++ b/nodedb/src/control/backup/restore/sections.rs @@ -108,6 +108,8 @@ pub(in crate::control::backup) fn append_snapshot( // A backup carries no Calvin cut: the restore re-applies its rows // as new writes on the destination. group_calvin: _, + // A backup carries no redo stream: its rows restore as new writes. + group_redo_streams: _, // A backup carries visible edge versions only. The restore re-applies // them at its own ordinal, so the source's cuts, hidden versions and // applied ordinals never reach the destination. diff --git a/nodedb/src/control/catalog_entry/post_apply/async_dispatch/array.rs b/nodedb/src/control/catalog_entry/post_apply/async_dispatch/array.rs index 447bfd416..d71b8f6bf 100644 --- a/nodedb/src/control/catalog_entry/post_apply/async_dispatch/array.rs +++ b/nodedb/src/control/catalog_entry/post_apply/async_dispatch/array.rs @@ -210,8 +210,8 @@ mod tests { payload: Payload::empty(), watermark_lsn: Lsn::ZERO, error_code: None, - read_set_valid: None, - read_version_lsn: Lsn::ZERO, + stage_vote: None, + read_versions: crate::types::ReadVersions::new(), write_set: Vec::new(), }, }) diff --git a/nodedb/src/control/catalog_entry/post_apply/async_dispatch/collection.rs b/nodedb/src/control/catalog_entry/post_apply/async_dispatch/collection.rs index 6cb9ce923..e80b09e54 100644 --- a/nodedb/src/control/catalog_entry/post_apply/async_dispatch/collection.rs +++ b/nodedb/src/control/catalog_entry/post_apply/async_dispatch/collection.rs @@ -105,12 +105,16 @@ pub(crate) async fn reclaim_collection_storage( // 1. Persist to redb (every node has its own catalog). A failure here // leaves no durable retry owner, so it is a `no_retry` failure: the caller // releases its lifecycle guard rather than leaking the drain. - crate::fail_point_err!("collection_reclaim::before_tombstone", |detail: String| { - ReclaimFailure::no_retry(crate::Error::Storage { - engine: "catalog".into(), - detail, - }) - }); + crate::fail_point_err!( + crate::fail_point::FailScope::Node(shared.node_id), + "collection_reclaim::before_tombstone", + |detail: String| { + ReclaimFailure::no_retry(crate::Error::Storage { + engine: "catalog".into(), + detail, + }) + } + ); let catalog = shared.credentials.catalog(); catalog .record_wal_tombstone(database_id, tenant_id, name, purge_lsn) diff --git a/nodedb/src/control/catalog_entry/post_apply/async_dispatch/core_fanout.rs b/nodedb/src/control/catalog_entry/post_apply/async_dispatch/core_fanout.rs index dfe5b6060..569aba8a0 100644 --- a/nodedb/src/control/catalog_entry/post_apply/async_dispatch/core_fanout.rs +++ b/nodedb/src/control/catalog_entry/post_apply/async_dispatch/core_fanout.rs @@ -146,6 +146,7 @@ pub(super) fn send_to_every_core( wal_lsn: None, resolved_now_ms: None, commit_hlc: None, + entry_version: None, admission: crate::bridge::envelope::Admission::Exempt( crate::bridge::envelope::ExemptReason::AlreadyOrdered, ), @@ -224,8 +225,8 @@ mod tests { payload: Payload::empty(), watermark_lsn: Lsn::ZERO, error_code: None, - read_set_valid: None, - read_version_lsn: Lsn::ZERO, + stage_vote: None, + read_versions: crate::types::ReadVersions::new(), write_set: Vec::new(), } } diff --git a/nodedb/src/control/catalog_entry/post_apply/async_dispatch/crdt_compact.rs b/nodedb/src/control/catalog_entry/post_apply/async_dispatch/crdt_compact.rs index 748909ca3..05c23a2ae 100644 --- a/nodedb/src/control/catalog_entry/post_apply/async_dispatch/crdt_compact.rs +++ b/nodedb/src/control/catalog_entry/post_apply/async_dispatch/crdt_compact.rs @@ -188,8 +188,8 @@ mod tests { payload: Payload::empty(), watermark_lsn: Lsn::ZERO, error_code, - read_set_valid: None, - read_version_lsn: Lsn::ZERO, + stage_vote: None, + read_versions: crate::types::ReadVersions::new(), write_set: Vec::new(), }, }) diff --git a/nodedb/src/control/change_stream/journal/codec.rs b/nodedb/src/control/change_stream/journal/codec.rs index b66e97ca6..1266ef63f 100644 --- a/nodedb/src/control/change_stream/journal/codec.rs +++ b/nodedb/src/control/change_stream/journal/codec.rs @@ -18,17 +18,14 @@ pub(super) const ROW_KEY_LEN: usize = PARTITION_KEY_LEN + POSITION_LEN; const FEED_LEN: usize = 2 * POSITION_LEN + 8; const TAG_GROUP: u8 = 0; -const TAG_CALVIN: u8 = 1; fn word(bytes: &[u8], at: usize) -> Option { Some(u64::from_be_bytes(bytes.get(at..at + 8)?.try_into().ok()?)) } pub(super) fn partition_key(partition: ChangePartition) -> [u8; PARTITION_KEY_LEN] { - let (tag, id) = match partition { - ChangePartition::Group(group) => (TAG_GROUP, group), - ChangePartition::Calvin(vshard) => (TAG_CALVIN, u64::from(vshard)), - }; + let ChangePartition(group) = partition; + let (tag, id) = (TAG_GROUP, group); let mut out = [0u8; PARTITION_KEY_LEN]; out[0] = tag; out[1..].copy_from_slice(&id.to_be_bytes()); @@ -38,8 +35,7 @@ pub(super) fn partition_key(partition: ChangePartition) -> [u8; PARTITION_KEY_LE pub(super) fn decode_partition(bytes: &[u8]) -> Option { let id = word(bytes, 1)?; match *bytes.first()? { - TAG_GROUP => Some(ChangePartition::Group(id)), - TAG_CALVIN => u32::try_from(id).ok().map(ChangePartition::Calvin), + TAG_GROUP => Some(ChangePartition(id)), _ => None, } } @@ -159,7 +155,7 @@ mod tests { #[test] fn keys_and_feeds_round_trip_and_sort_by_position() { - for partition in [ChangePartition::Group(7), ChangePartition::Calvin(3)] { + for partition in [ChangePartition(7), ChangePartition(3)] { assert_eq!(decode_partition(&partition_key(partition)), Some(partition)); let low = row_key(partition, CdcOffset::data_event(0, 5, 1)); let high = row_key(partition, CdcOffset::data_event(0, 5, 2)); diff --git a/nodedb/src/control/change_stream/journal/store.rs b/nodedb/src/control/change_stream/journal/store.rs index 255befe65..4ce448642 100644 --- a/nodedb/src/control/change_stream/journal/store.rs +++ b/nodedb/src/control/change_stream/journal/store.rs @@ -9,9 +9,8 @@ //! - A data group's changes are journaled when the apply loop saves the //! group's applied floor, before the floor: every entry at or below a //! saved floor has its changes on disk. Entries above it are delivered -//! again after a restart and publish again. -//! - A Calvin transaction's changes are journaled before its applied marker -//! is durable. +//! again after a restart and publish again. A committed Calvin slice +//! installs from a data-group entry, so its changes follow this rule. //! - A forwarded run is journaled before the receiver acks it. //! //! Each partition keeps `after`, the position above which it holds every @@ -199,7 +198,7 @@ mod tests { use crate::control::change_stream::{ChangeEvent, ChangeOperation}; use crate::types::{DatabaseId, Lsn, TenantId}; - const GROUP: ChangePartition = ChangePartition::Group(2); + const GROUP: ChangePartition = ChangePartition(2); fn event(index: u64) -> SequencedChangeEvent { SequencedChangeEvent::new( diff --git a/nodedb/src/control/change_stream/mod.rs b/nodedb/src/control/change_stream/mod.rs index 667101373..9bc5c58f8 100644 --- a/nodedb/src/control/change_stream/mod.rs +++ b/nodedb/src/control/change_stream/mod.rs @@ -6,9 +6,9 @@ pub mod stream; pub use journal::ChangeJournal; pub use live_set::LiveSubscriptionSet; +pub(crate) use stream::PositionedChange; pub use stream::{ ChangeCursor, ChangeEvent, ChangeOperation, ChangePartition, ChangeStream, ChangeStreamError, CursorParseError, CursorStep, ReplayError, ReplaySnapshot, ReplayStart, ReplayedChange, SequencedChangeEvent, Subscription, }; -pub(crate) use stream::{ChangeRun, PositionedChange}; diff --git a/nodedb/src/control/change_stream/stream/bus.rs b/nodedb/src/control/change_stream/stream/bus.rs index 5cc496569..6b489ddda 100644 --- a/nodedb/src/control/change_stream/stream/bus.rs +++ b/nodedb/src/control/change_stream/stream/bus.rs @@ -193,7 +193,7 @@ impl ChangeStream { } } let run = ChangeRun { - partition: ChangePartition::Group(group_id), + partition: ChangePartition(group_id), after: Some(CdcOffset::whole_index(first.saturating_sub(1))), through: CdcOffset::whole_index(last), changes, @@ -213,7 +213,7 @@ impl ChangeStream { .staged .retain(|&(group, index), _| group != group_id || index > last_included_index); state.ring.raise_floor( - ChangePartition::Group(group_id), + ChangePartition(group_id), CdcOffset::whole_index(last_included_index), ); } @@ -597,7 +597,7 @@ mod tests { let outsider = ChangeStream::new(16); let mut live = outsider.subscribe(None, Some(TenantId::new(1))); let mut cursor = live.start_cursor().clone(); - let group = ChangePartition::Group(3); + let group = ChangePartition(3); let deliver = |run: &AppendedRun| { outsider.deliver_remote_notify(&NotifyBroadcastMsg::from_run(1, run)) }; @@ -642,7 +642,7 @@ mod tests { let outsider = ChangeStream::new(16); let mut live = outsider.subscribe(None, Some(TenantId::new(1))); let mut cursor = live.start_cursor().clone(); - let group = ChangePartition::Group(5); + let group = ChangePartition(5); let deliver = |run: &AppendedRun| { outsider.deliver_remote_notify(&NotifyBroadcastMsg::from_run(1, run)) }; diff --git a/nodedb/src/control/change_stream/stream/cursor.rs b/nodedb/src/control/change_stream/stream/cursor.rs index db4236687..40870f97c 100644 --- a/nodedb/src/control/change_stream/stream/cursor.rs +++ b/nodedb/src/control/change_stream/stream/cursor.rs @@ -3,11 +3,9 @@ //! Resumable positions in the Control-Plane change stream. //! //! The stream is a set of partitions. Each partition is one totally ordered -//! feed that every node numbers alike: -//! -//! - `Group(g)`: the writes of data group `g`, at their Raft log index. -//! - `Calvin(v)`: the Calvin transactions vShard `v` applied, at their -//! sequencer position. +//! feed that every node numbers alike. `ChangePartition(g)` holds the writes +//! of data group `g`, at their Raft log index. A committed Calvin slice is one +//! of them: it installs from a data-group entry. //! //! A cursor holds, per partition, the position of the last event it //! consumed. A cursor taken on one node therefore resumes on any node that @@ -22,8 +20,7 @@ use crate::event::cdc::CdcOffset; use super::SequencedChangeEvent; const TOKEN_PREFIX: &str = "v2:"; -/// Partitions one cursor holds at most: every data group and every vShard's -/// Calvin feed of a large cluster. +/// Partitions one cursor holds at most: every data group of a large cluster. const MAX_ENTRIES: usize = 8192; /// Upper bound on one entry's text: a tag, a 20-digit id, and three /// 20-digit numbers with their separators. @@ -32,12 +29,10 @@ const MAX_TOKEN_LEN: usize = TOKEN_PREFIX.len() + MAX_ENTRIES * (MAX_ENTRY_LEN + /// One totally ordered feed of the change stream. #[derive(Clone, Copy, Debug, PartialEq, Eq, PartialOrd, Ord, Hash)] -pub enum ChangePartition { - /// A data group's writes, at their Raft log index. - Group(u64), - /// A vShard's Calvin transactions, at their sequencer position. - Calvin(u32), -} +pub struct ChangePartition( + /// The data group whose writes the partition holds, at their Raft log index. + pub u64, +); /// What a consumer does with the next live event. #[derive(Clone, Copy, Debug, PartialEq, Eq)] @@ -112,10 +107,8 @@ impl fmt::Display for ChangeCursor { if index > 0 { formatter.write_str(",")?; } - match partition { - ChangePartition::Group(group) => write!(formatter, "g{group}")?, - ChangePartition::Calvin(vshard) => write!(formatter, "c{vshard}")?, - } + let ChangePartition(group) = partition; + write!(formatter, "g{group}")?; write!( formatter, "@{}.{}.{}", @@ -167,10 +160,7 @@ impl FromStr for ChangeCursor { fn parse_entry(entry: &str) -> Result<(ChangePartition, CdcOffset), CursorParseError> { let (partition, position) = entry.split_once('@').ok_or(CursorParseError)?; let partition = match partition.as_bytes().first() { - Some(b'g') => ChangePartition::Group(number(&partition[1..])?), - Some(b'c') => ChangePartition::Calvin( - u32::try_from(number(&partition[1..])?).map_err(|_| CursorParseError)?, - ), + Some(b'g') => ChangePartition(number(&partition[1..])?), _ => return Err(CursorParseError), }; let mut parts = position.split('.'); @@ -228,11 +218,8 @@ mod tests { #[test] fn token_round_trips_and_is_strict() { let mut cursor = ChangeCursor::default(); - cursor.raise(ChangePartition::Group(3), CdcOffset::data_event(0, 42, 1)); - cursor.raise( - ChangePartition::Calvin(7), - CdcOffset::data_event_in(0, 9, 2, 1), - ); + cursor.raise(ChangePartition(3), CdcOffset::data_event(0, 42, 1)); + cursor.raise(ChangePartition(7), CdcOffset::data_event(0, 9, 1)); let token = cursor.to_string(); assert!(token.starts_with("v2:")); assert_eq!(token.parse::(), Ok(cursor)); @@ -245,7 +232,7 @@ mod tests { "v2:x1@0.1.2", "v2:l@0.1.2", "v2:l7@0.1.2", - "v2:c99999999999@0.1.2", + "v2:c1@0.1.2", "v2:g1@0.-1.2", ] { assert!(invalid.parse::().is_err(), "{invalid}"); @@ -254,7 +241,7 @@ mod tests { #[test] fn accept_delivers_new_events_and_skips_covered_ones() { - let group = ChangePartition::Group(1); + let group = ChangePartition(1); let mut cursor = ChangeCursor::default(); let first = event(group, CdcOffset::data_event(0, 5, 1), CdcOffset::ZERO); let second = event(group, CdcOffset::data_event(0, 6, 1), CdcOffset::ZERO); @@ -264,7 +251,7 @@ mod tests { assert!(cursor.covers(&second)); // Another partition's positions never compare with this one. let other = event( - ChangePartition::Group(2), + ChangePartition(2), CdcOffset::data_event(0, 1, 1), CdcOffset::ZERO, ); @@ -273,7 +260,7 @@ mod tests { #[test] fn a_hole_above_the_cursor_resets() { - let group = ChangePartition::Group(1); + let group = ChangePartition(1); let mut cursor = ChangeCursor::default(); cursor.raise(group, CdcOffset::whole_index(10)); let past_hole = event( diff --git a/nodedb/src/control/change_stream/stream/fanout.rs b/nodedb/src/control/change_stream/stream/fanout.rs index cef387379..79d854014 100644 --- a/nodedb/src/control/change_stream/stream/fanout.rs +++ b/nodedb/src/control/change_stream/stream/fanout.rs @@ -25,9 +25,7 @@ use tokio::sync::mpsc::error::TrySendError; use tracing::{trace, warn}; use crate::control::state::SharedState; -use crate::event::cross_shard::types::{ - NOTIFY_PARTITION_CALVIN, NOTIFY_PARTITION_GROUP, NotifyBroadcastMsg, NotifyChange, -}; +use crate::event::cross_shard::types::{NOTIFY_PARTITION_GROUP, NotifyBroadcastMsg, NotifyChange}; use super::ChangePartition; use super::ring::AppendedRun; @@ -51,13 +49,7 @@ impl ChangeFanout { return Vec::new(); }; let routing = routing.read().unwrap_or_else(|p| p.into_inner()); - let group = match partition { - ChangePartition::Group(group) => group, - ChangePartition::Calvin(vshard) => match routing.group_for_vshard(vshard) { - Ok(group) => group, - Err(_) => return Vec::new(), - }, - }; + let ChangePartition(group) = partition; let Some(info) = routing.group_info(group) else { return Vec::new(); }; @@ -176,10 +168,8 @@ fn spawn_peer( impl NotifyBroadcastMsg { /// The wire form of `run`. pub(crate) fn from_run(source_node: u64, run: &AppendedRun) -> Self { - let (partition_kind, partition_id) = match run.partition { - ChangePartition::Group(group) => (NOTIFY_PARTITION_GROUP, group), - ChangePartition::Calvin(vshard) => (NOTIFY_PARTITION_CALVIN, u64::from(vshard)), - }; + let ChangePartition(group) = run.partition; + let (partition_kind, partition_id) = (NOTIFY_PARTITION_GROUP, group); Self { source_node, partition_kind, @@ -206,10 +196,7 @@ impl NotifyBroadcastMsg { /// The partition the run belongs to, `None` for an unknown kind. pub fn partition(&self) -> Option { match self.partition_kind { - NOTIFY_PARTITION_GROUP => Some(ChangePartition::Group(self.partition_id)), - NOTIFY_PARTITION_CALVIN => u32::try_from(self.partition_id) - .ok() - .map(ChangePartition::Calvin), + NOTIFY_PARTITION_GROUP => Some(ChangePartition(self.partition_id)), _ => None, } } diff --git a/nodedb/src/control/change_stream/stream/journaling.rs b/nodedb/src/control/change_stream/stream/journaling.rs index 687042771..15f470bde 100644 --- a/nodedb/src/control/change_stream/stream/journaling.rs +++ b/nodedb/src/control/change_stream/stream/journaling.rs @@ -30,7 +30,7 @@ impl ChangeStream { /// feeds and are journaled with the next batch of their feed. pub fn attach_journal(&self, journal: Arc) -> crate::Result<()> { let feeds = journal.load()?; - let early_runs = { + { let mut state = self.lock(); let (early_events, early_feeds) = state.ring.drain(); for feed in feeds { @@ -41,7 +41,6 @@ impl ChangeStream { changes: feed.changes, }); } - let mut early_runs = Vec::new(); for (partition, floor, through) in early_feeds { let changes = early_events .iter() @@ -58,25 +57,19 @@ impl ChangeStream { through, changes, }); - match partition { - ChangePartition::Group(group_id) => state - .group_pending - .entry(group_id) - .or_default() - .extend(appended.events), - ChangePartition::Calvin(_) => early_runs.push(appended), - } + let ChangePartition(group_id) = partition; + state + .group_pending + .entry(group_id) + .or_default() + .extend(appended.events); } - early_runs - }; + } self.journal .set(journal) .map_err(|_| crate::Error::Internal { detail: "the change-feed journal was already attached".into(), })?; - for run in &early_runs { - self.journal_run(run); - } Ok(()) } @@ -98,7 +91,7 @@ impl ChangeStream { /// covers then has its changes on disk. pub(crate) fn journal_group(&self, group_id: u64, floor: u64) { let through = CdcOffset::whole_index(floor); - let partition = ChangePartition::Group(group_id); + let partition = ChangePartition(group_id); let batch = { let mut state = self.lock(); if self.journal.get().is_none() { @@ -120,8 +113,8 @@ impl ChangeStream { self.persist(batch); } - /// Journal a run a Calvin transaction or a forwarding leader produced, - /// before its publisher acknowledges it. + /// Journal a run a forwarding leader produced, before its publisher + /// acknowledges it. pub(super) fn journal_run(&self, run: &AppendedRun) { if self.journal.get().is_none() { return; diff --git a/nodedb/src/control/change_stream/stream/mod.rs b/nodedb/src/control/change_stream/stream/mod.rs index 755d42a82..fdf073252 100644 --- a/nodedb/src/control/change_stream/stream/mod.rs +++ b/nodedb/src/control/change_stream/stream/mod.rs @@ -12,7 +12,7 @@ pub mod types; pub use bus::ChangeStream; pub use cursor::{ChangeCursor, ChangePartition, CursorParseError, CursorStep}; pub use error::ChangeStreamError; -pub(crate) use ring::{ChangeRun, PositionedChange}; +pub(crate) use ring::PositionedChange; pub use ring::{ReplayError, ReplaySnapshot, ReplayStart, ReplayedChange}; pub use subscription::Subscription; pub use types::{ChangeEvent, ChangeOperation, SequencedChangeEvent}; diff --git a/nodedb/src/control/change_stream/stream/ring.rs b/nodedb/src/control/change_stream/stream/ring.rs index fe5b5084d..b33cc570a 100644 --- a/nodedb/src/control/change_stream/stream/ring.rs +++ b/nodedb/src/control/change_stream/stream/ring.rs @@ -352,7 +352,7 @@ mod tests { use crate::control::change_stream::ChangeOperation; use crate::types::Lsn; - const GROUP: ChangePartition = ChangePartition::Group(4); + const GROUP: ChangePartition = ChangePartition(4); fn change(index: u64, ordinal: u64) -> PositionedChange { PositionedChange { diff --git a/nodedb/src/control/checkpoint_manager.rs b/nodedb/src/control/checkpoint_manager.rs index 18a2f4652..40b0eabde 100644 --- a/nodedb/src/control/checkpoint_manager.rs +++ b/nodedb/src/control/checkpoint_manager.rs @@ -173,16 +173,18 @@ pub struct CheckpointCycleInputs<'a> { pub archiver: Option<&'a mut WalArchiver>, /// When present, the tombstone set is GC'd to the new truncation point. pub catalog: Option<&'a crate::control::security::catalog::SystemCatalog>, - /// The applied state of every Calvin scheduler on this node. Saved in - /// `catalog` before truncation deletes the applied markers it came from. - pub calvin_mirrors: Option<&'a crate::control::cluster::calvin::scheduler::AppliedMirrors>, + /// The Calvin applied ledger of every vShard on this node. Saved in + /// `catalog` before truncation deletes the stamped redo records it came + /// from. + pub calvin_ledgers: + Option<&'a crate::control::cluster::calvin::scheduler::CalvinAppliedLedgers>, } -/// The applied state of every mirror, as the catalog stores it. +/// The applied state of every ledger, as the catalog stores it. fn calvin_applied_states( - mirrors: &crate::control::cluster::calvin::scheduler::AppliedMirrors, + ledgers: &crate::control::cluster::calvin::scheduler::CalvinAppliedLedgers, ) -> Vec { - mirrors.snapshot_all() + ledgers.snapshot_all() } /// Save `states` in `catalog`. @@ -215,7 +217,7 @@ pub async fn run_checkpoint_cycle(inputs: CheckpointCycleInputs<'_>) -> Option) -> Option) -> Option) -> Option Error { match e { // A shard did not answer within its timeout: surface as a deterministic // deadline rather than an opaque internal error, matching the - // `TypedClusterError::DeadlineExceeded` mapping used elsewhere. - ClusterError::ShardTimeout { .. } => Error::DeadlineExceeded { - request_id: crate::types::RequestId::new(0), - }, + // `TypedClusterError::DeadlineExceeded` mapping used elsewhere. A + // sent request with no answer can have run, so it takes the same class. + ClusterError::ShardTimeout { .. } | ClusterError::Unanswered { .. } => { + Error::DeadlineExceeded { + request_id: crate::types::RequestId::new(0), + } + } // A shard's Data-Plane verdict keeps its code, so the statement // renders the SQLSTATE a single-node execution renders. ClusterError::DataPlane { code } => Error::DataPlane(code.into()), diff --git a/nodedb/src/control/cluster/array_executor/executor.rs b/nodedb/src/control/cluster/array_executor/executor.rs index b75c53d2b..2383e4f39 100644 --- a/nodedb/src/control/cluster/array_executor/executor.rs +++ b/nodedb/src/control/cluster/array_executor/executor.rs @@ -134,6 +134,7 @@ fn local_request( wal_lsn: None, resolved_now_ms: None, commit_hlc: None, + entry_version: None, admission: crate::bridge::envelope::Admission::Exempt( crate::bridge::envelope::ExemptReason::AlreadyOrdered, ), diff --git a/nodedb/src/control/cluster/array_executor/read.rs b/nodedb/src/control/cluster/array_executor/read.rs index 4ffdf3107..6b6334177 100644 --- a/nodedb/src/control/cluster/array_executor/read.rs +++ b/nodedb/src/control/cluster/array_executor/read.rs @@ -76,9 +76,12 @@ impl DataPlaneArrayExecutor { detail: format!("array slice response decode: {e}"), })?; let rows = split_msgpack_array_rows(&slice_resp.rows_msgpack)?; + // The shard's version rides back with its rows, even when it matched + // none, so a transaction validates the read on this vShard. Ok(ArraySliceExec { rows, truncated_before_horizon: slice_resp.truncated_before_horizon, + read_versions: resp.read_versions.to_wire(), }) } @@ -138,6 +141,7 @@ impl DataPlaneArrayExecutor { return Ok(ArrayAggExec { partials: Vec::new(), truncated_before_horizon: false, + read_versions: resp.read_versions.to_wire(), }); } @@ -152,6 +156,7 @@ impl DataPlaneArrayExecutor { Ok(ArrayAggExec { partials, truncated_before_horizon, + read_versions: resp.read_versions.to_wire(), }) } diff --git a/nodedb/src/control/cluster/array_executor/refusal.rs b/nodedb/src/control/cluster/array_executor/refusal.rs index fc3f23ad5..79e025fef 100644 --- a/nodedb/src/control/cluster/array_executor/refusal.rs +++ b/nodedb/src/control/cluster/array_executor/refusal.rs @@ -200,8 +200,8 @@ mod tests { payload: Payload::empty(), watermark_lsn: Lsn::ZERO, error_code: code.map(Box::new), - read_set_valid: None, - read_version_lsn: Lsn::ZERO, + stage_vote: None, + read_versions: crate::types::ReadVersions::new(), write_set: Vec::new(), } } diff --git a/nodedb/src/control/cluster/array_executor/write.rs b/nodedb/src/control/cluster/array_executor/write.rs index 1bd825663..9be65ad3b 100644 --- a/nodedb/src/control/cluster/array_executor/write.rs +++ b/nodedb/src/control/cluster/array_executor/write.rs @@ -124,10 +124,15 @@ impl DataPlaneArrayExecutor { detail: format!("{op_label}: plan is not encodable as a replicated entry"), })?; - let (apply_payload, _write_version) = - crate::control::wal_replication::propose_replicated_entry(&self.state, proposer, entry) - .await - .map_err(|e| execution_error(&format!("{op_label} raft propose"), e))?; + let (apply_payload, _write_versions) = + crate::control::wal_replication::propose_replicated_entry( + &self.state, + proposer, + entry, + crate::control::wal_replication::statement_propose_deadline(&self.state), + ) + .await + .map_err(|e| execution_error(&format!("{op_label} raft propose"), e))?; let affected = require_affected_count(&apply_payload).map_err(|e| ClusterError::Storage { detail: format!("{op_label}: {e}"), diff --git a/nodedb/src/control/cluster/calvin/mod.rs b/nodedb/src/control/cluster/calvin/mod.rs index 13c67536a..1f2051c19 100644 --- a/nodedb/src/control/cluster/calvin/mod.rs +++ b/nodedb/src/control/cluster/calvin/mod.rs @@ -5,6 +5,6 @@ pub mod scheduler; pub use executor::{OllpConfig, OllpError, OllpOrchestrator}; pub use scheduler::{ - CalvinReadResultProposal, RaftSequencerProposer, ReadResultEvent, Scheduler, SchedulerConfig, + BarrierEvent, CalvinReadResultProposal, RaftSequencerProposer, Scheduler, SchedulerConfig, SchedulerParams, SequencerProposer, propose_calvin_read_result, }; diff --git a/nodedb/src/control/cluster/calvin/scheduler/applied_ledger.rs b/nodedb/src/control/cluster/calvin/scheduler/applied_ledger.rs new file mode 100644 index 000000000..5a4f21b71 --- /dev/null +++ b/nodedb/src/control/cluster/calvin/scheduler/applied_ledger.rs @@ -0,0 +1,326 @@ +// SPDX-License-Identifier: BUSL-1.1 + +//! Which Calvin positions this node's replica of each vShard applied. +//! +//! One [`CalvinAppliedLedger`] per vShard answers whether this replica +//! applied `(epoch, position)`. Boot recovery fills it before the data-group +//! apply loop starts. A snapshot install replaces it. A scheduler seeds its +//! applied gate from it and marks each position it finishes. A checkpoint +//! saves it before WAL truncation deletes the records it came from. +//! +//! The ledger keeps the shape of the scheduler's gate: a fully-applied +//! watermark and the applied positions above it. The tail is pruned as the +//! watermark advances. +//! +//! A claim reserves a position while one stamped redo installs it. A second +//! copy of the same position finds the claim and installs nothing. A claim +//! is not an applied position: [`CalvinAppliedLedger::snapshot`] leaves it +//! out. + +use std::collections::{BTreeSet, HashMap}; +use std::sync::{Arc, Mutex}; + +use super::recovery::NOT_YET_APPLIED_EPOCH; +use crate::control::security::catalog::calvin_applied::StoredCalvinApplied; + +/// Why [`CalvinAppliedLedger::claim`] refused a position. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum ClaimRefusal { + /// The position is applied on this replica. + Applied, + /// An install of the position holds its claim. + Claimed, +} + +#[derive(Debug)] +struct LedgerState { + /// Every position of every epoch at or below this is applied. + /// [`NOT_YET_APPLIED_EPOCH`] means none is. + fully_applied_epoch: u64, + /// Applied positions of epochs above the watermark. + tail: BTreeSet<(u64, u32)>, + /// Positions an install holds and has not finished. + claims: BTreeSet<(u64, u32)>, +} + +impl LedgerState { + fn below_watermark(&self, epoch: u64) -> bool { + self.fully_applied_epoch != NOT_YET_APPLIED_EPOCH && epoch <= self.fully_applied_epoch + } + + fn is_applied(&self, epoch: u64, position: u32) -> bool { + self.below_watermark(epoch) || self.tail.contains(&(epoch, position)) + } + + fn insert_applied(&mut self, epoch: u64, position: u32) { + if !self.below_watermark(epoch) { + self.tail.insert((epoch, position)); + } + } +} + +/// Applied positions of one vShard on this node. +#[derive(Debug)] +pub struct CalvinAppliedLedger { + state: Mutex, +} + +impl CalvinAppliedLedger { + /// A ledger holding the watermark `fully_applied_epoch` and the applied + /// positions `tail` above it, with no claim. + pub fn new(fully_applied_epoch: u64, tail: BTreeSet<(u64, u32)>) -> Self { + Self { + state: Mutex::new(LedgerState { + fully_applied_epoch, + tail, + claims: BTreeSet::new(), + }), + } + } + + /// Reserve `(epoch, position)` for one install. Refuses an applied + /// position and a position another install holds. + pub fn claim(&self, epoch: u64, position: u32) -> Result<(), ClaimRefusal> { + let mut state = self.state.lock().unwrap_or_else(|p| p.into_inner()); + if state.is_applied(epoch, position) { + return Err(ClaimRefusal::Applied); + } + if !state.claims.insert((epoch, position)) { + return Err(ClaimRefusal::Claimed); + } + Ok(()) + } + + /// Drop the claim on `(epoch, position)`: its install did not apply, so a + /// later copy can install it. + pub fn release_claim(&self, epoch: u64, position: u32) { + let mut state = self.state.lock().unwrap_or_else(|p| p.into_inner()); + state.claims.remove(&(epoch, position)); + } + + /// Record that the install of `(epoch, position)` is durable. Drops its + /// claim. + pub fn mark_applied(&self, epoch: u64, position: u32) { + let mut state = self.state.lock().unwrap_or_else(|p| p.into_inner()); + state.claims.remove(&(epoch, position)); + state.insert_applied(epoch, position); + } + + /// Record that `(epoch, position)` finished with no install on this + /// vShard: an abort, or a slice that writes nothing. A claim on it stays + /// until its holder finishes. + pub fn mark_terminal(&self, epoch: u64, position: u32) { + let mut state = self.state.lock().unwrap_or_else(|p| p.into_inner()); + state.insert_applied(epoch, position); + } + + /// Record that every position of every epoch at or below `watermark` + /// applied. A lower watermark changes nothing. + pub fn fold(&self, watermark: u64) { + let mut state = self.state.lock().unwrap_or_else(|p| p.into_inner()); + if state.below_watermark(watermark) { + return; + } + state.fully_applied_epoch = watermark; + let above = (watermark.saturating_add(1), 0); + state.tail = state.tail.split_off(&above); + state.claims = state.claims.split_off(&above); + } + + /// The fully-applied watermark and the applied positions above it. + /// Claims are not applied positions, so they are absent. + pub fn snapshot(&self) -> (u64, BTreeSet<(u64, u32)>) { + let state = self.state.lock().unwrap_or_else(|p| p.into_inner()); + (state.fully_applied_epoch, state.tail.clone()) + } + + /// Whether this replica applied `(epoch, position)`. + pub fn is_applied(&self, epoch: u64, position: u32) -> bool { + let state = self.state.lock().unwrap_or_else(|p| p.into_inner()); + state.is_applied(epoch, position) + } + + /// The highest epoch with an applied position, or + /// [`NOT_YET_APPLIED_EPOCH`] when none is applied. + pub fn max_applied_epoch(&self) -> u64 { + let state = self.state.lock().unwrap_or_else(|p| p.into_inner()); + match state.tail.last() { + Some((epoch, _)) => *epoch, + None => state.fully_applied_epoch, + } + } +} + +/// The applied ledger of every vShard this node holds Calvin state for. +#[derive(Debug, Default)] +pub struct CalvinAppliedLedgers { + by_vshard: Mutex>>, +} + +impl CalvinAppliedLedgers { + /// Replace the ledger of `vshard_id` with one holding + /// `fully_applied_epoch` and `tail`. A holder of the old ledger keeps the + /// old one. + pub fn install( + &self, + vshard_id: u32, + fully_applied_epoch: u64, + tail: BTreeSet<(u64, u32)>, + ) -> Arc { + let ledger = Arc::new(CalvinAppliedLedger::new(fully_applied_epoch, tail)); + self.by_vshard + .lock() + .unwrap_or_else(|p| p.into_inner()) + .insert(vshard_id, Arc::clone(&ledger)); + ledger + } + + /// The ledger of `vshard_id`. A vShard with no ledger has no Calvin + /// history on this node, so it gets an empty one. + pub fn get_or_create(&self, vshard_id: u32) -> Arc { + let mut by_vshard = self.by_vshard.lock().unwrap_or_else(|p| p.into_inner()); + Arc::clone(by_vshard.entry(vshard_id).or_insert_with(|| { + Arc::new(CalvinAppliedLedger::new( + NOT_YET_APPLIED_EPOCH, + BTreeSet::new(), + )) + })) + } + + /// The ledger of `vshard_id`, when this node holds one. + pub fn get(&self, vshard_id: u32) -> Option> { + self.by_vshard + .lock() + .unwrap_or_else(|p| p.into_inner()) + .get(&vshard_id) + .cloned() + } + + /// Drop the ledger of `vshard_id`: this node left the vShard's group and + /// holds none of its applied positions. + pub fn remove(&self, vshard_id: u32) { + self.by_vshard + .lock() + .unwrap_or_else(|p| p.into_inner()) + .remove(&vshard_id); + } + + /// Every ledger's applied state, in the shape the catalog stores. + pub fn snapshot_all(&self) -> Vec { + let ledgers: Vec<(u32, Arc)> = self + .by_vshard + .lock() + .unwrap_or_else(|p| p.into_inner()) + .iter() + .map(|(vshard_id, ledger)| (*vshard_id, Arc::clone(ledger))) + .collect(); + ledgers + .into_iter() + .map(|(vshard_id, ledger)| { + let (fully_applied_epoch, tail) = ledger.snapshot(); + StoredCalvinApplied { + vshard_id, + fully_applied_epoch, + tail, + } + }) + .collect() + } +} + +#[cfg(test)] +mod tests { + use super::*; + + fn empty() -> CalvinAppliedLedger { + CalvinAppliedLedger::new(NOT_YET_APPLIED_EPOCH, BTreeSet::new()) + } + + #[test] + fn a_ledger_answers_like_the_gate() { + let ledger = empty(); + assert!(!ledger.is_applied(0, 0)); + ledger.mark_applied(3, 1); + assert!(ledger.is_applied(3, 1)); + assert!(!ledger.is_applied(3, 0)); + ledger.fold(3); + assert!(ledger.is_applied(3, 0)); + assert!(ledger.is_applied(2, 9)); + assert!(!ledger.is_applied(4, 0)); + // A mark at or below the watermark changes nothing. + ledger.mark_terminal(1, 0); + assert!(ledger.is_applied(1, 0)); + // A lower watermark never moves it back. + ledger.fold(1); + assert!(ledger.is_applied(3, 0)); + assert_eq!(ledger.max_applied_epoch(), 3); + } + + #[test] + fn a_claim_refuses_a_claimed_or_applied_position() { + let ledger = empty(); + assert_eq!(ledger.claim(2, 0), Ok(())); + assert_eq!(ledger.claim(2, 0), Err(ClaimRefusal::Claimed)); + ledger.mark_applied(2, 0); + assert_eq!(ledger.claim(2, 0), Err(ClaimRefusal::Applied)); + ledger.mark_terminal(2, 1); + assert_eq!(ledger.claim(2, 1), Err(ClaimRefusal::Applied)); + ledger.fold(5); + assert_eq!(ledger.claim(4, 7), Err(ClaimRefusal::Applied)); + assert_eq!(ledger.claim(6, 0), Ok(())); + } + + #[test] + fn a_released_claim_can_be_claimed_again() { + let ledger = empty(); + assert_eq!(ledger.claim(1, 3), Ok(())); + ledger.release_claim(1, 3); + assert!(!ledger.is_applied(1, 3)); + assert_eq!(ledger.claim(1, 3), Ok(())); + } + + #[test] + fn snapshot_names_only_applied_positions() { + let ledger = empty(); + ledger.claim(4, 0).expect("claim"); + ledger.claim(4, 1).expect("claim"); + ledger.mark_applied(4, 1); + ledger.mark_terminal(5, 2); + let (watermark, tail) = ledger.snapshot(); + assert_eq!(watermark, NOT_YET_APPLIED_EPOCH); + assert_eq!(tail, [(4, 1), (5, 2)].into_iter().collect()); + } + + #[test] + fn an_empty_ledger_has_no_applied_epoch() { + assert_eq!(empty().max_applied_epoch(), NOT_YET_APPLIED_EPOCH); + let folded = CalvinAppliedLedger::new(6, BTreeSet::new()); + assert_eq!(folded.max_applied_epoch(), 6); + } + + #[test] + fn an_installed_ledger_replaces_the_previous_one() { + let ledgers = CalvinAppliedLedgers::default(); + let first = ledgers.install(7, NOT_YET_APPLIED_EPOCH, BTreeSet::new()); + first.mark_applied(1, 0); + let second = ledgers.install(7, 4, BTreeSet::new()); + let current = ledgers.get(7).expect("ledger"); + assert!(Arc::ptr_eq(¤t, &second)); + assert!(current.is_applied(4, 0)); + assert!(ledgers.get(8).is_none()); + } + + #[test] + fn get_or_create_keeps_an_existing_ledger() { + let ledgers = CalvinAppliedLedgers::default(); + let recovered = ledgers.install(3, NOT_YET_APPLIED_EPOCH, [(2, 0)].into_iter().collect()); + let used = ledgers.get_or_create(3); + assert!(Arc::ptr_eq(&recovered, &used)); + assert!(used.is_applied(2, 0)); + let fresh = ledgers.get_or_create(9); + assert!(!fresh.is_applied(0, 0)); + assert_eq!(ledgers.snapshot_all().len(), 2); + ledgers.remove(9); + assert!(ledgers.get(9).is_none()); + } +} diff --git a/nodedb/src/control/cluster/calvin/scheduler/applied_mirror.rs b/nodedb/src/control/cluster/calvin/scheduler/applied_mirror.rs deleted file mode 100644 index 82ed6f408..000000000 --- a/nodedb/src/control/cluster/calvin/scheduler/applied_mirror.rs +++ /dev/null @@ -1,184 +0,0 @@ -// SPDX-License-Identifier: BUSL-1.1 - -//! A shared view of which Calvin positions this node's scheduler for one -//! vShard applied. -//! -//! The scheduler owns its [`super::AppliedGate`] and mutates it on its own -//! task. Other Control-Plane code needs one question answered: did this -//! node's replica of the vShard apply `(epoch, position)`? The scheduler -//! mirrors each applied position and each watermark advance here, so the -//! answer needs no message to the scheduler task. -//! -//! The mirror keeps the same shape as the gate: a fully-applied watermark and -//! the applied positions above it. The tail is pruned as the watermark -//! advances, so it stays as small as the gate's. - -use std::collections::{BTreeSet, HashMap}; -use std::sync::{Arc, Mutex}; - -use super::recovery::NOT_YET_APPLIED_EPOCH; -use crate::control::security::catalog::calvin_applied::StoredCalvinApplied; - -#[derive(Debug)] -struct MirrorState { - /// Every position of every epoch at or below this is applied. - /// [`NOT_YET_APPLIED_EPOCH`] means none is. - fully_applied_epoch: u64, - /// Applied positions of epochs above the watermark. - tail: BTreeSet<(u64, u32)>, -} - -/// Applied positions of one vShard's scheduler on this node. -#[derive(Debug)] -pub struct AppliedMirror { - state: Mutex, -} - -impl AppliedMirror { - /// A mirror seeded from the scheduler's recovery scan. - pub fn new(fully_applied_epoch: u64, tail: BTreeSet<(u64, u32)>) -> Self { - Self { - state: Mutex::new(MirrorState { - fully_applied_epoch, - tail, - }), - } - } - - /// Record that `(epoch, position)` applied. - pub fn mark(&self, epoch: u64, position: u32) { - let mut state = self.state.lock().unwrap_or_else(|p| p.into_inner()); - if state.fully_applied_epoch != NOT_YET_APPLIED_EPOCH && epoch <= state.fully_applied_epoch - { - return; - } - state.tail.insert((epoch, position)); - } - - /// Record that every position of every epoch at or below `watermark` - /// applied. - pub fn fold(&self, watermark: u64) { - let mut state = self.state.lock().unwrap_or_else(|p| p.into_inner()); - if state.fully_applied_epoch != NOT_YET_APPLIED_EPOCH - && watermark <= state.fully_applied_epoch - { - return; - } - state.fully_applied_epoch = watermark; - state.tail = state.tail.split_off(&(watermark.saturating_add(1), 0)); - } - - /// The mirror's state: the fully-applied watermark and the applied - /// positions above it. - pub fn snapshot(&self) -> (u64, BTreeSet<(u64, u32)>) { - let state = self.state.lock().unwrap_or_else(|p| p.into_inner()); - (state.fully_applied_epoch, state.tail.clone()) - } - - /// Whether this node's replica applied `(epoch, position)`. - pub fn is_applied(&self, epoch: u64, position: u32) -> bool { - let state = self.state.lock().unwrap_or_else(|p| p.into_inner()); - (state.fully_applied_epoch != NOT_YET_APPLIED_EPOCH && epoch <= state.fully_applied_epoch) - || state.tail.contains(&(epoch, position)) - } -} - -/// The applied mirror of every vShard scheduler on this node. -#[derive(Debug, Default)] -pub struct AppliedMirrors { - by_vshard: Mutex>>, -} - -impl AppliedMirrors { - /// Register the mirror of a scheduler starting for `vshard_id`. A - /// restarted scheduler replaces its predecessor's mirror. - pub fn register( - &self, - vshard_id: u32, - fully_applied_epoch: u64, - tail: &BTreeSet<(u64, u32)>, - ) -> Arc { - let mirror = Arc::new(AppliedMirror::new(fully_applied_epoch, tail.clone())); - self.by_vshard - .lock() - .unwrap_or_else(|p| p.into_inner()) - .insert(vshard_id, Arc::clone(&mirror)); - mirror - } - - /// Every registered mirror's state, in the shape the catalog stores. - pub fn snapshot_all(&self) -> Vec { - let mirrors: Vec<(u32, Arc)> = self - .by_vshard - .lock() - .unwrap_or_else(|p| p.into_inner()) - .iter() - .map(|(vshard_id, mirror)| (*vshard_id, Arc::clone(mirror))) - .collect(); - mirrors - .into_iter() - .map(|(vshard_id, mirror)| { - let (fully_applied_epoch, tail) = mirror.snapshot(); - StoredCalvinApplied { - vshard_id, - fully_applied_epoch, - tail, - } - }) - .collect() - } - - /// Drop the mirror of `vshard_id`: this node left the vShard's group and - /// holds none of its applied positions. - pub fn remove(&self, vshard_id: u32) { - self.by_vshard - .lock() - .unwrap_or_else(|p| p.into_inner()) - .remove(&vshard_id); - } - - /// The mirror of `vshard_id`, when this node runs its scheduler. - pub fn get(&self, vshard_id: u32) -> Option> { - self.by_vshard - .lock() - .unwrap_or_else(|p| p.into_inner()) - .get(&vshard_id) - .cloned() - } -} - -#[cfg(test)] -mod tests { - use super::*; - - #[test] - fn a_mirror_answers_like_the_gate() { - let mirror = AppliedMirror::new(NOT_YET_APPLIED_EPOCH, BTreeSet::new()); - assert!(!mirror.is_applied(0, 0)); - mirror.mark(3, 1); - assert!(mirror.is_applied(3, 1)); - assert!(!mirror.is_applied(3, 0)); - mirror.fold(3); - assert!(mirror.is_applied(3, 0)); - assert!(mirror.is_applied(2, 9)); - assert!(!mirror.is_applied(4, 0)); - // A mark at or below the watermark changes nothing. - mirror.mark(1, 0); - assert!(mirror.is_applied(1, 0)); - // A lower watermark never moves it back. - mirror.fold(1); - assert!(mirror.is_applied(3, 0)); - } - - #[test] - fn a_restarted_scheduler_replaces_its_mirror() { - let mirrors = AppliedMirrors::default(); - let first = mirrors.register(7, NOT_YET_APPLIED_EPOCH, &BTreeSet::new()); - first.mark(1, 0); - let second = mirrors.register(7, 4, &BTreeSet::new()); - let current = mirrors.get(7).expect("mirror"); - assert!(Arc::ptr_eq(¤t, &second)); - assert!(current.is_applied(4, 0)); - assert!(mirrors.get(8).is_none()); - } -} diff --git a/nodedb/src/control/cluster/calvin/scheduler/barrier_log.rs b/nodedb/src/control/cluster/calvin/scheduler/barrier_log.rs new file mode 100644 index 000000000..5b8c6223d --- /dev/null +++ b/nodedb/src/control/cluster/calvin/scheduler/barrier_log.rs @@ -0,0 +1,279 @@ +// SPDX-License-Identifier: BUSL-1.1 + +//! The dependent-read barrier events of one txn on one vShard, folded in the +//! order the vShard's data-group log holds them. +//! +//! Two kinds of entry reach a barrier through the log: a passive vShard's +//! read result, and the barrier's timeout. Every replica of the vShard +//! applies the log in one order, so every replica folds the same events into +//! the same [`BarrierLog`]: +//! +//! - A read result below the timeout entry counts. One above it does not. +//! - The first read result of a passive vShard counts. A later copy, from a +//! re-proposal, does not. +//! +//! [`BarrierLog::outcome`] is then the same on every replica: the barrier +//! completes when every passive vShard's result counts, and it times out +//! when the timeout entry came first. + +use std::collections::{BTreeMap, BTreeSet}; + +use nodedb_physical::physical_plan::meta::PassiveReadKeyId; +use nodedb_types::Value; + +/// Read values received from passive vShards, keyed by passive vShard. +/// `BTreeMap` for determinism. +pub type ReceivedReads = BTreeMap>; + +/// One barrier entry of the data-group log. +#[derive(Debug, Clone, PartialEq)] +pub enum BarrierEvent { + /// A passive vShard's read values. + Read { + passive_vshard: u32, + values: Vec<(PassiveReadKeyId, Value)>, + }, + /// The barrier's timeout. + Timeout, +} + +/// Where a barrier stands once its log is folded. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum BarrierOutcome { + /// Every passive vShard's result came before any timeout. + Complete, + /// The timeout came before a passive vShard's result. + TimedOut, + /// A passive vShard's result is still missing, and no timeout came. + Waiting, +} + +/// The barrier events of one txn, folded in log order. +#[derive(Debug, Clone, Default, PartialEq)] +pub struct BarrierLog { + reads: ReceivedReads, + timed_out: bool, +} + +impl BarrierLog { + /// Whether folding `event` changes this log: a timeout, or the first + /// result of a passive vShard, before any timeout. + pub fn accepts(&self, event: &BarrierEvent) -> bool { + if self.timed_out { + return false; + } + match event { + BarrierEvent::Read { passive_vshard, .. } => !self.reads.contains_key(passive_vshard), + BarrierEvent::Timeout => true, + } + } + + /// Fold `event`, the next barrier entry of the log. + pub fn note(&mut self, event: BarrierEvent) { + if !self.accepts(&event) { + return; + } + match event { + BarrierEvent::Read { + passive_vshard, + values, + } => { + self.reads.insert(passive_vshard, values); + } + BarrierEvent::Timeout => self.timed_out = true, + } + } + + /// Fold `later`, the events the log holds after every event of this one. + pub fn extend(&mut self, later: BarrierLog) { + if self.timed_out { + return; + } + for (passive_vshard, values) in later.reads { + self.reads.entry(passive_vshard).or_insert(values); + } + self.timed_out = later.timed_out; + } + + /// The events this log holds: one per counted result, one for a timeout. + pub fn event_count(&self) -> usize { + self.reads.len() + usize::from(self.timed_out) + } + + /// Where a barrier that waits for every vShard of `passive` stands. + pub fn outcome(&self, passive: &BTreeSet) -> BarrierOutcome { + if passive.iter().all(|vshard| self.reads.contains_key(vshard)) { + BarrierOutcome::Complete + } else if self.timed_out { + BarrierOutcome::TimedOut + } else { + BarrierOutcome::Waiting + } + } + + /// The vShards of `passive` whose result has not counted. + pub fn missing(&self, passive: &BTreeSet) -> Vec { + passive + .iter() + .copied() + .filter(|vshard| !self.reads.contains_key(vshard)) + .collect() + } + + /// Every counted read value, keyed by the row it names. `BTreeMap` + /// iterates in one order on every replica. + pub fn injected_reads(&self) -> BTreeMap { + self.reads + .values() + .flatten() + .map(|(key_id, value)| (key_id.clone(), value.clone())) + .collect() + } + + /// The log as the bytes a stored barrier row holds. + pub fn to_bytes(&self) -> crate::Result> { + let stored = StoredBarrierLog { + reads: self + .reads + .iter() + .map(|(vshard, values)| (*vshard, values.clone())) + .collect(), + timed_out: self.timed_out, + }; + zerompk::to_msgpack_vec(&stored).map_err(|e| crate::Error::Serialization { + format: "msgpack".into(), + detail: format!("encode calvin barrier log: {e}"), + }) + } + + /// The log a stored barrier row holds. + pub fn from_bytes(bytes: &[u8]) -> crate::Result { + let stored: StoredBarrierLog = + zerompk::from_msgpack(bytes).map_err(|e| crate::Error::Serialization { + format: "msgpack".into(), + detail: format!("decode calvin barrier log: {e}"), + })?; + Ok(Self { + reads: stored.reads.into_iter().collect(), + timed_out: stored.timed_out, + }) + } +} + +/// The stored form of a [`BarrierLog`]. +#[derive(zerompk::ToMessagePack, zerompk::FromMessagePack)] +#[msgpack(map)] +struct StoredBarrierLog { + reads: Vec<(u32, Vec<(PassiveReadKeyId, Value)>)>, + timed_out: bool, +} + +#[cfg(test)] +mod tests { + use nodedb_types::QualifiedCollection; + + use super::*; + + fn read(passive_vshard: u32, value: i64) -> BarrierEvent { + BarrierEvent::Read { + passive_vshard, + values: vec![( + PassiveReadKeyId::surrogate( + QualifiedCollection::from_stored(format!("c{passive_vshard}")), + 1, + ), + Value::Integer(value), + )], + } + } + + fn passive(vshards: &[u32]) -> BTreeSet { + vshards.iter().copied().collect() + } + + /// Every result before the timeout completes the barrier: the timeout + /// that follows changes nothing. + #[test] + fn results_before_the_timeout_complete_the_barrier() { + let mut log = BarrierLog::default(); + log.note(read(1, 10)); + log.note(read(2, 20)); + log.note(BarrierEvent::Timeout); + assert_eq!(log.outcome(&passive(&[1, 2])), BarrierOutcome::Complete); + assert_eq!(log.injected_reads().len(), 2); + } + + /// A result above the timeout does not count, so the barrier times out. + #[test] + fn a_result_after_the_timeout_does_not_count() { + let mut log = BarrierLog::default(); + log.note(read(1, 10)); + log.note(BarrierEvent::Timeout); + log.note(read(2, 20)); + assert_eq!(log.outcome(&passive(&[1, 2])), BarrierOutcome::TimedOut); + assert_eq!(log.missing(&passive(&[1, 2])), vec![2]); + } + + /// A barrier missing a result with no timeout waits. + #[test] + fn a_missing_result_waits() { + let mut log = BarrierLog::default(); + log.note(read(1, 10)); + assert_eq!(log.outcome(&passive(&[1, 2])), BarrierOutcome::Waiting); + } + + /// The first copy of a passive vShard's result counts. A re-proposed + /// copy does not replace it. + #[test] + fn the_first_copy_of_a_result_counts() { + let mut log = BarrierLog::default(); + log.note(read(1, 10)); + assert!(!log.accepts(&read(1, 99))); + log.note(read(1, 99)); + let values: Vec = log.injected_reads().into_values().collect(); + assert_eq!(values, vec![Value::Integer(10)]); + } + + /// Two logs folded one after the other give the outcome one log of + /// every event gives. + #[test] + fn extending_with_later_events_keeps_log_order() { + let mut earlier = BarrierLog::default(); + earlier.note(read(1, 10)); + let mut later = BarrierLog::default(); + later.note(BarrierEvent::Timeout); + later.note(read(2, 20)); + earlier.extend(later); + assert_eq!(earlier.outcome(&passive(&[1, 2])), BarrierOutcome::TimedOut); + + let mut timed_out = BarrierLog::default(); + timed_out.note(BarrierEvent::Timeout); + let mut late_reads = BarrierLog::default(); + late_reads.note(read(1, 10)); + timed_out.extend(late_reads); + assert_eq!(timed_out.outcome(&passive(&[1])), BarrierOutcome::TimedOut); + } + + /// A stored log decodes to the log it was encoded from. + #[test] + fn a_stored_log_round_trips() { + let mut log = BarrierLog::default(); + log.note(read(2, 20)); + log.note(read(1, 10)); + log.note(BarrierEvent::Timeout); + let bytes = log.to_bytes().expect("encode"); + assert_eq!(BarrierLog::from_bytes(&bytes).expect("decode"), log); + } + + /// The event count is what the log holds: counted results and a timeout. + #[test] + fn event_count_counts_held_events() { + let mut log = BarrierLog::default(); + assert_eq!(log.event_count(), 0); + log.note(read(1, 10)); + log.note(read(1, 11)); + assert_eq!(log.event_count(), 1); + log.note(BarrierEvent::Timeout); + assert_eq!(log.event_count(), 2); + } +} diff --git a/nodedb/src/control/cluster/calvin/scheduler/barrier_store.rs b/nodedb/src/control/cluster/calvin/scheduler/barrier_store.rs new file mode 100644 index 000000000..ff200246d --- /dev/null +++ b/nodedb/src/control/cluster/calvin/scheduler/barrier_store.rs @@ -0,0 +1,387 @@ +// SPDX-License-Identifier: BUSL-1.1 + +//! The stored dependent-read barrier log of each txn on each vShard. +//! +//! The data-group apply loop concludes a barrier entry as durably applied +//! only once its row in `_system.calvin_barrier_logs` holds the event. The +//! durable applied floor of a group asserts that every entry at or below it +//! is durable, and boot resumes Raft delivery above it. So an entry below +//! the floor is never delivered again, and its row is what a barrier of the +//! txn reads after a restart: +//! +//! - [`restore_at_boot`] marks the txn of every stored row stored in its +//! vShard's buffer, and removes the rows of finished txns; +//! - [`capture_group`] and [`install_group`] carry the rows of a data group's +//! unfinished txns in its snapshot, for the entries the snapshot covers; +//! - the scheduler removes a txn's row once the txn finished, and +//! [`sweep_finished`] removes on its stall tick each finished txn's row +//! that removal missed. + +use std::collections::{BTreeMap, BTreeSet}; + +use super::applied_ledger::CalvinAppliedLedger; +use super::barrier_log::{BarrierEvent, BarrierLog}; +use super::lock::TxnId; +use crate::control::security::catalog::SystemCatalog; +use crate::control::security::catalog::calvin_barrier_logs::StoredBarrierLog; +use crate::control::state::SharedState; +use crate::types::CutBarrierLog; + +/// Fold `event` into the stored row of `txn` on `vshard_id`. An event the +/// row's log does not take, a copy it holds or one after its timeout, +/// writes nothing. +pub fn save_event( + catalog: &SystemCatalog, + vshard_id: u32, + txn: TxnId, + event: &BarrierEvent, +) -> crate::Result<()> { + catalog.fold_calvin_barrier_log(vshard_id, txn.epoch, txn.position, |held| { + let mut log = match held { + Some(bytes) => BarrierLog::from_bytes(bytes)?, + None => BarrierLog::default(), + }; + if held.is_some() && !log.accepts(event) { + return Ok(None); + } + log.note(event.clone()); + log.to_bytes().map(Some) + }) +} + +/// The stored log of `txn` on `vshard_id`. A txn with no row has an empty +/// log. +pub fn load_log(catalog: &SystemCatalog, vshard_id: u32, txn: TxnId) -> crate::Result { + match catalog.load_calvin_barrier_log(vshard_id, txn.epoch, txn.position)? { + Some(bytes) => BarrierLog::from_bytes(&bytes), + None => Ok(BarrierLog::default()), + } +} + +/// Remove the stored row of `txn` on `vshard_id`, which finished. +pub fn remove_log(catalog: &SystemCatalog, vshard_id: u32, txn: TxnId) -> crate::Result<()> { + catalog.remove_calvin_barrier_logs(&[(vshard_id, txn.epoch, txn.position)]) +} + +/// Remove every stored row of `vshard_id` whose txn `ledger` marks applied. +/// Returns how many rows went. +/// +/// A finished txn's row stays when its removal at the finish failed, or +/// when its event was saved while the txn finished. The scheduler runs this +/// on its stall tick, so such a row goes while the node runs, and a failed +/// pass retries on the next tick. +pub fn sweep_finished( + catalog: &SystemCatalog, + vshard_id: u32, + ledger: &CalvinAppliedLedger, +) -> crate::Result { + let finished: Vec<(u32, u64, u32)> = catalog + .load_calvin_barrier_logs(Some(&BTreeSet::from([vshard_id])))? + .into_iter() + .filter(|row| ledger.is_applied(row.epoch, row.position)) + .map(|row| (row.vshard_id, row.epoch, row.position)) + .collect(); + catalog.remove_calvin_barrier_logs(&finished)?; + Ok(finished.len()) +} + +/// Whether `shared`'s ledger of `vshard_id` holds `(epoch, position)`. +fn is_applied(shared: &SharedState, vshard_id: u32, epoch: u64, position: u32) -> bool { + shared + .calvin + .applied + .get(vshard_id) + .is_some_and(|ledger| ledger.is_applied(epoch, position)) +} + +/// Mark the txn of every stored row stored in its vShard's buffer, and +/// remove the rows of finished txns. Boot runs this once the applied +/// ledgers are filled, before the data-group apply loop and the schedulers +/// start. +pub fn restore_at_boot(shared: &SharedState) -> crate::Result<()> { + let catalog = shared.credentials.catalog(); + let mut finished = Vec::new(); + let mut waiting: BTreeMap> = BTreeMap::new(); + for row in catalog.load_calvin_barrier_logs(None)? { + if is_applied(shared, row.vshard_id, row.epoch, row.position) { + finished.push((row.vshard_id, row.epoch, row.position)); + } else { + waiting + .entry(row.vshard_id) + .or_default() + .push(TxnId::new(row.epoch, row.position)); + } + } + catalog.remove_calvin_barrier_logs(&finished)?; + for (vshard_id, txns) in waiting { + shared + .calvin + .read_results + .vshard(vshard_id) + .mark_stored(txns); + } + Ok(()) +} + +/// The stored rows of the unfinished txns of `vshards`, for a snapshot of +/// their data group. The caller fences the group's apply, so the rows hold +/// exactly the barrier entries at or below the snapshot's index. +/// +/// Fails when a vShard holds an event its row lacks: the rows do not hold +/// every entry the snapshot covers. The build retries on the next +/// heartbeat. +pub fn capture_group( + shared: &SharedState, + vshards: &BTreeSet, +) -> crate::Result> { + if let Some(vshard_id) = vshards + .iter() + .copied() + .find(|vshard_id| shared.calvin.read_results.vshard(*vshard_id).has_unsaved()) + { + return Err(crate::Error::Internal { + detail: format!( + "snapshot build: vShard {vshard_id} holds a dependent-read barrier event its \ + stored row lacks; the build retries on the next heartbeat" + ), + }); + } + let rows = shared + .credentials + .catalog() + .load_calvin_barrier_logs(Some(vshards))?; + Ok(rows + .into_iter() + .filter(|row| !is_applied(shared, row.vshard_id, row.epoch, row.position)) + .map(|row| CutBarrierLog { + vshard_id: row.vshard_id, + epoch: row.epoch, + position: row.position, + log: row.log, + }) + .collect()) +} + +/// Install `logs`, the rows a snapshot of the data group of `vshards` +/// carries: they replace this node's rows of `vshards`, and each vShard's +/// buffer holds their txns as stored. +pub fn install_group( + shared: &SharedState, + vshards: &BTreeSet, + logs: Vec, +) -> crate::Result<()> { + let rows: Vec = logs + .into_iter() + .filter(|log| vshards.contains(&log.vshard_id)) + .map(|log| StoredBarrierLog { + vshard_id: log.vshard_id, + epoch: log.epoch, + position: log.position, + log: log.log, + }) + .collect(); + shared + .credentials + .catalog() + .replace_calvin_barrier_logs(vshards, &rows)?; + for vshard_id in vshards { + let stored: BTreeSet = rows + .iter() + .filter(|row| row.vshard_id == *vshard_id) + .map(|row| TxnId::new(row.epoch, row.position)) + .collect(); + shared + .calvin + .read_results + .vshard(*vshard_id) + .reset_to_stored(stored); + } + Ok(()) +} + +/// Remove every row of `vshard_id`, which left this node, and its buffer. +pub fn forget_vshard(shared: &SharedState, vshard_id: u32) -> crate::Result<()> { + shared.calvin.read_results.forget(vshard_id); + shared + .credentials + .catalog() + .replace_calvin_barrier_logs(&BTreeSet::from([vshard_id]), &[]) +} + +#[cfg(test)] +mod tests { + use std::sync::Arc; + + use nodedb_physical::physical_plan::meta::PassiveReadKeyId; + use nodedb_types::{QualifiedCollection, Value}; + + use super::*; + use crate::bridge::dispatch::Dispatcher; + use crate::control::cluster::calvin::scheduler::NOT_YET_APPLIED_EPOCH; + use crate::wal::WalManager; + + const VSHARD: u32 = 5; + + fn state() -> (Arc, tempfile::TempDir) { + let dir = tempfile::tempdir().expect("tempdir"); + let wal = Arc::new(WalManager::open_for_testing(&dir.path().join("wal")).expect("wal")); + let (dispatcher, _sides) = Dispatcher::new(1, 64); + ( + SharedState::new(dispatcher, wal).expect("shared state"), + dir, + ) + } + + fn read(passive_vshard: u32, value: i64) -> BarrierEvent { + BarrierEvent::Read { + passive_vshard, + values: vec![( + PassiveReadKeyId::kv( + QualifiedCollection::from_stored("items".to_owned()), + b"k".to_vec(), + ), + Value::Integer(value), + )], + } + } + + /// Saved events fold in order into the row, and the row reloads. + #[test] + fn saved_events_fold_into_the_row() { + let (state, _dir) = state(); + let catalog = state.credentials.catalog(); + let txn = TxnId::new(4, 1); + save_event(catalog, VSHARD, txn, &read(1, 10)).expect("save"); + save_event(catalog, VSHARD, txn, &BarrierEvent::Timeout).expect("save"); + save_event(catalog, VSHARD, txn, &read(2, 20)).expect("save"); + let mut expected = BarrierLog::default(); + expected.note(read(1, 10)); + expected.note(BarrierEvent::Timeout); + assert_eq!(load_log(catalog, VSHARD, txn).expect("load"), expected); + remove_log(catalog, VSHARD, txn).expect("remove"); + assert_eq!( + load_log(catalog, VSHARD, txn).expect("load"), + BarrierLog::default() + ); + } + + /// Boot rebuilds the buffer from the stored rows: an unfinished txn's + /// events wait as stored, and a finished txn's row is removed. + #[test] + fn boot_rebuilds_the_buffer_from_the_rows() { + let (state, _dir) = state(); + let catalog = state.credentials.catalog(); + let open = TxnId::new(7, 0); + let done = TxnId::new(6, 0); + save_event(catalog, VSHARD, open, &read(1, 10)).expect("save"); + save_event(catalog, VSHARD, done, &read(1, 11)).expect("save"); + state + .calvin + .applied + .install(VSHARD, NOT_YET_APPLIED_EPOCH, BTreeSet::from([(6, 0)])); + + restore_at_boot(&state).expect("restore"); + + let buffer = state.calvin.read_results.vshard(VSHARD); + assert_eq!(buffer.waiting_txns(), 1); + let events = buffer.take(open).expect("the open txn waits"); + assert!(events.stored); + let mut expected = BarrierLog::default(); + expected.note(read(1, 10)); + assert_eq!(load_log(catalog, VSHARD, open).expect("load"), expected); + assert!( + catalog + .load_calvin_barrier_log(VSHARD, done.epoch, done.position) + .expect("load") + .is_none(), + "the finished txn's row is removed" + ); + } + + /// A snapshot carries the rows of unfinished txns, and its install + /// replaces the target's rows and buffer with them. + #[test] + fn a_snapshot_carries_the_rows_of_unfinished_txns() { + let (builder, _builder_dir) = state(); + let open = TxnId::new(3, 2); + save_event(builder.credentials.catalog(), VSHARD, open, &read(1, 10)).expect("save"); + save_event( + builder.credentials.catalog(), + VSHARD, + TxnId::new(2, 0), + &read(1, 9), + ) + .expect("save"); + builder + .calvin + .applied + .install(VSHARD, NOT_YET_APPLIED_EPOCH, BTreeSet::from([(2, 0)])); + let vshards = BTreeSet::from([VSHARD]); + let logs = capture_group(&builder, &vshards).expect("capture"); + assert_eq!(logs.len(), 1); + assert_eq!((logs[0].epoch, logs[0].position), (3, 2)); + + let (target, _target_dir) = state(); + let stale = TxnId::new(1, 0); + save_event(target.credentials.catalog(), VSHARD, stale, &read(1, 1)).expect("save"); + target + .calvin + .read_results + .vshard(VSHARD) + .push(stale, read(1, 1), 8, false); + install_group(&target, &vshards, logs).expect("install"); + + let buffer = target.calvin.read_results.vshard(VSHARD); + assert!(!buffer.waits(stale)); + assert!(!buffer.has_unsaved()); + assert!(buffer.take(open).expect("installed txn").stored); + let mut expected = BarrierLog::default(); + expected.note(read(1, 10)); + assert_eq!( + load_log(target.credentials.catalog(), VSHARD, open).expect("load"), + expected + ); + assert_eq!( + load_log(target.credentials.catalog(), VSHARD, stale).expect("load"), + BarrierLog::default() + ); + } + + /// A leftover row of a finished txn goes on the sweep, with no restart. + /// The row of an unfinished txn stays. + #[test] + fn the_sweep_removes_a_finished_txns_leftover_row() { + let (state, _dir) = state(); + let catalog = state.credentials.catalog(); + let done = TxnId::new(8, 0); + let open = TxnId::new(9, 0); + save_event(catalog, VSHARD, done, &read(1, 1)).expect("save"); + save_event(catalog, VSHARD, open, &read(1, 2)).expect("save"); + save_event(catalog, VSHARD + 1, done, &read(1, 3)).expect("save"); + let ledger = state.calvin.applied.get_or_create(VSHARD); + ledger.mark_terminal(done.epoch, done.position); + + assert_eq!(sweep_finished(catalog, VSHARD, &ledger).expect("sweep"), 1); + + let left: Vec<(u32, u64, u32)> = catalog + .load_calvin_barrier_logs(None) + .expect("load") + .into_iter() + .map(|row| (row.vshard_id, row.epoch, row.position)) + .collect(); + assert_eq!(left, vec![(VSHARD, 9, 0), (VSHARD + 1, 8, 0)]); + assert_eq!(sweep_finished(catalog, VSHARD, &ledger).expect("sweep"), 0); + } + + /// A vShard holding an unsaved event refuses the capture. + #[test] + fn an_unsaved_event_refuses_the_capture() { + let (state, _dir) = state(); + state + .calvin + .read_results + .vshard(VSHARD) + .push(TxnId::new(1, 0), read(1, 1), 8, false); + assert!(capture_group(&state, &BTreeSet::from([VSHARD])).is_err()); + } +} diff --git a/nodedb/src/control/cluster/calvin/scheduler/caught_up.rs b/nodedb/src/control/cluster/calvin/scheduler/caught_up.rs new file mode 100644 index 000000000..1f9a24ba6 --- /dev/null +++ b/nodedb/src/control/cluster/calvin/scheduler/caught_up.rs @@ -0,0 +1,127 @@ +// SPDX-License-Identifier: BUSL-1.1 + +//! Whether each running Calvin scheduler on this node reached its rebuild +//! target. +//! +//! A scheduler starts with a rebuild target: the highest epoch its recovery +//! found applied for the vShard. It is caught up once its fully-applied +//! watermark reaches that target. A scheduler with no target is caught up +//! from the start. Startup holds the client gateway until every running +//! scheduler is caught up. +//! +//! Each scheduler holds a [`CaughtUpHandle`] for its vShard. The handle +//! removes its entry when the scheduler is dropped, so the registry lists +//! only running schedulers. A restarted scheduler replaces its predecessor's +//! entry, and the predecessor's drop leaves the new entry in place. + +use std::collections::BTreeMap; +use std::sync::atomic::{AtomicBool, Ordering}; +use std::sync::{Arc, Mutex}; + +type Entries = Arc>>>; + +/// Caught-up state of every running scheduler on this node, by vShard. +#[derive(Debug, Default)] +pub struct CaughtUpRegistry { + entries: Entries, +} + +impl CaughtUpRegistry { + /// Register the scheduler starting for `vshard_id`, not caught up yet. + pub fn register(&self, vshard_id: u32) -> CaughtUpHandle { + let flag = Arc::new(AtomicBool::new(false)); + self.entries + .lock() + .unwrap_or_else(|p| p.into_inner()) + .insert(vshard_id, Arc::clone(&flag)); + CaughtUpHandle { + vshard_id, + flag, + entries: Arc::clone(&self.entries), + } + } + + /// The vShards whose running scheduler is not caught up, in order. + pub fn lagging(&self) -> Vec { + self.entries + .lock() + .unwrap_or_else(|p| p.into_inner()) + .iter() + .filter(|(_, flag)| !flag.load(Ordering::Acquire)) + .map(|(vshard_id, _)| *vshard_id) + .collect() + } +} + +/// One scheduler's entry in the [`CaughtUpRegistry`]. +#[derive(Debug)] +pub struct CaughtUpHandle { + vshard_id: u32, + flag: Arc, + entries: Entries, +} + +impl CaughtUpHandle { + /// Whether this scheduler reported caught up. + pub fn is_caught_up(&self) -> bool { + self.flag.load(Ordering::Acquire) + } + + /// Report this scheduler caught up. The state never returns to lagging: + /// the fully-applied watermark only rises. + pub fn mark_caught_up(&self) { + self.flag.store(true, Ordering::Release); + } +} + +impl Drop for CaughtUpHandle { + fn drop(&mut self) { + let mut entries = self.entries.lock().unwrap_or_else(|p| p.into_inner()); + if entries + .get(&self.vshard_id) + .is_some_and(|flag| Arc::ptr_eq(flag, &self.flag)) + { + entries.remove(&self.vshard_id); + } + } +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn a_registered_scheduler_lags_until_it_reports() { + let registry = CaughtUpRegistry::default(); + let first = registry.register(3); + let second = registry.register(1); + assert_eq!(registry.lagging(), vec![1, 3]); + + second.mark_caught_up(); + assert!(second.is_caught_up()); + assert_eq!(registry.lagging(), vec![3]); + + first.mark_caught_up(); + assert!(registry.lagging().is_empty()); + } + + #[test] + fn a_dropped_scheduler_leaves_the_registry() { + let registry = CaughtUpRegistry::default(); + let handle = registry.register(5); + assert_eq!(registry.lagging(), vec![5]); + drop(handle); + assert!(registry.lagging().is_empty()); + } + + #[test] + fn a_predecessor_drop_keeps_its_successor_entry() { + let registry = CaughtUpRegistry::default(); + let predecessor = registry.register(7); + let successor = registry.register(7); + drop(predecessor); + assert_eq!(registry.lagging(), vec![7], "the successor still lags"); + successor.mark_caught_up(); + assert!(registry.lagging().is_empty()); + } +} diff --git a/nodedb/src/control/cluster/calvin/scheduler/cut_hold.rs b/nodedb/src/control/cluster/calvin/scheduler/cut_hold.rs new file mode 100644 index 000000000..aec4f2248 --- /dev/null +++ b/nodedb/src/control/cluster/calvin/scheduler/cut_hold.rs @@ -0,0 +1,129 @@ +// SPDX-License-Identifier: BUSL-1.1 + +//! The ordered cuts whose barrier one vShard's scheduler waits for. +//! +//! A cut marker that carries a barrier arrives between two epoch batches, as +//! every marker does. Every transaction of an epoch at or below the highest +//! epoch delivered before the marker came before it. Its slice installed +//! before the scheduler passed the marker, so its redo sits before the +//! group's barrier. Every transaction of a later epoch came after the +//! marker: its redo waits until this node applied the group's barrier. A +//! transaction before the marker never waits on it, so the hold never keeps +//! the marker from passing, nor the barrier from being placed. +//! +//! Every replica tracks the same cuts, so a new leader holds the same +//! transactions. A copy of a tracked cut, which the cut proposes again when +//! a leader change can drop its marker, changes nothing: the first copy +//! placed the cut. + +use crate::control::backup::cut_order::OrderedCut; + +/// A cut whose barrier the scheduler waits for. +#[derive(Debug, Clone, PartialEq, Eq)] +struct HeldCut { + cut: OrderedCut, + /// The highest epoch delivered before the cut's marker, `None` when none + /// was: then every transaction came after it. + through: Option, +} + +/// The cuts one scheduler holds later redo for, in arrival order. +#[derive(Debug, Default)] +pub struct CutHolds { + held: Vec, +} + +impl CutHolds { + /// Hold for `cut`, whose marker arrived after epochs up to `through`. + /// Returns `false` for a copy of a cut already held. + pub fn receive(&mut self, cut: OrderedCut, through: Option) -> bool { + let key = cut.key(); + if self.held.iter().any(|held| held.cut.key() == key) { + return false; + } + self.held.push(HeldCut { cut, through }); + true + } + + /// Whether a transaction of `epoch` waits for a held cut's barrier. + pub fn holds(&self, epoch: u64) -> bool { + self.held + .iter() + .any(|held| held.through.is_none_or(|through| through < epoch)) + } + + /// Stop holding for every cut `ended` names. Returns whether any ended. + pub fn end(&mut self, mut ended: impl FnMut(&OrderedCut) -> bool) -> bool { + let before = self.held.len(); + self.held.retain(|held| !ended(&held.cut)); + self.held.len() != before + } + + /// Every held cut, in arrival order. + pub fn cuts(&self) -> impl Iterator { + self.held.iter().map(|held| &held.cut) + } + + pub fn is_empty(&self) -> bool { + self.held.is_empty() + } +} + +#[cfg(test)] +mod tests { + use super::*; + + fn cut(hlc: u64) -> OrderedCut { + OrderedCut { + hlc, + restore_point: 0, + capture: None, + } + } + + #[test] + fn only_a_transaction_after_the_marker_waits() { + let mut holds = CutHolds::default(); + assert!(!holds.holds(9), "no cut, no hold"); + assert!(holds.receive(cut(100), Some(7))); + assert!(!holds.holds(7), "epoch 7 came before the marker"); + assert!(!holds.holds(3)); + assert!(holds.holds(8), "epoch 8 came after it"); + } + + #[test] + fn a_marker_before_any_epoch_holds_every_transaction() { + let mut holds = CutHolds::default(); + holds.receive(cut(100), None); + assert!(holds.holds(0)); + assert!(holds.holds(1)); + } + + /// A second copy of a held cut arrives later and keeps the first copy's + /// place: the transactions between the two copies still wait. + #[test] + fn a_copy_of_a_held_cut_keeps_the_first_place() { + let mut holds = CutHolds::default(); + assert!(holds.receive(cut(100), Some(7))); + assert!( + !holds.receive(cut(100), Some(12)), + "a copy is not a new cut" + ); + assert!(holds.holds(9), "epoch 9 came after the first copy"); + assert_eq!(holds.cuts().count(), 1); + } + + #[test] + fn an_ended_cut_releases_its_transactions() { + let mut holds = CutHolds::default(); + holds.receive(cut(100), Some(7)); + holds.receive(cut(200), Some(10)); + assert!(!holds.end(|_| false)); + assert!(holds.end(|ended| ended.hlc == 100)); + assert!(!holds.holds(9), "only the later cut holds, from epoch 11"); + assert!(holds.holds(11)); + assert!(holds.end(|_| true)); + assert!(holds.is_empty()); + assert!(!holds.holds(11)); + } +} diff --git a/nodedb/src/control/cluster/calvin/scheduler/driver/barrier.rs b/nodedb/src/control/cluster/calvin/scheduler/driver/barrier.rs index eee382569..708ee73ef 100644 --- a/nodedb/src/control/cluster/calvin/scheduler/driver/barrier.rs +++ b/nodedb/src/control/cluster/calvin/scheduler/driver/barrier.rs @@ -1,59 +1,42 @@ // SPDX-License-Identifier: BUSL-1.1 -//! Dependent-read barrier for Calvin scheduler. +//! Dependent-read barrier of the Calvin scheduler. //! -//! When a dependent-read Calvin txn arrives at an active vshard (one with -//! write keys), the scheduler inserts a [`PendingDependentBarrier`] and waits -//! for all passive vshards to deliver their read results via the per-vshard -//! Raft log (`ReplicatedWrite::CalvinReadResult`). +//! A dependent-read txn names the rows its passive vShards read. On an +//! active vShard (one the write set names) the data-group leader opens a +//! [`PendingDependentBarrier`] once the txn holds its locks. The barrier +//! waits for the read result of every passive vShard through the vShard's +//! data-group log (`ReplicatedWrite::CalvinReadResult`). //! -//! Once all passive vshards have delivered, the barrier assembles the -//! `injected_reads` map and dispatches `MetaOp::CalvinExecuteActive`. +//! The log alone decides the barrier, so every replica decides it alike. +//! The scheduler folds the log's barrier entries into a +//! [`crate::control::cluster::calvin::scheduler::BarrierLog`] per txn: //! -//! # Determinism +//! - Every result came first: the barrier completes. The leader compares the +//! values with the ones the coordinator read, and dispatches +//! `MetaOp::CalvinExecuteActive` or votes `PredictionDrift`. +//! - A `ReplicatedWrite::CalvinReadTimeout` came first: the barrier times +//! out, and the leader votes abort. //! -//! All maps here use `BTreeMap`/`BTreeSet` — never `HashMap`/`HashSet`. +//! The leader proposes the timeout entry once its barrier waited past +//! `tuning.calvin.dependent_read_passive_timeout_ms` on its own clock. The +//! clock only decides when the entry is proposed. Its place in the log +//! decides what it does. //! -//! # Timing +//! # Determinism //! -//! `Instant::now()` is used for `timeout_at` (observability / off-WAL path -//! only; barrier timeouts are never written to the WAL). +//! All maps here use `BTreeMap`/`BTreeSet` — never `HashMap`/`HashSet`. use std::collections::{BTreeMap, BTreeSet}; use std::time::Instant; -use nodedb_cluster::calvin::types::SequencedTxn; - -use super::super::lock_manager::TxnId; -use crate::types::TenantId; +use nodedb_cluster::calvin::types::{DependentReadSpec, SequencedTxn}; use nodedb_physical::physical_plan::meta::PassiveReadKeyId; use nodedb_types::Value; -// ── ReadResultEvent ──────────────────────────────────────────────────────────── - -/// A committed `CalvinReadResult` Raft entry delivered to the active vshard's -/// scheduler. -/// -/// Produced by the apply loop when a `ReplicatedWrite::CalvinReadResult` -/// entry commits on a vshard whose scheduler has an active dependent barrier. -#[derive(Debug)] -pub struct ReadResultEvent { - /// Sequencer epoch of the txn. - pub epoch: u64, - /// Position within the epoch batch. - pub position: u32, - /// Vshard that performed the read. - pub passive_vshard: u32, - /// Tenant scope. - pub tenant_id: TenantId, - /// Decoded read values from the passive participant. - pub values: Vec<(PassiveReadKeyId, Value)>, -} - -// ── PendingDependentBarrier ──────────────────────────────────────────────────── +use super::super::lock_manager::TxnId; -/// A dependent-read barrier: tracks which passive vshards have delivered -/// their read results and assembles the `injected_reads` map once all are in. +/// A dependent-read barrier the data-group leader holds open. pub struct PendingDependentBarrier { /// Original sequenced transaction. pub txn: SequencedTxn, @@ -61,42 +44,86 @@ pub struct PendingDependentBarrier { /// reservation owns the lock). Carried through the barrier so the eventual /// `dispatch_active_txn` call can park the correct id in `PendingTxn`. pub lock_owner: TxnId, - /// Passive vshards still waiting to deliver their read results. - /// Entries are removed as `ReadResultEvent`s arrive. - /// `BTreeSet` for determinism. - pub waiting_for: BTreeSet, - /// Received read values, keyed by passive vshard. - /// `BTreeMap` for determinism. - pub received: BTreeMap>, - /// Deadline for passive participant delivery. - /// - /// `Instant::now()` used for barrier timeout (observability / off-WAL path). - pub timeout_at: Instant, + /// The passive vShards whose read result the barrier waits for. + pub passive: BTreeSet, + /// When this leader proposes the barrier's timeout entry, on its own + /// clock. A proposal re-arms it, so a timeout entry a leader change lost + /// is proposed again. + pub timeout_due: Instant, +} + +/// The first passive row whose broadcast value differs from the value the +/// coordinator read, described. `None` when every row matches. +/// +/// A row reads back as its stored bytes, `Value::Bytes`, or `Value::Null` +/// when absent. A row no passive vShard returned, or one returned as any +/// other value, differs. +pub fn expected_reads_drift( + spec: &DependentReadSpec, + injected: &BTreeMap, +) -> Option { + spec.expected.iter().find_map(|(row, expected)| { + let found = match injected.get(row) { + Some(Value::Bytes(bytes)) => Some(bytes.as_slice()), + Some(Value::Null) => None, + Some(other) => { + return Some(format!( + "the passive read of {row:?} returned {other:?}, not stored bytes" + )); + } + None => return Some(format!("no passive vShard returned {row:?}")), + }; + (found != expected.as_deref()) + .then(|| format!("{row:?} changed since the coordinator read it")) + }) } -impl PendingDependentBarrier { - /// Whether all passive participants have delivered. - pub fn is_complete(&self) -> bool { - self.waiting_for.is_empty() +#[cfg(test)] +mod tests { + use nodedb_types::QualifiedCollection; + + use super::*; + + fn row() -> PassiveReadKeyId { + PassiveReadKeyId::kv( + QualifiedCollection::from_stored("items".to_owned()), + b"alice:sword".to_vec(), + ) } - /// Assemble the full `injected_reads` map from all received values. - /// - /// `BTreeMap` ensures deterministic iteration order across replicas. - pub fn assemble_injected_reads(&self) -> BTreeMap { - let mut out = BTreeMap::new(); - for values in self.received.values() { - for (key_id, value) in values { - out.insert(key_id.clone(), value.clone()); - } + fn spec(expected: Option<&[u8]>) -> DependentReadSpec { + DependentReadSpec { + passive_reads: BTreeMap::new(), + expected: BTreeMap::from([(row(), expected.map(<[u8]>::to_vec))]), } - out } - /// Whether this barrier has timed out. - /// - /// `Instant::now()` used for timeout check (observability; off-WAL path). - pub fn is_timed_out(&self) -> bool { - Instant::now() > self.timeout_at // no-determinism: scheduler barrier timeout check, not Calvin WAL data + #[test] + fn matching_bytes_report_no_drift() { + let injected = BTreeMap::from([(row(), Value::Bytes(b"v1".to_vec()))]); + assert_eq!(expected_reads_drift(&spec(Some(b"v1")), &injected), None); + } + + #[test] + fn changed_bytes_report_drift() { + let injected = BTreeMap::from([(row(), Value::Bytes(b"v2".to_vec()))]); + assert!(expected_reads_drift(&spec(Some(b"v1")), &injected).is_some()); + } + + #[test] + fn a_row_gone_since_the_read_reports_drift() { + let injected = BTreeMap::from([(row(), Value::Null)]); + assert!(expected_reads_drift(&spec(Some(b"v1")), &injected).is_some()); + } + + #[test] + fn an_absent_row_still_absent_reports_no_drift() { + let injected = BTreeMap::from([(row(), Value::Null)]); + assert_eq!(expected_reads_drift(&spec(None), &injected), None); + } + + #[test] + fn a_row_no_passive_returned_reports_drift() { + assert!(expected_reads_drift(&spec(Some(b"v1")), &BTreeMap::new()).is_some()); } } diff --git a/nodedb/src/control/cluster/calvin/scheduler/driver/config.rs b/nodedb/src/control/cluster/calvin/scheduler/driver/config.rs index a71084d62..89178f470 100644 --- a/nodedb/src/control/cluster/calvin/scheduler/driver/config.rs +++ b/nodedb/src/control/cluster/calvin/scheduler/driver/config.rs @@ -1,10 +1,17 @@ // SPDX-License-Identifier: BUSL-1.1 //! Configuration for the Calvin scheduler driver. +//! +//! The `[tuning.calvin]` section sets every knob except the epoch duration, +//! which comes from the sequencer config. use std::time::Duration; -use crate::bridge::dispatch::DATA_PLANE_QUEUE_CAPACITY; +use nodedb_cluster::calvin::SequencerConfig; +use nodedb_types::config::tuning::CalvinTuning; + +/// The longest wait between two restages of one committed txn. +pub const MAX_RESTAGE_BACKOFF: Duration = Duration::from_secs(60); /// Tuning parameters for a [`super::core::Scheduler`] instance. #[derive(Debug, Clone)] @@ -13,14 +20,16 @@ pub struct SchedulerConfig { pub channel_capacity: usize, /// Transaction deadline multiplier (× epoch_duration_ms). pub txn_deadline_multiplier: u32, - /// Epoch duration in milliseconds (used to compute deadlines). + /// Epoch duration in milliseconds (used to compute deadlines). Equal to + /// the sequencer's epoch duration. pub epoch_duration_ms: u64, - /// Timeout for passive participant responses in dependent-read txns. + /// Wait of a dependent-read barrier for its passive read results before + /// the data-group leader proposes the barrier's timeout entry. /// - /// Default: `epoch_duration_ms * 3` milliseconds. + /// Default: 5000 milliseconds, 250 default epochs. /// - /// `Instant::now()` is used for barrier timeouts (observability / off-WAL - /// path only; never influences WAL bytes). + /// The leader's clock decides only when it proposes the entry. The + /// entry's place in the log decides the barrier. pub dependent_read_passive_timeout_ms: u64, /// Stall deadline for a staged txn parked awaiting the durable global /// verdict. On expiry the scheduler re-probes the registry and, if the @@ -29,7 +38,7 @@ pub struct SchedulerConfig { /// cross-shard commit. This is purely a liveness/observability knob: it /// controls how often the stall is surfaced, never whether the txn aborts. /// - /// Default: `epoch_duration_ms * 250` milliseconds. + /// Default: 5000 milliseconds, 250 default epochs. pub verdict_stall_warn_ms: u64, /// In-flight backlog at which the scheduler stops taking new sequenced /// input. The backlog counts pending, blocked, and dependent-barrier txns. @@ -38,35 +47,54 @@ pub struct SchedulerConfig { /// input. A backlog of blocked txns alone keeps intake open, because a /// reservation release they wait on arrives as input. /// - /// Default: [`DATA_PLANE_QUEUE_CAPACITY`]. Every dispatch of one vShard - /// goes to one Data Plane core, whose queue holds that many requests. A - /// larger backlog cannot hold a dispatch slot per txn at once. + /// Default: [`crate::bridge::dispatch::DATA_PLANE_QUEUE_CAPACITY`]. Every + /// dispatch of one vShard goes to one Data Plane core, whose queue holds + /// that many requests. A larger backlog cannot hold a dispatch slot per + /// txn at once. pub max_inflight_backlog: usize, /// Most sequencer log entries one catch-up drain reads and replays. /// - /// Default: `channel_capacity`. A drop happens only when the fan-out - /// channel is full, so one window covers about one channel of missed - /// entries. Windows run back to back while intake is open. + /// Default: 512, the default `channel_capacity`. A drop happens only when + /// the fan-out channel is full, so one window covers about one channel of + /// missed entries. Windows run back to back while intake is open. pub catch_up_window: u64, + /// Most restages of a committed txn whose stage failed on this leader. + /// The scheduler halts once they run out. + /// + /// Default: 5. + pub restage_attempts: u32, + /// Wait before the first restage of a committed txn. Each later restage + /// waits twice as long. The wait only delays a retry of a decided + /// verdict, so it never changes what any replica applies. + /// + /// Default: 100 milliseconds, five default epochs. + pub restage_backoff_ms: u64, } impl Default for SchedulerConfig { fn default() -> Self { - let epoch_duration_ms = 20u64; - let channel_capacity = 512usize; - Self { - channel_capacity, - txn_deadline_multiplier: 3, - epoch_duration_ms, - dependent_read_passive_timeout_ms: epoch_duration_ms * 3, - verdict_stall_warn_ms: epoch_duration_ms * 250, - max_inflight_backlog: DATA_PLANE_QUEUE_CAPACITY, - catch_up_window: channel_capacity as u64, - } + Self::from_tuning(&CalvinTuning::default(), &SequencerConfig::default()) } } impl SchedulerConfig { + /// The scheduler config `tuning` sets, on the epoch duration of the + /// sequencer that feeds the scheduler. + pub fn from_tuning(tuning: &CalvinTuning, sequencer: &SequencerConfig) -> Self { + Self { + channel_capacity: tuning.channel_capacity, + txn_deadline_multiplier: tuning.txn_deadline_multiplier, + epoch_duration_ms: u64::try_from(sequencer.epoch_duration.as_millis()) + .unwrap_or(u64::MAX), + dependent_read_passive_timeout_ms: tuning.dependent_read_passive_timeout_ms, + verdict_stall_warn_ms: tuning.verdict_stall_warn_ms, + max_inflight_backlog: tuning.max_inflight_backlog, + catch_up_window: tuning.catch_up_window, + restage_attempts: tuning.restage_attempts, + restage_backoff_ms: tuning.restage_backoff_ms, + } + } + /// Passive timeout as a `Duration`. pub fn passive_timeout(&self) -> Duration { Duration::from_millis(self.dependent_read_passive_timeout_ms) @@ -76,4 +104,79 @@ impl SchedulerConfig { pub fn verdict_stall_warn(&self) -> Duration { Duration::from_millis(self.verdict_stall_warn_ms) } + + /// The wait before restage number `done + 1`, after `done` restages: + /// the base wait doubled once per restage done, at most + /// [`MAX_RESTAGE_BACKOFF`]. + pub fn restage_backoff(&self, done: u32) -> Duration { + let factor = 1u64.checked_shl(done).unwrap_or(u64::MAX); + Duration::from_millis(self.restage_backoff_ms.saturating_mul(factor)) + .min(MAX_RESTAGE_BACKOFF) + } +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::bridge::dispatch::DATA_PLANE_QUEUE_CAPACITY; + + #[test] + fn the_default_tuning_builds_the_shipped_scheduler_config() { + let config = SchedulerConfig::default(); + assert_eq!(config.channel_capacity, 512); + assert_eq!(config.txn_deadline_multiplier, 3); + assert_eq!(config.epoch_duration_ms, 20); + assert_eq!(config.dependent_read_passive_timeout_ms, 5_000); + assert_eq!(config.verdict_stall_warn_ms, 5_000); + assert_eq!(config.catch_up_window, 512); + assert_eq!(config.restage_attempts, 5); + assert_eq!(config.restage_backoff_ms, 100); + } + + /// Each restage waits twice as long as the one before, up to the cap. + /// A count past the shift width saturates instead of wrapping. + #[test] + fn the_restage_backoff_doubles_per_restage() { + let config = SchedulerConfig::default(); + assert_eq!(config.restage_backoff(0), Duration::from_millis(100)); + assert_eq!(config.restage_backoff(1), Duration::from_millis(200)); + assert_eq!(config.restage_backoff(4), Duration::from_millis(1_600)); + assert_eq!(config.restage_backoff(20), MAX_RESTAGE_BACKOFF); + assert_eq!(config.restage_backoff(64), MAX_RESTAGE_BACKOFF); + } + + /// The tuning default restates the dispatcher queue capacity, which the + /// tuning crate cannot import. The two must not drift apart. + #[test] + fn the_default_backlog_is_one_dispatcher_queue() { + assert_eq!( + CalvinTuning::default().max_inflight_backlog, + DATA_PLANE_QUEUE_CAPACITY + ); + assert_eq!( + SchedulerConfig::default().max_inflight_backlog, + DATA_PLANE_QUEUE_CAPACITY + ); + } + + #[test] + fn tuning_overrides_reach_the_scheduler_config() { + let tuning = CalvinTuning { + max_inflight_backlog: 128, + verdict_stall_warn_ms: 9_000, + restage_attempts: 2, + restage_backoff_ms: 7, + ..CalvinTuning::default() + }; + let sequencer = SequencerConfig { + epoch_duration: Duration::from_millis(50), + ..SequencerConfig::default() + }; + let config = SchedulerConfig::from_tuning(&tuning, &sequencer); + assert_eq!(config.max_inflight_backlog, 128); + assert_eq!(config.verdict_stall_warn(), Duration::from_millis(9_000)); + assert_eq!(config.epoch_duration_ms, 50); + assert_eq!(config.restage_attempts, 2); + assert_eq!(config.restage_backoff(0), Duration::from_millis(7)); + } } diff --git a/nodedb/src/control/cluster/calvin/scheduler/driver/core/apply_result.rs b/nodedb/src/control/cluster/calvin/scheduler/driver/core/apply_result.rs new file mode 100644 index 000000000..464500e25 --- /dev/null +++ b/nodedb/src/control/cluster/calvin/scheduler/driver/core/apply_result.rs @@ -0,0 +1,211 @@ +// SPDX-License-Identifier: BUSL-1.1 + +//! What a committed slice's install answers to its coordinator: the local +//! applied-result sidecar entry, and the result its `CompletionAck` carries. + +use crate::bridge::envelope::{Response, Status}; +use crate::control::cluster::calvin::scheduler::lock_manager::TxnId; +use crate::control::state::{AckSlice, CalvinAckResult, CalvinApplyResult}; + +use super::scheduler::Scheduler; + +impl Scheduler { + /// Deposit a primary slice's applied `reply` into this node's sidecar, + /// before its `CompletionAck` fires the coordinator's waiter. + /// + /// A multi-collection cross-shard COMMIT has many primary-write + /// participants, each a plain affected-count write, and they coalesce: + /// the first answer stands, with every participant's timeseries install + /// counts merged in. Only two participants that each carry RETURNING + /// rows conflict: a cross-shard RETURNING union is unsupported, so the + /// statement fails loudly rather than return one shard's rows. + pub(in crate::control::cluster::calvin::scheduler::driver::core) fn deposit_reply( + &self, + txn_id: TxnId, + reply: &Response, + returning: bool, + ) { + use std::collections::hash_map::Entry; + + let response = statement_reply(reply.clone()); + let key = nodedb_cluster::calvin::TxnId::new(txn_id.epoch, txn_id.position); + let vshard_id = self.vshard_id; + self.shared + .calvin + .apply_results + .deposit_with(key, |entry| match entry { + Entry::Vacant(slot) => { + slot.insert(CalvinApplyResult::Single { + response, + has_returning: returning, + }); + } + Entry::Occupied(mut slot) => { + let existing_returning = matches!( + slot.get(), + CalvinApplyResult::Single { + has_returning: true, + .. + } + ); + let already_conflict = matches!(slot.get(), CalvinApplyResult::Conflict); + if already_conflict { + // A RETURNING union was already recorded; it stays one. + } else if returning && existing_returning { + tracing::error!( + epoch = txn_id.epoch, + position = txn_id.position, + vshard = vshard_id, + "two RETURNING-bearing participants for one Calvin txn — cross-shard \ + RETURNING union unsupported" + ); + slot.insert(CalvinApplyResult::Conflict); + } else if returning { + // The incoming participant carries the rows; the held + // entry was a plain affected-count sibling. Rows win. + slot.insert(CalvinApplyResult::Single { + response, + has_returning: true, + }); + } else { + merge_install_counts(slot.get_mut(), &response); + } + } + }); + } + + /// The result a slice's `CompletionAck` carries for its applied `reply`: + /// the install's timeseries counts, and the rows of a RETURNING slice. + /// The rows are bounded by the query result limit and by the ack's + /// frame budget, whichever is lower. Every replica renders the same + /// reply, so every replica owes the same bytes. + pub(in crate::control::cluster::calvin::scheduler::driver::core) fn ack_result_of( + &self, + reply: &Response, + slice: AckSlice, + ) -> Vec { + let limit = self + .shared + .tuning + .network + .max_query_result_bytes + .min(crate::control::state::ACK_ROWS_FRAME_BUDGET); + CalvinAckResult::of(reply, slice, limit) + .to_bytes() + .unwrap_or_else(|error| { + tracing::warn!( + vshard_id = self.vshard_id, + %error, + "calvin: the completion ack carries no report" + ); + Vec::new() + }) + } +} + +/// Fold the timeseries install counts `incoming` answers into the plain +/// result `held`. A RETURNING result keeps its rows untouched. +fn merge_install_counts(held: &mut CalvinApplyResult, incoming: &Response) { + let CalvinApplyResult::Single { + response, + has_returning: false, + } = held + else { + return; + }; + if let Some(merged) = crate::engine::timeseries::install_counts::merge_count_payloads( + response.payload.as_bytes(), + incoming.payload.as_bytes(), + ) { + response.payload = crate::bridge::envelope::Payload::from_vec(merged); + } +} + +/// The response the statement drains. An install whose reply failed to +/// render answers `Ok` with the render error in `error_code`. The statement +/// reports that error. +fn statement_reply(mut response: Response) -> Response { + if response.status == Status::Ok && response.error_code.is_some() { + response.status = Status::Error; + response.payload = crate::bridge::envelope::Payload::empty(); + } + response +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::bridge::envelope::ErrorCode; + use crate::control::cluster::calvin::scheduler::driver::core::test_support::{ + error_response, staged_response, + }; + + fn installed_with_counts(collection: &str, accepted: u64, rejected: u64) -> Response { + use crate::engine::timeseries::install_counts::{TsInstallCount, TsInstallCounts}; + let mut response = staged_response(Status::Ok, None); + response.payload = crate::bridge::envelope::Payload::from_vec( + TsInstallCounts::new(vec![TsInstallCount { + collection: collection.into(), + accepted, + rejected, + }]) + .to_bytes() + .expect("encode install counts"), + ); + response + } + + /// Two plain participants of one Calvin transaction each installed a + /// timeseries batch. The coordinator's result carries both installs' + /// apply counts, so COMMIT reports every participant's rejected rows. A + /// RETURNING result keeps its rows. + #[test] + fn plain_participants_merge_their_install_counts() { + use crate::engine::timeseries::install_counts::TsInstallCounts; + let mut held = CalvinApplyResult::Single { + response: installed_with_counts("cpu", 2, 1), + has_returning: false, + }; + merge_install_counts(&mut held, &installed_with_counts("mem", 3, 2)); + let CalvinApplyResult::Single { response, .. } = &held else { + panic!("a merged result stays single"); + }; + let totals = TsInstallCounts::from_payload(response.payload.as_bytes()) + .expect("install counts") + .by_collection(); + assert_eq!(totals.get("cpu"), Some(&(2, 1))); + assert_eq!(totals.get("mem"), Some(&(3, 2))); + + let rows = crate::bridge::envelope::Payload::from_vec(vec![0x90]); + let mut returning = CalvinApplyResult::Single { + response: Response { + payload: rows, + ..installed_with_counts("cpu", 1, 0) + }, + has_returning: true, + }; + merge_install_counts(&mut returning, &installed_with_counts("mem", 1, 1)); + let CalvinApplyResult::Single { response, .. } = &returning else { + panic!("a RETURNING result stays single"); + }; + assert_eq!(response.payload.as_bytes(), &[0x90]); + } + + /// A render error on an installed slice reaches the statement as a + /// typed error. + #[test] + fn a_render_error_on_an_installed_slice_becomes_the_statement_error() { + let mut response = error_response(ErrorCode::Internal { + detail: "render".to_string(), + }); + response.status = Status::Ok; + + let reply = statement_reply(response); + + assert_eq!(reply.status, Status::Error); + assert!(matches!( + reply.error_code.as_deref(), + Some(ErrorCode::Internal { .. }) + )); + } +} diff --git a/nodedb/src/control/cluster/calvin/scheduler/driver/core/catch_up.rs b/nodedb/src/control/cluster/calvin/scheduler/driver/core/catch_up.rs index d02acd9a6..ce1ca86d1 100644 --- a/nodedb/src/control/cluster/calvin/scheduler/driver/core/catch_up.rs +++ b/nodedb/src/control/cluster/calvin/scheduler/driver/core/catch_up.rs @@ -253,6 +253,7 @@ mod tests { use nodedb_types::TenantId; use nodedb_types::id::DatabaseId; + use crate::control::cluster::calvin::scheduler::driver::core::test_proposer::lead_data_group; use crate::control::cluster::calvin::scheduler::driver::core::test_support::{ build_test_scheduler, build_test_scheduler_with_data_side, fill_tenant_inflight, make_sequenced_txn, make_validate_only_txn, test_coll_vshard, @@ -578,6 +579,7 @@ mod tests { let registry = CalvinCompletionRegistry::new_detached(); let (mut scheduler, _dir, mut data_side) = build_test_scheduler_with_data_side(test_coll_vshard(), registry); + lead_data_group(&mut scheduler); arm_two_dropped_stage_batches(&scheduler); let shared = std::sync::Arc::clone(&scheduler.shared); fill_tenant_inflight(&shared, &mut data_side, TenantId::new(1)); @@ -602,6 +604,7 @@ mod tests { let registry = CalvinCompletionRegistry::new_detached(); let (mut scheduler, _dir, mut data_side) = build_test_scheduler_with_data_side(test_coll_vshard(), registry); + lead_data_group(&mut scheduler); let (_idx0, idx1) = arm_two_dropped_stage_batches(&scheduler); let shared = std::sync::Arc::clone(&scheduler.shared); fill_tenant_inflight(&shared, &mut data_side, TenantId::new(1)); diff --git a/nodedb/src/control/cluster/calvin/scheduler/driver/core/commit_redo.rs b/nodedb/src/control/cluster/calvin/scheduler/driver/core/commit_redo.rs index 6ca3b632a..4cea57a79 100644 --- a/nodedb/src/control/cluster/calvin/scheduler/driver/core/commit_redo.rs +++ b/nodedb/src/control/cluster/calvin/scheduler/driver/core/commit_redo.rs @@ -1,39 +1,41 @@ // SPDX-License-Identifier: BUSL-1.1 -//! Redo-record resolution for a committed staged static Calvin transaction. +//! Redo resolution for a committed Calvin slice on the data-group leader. //! -//! A committed static Calvin dispatch stages its transaction on the Data -//! Plane (validate the read-set + buffer the plans, no base mutation). Once -//! the local commit vote is known (`resolve_staged_commit`), this module -//! drives the resolve step: dispatch `MetaOp::CalvinResolve` to reconstitute -//! the staged post-images as one replayable `RedoRecord`, WAL-append that -//! record (restoring restart durability for this vShard's slice of the -//! commit), then queue the flush for its sequencer-order turn -//! ([`super::flush_turn`]); `finish_resolved_commit` / `commit_apply_tail` -//! complete it. +//! Once the global verdict is COMMIT, the leader dispatches +//! `MetaOp::CalvinResolve` to reconstitute the staged post-images as one +//! replayable `RedoRecord`. It stamps the record with the slice's sequencer +//! position, attaches the transaction's messages and dedup key on the +//! participant that carries them, and proposes the record to the vShard's +//! data group (see [`super::redo_propose`]). Every replica, this one +//! included, installs it from the log. +use nodedb_physical::physical_plan::PhysicalPlan; +use nodedb_physical::physical_plan::meta::MetaOp; + +use super::super::types::CommitState; use super::deferred::{DispatchOutcome, DispatchStep}; use super::halt::{HaltReason, HaltStep, error_response_text}; +use super::redo_propose::OwedRedo; use super::scheduler::Scheduler; use crate::bridge::envelope::{Response, Status}; use crate::control::cluster::calvin::scheduler::lock_manager::TxnId; -use crate::control::server::dispatch_utils::MintedRecords; +use crate::control::wal_replication::CollectionIncarnation; +use crate::control::wal_replication::encode::{CalvinEntryStamps, RedoEntryTarget}; +use crate::control::wal_replication::transaction_redo::{CalvinSlice, TransactionRedoPayload}; +use crate::control::wal_replication::types::CalvinRedoMeta; use crate::types::VShardId; use crate::wal::{CalvinStamp, RedoRecord}; -use nodedb_physical::physical_plan::PhysicalPlan; -use nodedb_physical::physical_plan::meta::MetaOp; impl Scheduler { /// Handle the `MetaOp::CalvinResolve` response: decode the resolved - /// `RedoRecord`, attach the transaction's messages on the participant that - /// carries them, WAL-append it (unless it holds no op and no message), - /// then dispatch the flush that installs it, stamped with that record's - /// LSN. + /// `RedoRecord`, stamp it, attach the transaction's messages and dedup + /// key on the participant that carries them, and propose it. /// - /// The verdict is already COMMIT, so a skipped resolve will tear the - /// committed txn on this replica. A non-`Ok` response, a decode failure, - /// or a WAL-append failure halts the scheduler: the txn keeps its - /// `pending` entry and locks, and its position stays unapplied. + /// The verdict is already COMMIT, so a skipped resolve tears the + /// committed txn. A non-`Ok` response or a record that cannot be read + /// halts the scheduler: the txn keeps its `pending` entry and locks, and + /// its position stays unapplied. pub(in crate::control::cluster::calvin::scheduler::driver::core) fn finish_redo_resolve( &mut self, txn_id: TxnId, @@ -48,214 +50,134 @@ impl Scheduler { ); return; } - - let mut redo = match RedoRecord::from_bytes(response.payload.as_bytes()) { - Ok(r) => r, - Err(e) => { - self.halt_apply( - txn_id, - HaltReason::ResolveFailed, - HaltStep::Resolve, - format!("CalvinResolve redo record decode failed: {e}"), - ); - return; + match self.owed_redo(txn_id, &response) { + Ok(owed) => { + if let Some(pending) = self.pending.get_mut(&txn_id) { + pending.redo = Some(owed); + pending.commit_state = CommitState::AwaitingRedoApply { proposed: None }; + } + self.propose_calvin_redo(txn_id); } - }; - let Some(pending) = self.pending.get(&txn_id) else { - // Txn state was reclaimed out from under us (must not happen — - // locks are held until `on_txn_complete`); complete defensively. - self.metrics.record_completed(); - self.on_txn_complete(txn_id); - return; - }; - let tenant_id = pending.txn.tx_class.tenant_id; - let database_id = pending.txn.tx_class.database_id; - let event_source = - super::request::slice_event_source(&pending.txn.tx_class, &pending.flush_scope); - let commit_hlc = self - .cut_floors - .commit_hlc(pending.txn.epoch, pending.txn.epoch_system_ms); - // The stamp carries what the slice folds, so the live install and - // restart replay fold at this record's LSN. + Err(error) => self.halt_apply( + txn_id, + HaltReason::ResolveFailed, + HaltStep::Resolve, + error.to_string(), + ), + } + } + + /// The stamped redo the committed slice `txn_id` proposes, from its + /// resolve answer `response`. + fn owed_redo(&self, txn_id: TxnId, response: &Response) -> crate::Result { + // The answer carries the slice's staged reply beside the record. Every + // replica renders the reply after its install. + let resolved = zerompk::from_msgpack::( + response.payload.as_bytes(), + ) + .map_err(|e| crate::Error::Serialization { + format: "msgpack".into(), + detail: format!("CalvinResolve answer: {e}"), + })?; + let mut redo = RedoRecord::from_bytes(&resolved.redo)?; + let pending = self + .pending + .get(&txn_id) + .ok_or_else(|| missing_pending_error(txn_id))?; + let tx_class = &pending.txn.tx_class; + // The stamp names the position every replica claims at install. redo.calvin_stamp = Some(CalvinStamp { epoch: txn_id.epoch, position: txn_id.position, vshard_id: self.vshard_id, - collections: pending.flush_scope.collections.clone(), - sum_targets: pending.flush_scope.sum_targets.clone(), }); // The participants whose records carry the messages and the applied - // key. A class whose write vShards cannot be derived halts the resolve. - let tx_class = &pending.txn.tx_class; - let homes = tx_class + // key. A class whose write vShards cannot be derived fails. + let publish_home = tx_class .publish_vshard() - .and_then(|publish| Ok((publish, tx_class.applied_key_home()?))); - let (publish_home, applied_key_home) = match homes { - Ok(homes) => homes, - Err(e) => { - self.halt_apply( - txn_id, - HaltReason::ResolveFailed, - HaltStep::Resolve, - format!("Calvin transaction write vShards underivable: {e}"), - ); - return; - } - }; + .map_err(|e| crate::Error::Internal { + detail: format!("Calvin transaction write vShards underivable: {e}"), + })?; + let applied_key_home = tx_class + .applied_key_home() + .map_err(|e| crate::Error::Internal { + detail: format!("Calvin transaction write vShards underivable: {e}"), + })?; // One participant's record carries the messages the transaction's // trigger bodies published, so they commit once, with its writes. + // Every replica stamps their position from the entry's log place. if publish_home == Some(self.vshard_id) { - match crate::wal::RedoPublish::decode_all(&pending.txn.tx_class.publishes) { - Ok(mut publishes) => { - // Named by the transaction's sequencer position on its - // vShard's Calvin partition, the same on every replica. - crate::wal::RedoPublish::stamp_all( - &mut publishes, - crate::wal::PublishPosition { - partition: crate::event::cdc::position::calvin_partition( - self.vshard_id, - ), - epoch: 0, - index: txn_id.epoch, - base: u64::from(txn_id.position), - }, - ); - redo.publishes = publishes; - } - Err(e) => { - self.halt_apply( - txn_id, - HaltReason::ResolveFailed, - HaltStep::Resolve, - format!("Calvin transaction publishes decode failed: {e}"), - ); - return; - } - } + redo.publishes = crate::wal::RedoPublish::decode_all(&tx_class.publishes)?; } // One participant's record carries the dedup key of the cross-shard // request the transaction applies, so the key is durable exactly when // the request's writes are. if applied_key_home == Some(self.vshard_id) { - match zerompk::from_msgpack::( - &pending.txn.tx_class.applied_key, - ) { - Ok(key) => redo.cross_shard_applied = Some(key), - Err(e) => { - self.halt_apply( - txn_id, - HaltReason::ResolveFailed, - HaltStep::Resolve, - format!("Calvin transaction applied key decode failed: {e}"), - ); - return; - } - } + let key = + zerompk::from_msgpack::(&tx_class.applied_key) + .map_err(|e| crate::Error::Serialization { + format: "msgpack".into(), + detail: format!("Calvin transaction applied key: {e}"), + })?; + redo.cross_shard_applied = Some(key); } - let installs_nothing = - redo.ops.is_empty() && redo.publishes.is_empty() && redo.cross_shard_applied.is_none(); - - // The flush installs these exact bytes, the payload of the record - // appended below. - let redo_bytes = if installs_nothing { - Vec::new() - } else { - match redo.to_bytes() { - Ok(bytes) => bytes, - Err(e) => { - self.halt_apply( - txn_id, - HaltReason::ResolveFailed, - HaltStep::Resolve, - format!("CalvinResolve redo record encode failed: {e}"), - ); - return; - } - } - }; - - // The record's outcome-floor window opens before the append. It stays - // with the pending txn until the flush completes. - let (redo_lsn, redo_records) = if installs_nothing { - (None, None) - } else { - let records = MintedRecords::open(&self.shared.outcome_floor); - // The flush reports no rows beyond the record's own: restart - // replay folds the sum targets from the record's stamp. The record - // is therefore whole at append, and no part follows it. - let appended = records - .appender(&self.shared.wal, crate::wal::manager::NO_APPLY_KEY) - .with_event_source(event_source) - .with_commit_hlc(commit_hlc) - .append_whole_transaction_redo( - tenant_id, - VShardId::new(self.vshard_id), - database_id, - &redo, - ); - match appended { - Ok(lsn) => { - // The txn committed, so its redo record is never - // cancelled. The flush closes it from its outcome. - records.mark_sent(); - // Before the flush, so the position exists before any of - // the txn's change events reach the Event Plane. - self.shared.cdc_router.positions().record_calvin( - lsn.as_u64(), - crate::event::cdc::position::CalvinPosition { - sequencer_epoch: txn_id.epoch, - position: txn_id.position, - }, - ); - self.shared - .cdc_router - .positions() - .record_commit_hlc(lsn.as_u64(), commit_hlc); - (Some(lsn), Some(records)) - } - Err(e) => { - // The txn stays pending and unapplied, and a failed append - // leaves no record for restart replay to reach. - records.settle(); - self.halt_apply( - txn_id, - HaltReason::WalAppendFailed, - HaltStep::RedoAppend, - format!("TransactionRedo WAL append failed: {e}"), - ); - return; - } - } - }; - if let Some(pending) = self.pending.get_mut(&txn_id) { - pending.redo_records = redo_records; - // The commit publishes the rows its redo installs, as a - // data-group `TransactionRedo` apply does. A rolled-back - // transaction never resolves, so it publishes nothing. - pending.change_sets = if redo_bytes.is_empty() { - Vec::new() - } else { - vec![crate::control::server::dispatch_utils::redo_change_set( - &redo_bytes, - )] - }; - pending.flush_scope.redo = redo_bytes; - } - - // The flush runs in sequencer order, once every lower txn finished. - self.queue_flush(txn_id, redo_lsn); + let scope = &pending.scope; + let payload = TransactionRedoPayload::from_calvin( + redo, + CalvinSlice { + collections: scope.collections.clone(), + sum_targets: scope.sum_targets.clone(), + identities: scope.identities.clone(), + event_source: super::request::slice_event_source(tx_class, scope), + origin: scope.origin, + meta: CalvinRedoMeta { + epoch_system_ms: pending.txn.epoch_system_ms, + reply: resolved.reply, + primary_write: pending.has_primary_write, + user_write: pending.raises_write_mark, + returning: pending.has_returning, + }, + }, + ); + let incarnations = scope + .collections + .iter() + .map(|collection| CollectionIncarnation { + collection: collection.clone(), + incarnation: tx_class + .incarnations + .iter() + .find(|named| named.collection == *collection) + .map_or(nodedb_types::Hlc::ZERO, |named| named.incarnation), + }) + .collect(); + Ok(OwedRedo { + payload, + stamps: CalvinEntryStamps { + write_hlc: self + .cut_floors + .commit_hlc(pending.txn.epoch, pending.txn.epoch_system_ms), + restore_id: tx_class.restore_id, + metadata_floor: tx_class.metadata_floor, + incarnations, + }, + target: RedoEntryTarget { + tenant_id: tx_class.tenant_id, + database_id: tx_class.database_id, + vshard_id: VShardId::new(self.vshard_id), + }, + attempts: 0, + }) } /// Dispatch `MetaOp::CalvinResolve` to this vShard's core, registering a /// response bridge so the resolve response re-enters the completion loop /// under `CommitState::AwaitingRedoResolve`. /// - /// Mirrors `dispatch_commit_resolution`'s exempt, no-WAL-LSN dispatch - /// shape — a resolve reads the staged overlay and writes nothing. - /// - /// A capacity refusal returns [`DispatchOutcome::Deferred`]: the resolve - /// is parked for re-send and the txn stays in flight. A txn with no - /// `pending` entry returns [`DispatchOutcome::Failed`]. + /// A resolve reads the staged overlay and writes nothing, so no WAL LSN + /// rides on it. A capacity refusal returns [`DispatchOutcome::Deferred`]: + /// the resolve is parked for re-send and the txn stays in flight. A txn + /// with no `pending` entry returns [`DispatchOutcome::Failed`]. pub(in crate::control::cluster::calvin::scheduler::driver::core) fn dispatch_calvin_resolve( &mut self, txn_id: TxnId, @@ -265,23 +187,18 @@ impl Scheduler { }; let tenant_id = pending.txn.tx_class.tenant_id; let database_id = pending.txn.tx_class.database_id; - let epoch = txn_id.epoch; - let position = txn_id.position; - let request_id = self.next_request_id(); - let plan = PhysicalPlan::Meta(MetaOp::CalvinResolve { epoch, position }); - // A resolve reads the staged overlay only; it writes no WAL record - // itself, so no committed LSN rides on this envelope. - let request = self.build_exempt_request(request_id, tenant_id, database_id, plan, None); - - // The resolve response re-enters the completion loop under the SAME - // txn_id, now in `AwaitingRedoResolve`, where `finish_redo_resolve` runs. + let plan = PhysicalPlan::Meta(MetaOp::CalvinResolve { + epoch: txn_id.epoch, + position: txn_id.position, + }); + let request = self.build_exempt_request(request_id, tenant_id, database_id, plan); self.dispatch_sequenced(txn_id, DispatchStep::Resolve, request) } } -/// The terminal error for a commit-resolution dispatch whose txn has no -/// `pending` entry to build the request from. +/// The terminal error for a dispatch whose txn has no `pending` entry to +/// build the request from. pub(in crate::control::cluster::calvin::scheduler::driver::core) fn missing_pending_error( txn_id: TxnId, ) -> crate::Error { @@ -345,9 +262,9 @@ mod tests { Some(HaltReason::ResolveFailed) ); assert_eq!( - scheduler.pending.get(&txn_id).and_then(|p| p.commit_state), + scheduler.pending.get(&txn_id).map(|p| p.commit_state), Some(CommitState::AwaitingRedoResolve), - "no flush is dispatched" + "no redo is proposed" ); } } diff --git a/nodedb/src/control/cluster/calvin/scheduler/driver/core/commit_resolution_dispatch.rs b/nodedb/src/control/cluster/calvin/scheduler/driver/core/commit_resolution_dispatch.rs deleted file mode 100644 index 228f0a329..000000000 --- a/nodedb/src/control/cluster/calvin/scheduler/driver/core/commit_resolution_dispatch.rs +++ /dev/null @@ -1,104 +0,0 @@ -// SPDX-License-Identifier: BUSL-1.1 - -//! Dispatch of staged Calvin flush/drop resolution operations. - -use nodedb_physical::physical_plan::PhysicalPlan; -use nodedb_physical::physical_plan::meta::MetaOp; - -use super::commit_redo::missing_pending_error; -use super::deferred::{DispatchOutcome, DispatchStep}; -use super::scheduler::Scheduler; -use crate::bridge::dispatch::JournalGroup; -use crate::control::cluster::calvin::scheduler::lock_manager::TxnId; -use crate::types::Lsn; - -/// How a staged transaction resolves on this vShard. -pub(in crate::control::cluster::calvin::scheduler::driver::core) enum CommitResolution { - /// Install the committed redo record the scheduler appended at - /// `redo_lsn`. The flush scope holds its bytes. Both are empty when the - /// transaction wrote nothing on this vShard. - Flush { redo_lsn: Option }, - /// Discard the staged state under an abort verdict. - Drop, -} - -impl Scheduler { - /// Dispatch a flush or drop of a staged transaction's commit-pending buffer. - /// - /// A flush carries the redo record, the collections the local plans - /// write, and their materialized-sum targets, so the Data Plane installs - /// the record the way every committed transaction installs. - /// - /// A capacity refusal returns [`DispatchOutcome::Deferred`]: the flush or - /// drop is parked for re-send and the txn stays in flight. A txn with no - /// `pending` entry returns [`DispatchOutcome::Failed`]. The flush takes its - /// collections and sum targets from the scope derived at stage time. A - /// flush whose gates a halt released takes them back first, and fails when - /// a collection it writes no longer holds its planned incarnation. - pub(in crate::control::cluster::calvin::scheduler::driver::core) fn dispatch_commit_resolution( - &mut self, - txn_id: TxnId, - resolution: CommitResolution, - ) -> DispatchOutcome { - // A flush writes the collections: it holds their gates while it runs. - if matches!(resolution, CommitResolution::Flush { .. }) - && let Err(error) = self.regate(txn_id) - { - return DispatchOutcome::Failed(error); - } - let Some(pending) = self.pending.get_mut(&txn_id) else { - return DispatchOutcome::Failed(missing_pending_error(txn_id)); - }; - let tenant_id = pending.txn.tx_class.tenant_id; - let database_id = pending.txn.tx_class.database_id; - let event_source = - super::request::slice_event_source(&pending.txn.tx_class, &pending.flush_scope); - let commit_hlc = self - .cut_floors - .commit_hlc(pending.txn.epoch, pending.txn.epoch_system_ms); - let epoch = txn_id.epoch; - let position = txn_id.position; - let (plan, step, wal_lsn, journal) = match resolution { - CommitResolution::Flush { redo_lsn } => { - pending.flush_scope.sends = pending.flush_scope.sends.saturating_add(1); - let scope = &pending.flush_scope; - // The core stores the flush's append inputs beside its - // effects for boot. The record is whole at append, so no part - // follows it. - let journal = redo_lsn.map(|origin| JournalGroup { - origin, - collection: scope.collections.first().cloned().unwrap_or_default(), - apply_key: crate::wal::manager::NO_APPLY_KEY, - commit_hlc: Some(commit_hlc), - change_position: None, - }); - ( - PhysicalPlan::Meta(MetaOp::CalvinFlush { - epoch, - position, - redo: scope.redo.clone(), - collections: scope.collections.clone(), - sum_targets: scope.sum_targets.clone(), - }), - DispatchStep::Flush, - redo_lsn, - journal, - ) - } - CommitResolution::Drop => ( - PhysicalPlan::Meta(MetaOp::CalvinDrop { epoch, position }), - DispatchStep::Drop, - None, - None, - ), - }; - let request_id = self.next_request_id(); - let mut request = - self.build_exempt_request(request_id, tenant_id, database_id, plan, wal_lsn); - // The flush emits the transaction's events, under its own source and - // dated by its commit HLC. - request.event_source = event_source; - request.commit_hlc = Some(commit_hlc); - self.dispatch_sequenced_journalled(txn_id, step, request, journal) - } -} diff --git a/nodedb/src/control/cluster/calvin/scheduler/driver/core/commit_resolve/apply_tail.rs b/nodedb/src/control/cluster/calvin/scheduler/driver/core/commit_resolve/apply_tail.rs deleted file mode 100644 index 940ba2974..000000000 --- a/nodedb/src/control/cluster/calvin/scheduler/driver/core/commit_resolve/apply_tail.rs +++ /dev/null @@ -1,661 +0,0 @@ -// SPDX-License-Identifier: BUSL-1.1 - -//! Commit tail: runs once a flush/drop response has returned, depositing the -//! applied result, marking the apply durable, recording write versions, and -//! proposing the `CompletionAck`. - -use crate::bridge::envelope::{ErrorCode, Response, Status}; -use crate::control::cluster::calvin::scheduler::driver::core::commit_resolution_dispatch::CommitResolution; -use crate::control::cluster::calvin::scheduler::driver::core::deferred::{ - DispatchOutcome, DispatchStep, -}; -use crate::control::cluster::calvin::scheduler::driver::core::halt::{ - HaltReason, HaltStep, error_response_text, -}; -use crate::control::cluster::calvin::scheduler::driver::core::owed::SchedulerProposal; -use crate::control::cluster::calvin::scheduler::driver::core::scheduler::Scheduler; -use crate::control::cluster::calvin::scheduler::lock_manager::TxnId; -use crate::control::cluster::calvin::scheduler::metrics::infra_abort_reason; - -/// The most flushes one committed txn sends. Each refused install rolled -/// every write back, so a resend is safe. A refusal that outlasts the bound -/// halts the scheduler. -pub(in crate::control::cluster::calvin::scheduler::driver::core) const MAX_FLUSH_SENDS: u32 = 8; - -impl Scheduler { - /// Run the commit tail once a flush/drop response has returned. - /// - /// On a successful flush the full commit tail runs (deposit applied result, - /// `CalvinApplied` WAL + write-version recording, `CompletionAck`). On a - /// successful drop only the `CompletionAck` is proposed — the coordinator's - /// completion waiter still fires and the epoch advances, but nothing was - /// written so there is no result to deposit, no apply LSN, and no versions - /// to record. - /// - /// A flush refused with `RetryableRefusal` rolled its install back and - /// kept the staged buffer, so the scheduler sends the same flush again, up - /// to [`MAX_FLUSH_SENDS`] sends. Any other non-`Ok` flush, or a refusal - /// past the bound, halts the scheduler: a skipped flush tears the - /// committed txn on this replica. A non-`Ok` drop completes the txn: - /// under an abort verdict no replica writes anything. - pub(in crate::control::cluster::calvin::scheduler::driver::core) async fn finish_resolved_commit( - &mut self, - txn_id: TxnId, - response: Response, - committed: bool, - redo_lsn: Option, - ) { - if response.status != Status::Ok { - if committed && self.resend_refused_flush(txn_id, &response, redo_lsn) { - return; - } - if committed { - self.halt_apply( - txn_id, - HaltReason::FlushFailed, - HaltStep::Flush, - error_response_text("CalvinFlush", &response), - ); - return; - } - tracing::error!( - vshard_id = self.vshard_id, - epoch = txn_id.epoch, - position = txn_id.position, - "calvin: drop response was not Ok under an abort verdict; completing the \ - aborted txn, since no replica writes it" - ); - self.metrics.record_executor_error(); - self.metrics - .record_infra_abort(infra_abort_reason::IO_ERROR); - self.metrics.record_completed(); - self.on_txn_complete(txn_id); - return; - } - - let completed = if committed { - self.commit_apply_tail(txn_id, response, redo_lsn).await - } else { - // A dropped txn installed nothing, so its ack reports nothing. - self.propose_sequencer_entry( - txn_id, - SchedulerProposal::CompletionAck { result: Vec::new() }, - ); - true - }; - // `false` means the commit tail halted the scheduler: the txn stays - // pending and unapplied. - if completed { - self.metrics.record_completed(); - self.on_txn_complete(txn_id); - } - } - - /// Send the flush of `txn_id` again when the install refused it as - /// retryable and the send bound allows another. Returns whether the - /// refusal is handled: the flush went out again, or its dispatch failed - /// and the step's terminal handling ran. - fn resend_refused_flush( - &mut self, - txn_id: TxnId, - response: &Response, - redo_lsn: Option, - ) -> bool { - if !matches!( - response.error_code.as_deref(), - Some(ErrorCode::RetryableRefusal { .. }) - ) { - return false; - } - let sends = self - .pending - .get(&txn_id) - .map_or(MAX_FLUSH_SENDS, |pending| pending.flush_scope.sends); - if sends >= MAX_FLUSH_SENDS { - return false; - } - tracing::warn!( - vshard_id = self.vshard_id, - epoch = txn_id.epoch, - position = txn_id.position, - sends, - "calvin: the flush install was refused as retryable; sending it again" - ); - if let DispatchOutcome::Failed(error) = - self.dispatch_commit_resolution(txn_id, CommitResolution::Flush { redo_lsn }) - { - self.fail_dispatch_step(txn_id, DispatchStep::Flush, error); - } - true - } - - /// Deposit the applied result, durably mark the apply, record the apply's - /// write versions, and propose the `CompletionAck`. - /// - /// Shared by the flush-completion path and the direct-apply (dependent / - /// active) apply path. - /// - /// Returns `false` once a failed `CalvinApplied` WAL append halted the - /// scheduler: the position must not be marked applied without its marker, - /// so the caller leaves the txn pending. - /// - /// `redo_lsn` is `Some(lsn)` when a `TransactionRedo` record was already - /// WAL-appended for this commit's non-empty write set (`finish_redo_resolve`) - /// — that record already IS the durable applied marker, so only write - /// versions are recorded at it. `None` (a drop, an empty-ops staged commit, - /// or the direct-apply dependent/active path, which carries no redo record) - /// falls back to appending a `CalvinApplied` marker here, exactly as before - /// this record existed. - pub(in crate::control::cluster::calvin::scheduler::driver::core) async fn commit_apply_tail( - &mut self, - txn_id: TxnId, - response: Response, - redo_lsn: Option, - ) -> bool { - // The install folded materialized sums into target rows no redo - // sub-record names. The redo record's stamp carries the sum targets, - // so restart replay folds at the same LSN. - // Deposit the FULL applied Response (affected-count + watermark + any - // RETURNING rows) into the local sidecar BEFORE proposing the replicated - // CompletionAck. The ack fires the coordinator's completion oneshot on - // every sequencer member, so depositing first guarantees the result is - // present by the time the coordinator drains it — no lost result, no - // race. - // - // Gated on the PRIMARY-WRITE participant: any participant whose slice - // carries the user's non-edge DML (Document/KV/Vector/etc.), as opposed - // to the implicit graph-edge cleanup that dual-homes alongside it. A - // multi-collection cross-shard COMMIT has MANY primary-write - // participants — each a plain affected-count write — and they coalesce: - // the first applied response stands for the coordinator (which discards - // it for a COMMIT tag anyway), and the plain-write siblings do not - // conflict. Only a genuine cross-shard RETURNING union — two - // participants each carrying RETURNING rows — records `Conflict`. - // The primary slice's answer, its RETURNING rows or its affected - // count, and the timeseries install counts also ride the replicated - // CompletionAck, so a coordinator on a node with no replica of this - // vShard reads them. - let (has_primary_write, has_returning) = self - .pending - .get(&txn_id) - .map(|p| (p.has_primary_write, p.has_returning)) - .unwrap_or((false, false)); - let ack_result = self.ack_result_of( - &response, - crate::control::state::AckSlice { - primary_write: has_primary_write, - returning: has_returning, - }, - ); - if has_primary_write { - use std::collections::hash_map::Entry; - - let response = statement_reply(response); - - use crate::control::state::CalvinApplyResult; - - let key = nodedb_cluster::calvin::TxnId::new(txn_id.epoch, txn_id.position); - let vshard_id = self.vshard_id; - self.shared.calvin.apply_results.deposit_with(key, |entry| match entry { - Entry::Vacant(slot) => { - slot.insert(CalvinApplyResult::Single { - response, - has_returning, - }); - } - Entry::Occupied(mut slot) => { - // Derive both facts from the existing entry BEFORE any - // insert, so the immutable borrow does not outlive the - // mutable one. - let existing_returning = matches!( - slot.get(), - CalvinApplyResult::Single { - has_returning: true, - .. - } - ); - let already_conflict = matches!(slot.get(), CalvinApplyResult::Conflict); - - if already_conflict { - // A RETURNING union was already recorded; stays Conflict. - } else if has_returning && existing_returning { - // Two RETURNING-bearing participants for one Calvin txn: - // a cross-shard RETURNING union, which is unsupported. - // Record Conflict so the coordinator fails the statement - // loudly rather than returning one shard's partial rows. - tracing::error!( - epoch = txn_id.epoch, - position = txn_id.position, - vshard = vshard_id, - "two RETURNING-bearing participants for one Calvin txn — cross-shard \ - RETURNING union unsupported" - ); - slot.insert(CalvinApplyResult::Conflict); - } else if has_returning { - // The incoming participant carries the rows; the existing - // entry was a plain affected-count sibling. Rows win. - slot.insert(CalvinApplyResult::Single { - response, - has_returning: true, - }); - } else { - // Incoming is a plain write; keep the existing entry — a - // multi-collection cross-shard COMMIT coalesces (the - // coordinator discards it for a COMMIT tag anyway). - // The timeseries install counts of both merge, so the - // COMMIT reports every participant's apply count. - merge_install_counts(slot.get_mut(), &response); - } - } - }); - } - let applied_lsn = match redo_lsn { - // The TransactionRedo record already durably marks this apply — the - // SAME shard-local WAL-LSN space fast-path writes and read - // watermarks use. Record the apply's per-key write versions at it; - // no second (CalvinApplied) marker is written. - Some(lsn) => { - self.record_calvin_write_versions(txn_id, lsn); - Some(lsn) - } - None => match self - .shared - .wal - .appender(crate::wal::manager::NO_APPLY_KEY) - .with_commit_hlc(self.txn_commit_hlc(txn_id)) - .append_calvin_applied( - crate::types::VShardId::new(self.vshard_id), - txn_id.epoch, - txn_id.position, - ) { - // The CalvinApplied WAL LSN is the committed write-LSN for this - // apply — the SAME shard-local WAL-LSN space fast-path writes and - // read watermarks use. Record the apply's per-key write versions - // at it once it exists; it does not exist yet at dispatch time. - Ok(applied_lsn) => { - self.record_calvin_write_versions(txn_id, applied_lsn); - Some(applied_lsn) - } - Err(e) => { - self.halt_apply( - txn_id, - HaltReason::WalAppendFailed, - HaltStep::AppliedMarker, - format!("CalvinApplied WAL append failed: {e}"), - ); - None - } - }, - }; - let Some(lsn) = applied_lsn else { - // The apply cannot be acknowledged without a durable participant - // LSN: CDC and write-version consumers will otherwise observe a - // successful commit with no authoritative ordering point. The - // scheduler halted above. - return false; - }; - // The record is whole at append: the flush reports no rows, so no - // part follows it. A record split over the WAL record limit is - // durable once its last continuation is. - let durable_through = self - .pending - .get(&txn_id) - .and_then(|pending| pending.redo_records.as_ref()) - .and_then(|records| records.last_lsn()) - .map_or(lsn, |last| last.max(lsn)); - // Control change-stream events are distinct from Data-Plane - // WriteEvents. Every replica publishes the rows this participant's - // redo installs at the transaction's sequencer position, which every - // replica shares; the vShard's leader forwards them to the nodes - // that do not replicate it. The publish journals them durably before - // the applied marker's fsync below, so a restart that finds the - // position applied also finds its changes. - if let Some(pending) = self.pending.get_mut(&txn_id) { - let calvin = crate::control::server::dispatch_utils::CalvinApply { - tenant_id: pending.txn.tx_class.tenant_id, - database_id: pending.txn.tx_class.database_id, - vshard: self.vshard_id, - sequencer_epoch: txn_id.epoch, - position: txn_id.position, - commit_hlc: self - .cut_floors - .commit_hlc(pending.txn.epoch, pending.txn.epoch_system_ms), - }; - let change_sets = std::mem::take(&mut pending.change_sets); - crate::control::server::dispatch_utils::publish_calvin_change_sets( - &self.shared, - calvin, - change_sets, - lsn, - ); - } - // The record at `lsn` is this position's only applied marker. An - // append only buffers it, so it is durable before the mark and the - // ack: a restart that lost it will take the position for unapplied, - // run the transaction again, and never settle its ack. The wait joins - // the WAL group commit, so concurrent Calvin commits share one fsync. - if let Err(e) = self.shared.wal.wait_durable(durable_through).await { - self.halt_apply( - txn_id, - HaltReason::WalAppendFailed, - HaltStep::AppliedMarker, - format!("applied marker fsync at lsn {} failed: {e}", lsn.as_u64()), - ); - return false; - } - if let Some(origin) = redo_lsn { - self.note_flush_settled(origin); - self.record_applied_key(txn_id); - } - // The commit's mark lands before the ack, as a write through the - // funnel records its mark before its response returns. - self.record_calvin_write_mark(txn_id); - self.propose_sequencer_entry( - txn_id, - SchedulerProposal::CompletionAck { result: ack_result }, - ); - true - } -} - -impl Scheduler { - /// Record the dedup key this participant's redo record carries, once the - /// record is durable and installed. Every replica records it, as a - /// Raft-applied redo's key is, so a request or trigger body sent again to - /// any replica applies once. - fn record_applied_key(&self, txn_id: TxnId) { - let Some(pending) = self.pending.get(&txn_id) else { - return; - }; - let tx_class = &pending.txn.tx_class; - if tx_class.applied_key.is_empty() - || tx_class.applied_key_home().ok().flatten() != Some(self.vshard_id) - { - return; - } - let Some(dedup) = self.shared.cross_shard_dedup.get() else { - return; - }; - match zerompk::from_msgpack::(&tx_class.applied_key) { - Ok(key) => { - if let Err(error) = dedup.record_applied(&key) { - tracing::error!( - vshard_id = self.vshard_id, - origin = %key.origin, - %error, - "calvin: dedup key held in memory only; the WAL restores it on restart" - ); - } - } - Err(error) => tracing::error!( - vshard_id = self.vshard_id, - %error, - "calvin: the applied key does not decode; it is not recorded" - ), - } - } - - /// The report the flush `response` owes the coordinator, encoded for the - /// replicated `CompletionAck`: the apply's timeseries install counts, and - /// the rows of a `returning` slice. The rows are bounded by the query - /// result limit and by the ack's frame budget, whichever is lower. - fn ack_result_of( - &self, - response: &Response, - slice: crate::control::state::AckSlice, - ) -> Vec { - let limit = self - .shared - .tuning - .network - .max_query_result_bytes - .min(crate::control::state::ACK_ROWS_FRAME_BUDGET); - crate::control::state::CalvinAckResult::of(response, slice, limit) - .to_bytes() - .unwrap_or_else(|error| { - tracing::warn!( - vshard_id = self.vshard_id, - %error, - "calvin: the completion ack carries no report" - ); - Vec::new() - }) - } -} - -/// The response the statement drains. A flush whose install succeeded but -/// whose reply failed to render answers `Ok` with the render error in -/// `error_code`. The statement reports that error. -/// Fold the timeseries install counts `incoming` answers into the plain -/// result `held`. A RETURNING result keeps its rows untouched. -fn merge_install_counts(held: &mut crate::control::state::CalvinApplyResult, incoming: &Response) { - let crate::control::state::CalvinApplyResult::Single { - response, - has_returning: false, - } = held - else { - return; - }; - if let Some(merged) = crate::engine::timeseries::install_counts::merge_count_payloads( - response.payload.as_bytes(), - incoming.payload.as_bytes(), - ) { - response.payload = crate::bridge::envelope::Payload::from_vec(merged); - } -} - -fn statement_reply(mut response: Response) -> Response { - if response.status == Status::Ok && response.error_code.is_some() { - response.status = Status::Error; - response.payload = crate::bridge::envelope::Payload::empty(); - } - response -} - -#[cfg(test)] -mod tests { - use super::*; - use crate::control::cluster::calvin::scheduler::driver::core::test_support::{ - error_response, scheduler_with_pending, staged_response, - }; - use crate::control::cluster::calvin::scheduler::driver::types::CommitState; - - fn internal_error() -> Response { - error_response(ErrorCode::Internal { - detail: "core failed".to_string(), - }) - } - - fn flushed_with_counts(collection: &str, accepted: u64, rejected: u64) -> Response { - use crate::engine::timeseries::install_counts::{TsInstallCount, TsInstallCounts}; - let mut response = staged_response(Status::Ok, None); - response.payload = crate::bridge::envelope::Payload::from_vec( - TsInstallCounts::new(vec![TsInstallCount { - collection: collection.into(), - accepted, - rejected, - }]) - .to_bytes() - .expect("encode install counts"), - ); - response - } - - /// Two plain participants of one Calvin transaction each installed a - /// timeseries batch. The coordinator's result carries both installs' - /// apply counts, so COMMIT reports every participant's rejected rows. A - /// RETURNING result keeps its rows. - #[test] - fn plain_participants_merge_their_install_counts() { - use crate::control::state::CalvinApplyResult; - use crate::engine::timeseries::install_counts::TsInstallCounts; - let mut held = CalvinApplyResult::Single { - response: flushed_with_counts("cpu", 2, 1), - has_returning: false, - }; - merge_install_counts(&mut held, &flushed_with_counts("mem", 3, 2)); - let CalvinApplyResult::Single { response, .. } = &held else { - panic!("a merged result stays single"); - }; - let totals = TsInstallCounts::from_payload(response.payload.as_bytes()) - .expect("install counts") - .by_collection(); - assert_eq!(totals.get("cpu"), Some(&(2, 1))); - assert_eq!(totals.get("mem"), Some(&(3, 2))); - - let rows = crate::bridge::envelope::Payload::from_vec(vec![0x90]); - let mut returning = CalvinApplyResult::Single { - response: Response { - payload: rows, - ..flushed_with_counts("cpu", 1, 0) - }, - has_returning: true, - }; - merge_install_counts(&mut returning, &flushed_with_counts("mem", 1, 1)); - let CalvinApplyResult::Single { response, .. } = &returning else { - panic!("a RETURNING result stays single"); - }; - assert_eq!(response.payload.as_bytes(), &[0x90]); - } - - /// A flush that returns an error under a COMMIT verdict holds the txn - /// unapplied and halts: a second flush will apply nothing. - #[tokio::test] - async fn flush_error_response_holds_committed_txn_unapplied() { - let txn_id = TxnId::new(9, 2); - let (mut scheduler, _dir) = scheduler_with_pending( - txn_id, - CommitState::AwaitingResolve { - committed: true, - redo_lsn: None, - }, - ); - - scheduler - .finish_resolved_commit(txn_id, internal_error(), true, None) - .await; - - assert!(!scheduler.applied.is_applied(9, 2)); - assert!(scheduler.pending.contains_key(&txn_id)); - assert_eq!( - scheduler.apply_halt().map(|h| h.reason), - Some(HaltReason::FlushFailed) - ); - assert!(scheduler.shared.sequencer_halt.apply_halt().is_halted()); - } - - /// A drop that returns an error under an abort verdict still completes - /// the txn: no replica writes an aborted txn. - #[tokio::test] - async fn drop_error_response_under_abort_completes_txn() { - let txn_id = TxnId::new(9, 2); - let (mut scheduler, _dir) = scheduler_with_pending( - txn_id, - CommitState::AwaitingResolve { - committed: false, - redo_lsn: None, - }, - ); - - scheduler - .finish_resolved_commit(txn_id, internal_error(), false, None) - .await; - - assert!(scheduler.applied.is_applied(9, 2)); - assert!(!scheduler.pending.contains_key(&txn_id)); - assert!(!scheduler.is_apply_halted()); - } - fn retryable_refusal() -> Response { - error_response(ErrorCode::RetryableRefusal { - reason: "install rolled back".to_string(), - }) - } - - /// A flush refused as retryable reaches the Data Plane again with the - /// same redo record, and the txn stays pending and unhalted. - #[tokio::test] - async fn retryable_flush_refusal_resends_the_same_flush() { - use crate::control::cluster::calvin::scheduler::driver::core::test_support::{ - await_data_plane_request, build_test_scheduler_with_data_side, make_sequenced_txn, - staged_pending, - }; - use nodedb_physical::physical_plan::PhysicalPlan; - use nodedb_physical::physical_plan::meta::MetaOp; - - let txn_id = TxnId::new(9, 2); - let registry = nodedb_cluster::calvin::CalvinCompletionRegistry::new_detached(); - let (mut scheduler, _dir, mut data_side) = build_test_scheduler_with_data_side(7, registry); - let mut pending = staged_pending(make_sequenced_txn(9, 2), txn_id); - pending.commit_state = Some(CommitState::AwaitingResolve { - committed: true, - redo_lsn: None, - }); - pending.flush_scope.redo = vec![7, 7, 7]; - pending.flush_scope.sends = 1; - scheduler.pending.insert(txn_id, pending); - - scheduler - .finish_resolved_commit(txn_id, retryable_refusal(), true, None) - .await; - - assert!(!scheduler.is_apply_halted()); - assert!(!scheduler.applied.is_applied(9, 2)); - assert_eq!( - scheduler.pending.get(&txn_id).map(|p| p.flush_scope.sends), - Some(2) - ); - assert!( - await_data_plane_request(&mut data_side, |plan| matches!( - plan, - PhysicalPlan::Meta(MetaOp::CalvinFlush { epoch: 9, position: 2, redo, .. }) - if redo == &vec![7, 7, 7] - )) - .await, - "the resent flush carries the same redo record" - ); - } - - /// A retryable refusal past the send bound halts like any flush error. - #[tokio::test] - async fn retryable_flush_refusal_past_the_bound_halts() { - let txn_id = TxnId::new(9, 2); - let (mut scheduler, _dir) = scheduler_with_pending( - txn_id, - CommitState::AwaitingResolve { - committed: true, - redo_lsn: None, - }, - ); - if let Some(pending) = scheduler.pending.get_mut(&txn_id) { - pending.flush_scope.sends = MAX_FLUSH_SENDS; - } - - scheduler - .finish_resolved_commit(txn_id, retryable_refusal(), true, None) - .await; - - assert!(scheduler.pending.contains_key(&txn_id)); - assert_eq!( - scheduler.apply_halt().map(|h| h.reason), - Some(HaltReason::FlushFailed) - ); - } - - /// A render error on an installed flush reaches the statement as a typed - /// error, and the txn completes. - #[test] - fn a_render_error_on_an_installed_flush_becomes_the_statement_error() { - let mut response = error_response(ErrorCode::Internal { - detail: "render".to_string(), - }); - response.status = Status::Ok; - - let reply = statement_reply(response); - - assert_eq!(reply.status, Status::Error); - assert!(matches!( - reply.error_code.as_deref(), - Some(ErrorCode::Internal { .. }) - )); - } -} diff --git a/nodedb/src/control/cluster/calvin/scheduler/driver/core/commit_resolve/drop_tail.rs b/nodedb/src/control/cluster/calvin/scheduler/driver/core/commit_resolve/drop_tail.rs new file mode 100644 index 000000000..2d1d3786c --- /dev/null +++ b/nodedb/src/control/cluster/calvin/scheduler/driver/core/commit_resolve/drop_tail.rs @@ -0,0 +1,90 @@ +// SPDX-License-Identifier: BUSL-1.1 + +//! The end of a staged slice that commits no log entry: an aborted txn, or a +//! committed slice with no local write. Its drop answered, so it owes an +//! empty `CompletionAck` and completes. + +use crate::bridge::envelope::{Response, Status}; +use crate::control::cluster::calvin::scheduler::driver::core::scheduler::Scheduler; +use crate::control::cluster::calvin::scheduler::lock_manager::TxnId; +use crate::control::cluster::calvin::scheduler::metrics::infra_abort_reason; + +impl Scheduler { + /// Complete `txn_id` once its drop answered. + /// + /// A drop that did not answer `Ok` still completes the txn: no replica + /// writes it, whatever the drop did on this core. + pub(in crate::control::cluster::calvin::scheduler::driver::core) fn finish_drop( + &mut self, + txn_id: TxnId, + response: &Response, + ) { + if response.status != Status::Ok { + tracing::error!( + vshard_id = self.vshard_id, + epoch = txn_id.epoch, + position = txn_id.position, + error = ?response.error_code.as_deref(), + "calvin: drop answered with an error; completing the txn, since no replica \ + writes it" + ); + self.metrics.record_executor_error(); + self.metrics + .record_infra_abort(infra_abort_reason::IO_ERROR); + } + self.complete_without_entry(txn_id); + } +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::bridge::envelope::ErrorCode; + use crate::control::cluster::calvin::scheduler::driver::core::owed::OwedKind; + use crate::control::cluster::calvin::scheduler::driver::core::test_support::{ + error_response, scheduler_with_pending, staged_response, + }; + use crate::control::cluster::calvin::scheduler::driver::types::CommitState; + + /// A drop that answers `Ok` completes the txn and owes its ack. + #[tokio::test] + async fn an_answered_drop_completes_the_txn_and_owes_its_ack() { + let txn_id = TxnId::new(9, 1); + let (mut scheduler, _dir) = scheduler_with_pending(txn_id, CommitState::AwaitingDrop); + + scheduler.finish_drop(txn_id, &staged_response(Status::Ok, None)); + + assert!(scheduler.applied.is_applied(9, 1)); + assert!(scheduler.ledger.is_applied(9, 1), "no log entry marks it"); + assert!(!scheduler.pending.contains_key(&txn_id)); + assert!( + scheduler + .owed + .contains_key(&(txn_id, OwedKind::CompletionAck)) + ); + } + + /// A drop that returns an error still completes the txn and owes its + /// ack: no replica writes a txn that ends with no log entry. + #[tokio::test] + async fn a_drop_error_still_completes_the_txn() { + let txn_id = TxnId::new(9, 2); + let (mut scheduler, _dir) = scheduler_with_pending(txn_id, CommitState::AwaitingDrop); + + scheduler.finish_drop( + txn_id, + &error_response(ErrorCode::Internal { + detail: "core failed".to_string(), + }), + ); + + assert!(scheduler.applied.is_applied(9, 2)); + assert!(!scheduler.pending.contains_key(&txn_id)); + assert!(!scheduler.is_apply_halted()); + assert!( + scheduler + .owed + .contains_key(&(txn_id, OwedKind::CompletionAck)) + ); + } +} diff --git a/nodedb/src/control/cluster/calvin/scheduler/driver/core/commit_resolve/flush_parts.rs b/nodedb/src/control/cluster/calvin/scheduler/driver/core/commit_resolve/flush_parts.rs deleted file mode 100644 index 7a830fc08..000000000 --- a/nodedb/src/control/cluster/calvin/scheduler/driver/core/commit_resolve/flush_parts.rs +++ /dev/null @@ -1,28 +0,0 @@ -// SPDX-License-Identifier: BUSL-1.1 - -//! The stored write set of a committed Calvin flush. -//! -//! A committed Calvin record is whole at append: its flush reports no rows, -//! so no part follows it. The core still runs the flush journalled, which -//! stores the flush's append inputs beside its effects for boot. Once the -//! record is durable, the core drops what it stored. - -use crate::control::cluster::calvin::scheduler::driver::core::scheduler::Scheduler; -use crate::types::{Lsn, VShardId}; - -impl Scheduler { - /// Note that the record at `origin` is durable, so the core drops the - /// write set it stored. - pub(in crate::control::cluster::calvin::scheduler::driver::core) fn note_flush_settled( - &self, - origin: Lsn, - ) { - let vshard_id = VShardId::new(self.vshard_id); - match self.shared.dispatcher.lock() { - Ok(mut d) => d.note_write_set_settled(vshard_id, origin), - Err(poisoned) => poisoned - .into_inner() - .note_write_set_settled(vshard_id, origin), - } - } -} diff --git a/nodedb/src/control/cluster/calvin/scheduler/driver/core/commit_resolve/mod.rs b/nodedb/src/control/cluster/calvin/scheduler/driver/core/commit_resolve/mod.rs index 69168ac82..309b4965d 100644 --- a/nodedb/src/control/cluster/calvin/scheduler/driver/core/commit_resolve/mod.rs +++ b/nodedb/src/control/cluster/calvin/scheduler/driver/core/commit_resolve/mod.rs @@ -1,18 +1,15 @@ // SPDX-License-Identifier: BUSL-1.1 -//! Verdict-driven commit resolution for staged static Calvin transactions. +//! Verdict-driven commit resolution for a staged Calvin transaction on the +//! data-group leader, and verdict handling for a follower's held txn. //! -//! A static Calvin dispatch STAGES its transaction on the Data Plane (validate -//! the read-set + buffer the plans, no base mutation). Its executor response -//! carries the local commit vote on `read_set_valid`. This module drives the -//! final step: dispatch a flush (commit, after `commit_redo` has WAL-appended -//! the resolved `TransactionRedo`) or drop (abort) of the staged buffer, wait -//! for its response, then run the commit tail (deposit applied result, record -//! write versions — plus a `CalvinApplied` WAL fallback when no redo record -//! was appended — propose `CompletionAck`) for a flush, or ack-only for a -//! drop. +//! A static or active dispatch STAGES its transaction on the Data Plane +//! (validate the read-set + buffer the plans, no base mutation). Its executor +//! response carries the local commit vote on `stage_vote`. The leader votes, +//! waits for the global verdict, then resolves a committed write slice into +//! its redo and proposes it (see `super::commit_redo`), or drops the staged +//! state of a slice that ends with no log entry. -mod apply_tail; -mod flush_parts; +mod drop_tail; mod verdict; mod vote; diff --git a/nodedb/src/control/cluster/calvin/scheduler/driver/core/commit_resolve/verdict.rs b/nodedb/src/control/cluster/calvin/scheduler/driver/core/commit_resolve/verdict.rs index 2168b5c19..ebfb477c5 100644 --- a/nodedb/src/control/cluster/calvin/scheduler/driver/core/commit_resolve/verdict.rs +++ b/nodedb/src/control/cluster/calvin/scheduler/driver/core/commit_resolve/verdict.rs @@ -1,148 +1,178 @@ // SPDX-License-Identifier: BUSL-1.1 //! Resume-on-verdict, verdict-signal handling, and the stall re-probe sweep -//! for a staged Calvin transaction parked on the cross-shard commit barrier. +//! for a granted Calvin transaction waiting on the cross-shard commit +//! barrier. use std::sync::atomic::Ordering; use std::time::Instant; use nodedb_cluster::calvin::VerdictSignal; -use crate::control::cluster::calvin::scheduler::driver::core::commit_resolution_dispatch::CommitResolution; use crate::control::cluster::calvin::scheduler::driver::core::deferred::{ DispatchOutcome, DispatchStep, }; -use crate::control::cluster::calvin::scheduler::driver::core::halt::{HaltReason, HaltStep}; +use crate::control::cluster::calvin::scheduler::driver::core::owed::SchedulerProposal; +use crate::control::cluster::calvin::scheduler::driver::core::process::LedgerMark; use crate::control::cluster::calvin::scheduler::driver::core::scheduler::Scheduler; use crate::control::cluster::calvin::scheduler::driver::types::CommitState; use crate::control::cluster::calvin::scheduler::lock_manager::TxnId; impl Scheduler { - /// Resume a txn parked in [`CommitState::AwaitingVerdict`] once the durable - /// GLOBAL verdict is known: dispatch its flush (commit) or drop (abort). + /// Act on the durable GLOBAL verdict of `txn_id`. /// - /// `committed` is the authoritative cross-shard verdict — NOT this shard's - /// local vote. On commit, dispatches `MetaOp::CalvinResolve` and moves the - /// txn to [`CommitState::AwaitingRedoResolve`] (the resolved redo is - /// WAL-appended and the flush dispatched from [`Self::finish_redo_resolve`]). - /// On abort, dispatches the drop directly and moves the txn to - /// [`CommitState::AwaitingResolve`]. Bumps the flushed / dropped counter. The - /// commit tail runs later in [`Self::finish_resolved_commit`], once the - /// flush/drop response arrives. + /// `committed` is the authoritative cross-shard verdict, never this + /// shard's local vote. /// - /// Double-resume guard: the verdict push and the probe-on-park (and the - /// stall re-probe sweep) can all fire for one txn, so this first confirms the - /// txn is still `Some(AwaitingVerdict)` — if it already transitioned out - /// (resolve/drop dispatched, or completed), this is a no-op. This guarantees - /// the flush/drop is dispatched exactly once. + /// A leader's txn parked in [`CommitState::AwaitingVerdict`]: + /// - abort: dispatch the drop of its staged state; + /// - COMMIT for a slice with no write: drop the staged state too, since + /// nothing installs; + /// - COMMIT for a write slice: resolve its staged post-images, at once, + /// or at its turn for a whole-collection resolve. The resolved redo is + /// proposed from [`Self::finish_redo_resolve`]. /// - /// A COMMIT verdict for a txn this replica failed to stage halts the - /// scheduler: the txn stays parked with its locks, its stall deadline - /// cleared, and its position unapplied. + /// A follower's [`CommitState::Following`] txn completes at an abort or + /// at a COMMIT for a slice with no write: no log entry follows either. + /// A COMMIT for a write slice waits for the slice's redo to apply. + /// + /// Any other state already left the barrier, so a duplicate push, probe + /// or sweep is a no-op. + /// + /// A COMMIT verdict for a txn this leader failed to stage restages it + /// (see `super::super::restage`). The txn keeps its locks and its + /// position stays unapplied meanwhile. pub(in crate::control::cluster::calvin::scheduler::driver::core) fn resume_on_verdict( &mut self, txn_id: TxnId, committed: bool, ) { - // Guard: only a still-parked txn resumes. Mirrors `handle_completion`'s - // state-match so a duplicate push/probe/timeout is idempotent. - if !matches!( - self.pending.get(&txn_id).and_then(|p| p.commit_state), - Some(CommitState::AwaitingVerdict) - ) { - return; - } - - if committed - && let Some(pending) = self.pending.get_mut(&txn_id) - && let Some(stage_error) = pending.stage_error.clone() - { - pending.verdict_deadline = None; - self.halt_apply( + // An abort verdict ends a txn still waiting at its barrier: it + // stages nothing, and its locks free at once. + if !committed && self.dependent_barrier.contains_key(&txn_id) { + self.abort_open_barrier( txn_id, - HaltReason::LocalStageFailed, - HaltStep::Stage, - format!("COMMIT verdict for a txn this replica did not stage: {stage_error}"), + nodedb_cluster::calvin::AbortReason::ParticipantError, + "an abort verdict reached the txn's open dependent-read barrier".to_owned(), ); return; } + let Some((state, writes)) = self + .pending + .get(&txn_id) + .map(|p| (p.commit_state, p.scope.writes)) + else { + return; + }; + match state { + CommitState::Following => { + if !committed || !writes { + self.complete_without_entry(txn_id); + } + } + CommitState::AwaitingVerdict => self.resume_parked(txn_id, committed, writes), + CommitState::Staged + | CommitState::ReadingPassive + | CommitState::AwaitingResolveTurn + | CommitState::AwaitingRedoResolve + | CommitState::AwaitingRedoApply { .. } + | CommitState::AwaitingDrop + | CommitState::AwaitingRestage => {} + } + } - // A slice that truncates rows resolves at its turn, once every lower - // txn of this vShard finished. + /// Resume a leader's txn parked on the barrier. + fn resume_parked(&mut self, txn_id: TxnId, committed: bool, writes: bool) { + // An earlier leader of the vShard voted COMMIT, and this leader's + // stage failed: the txn cannot drop while its peers apply it. if committed - && let Some(pending) = self.pending.get_mut(&txn_id) - && pending.flush_scope.resolve_at_turn + && self + .pending + .get(&txn_id) + .is_some_and(|pending| pending.stage_error.is_some()) { - pending.commit_state = Some(CommitState::AwaitingResolveTurn); - pending.verdict_deadline = None; - self.shared - .calvin - .counters - .commits_flushed - .fetch_add(1, Ordering::Relaxed); - self.pump_flush_turn(); + self.restage_or_halt(txn_id); return; } - - let (outcome, step) = if committed { - // Resolve the staged post-images into a replayable `RedoRecord` - // first; the redo is WAL-appended (in `finish_redo_resolve`) before - // the flush is dispatched, restoring restart durability for this - // vShard's slice of a multi-shard Calvin commit. - (self.dispatch_calvin_resolve(txn_id), DispatchStep::Resolve) + if let Some(pending) = self.pending.get_mut(&txn_id) { + // No longer parked: clear the stall deadline. + pending.verdict_deadline = None; + } + let counter = if committed { + &self.shared.calvin.counters.commits_flushed } else { - ( - self.dispatch_commit_resolution(txn_id, CommitResolution::Drop), - DispatchStep::Drop, - ) + &self.shared.calvin.counters.commits_dropped }; - if let DispatchOutcome::Failed(error) = outcome { - // Terminal resolve/drop refusal: the scheduler halts and holds - // the txn parked with its locks and staged buffer. The cleared - // deadline keeps the stall sweep from re-sending it. + counter.fetch_add(1, Ordering::Relaxed); + + if !committed || !writes { + self.drop_staged_slice(txn_id); + return; + } + // A slice that truncates rows resolves at its turn, once every lower + // txn of this vShard finished. + let at_turn = self + .pending + .get(&txn_id) + .is_some_and(|pending| pending.scope.resolve_at_turn); + if at_turn { if let Some(pending) = self.pending.get_mut(&txn_id) { - pending.verdict_deadline = None; + pending.commit_state = CommitState::AwaitingResolveTurn; } - self.fail_dispatch_step(txn_id, step, error); + self.pump_resolve_turn(); return; } + self.start_resolve(txn_id); + } - // Sent or parked for re-send at capacity: either way the txn awaits - // this step's response, so a duplicate verdict push or probe is a - // no-op under the guard above. + /// Dispatch the resolve of a committed slice, and wait for its answer. + pub(in crate::control::cluster::calvin::scheduler::driver::core) fn start_resolve( + &mut self, + txn_id: TxnId, + ) { if let Some(pending) = self.pending.get_mut(&txn_id) { - pending.commit_state = Some(if committed { - CommitState::AwaitingRedoResolve - } else { - CommitState::AwaitingResolve { - committed: false, - redo_lsn: None, - } - }); - // No longer parked: clear the stall deadline. - pending.verdict_deadline = None; + pending.commit_state = CommitState::AwaitingRedoResolve; + } + // Sent or parked for re-send at capacity: either way the txn awaits + // the resolve's answer. + if let DispatchOutcome::Failed(error) = self.dispatch_calvin_resolve(txn_id) { + self.fail_dispatch_step(txn_id, DispatchStep::Resolve, error); } + } - if committed { - self.shared - .calvin - .counters - .commits_flushed - .fetch_add(1, Ordering::Relaxed); - } else { - self.shared - .calvin - .counters - .commits_dropped - .fetch_add(1, Ordering::Relaxed); + /// Dispatch the drop of a staged slice that ends with no log entry, and + /// wait for its answer. + fn drop_staged_slice(&mut self, txn_id: TxnId) { + if let Some(pending) = self.pending.get_mut(&txn_id) { + pending.commit_state = CommitState::AwaitingDrop; + } + if let DispatchOutcome::Failed(error) = self.dispatch_drop(txn_id, DispatchStep::Drop) { + // Terminal refusal: the scheduler halts and holds the txn with + // its locks and staged buffer. + self.fail_dispatch_step(txn_id, DispatchStep::Drop, error); } } + /// Complete `txn_id`, which ends with no log entry on this vShard. It + /// owes an empty `CompletionAck`: the coordinator's outcome waits for + /// every participant's. + pub(in crate::control::cluster::calvin::scheduler::driver::core) fn complete_without_entry( + &mut self, + txn_id: TxnId, + ) { + self.propose_sequencer_entry( + txn_id, + SchedulerProposal::CompletionAck { result: Vec::new() }, + ); + self.metrics.record_completed(); + self.on_txn_complete(txn_id, LedgerMark::Terminal); + } + /// Handle a pushed [`VerdictSignal`] from this node's completion registry. /// - /// Matches the signal to the parked txn by `(epoch, position)` and resumes - /// it. A signal for a txn this scheduler does not host, or one that already - /// resumed, is a harmless no-op (the double-resume guard covers the latter). + /// Matches the signal to the txn by `(epoch, position)` and resumes it. A + /// signal for a txn this scheduler does not host, or one that already + /// resumed, is a harmless no-op. pub(in crate::control::cluster::calvin::scheduler::driver::core) fn handle_verdict_signal( &mut self, signal: VerdictSignal, @@ -159,10 +189,14 @@ impl Scheduler { /// emit a stall metric + warning, and re-arm the deadline so the warning is /// rate-limited rather than per-iteration. It NEVER releases locks and NEVER /// unilaterally aborts: a participant cannot know whether a peer already - /// flushed a COMMIT, so aborting one side while a peer committed will tear + /// committed, so aborting one side while a peer committed will tear /// the transaction. The verdict is guaranteed to arrive eventually — a /// post-failover leader re-aggregates the replicated votes (seeded on every /// replica) into the same verdict — so waiting is always the safe action. + /// + /// The same stall tick re-proposes this vShard's vote while it is owed + /// (`retry_owed_sequencer_entries`). The warning names whether the vote is + /// still owed, so a stall that waits on another participant shows apart. pub(in crate::control::cluster::calvin::scheduler::driver::core) fn check_awaiting_verdict_stalls( &mut self, ) { @@ -171,7 +205,7 @@ impl Scheduler { let stalled: Vec = self .pending .iter() - .filter(|(_, p)| matches!(p.commit_state, Some(CommitState::AwaitingVerdict))) + .filter(|(_, p)| p.commit_state == CommitState::AwaitingVerdict) .filter(|(_, p)| p.verdict_deadline.is_some_and(|d| now >= d)) .map(|(id, _)| *id) .collect(); @@ -187,13 +221,18 @@ impl Scheduler { // Verdict still unknown: keep waiting, hold locks, never abort. self.metrics.record_verdict_stall(); + let vote_owed = self.owed.contains_key(&( + txn_id, + crate::control::cluster::calvin::scheduler::driver::core::owed::OwedKind::Vote, + )); tracing::warn!( vshard_id = self.vshard_id, epoch = txn_id.epoch, position = txn_id.position, + vote_owed, "calvin: staged txn still awaiting the cross-shard verdict past its stall \ deadline; HOLDING locks and waiting (never aborting — a peer may have already \ - flushed a commit). The verdict is guaranteed to arrive." + committed). The verdict is guaranteed to arrive." ); if let Some(pending) = self.pending.get_mut(&txn_id) { pending.verdict_deadline = Some(now + self.config.verdict_stall_warn()); @@ -216,12 +255,12 @@ mod tests { use super::*; use crate::bridge::dispatch::CoreChannelDataSide; use crate::bridge::envelope::ErrorCode; - use crate::bridge::envelope::{Payload, Status}; - use crate::control::cluster::calvin::scheduler::driver::core::halt::HaltReason; + use crate::bridge::envelope::{Payload, StageVote, Status}; + use crate::control::cluster::calvin::scheduler::driver::core::test_proposer::lead_data_group; use crate::control::cluster::calvin::scheduler::driver::core::test_support::{ - await_data_plane_request, build_test_scheduler_with_data_side, error_response, - fill_tenant_inflight, make_sequenced_txn, release_filler, spawn_scheduler_loop, - staged_pending, staged_response, + await_data_plane_request, build_test_scheduler, build_test_scheduler_with_data_side, + error_response, fill_tenant_inflight, following_pending, make_sequenced_txn, + release_filler, spawn_scheduler_loop, staged_pending, staged_response, }; use crate::control::state::SharedState; use crate::types::RequestId; @@ -242,7 +281,7 @@ mod tests { let registry = CalvinCompletionRegistry::new_detached(); let (mut scheduler, dir, mut data_side) = build_test_scheduler_with_data_side(7, registry); let mut pending = staged_pending(make_sequenced_txn(txn_id.epoch, txn_id.position), txn_id); - pending.commit_state = Some(state); + pending.commit_state = state; scheduler.pending.insert(txn_id, pending); let shared = Arc::clone(&scheduler.shared); let fillers = fill_tenant_inflight(&shared, &mut data_side, TenantId::new(1)); @@ -255,8 +294,7 @@ mod tests { } } - /// An Ok resolve response whose redo record carries no ops, so the flush - /// dispatch follows at once with no WAL append. + /// An Ok resolve response whose redo record carries no ops. fn empty_redo_response() -> crate::bridge::envelope::Response { let redo = RedoRecord { version: 1, @@ -268,7 +306,12 @@ mod tests { row_changes: Vec::new(), }; let mut response = staged_response(Status::Ok, None); - response.payload = Payload::from_vec(redo.to_bytes().expect("encode empty redo record")); + let resolved = nodedb_physical::physical_plan::CalvinResolved { + redo: redo.to_bytes().expect("encode empty redo record"), + reply: nodedb_physical::physical_plan::CalvinReplySpec::Count(Vec::new()), + }; + response.payload = + Payload::from_vec(zerompk::to_msgpack_vec(&resolved).expect("encode resolved answer")); response } @@ -292,24 +335,28 @@ mod tests { ); } - /// A refused flush dispatch after the redo resolves does not complete the - /// txn: its position stays unapplied and its pending entry stays. + /// A resolved redo is proposed to the data group, and the txn waits for + /// its apply: its position stays unapplied and its pending entry stays. #[tokio::test] - async fn resolved_redo_flush_refused_at_capacity_does_not_complete_txn() { + async fn a_resolved_redo_is_proposed_and_waits_for_its_apply() { let txn_id = TxnId::new(14, 2); - let mut parked = parked_at_capacity(txn_id, CommitState::AwaitingRedoResolve); - let scheduler = &mut parked.scheduler; + let (mut scheduler, _dir) = build_test_scheduler(7); + lead_data_group(&mut scheduler); + let mut pending = staged_pending(make_sequenced_txn(14, 2), txn_id); + pending.commit_state = CommitState::AwaitingRedoResolve; + scheduler.pending.insert(txn_id, pending); scheduler.finish_redo_resolve(txn_id, empty_redo_response()); + assert!(matches!( + scheduler.pending.get(&txn_id).map(|p| p.commit_state), + Some(CommitState::AwaitingRedoApply { proposed: Some(_) }) + )); assert!( !scheduler.applied.is_applied(14, 2), - "a refused flush must not mark the position applied" - ); - assert!( - scheduler.pending.contains_key(&txn_id), - "a refused flush must keep the txn's pending entry" + "a proposed redo must not mark the position applied" ); + assert!(!scheduler.is_apply_halted()); } /// Under an ABORT verdict, a refused drop dispatch does not complete the @@ -367,10 +414,10 @@ mod tests { ); } - /// Once a Data Plane response frees tenant capacity, a refused flush + /// Once a Data Plane response frees tenant capacity, a refused drop /// reaches the Data Plane. #[tokio::test] - async fn refused_flush_reaches_data_plane_after_capacity_frees() { + async fn refused_drop_reaches_data_plane_after_capacity_frees() { let txn_id = TxnId::new(14, 2); let ParkedAtCapacity { mut scheduler, @@ -378,19 +425,18 @@ mod tests { mut data_side, shared, fillers, - } = parked_at_capacity(txn_id, CommitState::AwaitingRedoResolve); + } = parked_at_capacity(txn_id, CommitState::AwaitingVerdict); - scheduler.finish_redo_resolve(txn_id, empty_redo_response()); + scheduler.resume_on_verdict(txn_id, false); let running = spawn_scheduler_loop(scheduler); release_filler(&shared, &mut data_side, fillers[0]); let arrived = await_data_plane_request(&mut data_side, |plan| { matches!( plan, - PhysicalPlan::Meta(MetaOp::CalvinFlush { + PhysicalPlan::Meta(MetaOp::CalvinDrop { epoch: 14, position: 2, - .. }) ) }) @@ -399,14 +445,14 @@ mod tests { assert!( arrived, - "the refused flush must reach the Data Plane once capacity frees" + "the refused drop must reach the Data Plane once capacity frees" ); } /// A false vote from either participant makes the only global verdict abort; /// applying that durable verdict broadcasts the abort to every parked local /// participant. The scheduler's `resume_on_verdict(false)` then dispatches a - /// drop, never a resolve/flush, on each recipient. + /// drop, never a resolve, on each recipient. #[tokio::test] async fn two_participant_false_vote_broadcasts_global_abort_to_every_scheduler() { let registry = CalvinCompletionRegistry::new_detached(); @@ -425,7 +471,10 @@ mod tests { // Local staging votes only park their own staged slices; neither the // affirmative nor the failed participant can resolve or drop unilaterally. - first_scheduler.resolve_staged_commit(txn_id, &staged_response(Status::Ok, Some(true))); + first_scheduler.resolve_staged_commit( + txn_id, + &staged_response(Status::Ok, Some(StageVote::Commit)), + ); second_scheduler.resolve_staged_commit(txn_id, &staged_response(Status::Error, None)); for (scheduler, data_side) in [ (&first_scheduler, &mut first_data), @@ -435,7 +484,7 @@ mod tests { scheduler .pending .get(&txn_id) - .and_then(|pending| pending.commit_state), + .map(|pending| pending.commit_state), Some(CommitState::AwaitingVerdict) )); assert!(data_side.request_rx.try_pop().is_err()); @@ -483,11 +532,8 @@ mod tests { scheduler .pending .get(&txn_id) - .and_then(|pending| pending.commit_state), - Some(CommitState::AwaitingResolve { - committed: false, - redo_lsn: None - }) + .map(|pending| pending.commit_state), + Some(CommitState::AwaitingDrop) )); let request = data_side .request_rx @@ -504,16 +550,12 @@ mod tests { } } - /// A follower scheduler whose stage failed, parked on the verdict barrier. - fn follower_with_failed_stage( + /// A leader scheduler whose stage failed, parked on the verdict barrier. + fn leader_with_failed_stage( txn_id: TxnId, ) -> (Scheduler, tempfile::TempDir, CoreChannelDataSide) { let registry = CalvinCompletionRegistry::new_detached(); let (mut scheduler, dir, data_side) = build_test_scheduler_with_data_side(7, registry); - assert!( - !scheduler.is_group_leader(), - "the fixture hosts no data group" - ); scheduler.pending.insert( txn_id, staged_pending(make_sequenced_txn(txn_id.epoch, txn_id.position), txn_id), @@ -527,13 +569,14 @@ mod tests { (scheduler, dir, data_side) } - /// A COMMIT verdict for a txn this follower failed to stage halts the - /// scheduler: the leader staged and voted commit, so this replica cannot - /// apply it. No resolve is dispatched, and the stall sweep stops. + /// A COMMIT verdict for a txn this leader failed to stage holds it for a + /// restage: an earlier leader staged and voted commit, so the txn cannot + /// drop. The failed stage's state is discarded, no resolve is + /// dispatched, and the position stays unapplied. #[tokio::test] - async fn commit_verdict_after_local_stage_error_halts_unapplied() { + async fn commit_verdict_after_local_stage_error_awaits_a_restage() { let txn_id = TxnId::new(14, 2); - let (mut scheduler, _dir, mut data_side) = follower_with_failed_stage(txn_id); + let (mut scheduler, _dir, mut data_side) = leader_with_failed_stage(txn_id); scheduler.resume_on_verdict(txn_id, true); @@ -542,28 +585,43 @@ mod tests { .pending .get(&txn_id) .expect("the txn stays pending"); - assert_eq!(pending.commit_state, Some(CommitState::AwaitingVerdict)); + assert_eq!(pending.commit_state, CommitState::AwaitingRestage); assert_eq!(pending.verdict_deadline, None); - assert_eq!( - scheduler.apply_halt().map(|h| h.reason), - Some(HaltReason::LocalStageFailed) - ); + assert!(!scheduler.is_apply_halted()); + let restage = scheduler + .restages + .get(&txn_id) + .expect("the txn waits for its first restage"); + assert_eq!(restage.attempts, 0); + assert!(restage.due.is_some()); + let request = data_side + .request_rx + .try_pop() + .expect("the failed stage's state is discarded"); + assert!(matches!( + request.inner.plan, + PhysicalPlan::Meta(MetaOp::CalvinDrop { + epoch: 14, + position: 2 + }) + )); assert!( data_side.request_rx.try_pop().is_err(), "no resolve reaches the Data Plane" ); } - /// An abort verdict for a txn this replica failed to stage drops it as + /// An abort verdict for a txn this leader failed to stage drops it as /// usual: every replica reaches the same abort. #[tokio::test] async fn abort_verdict_after_local_stage_error_drops_without_halting() { let txn_id = TxnId::new(14, 2); - let (mut scheduler, _dir, mut data_side) = follower_with_failed_stage(txn_id); + let (mut scheduler, _dir, mut data_side) = leader_with_failed_stage(txn_id); scheduler.resume_on_verdict(txn_id, false); assert!(!scheduler.is_apply_halted()); + assert!(scheduler.restages.is_empty(), "an abort never restages"); let request = data_side .request_rx .try_pop() @@ -576,4 +634,38 @@ mod tests { }) )); } + + /// A follower's txn completes at an abort verdict with no log entry: it + /// owes its ack and marks its position applied. A COMMIT verdict for a + /// write slice leaves it waiting for its redo. + #[tokio::test] + async fn a_following_txn_completes_at_an_abort_and_waits_at_a_commit() { + use crate::control::cluster::calvin::scheduler::driver::core::owed::OwedKind; + let (mut scheduler, _dir) = build_test_scheduler(7); + let aborted = TxnId::new(14, 2); + let committed = TxnId::new(15, 0); + for txn_id in [aborted, committed] { + scheduler.pending.insert( + txn_id, + following_pending(make_sequenced_txn(txn_id.epoch, txn_id.position), txn_id), + ); + } + + scheduler.resume_on_verdict(aborted, false); + scheduler.resume_on_verdict(committed, true); + + assert!(!scheduler.pending.contains_key(&aborted)); + assert!(scheduler.applied.is_applied(14, 2)); + assert!( + scheduler + .owed + .contains_key(&(aborted, OwedKind::CompletionAck)) + ); + assert_eq!( + scheduler.pending.get(&committed).map(|p| p.commit_state), + Some(CommitState::Following), + "a committed write slice waits for its redo" + ); + assert!(!scheduler.applied.is_applied(15, 0)); + } } diff --git a/nodedb/src/control/cluster/calvin/scheduler/driver/core/commit_resolve/vote.rs b/nodedb/src/control/cluster/calvin/scheduler/driver/core/commit_resolve/vote.rs index 2a837fe53..27eb09f90 100644 --- a/nodedb/src/control/cluster/calvin/scheduler/driver/core/commit_resolve/vote.rs +++ b/nodedb/src/control/cluster/calvin/scheduler/driver/core/commit_resolve/vote.rs @@ -6,25 +6,27 @@ use std::sync::atomic::Ordering; use std::time::Instant; use crate::bridge::envelope::Response; -use crate::control::cluster::calvin::scheduler::driver::core::halt::error_response_text; +use crate::control::cluster::calvin::scheduler::driver::core::halt::{ + HaltReason, HaltStep, error_response_text, +}; use crate::control::cluster::calvin::scheduler::driver::core::owed::SchedulerProposal; use crate::control::cluster::calvin::scheduler::driver::core::scheduler::Scheduler; use crate::control::cluster::calvin::scheduler::driver::core::staged_vote::{ - StagedVote, staged_commit_vote, + StageVoteError, StagedVote, staged_commit_vote, }; use crate::control::cluster::calvin::scheduler::driver::types::CommitState; use crate::control::cluster::calvin::scheduler::lock_manager::TxnId; impl Scheduler { - /// Cast this participant's local commit vote for a staged transaction, then - /// PARK it on the cross-shard commit barrier awaiting the durable GLOBAL - /// verdict — it does NOT self-decide flush-or-drop on its local vote. + /// Cast this leader's commit vote for a staged transaction, then PARK it + /// on the cross-shard commit barrier awaiting the durable GLOBAL verdict. + /// It never self-decides commit or abort on its local vote. /// - /// The staged executor response is validate-only: its `read_set_valid` is - /// this shard's local commit vote (`Some(true)` => commit, `Some(false)` => - /// abort; a `None` from the active/dependent path is treated as commit). The - /// leader proposes that vote via the sequencer Raft group; the sequencer - /// aggregates all participants' votes into a single authoritative + /// The staged executor response is validate-only: its `stage_vote` is this + /// shard's local commit vote, read by [`staged_commit_vote`]. A stage + /// response the scheduler cannot read a vote from halts apply. It never + /// counts as a commit. The vote travels through the sequencer Raft group, + /// which aggregates every participant's vote into one authoritative /// `SequencerEntry::Verdict`, applied on every replica. /// /// This method moves the txn to [`CommitState::AwaitingVerdict`] WITHOUT @@ -33,18 +35,15 @@ impl Scheduler { /// push we raced) it resumes at once via [`Self::resume_on_verdict`]; /// otherwise it stays parked, holding locks and its staged buffer, until the /// verdict push, a later probe, or the stall re-probe sweep delivers the - /// verdict. Resuming (in `resume_on_verdict`) is where the flush/drop is - /// dispatched and the flushed/dropped counters bump — using the GLOBAL - /// verdict, never the local vote. + /// verdict. pub(in crate::control::cluster::calvin::scheduler::driver::core) fn resolve_staged_commit( &mut self, txn_id: TxnId, staged_response: &Response, ) { - // A staged error is always an abort vote. Only successful staged - // responses can use `None` for the dependent-read path; accepting an - // error-plus-None as commit will let a failed participant flush after - // its peers received a global commit verdict. + // A staged error is always an abort vote. A response with no readable + // vote halts: counting it as a commit will let a failed participant + // commit after its peers received a global commit verdict. let superseded = self .pending .get(&txn_id) @@ -53,41 +52,46 @@ impl Scheduler { StagedVote::CollectionSuperseded } else { match staged_commit_vote(staged_response) { - // A read another node served is numbered in that node's WAL: - // this node's versions cannot show it still current. - StagedVote::Commit if self.validates_read_served_elsewhere(txn_id) => { - StagedVote::SerializationConflict + Ok(vote) => vote, + Err(error) => { + self.metrics.record_executor_error(); + let reason = match error { + StageVoteError::CoreFailStopped { .. } => HaltReason::CoreFailStopped, + StageVoteError::Missing { .. } + | StageVoteError::CommitWithoutSuccess { .. } => { + HaltReason::StageVoteInvalid + } + }; + self.halt_apply( + txn_id, + reason, + HaltStep::Stage, + format!("{error}: {}", error_response_text("stage", staged_response)), + ); + return; } - vote => vote, } }; - // Durably propose this participant's commit vote via the sequencer - // Raft group, leader-guarded: only the data-group leader ran - // read-set validation, so only a leader's vote is - // authoritative. The sequencer aggregates every participant's vote into - // the single global verdict this txn parks on below. An abort travels as + // Durably propose this leader's commit vote via the sequencer Raft + // group. Only the leader stages, so its vote is the vShard's. The + // sequencer aggregates every participant's vote into the single + // global verdict this txn parks on below. An abort travels as // `AbortVote` so its cause survives to the coordinator. The vote stays // owed until the tally holds it, so a refused or dropped proposal is - // proposed again rather than lost. - // - // A follower owes an abort vote and proposes it only if it comes to - // lead before any vote of this vShard applies: the previous leader - // lost its leadership, or its scheduler cannot run. A follower ran - // none of the leader-only checks, so abort is the only vote it can - // cast. The txn fails with a retryable conflict and never hangs. - let abort = if self.is_group_leader() { - vote.abort_reason() - } else { - Some(nodedb_cluster::calvin::AbortReason::SerializationConflict) - }; - self.propose_sequencer_entry(txn_id, SchedulerProposal::Vote { abort }); + // proposed again rather than lost. A promoted leader's vote for a + // txn whose vote applied already changes nothing: the first counts. + self.propose_sequencer_entry( + txn_id, + SchedulerProposal::Vote { + abort: vote.abort_reason(), + }, + ); if vote == StagedVote::SerializationConflict { - // The staged slice's read-set was no longer current: observe it, the - // same node-global signal the direct-apply path records. A - // participant error never validated a read-set, so it must not count - // here. + // The staged slice's read-set was no longer current: observe it on + // the node-global counter. A participant error never validated a + // read-set, so it must not count here. self.shared .calvin .counters @@ -100,14 +104,14 @@ impl Scheduler { // this local vote, decides. If the txn already vanished (torn down // elsewhere), there is nothing to park. // - // A stage error parks too. A deterministic error fails on every - // replica, the leader votes abort, and every replica drops. A local - // error on a follower leaves the leader's commit vote standing: - // `resume_on_verdict` halts on that COMMIT verdict. + // A stage error parks too. The leader votes abort, and every + // participant drops at the abort verdict. A COMMIT verdict for it + // can follow only a vote an earlier leader cast: `resume_on_verdict` + // restages the txn. match self.pending.get_mut(&txn_id) { Some(pending) => { - pending.commit_state = Some(CommitState::AwaitingVerdict); - // A superseded slice cannot flush here: its collection is gone. + pending.commit_state = CommitState::AwaitingVerdict; + // A superseded slice cannot commit here: its collection is gone. pending.stage_error = match vote { StagedVote::ParticipantError | StagedVote::PredictionDrift => { Some(error_response_text("stage", staged_response)) @@ -136,20 +140,185 @@ impl Scheduler { self.resume_on_verdict(txn_id, verdict); } } +} - /// Whether a read this vShard validates for `txn_id` was served by a node - /// other than this one. Its `read_lsn` is a position in that node's WAL. - fn validates_read_served_elsewhere(&self, txn_id: TxnId) -> bool { - let Some(pending) = self.pending.get(&txn_id) else { - return false; - }; - let tx_class = &pending.txn.tx_class; - tx_class.versioned_reads.iter().any(|entry| { - super::super::routing::versioned_read_homes_on( - entry, - tx_class.database_id, - self.vshard_id, - ) && entry.served_by != self.shared.node_id - }) +#[cfg(test)] +mod tests { + use nodedb_cluster::calvin::{AbortReason, SequencerEntry}; + + use super::*; + use crate::bridge::envelope::ErrorCode; + use crate::bridge::envelope::{StageVote, Status}; + use crate::control::cluster::calvin::scheduler::driver::core::owed::OwedKind; + use crate::control::cluster::calvin::scheduler::driver::core::test_proposer::{ + CapturingProposer, lead_data_group, + }; + use crate::control::cluster::calvin::scheduler::driver::core::test_support::{ + build_test_scheduler, error_response, make_sequenced_txn, staged_pending, staged_response, + }; + + const VSHARD: u32 = 7; + + /// A data-group leader holding `txn_id` staged, with a proposer that + /// accepts every proposal. + fn staged_leader( + txn_id: TxnId, + ) -> ( + Scheduler, + tempfile::TempDir, + std::sync::Arc, + ) { + let (mut scheduler, dir) = build_test_scheduler(VSHARD); + let proposer = CapturingProposer::accepting(); + scheduler.sequencer_proposer = proposer.clone(); + lead_data_group(&mut scheduler); + scheduler.pending.insert( + txn_id, + staged_pending(make_sequenced_txn(txn_id.epoch, txn_id.position), txn_id), + ); + (scheduler, dir, proposer) + } + + #[tokio::test] + async fn each_stage_vote_proposes_its_own_vote() { + let cases = [ + (Status::Ok, StageVote::Commit, None), + ( + Status::Ok, + StageVote::SerializationConflict, + Some(AbortReason::SerializationConflict), + ), + ( + Status::Error, + StageVote::PredictionDrift, + Some(AbortReason::PredictionDrift), + ), + ( + Status::Error, + StageVote::ParticipantError, + Some(AbortReason::ParticipantError), + ), + ]; + for (status, vote, abort) in cases { + let txn_id = TxnId::new(30, 1); + let (mut scheduler, _dir, proposer) = staged_leader(txn_id); + + scheduler.resolve_staged_commit(txn_id, &staged_response(status, Some(vote))); + + let expected = match abort { + None => SequencerEntry::Vote { + epoch: 30, + position: 1, + vshard: VSHARD, + }, + Some(reason) => SequencerEntry::AbortVote { + epoch: 30, + position: 1, + vshard: VSHARD, + reason, + }, + }; + assert_eq!(proposer.accepted(), vec![expected], "{vote:?}"); + assert_eq!( + scheduler.pending.get(&txn_id).map(|p| p.commit_state), + Some(CommitState::AwaitingVerdict), + "{vote:?}" + ); + assert!(scheduler.apply_halt().is_none(), "{vote:?}"); + } + } + + #[tokio::test] + async fn a_successful_stage_response_without_a_vote_halts_and_never_votes() { + let txn_id = TxnId::new(31, 0); + let (mut scheduler, _dir, proposer) = staged_leader(txn_id); + + scheduler.resolve_staged_commit(txn_id, &staged_response(Status::Ok, None)); + + assert_eq!(proposer.attempt_count(), 0, "no vote is proposed"); + assert_eq!( + scheduler.apply_halt().map(|h| h.reason), + Some(HaltReason::StageVoteInvalid) + ); + assert_eq!( + scheduler.pending.get(&txn_id).map(|p| p.commit_state), + Some(CommitState::Staged), + "the txn does not park on the verdict barrier" + ); + assert!(!scheduler.applied.is_applied(31, 0)); + } + + #[tokio::test] + async fn a_commit_vote_on_an_error_response_halts_and_never_votes() { + let txn_id = TxnId::new(32, 0); + let (mut scheduler, _dir, proposer) = staged_leader(txn_id); + + scheduler.resolve_staged_commit( + txn_id, + &staged_response(Status::Error, Some(StageVote::Commit)), + ); + + assert_eq!(proposer.attempt_count(), 0, "no vote is proposed"); + assert_eq!( + scheduler.apply_halt().map(|h| h.reason), + Some(HaltReason::StageVoteInvalid) + ); + } + + /// A fail-stopped core refuses the stage and every later request. The + /// leader halts on it and votes nothing. + #[tokio::test] + async fn a_fail_stopped_core_halts_the_stage_and_never_votes() { + let txn_id = TxnId::new(33, 0); + let (mut scheduler, _dir, proposer) = staged_leader(txn_id); + + scheduler.resolve_staged_commit( + txn_id, + &error_response(ErrorCode::CoreFailStopped { + core_id: 0, + detail: "rollback failed".to_string(), + }), + ); + + assert_eq!(proposer.attempt_count(), 0, "no vote is proposed"); + assert_eq!( + scheduler.apply_halt().map(|h| h.reason), + Some(HaltReason::CoreFailStopped) + ); + assert!(!scheduler.applied.is_applied(33, 0)); + } + + /// The leader's vote is its real one, never a forced abort. A node that + /// lost its Raft leadership after it staged keeps the vote owed and + /// proposes nothing. + #[tokio::test] + async fn a_vote_is_proposed_only_while_this_node_leads() { + let txn_id = TxnId::new(34, 0); + let (mut scheduler, _dir) = build_test_scheduler(VSHARD); + let proposer = CapturingProposer::accepting(); + scheduler.sequencer_proposer = proposer.clone(); + scheduler.pending.insert( + txn_id, + staged_pending(make_sequenced_txn(txn_id.epoch, txn_id.position), txn_id), + ); + + scheduler.resolve_staged_commit( + txn_id, + &staged_response(Status::Ok, Some(StageVote::Commit)), + ); + + assert_eq!(proposer.attempt_count(), 0, "a non-leader proposes nothing"); + assert!(scheduler.owed.contains_key(&(txn_id, OwedKind::Vote))); + lead_data_group(&mut scheduler); + scheduler.retry_owed_sequencer_entries(); + assert_eq!( + proposer.accepted(), + vec![SequencerEntry::Vote { + epoch: 34, + position: 0, + vshard: VSHARD, + }], + "the vote is the stage's commit, not a forced abort" + ); } } diff --git a/nodedb/src/control/cluster/calvin/scheduler/driver/core/completion_route.rs b/nodedb/src/control/cluster/calvin/scheduler/driver/core/completion_route.rs index aee1145f2..8e13595db 100644 --- a/nodedb/src/control/cluster/calvin/scheduler/driver/core/completion_route.rs +++ b/nodedb/src/control/cluster/calvin/scheduler/driver/core/completion_route.rs @@ -3,16 +3,14 @@ //! Executor-response routing for the Calvin scheduler. //! //! [`Scheduler::handle_completion`] is the `completion_rx` arm of the main -//! `select!` loop: it classifies each executor response (disconnect, OLLP -//! mismatch, staged commit-resolution state, or direct apply) and routes it to -//! the matching handler. Staged transactions thread through the commit-barrier -//! states in [`super::commit_resolve`]; a txn parked in -//! [`CommitState::AwaitingVerdict`] has no outstanding bridge, so a completion -//! for it is a no-op that keeps it parked. +//! `select!` loop: it classifies each executor response (disconnect, or the +//! txn's commit-resolution state) and routes it to the matching handler. +//! Only the answer to the request a txn waits for drives it. An answer to a +//! step the txn left, after a demotion or an install from another copy, is +//! ignored. use super::super::types::CommitState; use super::halt::{HaltReason, HaltStep}; -use super::owed::SchedulerProposal; use super::scheduler::Scheduler; use crate::bridge::envelope::Response; use crate::control::cluster::calvin::scheduler::lock_manager::TxnId; @@ -22,12 +20,28 @@ impl Scheduler { /// Process a completed executor response (or disconnected channel). /// /// Called from the `completion_rx` arm of the main `select!` loop. - pub(in crate::control::cluster::calvin::scheduler::driver::core) async fn handle_completion( + pub(in crate::control::cluster::calvin::scheduler::driver::core) fn handle_completion( &mut self, txn_id: TxnId, request_id: RequestId, resp_opt: Option, ) { + if !self.awaits(txn_id, request_id) { + tracing::debug!( + vshard_id = self.vshard_id, + request_id = request_id.as_u64(), + epoch = txn_id.epoch, + position = txn_id.position, + "calvin: executor answer to a step the txn left; ignoring" + ); + return; + } + let Some(commit_state) = self.pending.get(&txn_id).map(|p| p.commit_state) else { + return; + }; + if let Some(pending) = self.pending.get_mut(&txn_id) { + pending.awaiting = None; + } let response = match resp_opt { Some(r) => r, None => { @@ -35,11 +49,10 @@ impl Scheduler { // so the request's outcome on this replica is unknown. Hold // the txn unapplied and halt. self.metrics.record_executor_error(); - let state = self.pending.get(&txn_id).and_then(|p| p.commit_state); self.halt_apply( txn_id, HaltReason::ResponseDisconnected, - HaltStep::awaited_by(state), + HaltStep::awaited_by(commit_state), format!( "executor response channel for request {} closed before a response", request_id.as_u64() @@ -56,156 +69,39 @@ impl Scheduler { .unwrap_or(0); self.metrics.record_executor_txn_duration_ms(elapsed_ms); - // A staged transaction resolves through its commit-barrier state, OLLP - // answer included. The flush installs a resolved redo record and runs - // no OLLP check, so an OLLP answer under a commit state is a failed - // step that halts and holds. It never settles as a retry. - let commit_state = self.pending.get(&txn_id).and_then(|p| p.commit_state); - - // OLLP mismatch: the active executor detected predicate drift and returned - // OllpRetryRequired without writing. The retry loop is COORDINATOR-owned - // (`run_dependent_with_retry`): the scheduler must NOT re-submit a stale - // prediction. Instead it (1) releases the aborted attempt's locks and - // (2) signals the coordinator's completion waiter via the registry so it - // can run a FRESH reconnaissance and resubmit. - if commit_state.is_none() - && response.status == crate::bridge::envelope::Status::Error - && response.error_code.as_deref() - == Some(&crate::bridge::envelope::ErrorCode::OllpRetryRequired) - { - // A mismatch is a normal OLLP retry signal, not a failure: the executor - // correctly detected predicate drift and declined to write. Count it as - // a received executor response, but NOT as an executor error or infra - // abort — those will inflate failure metrics on every routine retry. - self.metrics.record_completed(); - // (1) Release the aborted attempt's locks and clean up pending state. - // A retry that left the aborted attempt's locks held will leak them. - self.on_txn_complete(txn_id); - // OLLP mismatch broadcast is LEADER-ONLY. The optimistic-lock - // verification runs only on the data-group leader (the data plane - // skips it on followers — see the `ollp_is_group_leader` gate in the - // bulk-DML handlers), so by construction only a leader's executor can - // return `OllpRetryRequired`. This guard is defense-in-depth: a - // non-leader scheduler must never broadcast a mismatch — a lagging - // follower can otherwise poison an attempt the leader already - // completed, exhausting retries on a static dataset. A non-leader - // releases locks (done above) and returns; the leader owns the - // single mismatch signal that the completion registry observes. - if !self.is_group_leader() { - tracing::debug!( - vshard_id = self.vshard_id, - epoch = txn_id.epoch, - position = txn_id.position, - "calvin: OllpRetryRequired observed on non-leader; NOT broadcasting mismatch \ - (leader owns the verification decision)" - ); - return; - } - // (2) Broadcast the mismatch signal via the sequencer-group Raft so that - // the coordinator's CalvinCompletionRegistry fires on EVERY replica, - // including remote nodes. The SequencerStateMachine::apply() arm for - // OllpMismatch calls note_ollp_mismatch, waking the coordinator's - // retry-loop waiter wherever it is — mirrors how CompletionAck is - // delivered to remote coordinators. - self.propose_sequencer_entry(txn_id, SchedulerProposal::OllpMismatch); - return; - } - - // Staged static Calvin apply: a STAGED response carries the local commit - // vote; drive the vote-and-park and let the verdict resume the - // flush-or-drop. Dependent / active txns carry no `commit_state` and apply - // directly below. match commit_state { - Some(CommitState::Staged) => { + CommitState::Staged => { // Stage failures remain participants in the barrier: - // `resolve_staged_commit` turns any `Status::Error` into an - // explicit false vote, parks locally, and waits for the - // durable global verdict before issuing a drop. + // `resolve_staged_commit` proposes the response's abort vote, + // parks locally, and waits for the durable global verdict + // before it issues a drop. self.resolve_staged_commit(txn_id, &response); - return; } - Some(CommitState::AwaitingRedoResolve) => { + CommitState::AwaitingRedoResolve => { self.finish_redo_resolve(txn_id, response); - return; } - Some(CommitState::AwaitingResolve { - committed, - redo_lsn, - }) => { - self.finish_resolved_commit(txn_id, response, committed, redo_lsn) - .await; - return; + CommitState::AwaitingDrop => { + self.finish_drop(txn_id, &response); } - Some(CommitState::AwaitingFlushTurn { .. } | CommitState::AwaitingResolveTurn) => { - // A txn waiting for its flush or resolve turn has no request - // out. - tracing::warn!( - vshard_id = self.vshard_id, - request_id = request_id.as_u64(), - epoch = txn_id.epoch, - position = txn_id.position, - "calvin: executor completion for a txn waiting for its flush turn; ignoring" - ); - return; + CommitState::ReadingPassive => { + self.finish_passive_read(txn_id, &response); } - Some(CommitState::AwaitingVerdict) => { - // A parked txn dispatched no executor request (it is waiting on - // the cross-shard verdict, not the Data Plane), so no response - // bridge is outstanding for it — a completion here indicates a - // logic error, not a real Data-Plane reply. Do NOT run the commit - // tail or release locks (that can tear the transaction); - // remain parked and let the verdict path resume it. + CommitState::Following + | CommitState::AwaitingVerdict + | CommitState::AwaitingResolveTurn + | CommitState::AwaitingRedoApply { .. } + | CommitState::AwaitingRestage => { + // A txn in these states sends no request it waits for. A + // recorded request names a step it left. tracing::warn!( vshard_id = self.vshard_id, request_id = request_id.as_u64(), epoch = txn_id.epoch, position = txn_id.position, - "calvin: executor completion for a txn parked on the commit barrier; \ - ignoring (no bridge is outstanding while AwaitingVerdict)" + ?commit_state, + "calvin: executor completion for a txn with no request out; ignoring" ); - return; } - None => {} - } - - if response.status != crate::bridge::envelope::Status::Ok { - // A failed direct apply must not leave the txn parked with its locks - // held: that wedges every txn queued behind those keys and freezes - // this vShard's epoch watermark, and no sweep re-drives the entry. - // Surface the infra abort and force completion. - tracing::error!( - vshard_id = self.vshard_id, - epoch = txn_id.epoch, - position = txn_id.position, - "calvin: executor response was not Ok; forcing infra-abort completion so locks \ - release and the epoch advances" - ); - self.metrics.record_executor_error(); - self.metrics.record_infra_abort( - crate::control::cluster::calvin::scheduler::metrics::infra_abort_reason::IO_ERROR, - ); - self.metrics.record_completed(); - self.on_txn_complete(txn_id); - return; - } - - // Observe whether the applying participant reported its slice of the - // transaction's reads as no longer current against the local write - // versions. Direct-apply (dependent/active) observation only: the - // staged path folds this into its commit vote instead. `None` means - // no read-set was checked. - if response.read_set_valid == Some(false) { - self.shared - .calvin - .counters - .read_set_validation_failures - .fetch_add(1, std::sync::atomic::Ordering::Relaxed); - } - // `false` means the commit tail halted the scheduler: the txn stays - // pending and unapplied. - if self.commit_apply_tail(txn_id, response, None).await { - self.metrics.record_completed(); - self.on_txn_complete(txn_id); } } } @@ -224,20 +120,13 @@ mod tests { /// A response channel that closes before a response leaves the txn's /// outcome unknown: the txn stays pending and unapplied, the scheduler /// halts, and the node marker names the txn. - #[tokio::test] - async fn disconnected_response_holds_txn_unapplied_and_sets_node_marker() { + #[test] + fn disconnected_response_holds_txn_unapplied_and_sets_node_marker() { let txn_id = TxnId::new(5, 1); - let (mut scheduler, _dir) = scheduler_with_pending( - txn_id, - CommitState::AwaitingResolve { - committed: true, - redo_lsn: None, - }, - ); + let (mut scheduler, _dir) = + scheduler_with_pending(txn_id, CommitState::AwaitingRedoResolve); - scheduler - .handle_completion(txn_id, RequestId::new(9), None) - .await; + scheduler.handle_completion(txn_id, RequestId::new(9), None); assert!( !scheduler.applied.is_applied(5, 1), @@ -251,7 +140,7 @@ mod tests { let marker = scheduler.shared.sequencer_halt.apply_halt().report(); assert_eq!( marker.map(|h| (h.vshard_id, h.epoch, h.position, h.step)), - Some((7, 5, 1, "flush")) + Some((7, 5, 1, "resolve")) ); assert_eq!(scheduler.metrics.apply_halted.load(Ordering::Relaxed), 1); assert_eq!( @@ -261,21 +150,18 @@ mod tests { } /// A second disconnect keeps the first cause and holds its txn too. - #[tokio::test] - async fn second_halt_keeps_the_first_cause() { + #[test] + fn second_halt_keeps_the_first_cause() { let first = TxnId::new(5, 1); let second = TxnId::new(6, 0); let (mut scheduler, _dir) = scheduler_with_pending(first, CommitState::Staged); let mut pending = staged_pending(make_sequenced_txn(6, 0), second); - pending.commit_state = Some(CommitState::AwaitingRedoResolve); + pending.commit_state = CommitState::AwaitingRedoResolve; + pending.awaiting = Some(RequestId::new(10)); scheduler.pending.insert(second, pending); - scheduler - .handle_completion(first, RequestId::new(9), None) - .await; - scheduler - .handle_completion(second, RequestId::new(10), None) - .await; + scheduler.handle_completion(first, RequestId::new(9), None); + scheduler.handle_completion(second, RequestId::new(10), None); assert!(!scheduler.applied.is_applied(6, 0)); assert!(scheduler.pending.contains_key(&second)); @@ -286,30 +172,23 @@ mod tests { ); } - /// A committed flush that answers `OllpRetryRequired` wrote nothing on - /// this replica. The txn stays pending and unapplied, its redo records - /// stay open, and the scheduler halts on the flush. - #[tokio::test] - async fn an_ollp_answer_to_a_committed_flush_halts_and_holds() { + /// A refused resolve of a committed txn leaves no redo to propose. The + /// txn stays pending and unapplied, and the scheduler halts on the + /// resolve. + #[test] + fn a_refused_resolve_of_a_committed_txn_halts_and_holds() { let txn_id = TxnId::new(5, 1); - let (mut scheduler, _dir) = scheduler_with_pending( + let (mut scheduler, _dir) = + scheduler_with_pending(txn_id, CommitState::AwaitingRedoResolve); + + scheduler.handle_completion( txn_id, - CommitState::AwaitingResolve { - committed: true, - redo_lsn: None, - }, + RequestId::new(9), + Some(error_response( + crate::bridge::envelope::ErrorCode::OllpRetryRequired, + )), ); - scheduler - .handle_completion( - txn_id, - RequestId::new(9), - Some(error_response( - crate::bridge::envelope::ErrorCode::OllpRetryRequired, - )), - ) - .await; - assert!(!scheduler.applied.is_applied(5, 1)); assert!( scheduler.pending.contains_key(&txn_id), @@ -317,7 +196,28 @@ mod tests { ); assert_eq!( scheduler.apply_halt().map(|h| h.reason), - Some(HaltReason::FlushFailed) + Some(HaltReason::ResolveFailed) + ); + } + + /// An answer to a request the txn no longer waits for, as after a + /// demotion, changes nothing. + #[test] + fn an_answer_to_a_step_the_txn_left_is_ignored() { + let txn_id = TxnId::new(5, 1); + let (mut scheduler, _dir) = + scheduler_with_pending(txn_id, CommitState::AwaitingRedoResolve); + + scheduler.handle_completion(txn_id, RequestId::new(8), None); + + assert!(scheduler.apply_halt().is_none()); + assert_eq!( + scheduler.pending.get(&txn_id).map(|p| p.commit_state), + Some(CommitState::AwaitingRedoResolve) + ); + assert_eq!( + scheduler.pending.get(&txn_id).and_then(|p| p.awaiting), + Some(RequestId::new(9)) ); } } diff --git a/nodedb/src/control/cluster/calvin/scheduler/driver/core/cut_marker.rs b/nodedb/src/control/cluster/calvin/scheduler/driver/core/cut_marker.rs index 00565b04e..d4b954665 100644 --- a/nodedb/src/control/cluster/calvin/scheduler/driver/core/cut_marker.rs +++ b/nodedb/src/control/cluster/calvin/scheduler/driver/core/cut_marker.rs @@ -1,30 +1,48 @@ // SPDX-License-Identifier: BUSL-1.1 -//! A backup's cut marker on this vShard's scheduler, and the tenant write -//! mark a committed Calvin transaction records. +//! A backup's cut marker on this vShard's scheduler. //! //! The scheduler reports a marker to the node's cut registry once every -//! transaction delivered before it finished: installed by its flush, or -//! dropped under an abort verdict. The backup waiting on the marker then +//! transaction delivered before it finished: installed from its stamped +//! redo, or ended with no log entry. The backup waiting on the marker then //! snapshots a vShard that holds every one of them. Every transaction //! delivered after the marker records a commit HLC above the marker's -//! watermark, so a restore of that backup refuses it. +//! watermark, so a restore of that backup refuses it. The data-group apply +//! raises the tenant's write mark from the redo entry's commit HLC. +//! +//! A marker that carries a barrier orders the data groups too (see +//! [`crate::control::backup::cut_order`]). The scheduler holds the redo of +//! every transaction delivered after it until this node applied the cut's +//! barrier in the vShard's group, or the cut's window closed. It starts the +//! node's driver of the cut, which places the barrier while this node leads +//! the group. + +use std::sync::Arc; + +use nodedb_cluster::calvin::CutBarrierWire; use super::scheduler::Scheduler; -use crate::control::cluster::calvin::scheduler::lock_manager::TxnId; -use crate::control::security::auth_fence::cluster::group_of_vshard; -use crate::control::state::tenant_marks::MarkSite; +use crate::control::backup::cut_order::driver::ensure_driver; +use crate::control::backup::cut_order::{CutWindow, OrderedCut}; +use crate::control::state::SharedState; impl Scheduler { - /// Receive a backup's cut marker carrying `hlc`, in input order. + /// Receive a backup's cut marker carrying `hlc`, in input order. A + /// `barrier` orders the cut's barriers with the marker. pub(in crate::control::cluster::calvin::scheduler::driver::core) fn receive_cut_marker( &mut self, hlc: u64, + restore_point: u64, + barrier: Option>, ) { - if self - .cut_floors - .receive(hlc, self.applied.highest_seen_epoch()) - { + let highest_seen = self.applied.highest_seen_epoch(); + if let Some(barrier) = barrier { + self.hold_for_cut( + OrderedCut::from_marker(hlc, restore_point, &barrier), + highest_seen, + ); + } + if self.cut_floors.receive(hlc, highest_seen) { self.shared.calvin.cuts.note_passed(self.vshard_id, hlc); } self.report_passed_cuts(); @@ -45,93 +63,241 @@ impl Scheduler { self.cut_floors.fold(self.applied.fully_applied_epoch()); } - /// The commit HLC of the transaction `txn_id`: the instant every WAL - /// record of its install carries. A transaction whose state is gone takes - /// the node's HLC now. - pub(in crate::control::cluster::calvin::scheduler::driver::core) fn txn_commit_hlc( - &self, - txn_id: TxnId, - ) -> u64 { - match self.pending.get(&txn_id) { - Some(pending) => self - .cut_floors - .commit_hlc(pending.txn.epoch, pending.txn.epoch_system_ms), - None => self.shared.hlc_clock.now().wall_ns, + /// Hold the redo of every transaction after the marker of `cut`, which + /// arrived after epochs up to `through`, and start the cut's driver. + fn hold_for_cut(&mut self, cut: OrderedCut, through: Option) { + if window_closed(&self.shared, &cut) { + // A marker this scheduler receives again after a restart, long + // after its cut ended. Its cut placed every barrier it ever will. + return; + } + if self.cut_holds.receive(cut.clone(), through) { + ensure_driver(&self.shared, &cut); } } - /// Record the commit HLC of the committed transaction `txn_id` on its - /// tenant's observed write high-water, before its `CompletionAck` lets - /// the coordinator acknowledge the COMMIT. - /// - /// Only a slice that carries a primary user data write records it. The - /// implicit edge cleanup that dual-homes alongside one writes no user row - /// of its own. - /// - /// A transaction that re-issues a RESTORE raises the tenant's restore - /// mark under its restore id instead, so a retry of that RESTORE finds - /// only its own marks. Its commit HLC still folds into this node's clock, - /// so a later backup here stamps a newer watermark. - pub(in crate::control::cluster::calvin::scheduler::driver::core) fn record_calvin_write_mark( + /// Whether the redo of a transaction of `epoch` waits for an ordered + /// cut's barrier. + pub(in crate::control::cluster::calvin::scheduler::driver::core) fn held_for_cut( &self, - txn_id: TxnId, - ) { - let Some(pending) = self.pending.get(&txn_id) else { - return; - }; - if !pending.has_primary_write { - return; - } - let tx_class = &pending.txn.tx_class; - let tenant_id = tx_class.tenant_id.as_u64(); - let restore_id = tx_class.restore_id; - let collection = tx_class.write_set.0.first().map(|set| set.collection()); - let commit_hlc = self - .cut_floors - .commit_hlc(pending.txn.epoch, pending.txn.epoch_system_ms); - if restore_id == 0 { - self.shared - .advance_tenant_write_hlc(tenant_id, commit_hlc, "calvin flush", collection); - } else { - self.shared - .hlc_clock - .update(nodedb_types::Hlc::new(commit_hlc, 0)); - } + epoch: u64, + ) -> bool { + self.cut_holds.holds(epoch) + } - // The durable mark, in the data group that homes this vShard, lands - // before the ack too: RESTORE's guard reads it on every node, after a - // restart as well. - let group_id = match group_of_vshard(&self.shared, self.vshard_id) { - Ok(group_id) => group_id, - Err(error) => { - tracing::error!( - vshard_id = self.vshard_id, - %error, - "calvin: no data group for this vShard; the commit's write mark stays \ - in memory only" + /// End the hold of every cut whose barrier this node applied in the + /// vShard's group, or whose window closed. Start the driver again of a + /// cut still held whose driver stopped. Returns whether a hold ended: the + /// held redo can then be proposed. + pub(in crate::control::cluster::calvin::scheduler::driver::core) fn release_cut_holds( + &mut self, + ) -> bool { + if self.cut_holds.is_empty() { + return false; + } + let group_id = self.data_group(); + let shared = Arc::clone(&self.shared); + let vshard_id = self.vshard_id; + let ended = self.cut_holds.end(|cut| { + let applied = group_id.is_some_and(|group_id| { + shared + .calvin + .cut_barriers + .applied(cut.key(), group_id) + .is_some() + }); + if applied { + return true; + } + let closed = window_closed(&shared, cut); + if closed { + tracing::warn!( + vshard_id, + ?group_id, + hlc = cut.hlc, + "calvin scheduler: a cut's window closed before its barrier applied here; \ + releasing the redo held for it" ); - return; } - }; - let marks = &self.shared.tenant_marks; - if restore_id == 0 { - marks.raise( - group_id, - tenant_id, - commit_hlc, - MarkSite::CalvinFlush, - collection, - ); - } else { - marks.raise_restore(group_id, tenant_id, commit_hlc, collection, restore_id); + closed + }); + for cut in self.cut_holds.cuts() { + // no-determinism: the window bounds only when this node proposes. + if shared.hlc_clock.now().wall_ns <= CutWindow::on(&shared, cut.hlc).propose_until { + ensure_driver(&shared, cut); + } } - if let Err(error) = marks.persist(self.shared.credentials.catalog()) { - // The mark stays pending; the next persist on this node writes it. - tracing::error!( - vshard_id = self.vshard_id, - %error, - "calvin: the commit's write mark did not persist yet" - ); + ended + } + + /// The data group that homes this vShard. + fn data_group(&self) -> Option { + let mr = self.multi_raft.lock().unwrap_or_else(|p| p.into_inner()); + mr.routing() + .read() + .unwrap_or_else(|p| p.into_inner()) + .group_for_vshard(self.vshard_id) + .ok() + } +} + +/// Whether `cut`'s hold window closed on `state`'s clock. +/// +/// no-determinism: the window bounds only how long this node holds redo. +/// Every replica installs the redo at its place in the log. +fn window_closed(state: &SharedState, cut: &OrderedCut) -> bool { + state.hlc_clock.now().wall_ns > CutWindow::on(state, cut.hlc).hold_until +} + +#[cfg(test)] +mod tests { + use nodedb_cluster::calvin::types::SchedulerInput; + + use super::*; + use crate::control::cluster::calvin::scheduler::driver::core::test_support::{ + build_test_scheduler, make_validate_only_txn, test_coll_vshard, + }; + + /// A cut whose watermark is now, so its window is open. + fn fresh_cut(scheduler: &Scheduler) -> OrderedCut { + OrderedCut { + hlc: scheduler.shared.hlc_clock.now().wall_ns, + restore_point: 0, + capture: None, } } + + fn marker(cut: &OrderedCut) -> SchedulerInput { + SchedulerInput::CutMarker { + hlc: cut.hlc, + restore_point: cut.restore_point, + barrier: Some(Arc::new(cut.to_wire())), + } + } + + /// Only a transaction delivered after a barrier-carrying marker waits, + /// and the marker still passes once the earlier ones finished. + #[tokio::test] + async fn a_marker_with_a_barrier_holds_only_later_redo() { + let (mut scheduler, _dir) = build_test_scheduler(test_coll_vshard()); + scheduler + .process_scheduler_input(SchedulerInput::Txn(Box::new(make_validate_only_txn(4, 0)))); + let cut = fresh_cut(&scheduler); + scheduler.process_scheduler_input(marker(&cut)); + + assert!(!scheduler.held_for_cut(4), "epoch 4 came before the marker"); + assert!(scheduler.held_for_cut(5), "epoch 5 came after it"); + assert!( + scheduler + .shared + .calvin + .cut_barriers + .cut(cut.key()) + .is_some(), + "the cut's driver tracks it" + ); + } + + /// The hold ends once this node applied the cut's barrier in the + /// vShard's group, and not before. A barrier of another group ends + /// nothing. + #[tokio::test] + async fn the_hold_ends_when_the_group_applied_the_barrier() { + let (mut scheduler, _dir) = build_test_scheduler(test_coll_vshard()); + let cut = fresh_cut(&scheduler); + scheduler.process_scheduler_input(marker(&cut)); + assert!( + scheduler.held_for_cut(0), + "a marker before any epoch holds all" + ); + let group_id = scheduler + .data_group() + .expect("the fixture routes every vShard"); + + assert!(!scheduler.release_cut_holds()); + scheduler + .shared + .calvin + .cut_barriers + .note_applied(&cut, group_id.wrapping_add(1), 9); + assert!(!scheduler.release_cut_holds(), "another group's barrier"); + assert!(scheduler.held_for_cut(0)); + + scheduler + .shared + .calvin + .cut_barriers + .note_applied(&cut, group_id, 12); + assert!(scheduler.release_cut_holds()); + assert!(!scheduler.held_for_cut(0)); + assert!(!scheduler.release_cut_holds(), "nothing left to end"); + } + + /// A second copy of the marker keeps the first copy's place, and a copy + /// of a cut whose barrier applied holds nothing for long. + #[tokio::test] + async fn a_marker_copy_keeps_the_first_place() { + let (mut scheduler, _dir) = build_test_scheduler(test_coll_vshard()); + scheduler + .process_scheduler_input(SchedulerInput::Txn(Box::new(make_validate_only_txn(4, 0)))); + let cut = fresh_cut(&scheduler); + scheduler.process_scheduler_input(marker(&cut)); + scheduler + .process_scheduler_input(SchedulerInput::Txn(Box::new(make_validate_only_txn(6, 0)))); + scheduler.process_scheduler_input(marker(&cut)); + assert!( + scheduler.held_for_cut(5), + "epoch 5 came after the first copy, so it waits" + ); + assert_eq!(scheduler.cut_holds.cuts().count(), 1); + + let group_id = scheduler + .data_group() + .expect("the fixture routes every vShard"); + scheduler + .shared + .calvin + .cut_barriers + .note_applied(&cut, group_id, 30); + assert!(scheduler.release_cut_holds()); + scheduler.process_scheduler_input(marker(&cut)); + assert!( + scheduler.release_cut_holds(), + "a late copy finds the barrier applied" + ); + assert!(!scheduler.held_for_cut(7)); + } + + /// A marker received long after its cut's window closed holds nothing: + /// a replay after a restart never blocks redo on a cut that ended. + #[tokio::test] + async fn a_stale_marker_holds_nothing() { + let (mut scheduler, _dir) = build_test_scheduler(test_coll_vshard()); + let stale = OrderedCut { + hlc: 1, + restore_point: 0, + capture: None, + }; + scheduler.process_scheduler_input(marker(&stale)); + assert!(!scheduler.held_for_cut(0)); + assert!(scheduler.cut_holds.is_empty()); + } + + /// A cut whose window closes with no barrier ends its hold. + #[tokio::test] + async fn a_closed_window_ends_the_hold() { + let (mut scheduler, _dir) = build_test_scheduler(test_coll_vshard()); + let cut = fresh_cut(&scheduler); + scheduler.process_scheduler_input(marker(&cut)); + assert!(scheduler.held_for_cut(1)); + let past_window = CutWindow::on(&scheduler.shared, cut.hlc) + .hold_until + .saturating_add(1); + scheduler + .shared + .hlc_clock + .update(nodedb_types::Hlc::new(past_window, 0)); + assert!(scheduler.release_cut_holds()); + assert!(!scheduler.held_for_cut(1)); + } } diff --git a/nodedb/src/control/cluster/calvin/scheduler/driver/core/deferred.rs b/nodedb/src/control/cluster/calvin/scheduler/driver/core/deferred.rs index a42623686..fe7766f9b 100644 --- a/nodedb/src/control/cluster/calvin/scheduler/driver/core/deferred.rs +++ b/nodedb/src/control/cluster/calvin/scheduler/driver/core/deferred.rs @@ -10,15 +10,11 @@ //! refusal halts the scheduler (see [`super::halt`]). use std::collections::VecDeque; -use std::sync::atomic::Ordering; use std::time::Instant; -use nodedb_physical::physical_plan::PhysicalPlan; -use nodedb_physical::physical_plan::meta::MetaOp; - use super::halt::{HaltReason, HaltStep}; use super::scheduler::Scheduler; -use crate::bridge::dispatch::{DispatchRefusal, JournalGroup}; +use crate::bridge::dispatch::DispatchRefusal; use crate::bridge::envelope::Request; use crate::control::cluster::calvin::scheduler::lock_manager::TxnId; use crate::types::RequestId; @@ -30,14 +26,31 @@ pub(in crate::control::cluster::calvin::scheduler::driver::core) enum DispatchSt StageStatic, /// `CalvinExecuteActive` stage of a dependent-read txn. StageActive, + /// `CalvinExecutePassive` read of a dependent-read txn. + ReadPassive, /// `CalvinResolve` of a committed staged txn. Resolve, - /// `CalvinFlush` of a committed staged txn. - Flush, - /// `CalvinDrop` of an aborted staged txn. + /// `CalvinDrop` of a staged txn that ends with no log entry. The txn + /// completes on its answer. Drop, - /// One-way `RecordCalvinWriteVersions` of a committed txn. - WriteVersionRecord, + /// `CalvinDrop` of staged state the txn no longer needs: a demoted + /// leader's, a failed stage a restage replaces, or a restaged slice + /// whose redo applied. Its answer is drained and read by nothing. + Discard, +} + +impl DispatchStep { + /// Whether the txn waits for this step's answer. + fn is_awaited(self) -> bool { + match self { + Self::StageStatic + | Self::StageActive + | Self::ReadPassive + | Self::Resolve + | Self::Drop => true, + Self::Discard => false, + } + } } /// Result of [`Scheduler::dispatch_sequenced`]. @@ -57,8 +70,6 @@ pub(in crate::control::cluster::calvin::scheduler::driver::core) struct Deferred txn_id: TxnId, step: DispatchStep, request: Request, - /// The record group the request's write set journals into. - journal: Option, } /// FIFO of requests refused at capacity, in refusal order. @@ -77,6 +88,9 @@ impl Scheduler { /// Register `request` in the tracker, dispatch it, and cancel the /// registration on refusal. /// + /// An awaited step records the request on the txn's pending entry, so + /// only this request's answer drives the txn. + /// /// A capacity refusal appends the request to the deferred FIFO and /// returns [`DispatchOutcome::Deferred`]. The caller keeps the txn in /// flight. Any other refusal returns [`DispatchOutcome::Failed`], and the @@ -87,19 +101,12 @@ impl Scheduler { step: DispatchStep, request: Request, ) -> DispatchOutcome { - self.dispatch_sequenced_journalled(txn_id, step, request, None) - } - - /// [`Self::dispatch_sequenced`] for a request whose write set journals - /// into `journal`. - pub(in crate::control::cluster::calvin::scheduler::driver::core) fn dispatch_sequenced_journalled( - &mut self, - txn_id: TxnId, - step: DispatchStep, - request: Request, - journal: Option, - ) -> DispatchOutcome { - match self.send_once(txn_id, step, request, journal) { + if step.is_awaited() + && let Some(pending) = self.pending.get_mut(&txn_id) + { + pending.awaiting = Some(request.request_id); + } + match self.send_once(txn_id, step, request) { Attempt::Sent => DispatchOutcome::Sent, Attempt::Capacity(parked) => { self.deferred.push_back(*parked); @@ -118,20 +125,20 @@ impl Scheduler { !self.deferred.is_empty() } - /// Whether the run loop re-sends parked requests: some wait, and the - /// scheduler has not halted. - pub(in crate::control::cluster::calvin::scheduler::driver::core) fn resends_deferred( + /// Whether a refused request of `txn_id` waits for capacity. + pub(in crate::control::cluster::calvin::scheduler::driver::core) fn has_deferred_for( &self, + txn_id: TxnId, ) -> bool { - self.has_deferred_dispatch() && !self.is_apply_halted() + self.deferred.iter().any(|parked| parked.txn_id == txn_id) } - /// Whether a refused request of `txn_id` waits for capacity. - pub(in crate::control::cluster::calvin::scheduler::driver::core) fn has_parked_dispatch( + /// Whether the run loop re-sends parked requests: some wait, and the + /// scheduler has not halted. + pub(in crate::control::cluster::calvin::scheduler::driver::core) fn resends_deferred( &self, - txn_id: TxnId, ) -> bool { - self.deferred.iter().any(|parked| parked.txn_id == txn_id) + self.has_deferred_dispatch() && !self.is_apply_halted() } /// Number of refused requests waiting for capacity. @@ -146,6 +153,10 @@ impl Scheduler { /// Stops at the first capacity refusal, which goes back to the FIFO /// front. A terminal refusal runs the step's terminal handling, and stops /// the pass once the scheduler halted. + /// + /// A request of a txn that left `pending`, or that waits for another + /// request now, is dropped. A discard still goes out: it clears staged + /// state the txn left on its core. pub(in crate::control::cluster::calvin::scheduler::driver::core) fn redispatch_deferred( &mut self, ) { @@ -154,20 +165,19 @@ impl Scheduler { txn_id, step, mut request, - journal, } = parked; - if step != DispatchStep::WriteVersionRecord && !self.pending.contains_key(&txn_id) { - tracing::error!( + if step.is_awaited() && !self.awaits(txn_id, request.request_id) { + tracing::debug!( vshard_id = self.vshard_id, epoch = txn_id.epoch, position = txn_id.position, ?step, - "calvin: parked dispatch for a txn no longer pending; one step per txn is broken, discarding" + "calvin: parked dispatch of a step the txn left; discarding" ); continue; } - self.refresh_deferred_request(step, &mut request); - match self.send_once(txn_id, step, request, journal) { + request.deadline = self.request_deadline(); + match self.send_once(txn_id, step, request) { Attempt::Sent => {} Attempt::Capacity(parked) => { self.deferred.push_front(*parked); @@ -185,14 +195,23 @@ impl Scheduler { .set_dispatch_deferred_depth(self.deferred.len()); } + /// Whether `txn_id` is pending and waits for the answer to `request_id`. + pub(in crate::control::cluster::calvin::scheduler::driver::core) fn awaits( + &self, + txn_id: TxnId, + request_id: RequestId, + ) -> bool { + self.pending + .get(&txn_id) + .is_some_and(|pending| pending.awaiting == Some(request_id)) + } + /// Run the terminal handling of a dispatch the dispatcher refused for a /// reason other than capacity. /// - /// A stage, resolve, flush, or drop refusal halts the scheduler: the txn - /// keeps its `pending` entry and locks, and its position stays unapplied. - /// A refusal during shutdown holds the txn the same way. A write-version - /// record is dropped with a warning, because the commit does not depend on - /// it. + /// Every refusal halts the scheduler: the txn keeps its `pending` entry + /// and locks, and its position stays unapplied. A refusal during shutdown + /// holds the txn the same way. pub(in crate::control::cluster::calvin::scheduler::driver::core) fn fail_dispatch_step( &mut self, txn_id: TxnId, @@ -201,19 +220,9 @@ impl Scheduler { ) { let halt_step = match step { DispatchStep::StageStatic | DispatchStep::StageActive => HaltStep::Stage, + DispatchStep::ReadPassive => HaltStep::PassiveRead, DispatchStep::Resolve => HaltStep::Resolve, - DispatchStep::Flush => HaltStep::Flush, - DispatchStep::Drop => HaltStep::Drop, - DispatchStep::WriteVersionRecord => { - tracing::warn!( - vshard_id = self.vshard_id, - epoch = txn_id.epoch, - position = txn_id.position, - %error, - "calvin: write-version record dispatch failed" - ); - return; - } + DispatchStep::Drop | DispatchStep::Discard => HaltStep::Drop, }; self.halt_apply( txn_id, @@ -224,38 +233,12 @@ impl Scheduler { } /// One send attempt: register, dispatch, and cancel on refusal. - fn send_once( - &mut self, - txn_id: TxnId, - step: DispatchStep, - request: Request, - journal: Option, - ) -> Attempt { - // A crash test holds one collection's flush here: the redo record is - // appended and no core holds the flush. The flush waits in the - // re-send queue, as at capacity, so the scheduler keeps running. - #[cfg(feature = "failpoints")] - if step == DispatchStep::Flush && crate::control::fail_gate::holds_flush(&request.plan) { - tracing::info!( - vshard_id = self.vshard_id, - epoch = txn_id.epoch, - position = txn_id.position, - "calvin: flush held at a fail point" - ); - return Attempt::Capacity(Box::new(DeferredDispatch { - txn_id, - step, - request, - journal, - })); - } + fn send_once(&mut self, txn_id: TxnId, step: DispatchStep, request: Request) -> Attempt { let request_id = request.request_id; let resp_rx = self.shared.tracker.register(request_id); let result = match self.shared.dispatcher.lock() { - Ok(mut dispatcher) => dispatcher.try_dispatch_journalled(request, journal.clone()), - Err(poisoned) => poisoned - .into_inner() - .try_dispatch_journalled(request, journal.clone()), + Ok(mut dispatcher) => dispatcher.try_dispatch(request), + Err(poisoned) => poisoned.into_inner().try_dispatch(request), }; let refusal = match result { Ok(()) => { @@ -281,7 +264,6 @@ impl Scheduler { txn_id, step, request, - journal, })) } other => Attempt::Failed(other), @@ -305,46 +287,18 @@ impl Scheduler { } self.spawn_response_bridge(txn_id, request_id, resp_rx); } - DispatchStep::Resolve | DispatchStep::Flush | DispatchStep::Drop => { + DispatchStep::ReadPassive | DispatchStep::Resolve | DispatchStep::Drop => { self.spawn_response_bridge(txn_id, request_id, resp_rx); } - DispatchStep::WriteVersionRecord => { - // One-way record: drain the response so it routes to a live - // receiver, then discard it. + DispatchStep::Discard => { + // Drain the answer so it routes to a live receiver. tokio::spawn(async move { let mut rx = resp_rx; let _ = rx.recv().await; }); - self.shared - .calvin - .counters - .write_versions_recorded - .fetch_add(1, Ordering::Relaxed); } } } - - /// Refresh the dispatch-time fields of a parked request before a re-send. - /// - /// The deadline restarts from now. A stage request re-reads group - /// leadership, which the Data Plane uses to gate OLLP verification. - fn refresh_deferred_request(&self, step: DispatchStep, request: &mut Request) { - request.deadline = self.request_deadline(); - if !matches!(step, DispatchStep::StageStatic | DispatchStep::StageActive) { - return; - } - if let PhysicalPlan::Meta( - MetaOp::CalvinExecuteStatic { - is_group_leader, .. - } - | MetaOp::CalvinExecuteActive { - is_group_leader, .. - }, - ) = &mut request.plan - { - *is_group_leader = self.is_group_leader(); - } - } } #[cfg(test)] @@ -358,6 +312,7 @@ mod tests { use super::*; use crate::control::cluster::calvin::scheduler::driver::core::halt::HaltReason; use crate::control::cluster::calvin::scheduler::driver::core::intake::IntakeClosure; + use crate::control::cluster::calvin::scheduler::driver::core::test_proposer::lead_data_group; use crate::control::cluster::calvin::scheduler::driver::core::test_support::{ begin_data_plane_drain, build_test_scheduler_with_data_side, fill_tenant_inflight, make_validate_only_txn, test_coll_vshard, @@ -371,6 +326,7 @@ mod tests { let registry = CalvinCompletionRegistry::new_detached(); let (mut scheduler, _dir, _data_side) = build_test_scheduler_with_data_side(test_coll_vshard(), registry); + lead_data_group(&mut scheduler); begin_data_plane_drain(&scheduler.shared); let txn_id = TxnId::new(3, 0); @@ -409,6 +365,7 @@ mod tests { let registry = CalvinCompletionRegistry::new_detached(); let (mut scheduler, _dir, mut data_side) = build_test_scheduler_with_data_side(test_coll_vshard(), registry); + lead_data_group(&mut scheduler); let shared = Arc::clone(&scheduler.shared); fill_tenant_inflight(&shared, &mut data_side, TenantId::new(1)); let txn_id = TxnId::new(3, 0); diff --git a/nodedb/src/control/cluster/calvin/scheduler/driver/core/dispatch/active_dispatch.rs b/nodedb/src/control/cluster/calvin/scheduler/driver/core/dispatch/active_dispatch.rs index 27058bed9..7e1ad7cf8 100644 --- a/nodedb/src/control/cluster/calvin/scheduler/driver/core/dispatch/active_dispatch.rs +++ b/nodedb/src/control/cluster/calvin/scheduler/driver/core/dispatch/active_dispatch.rs @@ -1,12 +1,10 @@ // SPDX-License-Identifier: BUSL-1.1 -//! Active (dependent-read) transaction dispatch: submits a -//! `CalvinExecuteActive` task once all passive results have landed. +//! Active (dependent-read) transaction dispatch on the data-group leader: +//! submits a `CalvinExecuteActive` task once all passive results have landed. use std::time::Instant; -use tracing::error; - use nodedb_cluster::calvin::types::SequencedTxn; use nodedb_physical::physical_plan::PhysicalPlan; use nodedb_physical::physical_plan::meta::MetaOp; @@ -14,7 +12,8 @@ use nodedb_physical::physical_plan::meta::MetaOp; use super::super::deferred::{DispatchOutcome, DispatchStep}; use super::super::scheduler::Scheduler; use super::primary_write::{ - plans_have_primary_write, plans_have_returning, txn_has_non_derived_write, + plans_have_primary_write, plans_have_returning, plans_raise_write_mark, + txn_has_non_derived_write, }; use crate::control::cluster::calvin::scheduler::lock_manager::TxnId; @@ -38,14 +37,7 @@ impl Scheduler { let plans = match super::super::super::helpers::decode_plans(&txn.tx_class.plans) { Ok(p) => p, Err(e) => { - error!( - vshard_id = self.vshard_id, - epoch, - position, - error = %e, - "calvin scheduler: active plan decode failed; releasing locks" - ); - self.on_unpending_txn_complete(txn_id, lock_owner); + self.reject_plan(txn, txn_id, lock_owner, &e); return; } }; @@ -59,12 +51,11 @@ impl Scheduler { match self.local_calvin_plans(plans, txn.tx_class.database_id, epoch, position) { Ok(p) if !p.is_empty() => p, Ok(_) => { - // A dependent-read active txn dispatched here always carries a - // local write slice (the OLLP orchestrator only routes the write - // participant through this path). An empty local slice is a - // routing bug, not a read-only participant — surface it as a - // terminal routing failure rather than dispatching an - // active task with nothing to apply. + // Only a vShard the write set names opens a barrier, so an + // active txn dispatched here carries a local write slice. An + // empty local slice means the plans and the write set + // disagree, so it rejects the plans rather than dispatching + // an active task with nothing to apply. let e = crate::Error::Internal { detail: format!( "calvin active txn {epoch}/{position} homes no local write plans \ @@ -72,36 +63,24 @@ impl Scheduler { self.vshard_id ), }; - error!( - vshard_id = self.vshard_id, - epoch, - position, - error = %e, - "calvin scheduler: active txn homes no local writes; releasing locks" - ); - self.propose_routing_failure(txn_id, &e); - self.on_unpending_txn_complete(txn_id, lock_owner); + self.reject_plan(txn, txn_id, lock_owner, &e); return; } Err(e) => { - error!( - vshard_id = self.vshard_id, - epoch, - position, - error = %e, - "calvin scheduler: active txn routing failed; releasing locks" - ); - self.propose_routing_failure(txn_id, &e); - self.on_unpending_txn_complete(txn_id, lock_owner); + self.reject_plan(txn, txn_id, lock_owner, &e); return; } }; - if !self.bind_local_identities(&mut plans, txn.tx_class.database_id, tenant_id, txn_id) { + let Some(identities) = + self.bind_local_identities(&mut plans, txn.tx_class.database_id, tenant_id, txn_id) + else { return; - } + }; let has_primary_write = plans_have_primary_write(&plans, has_non_derived_write); + let raises_write_mark = plans_raise_write_mark(&plans, has_primary_write); let has_returning = plans_have_returning(&plans); - let flush_scope = super::super::super::types::FlushScope::of_plans(&plans); + let mut scope = super::super::super::types::SliceScope::of_plans(&plans); + scope.identities = identities; let plan = PhysicalPlan::Meta(MetaOp::CalvinExecuteActive { epoch, position, @@ -109,18 +88,15 @@ impl Scheduler { plans, injected_reads, epoch_system_ms: txn.epoch_system_ms, - is_group_leader: self.is_group_leader(), }); - // Calvin allocates the CalvinApplied WAL LSN post-apply (in the - // scheduler's response handler), so no committed LSN is known at - // dispatch time to stamp here. + // A stage writes no WAL record, so no committed LSN rides on it. let database_id = txn.tx_class.database_id; - let request = self.build_exempt_request(request_id, tenant_id, database_id, plan, None); + let request = self.build_exempt_request(request_id, tenant_id, database_id, plan); // The txn enters `pending` before the dispatch, so a stage refused at // capacity stays in flight with its locks until the re-send. - // Every replica checks the transaction's collection incarnations. + // The leader checks the transaction's collection incarnations. let (superseded, gates) = self.check_incarnations(&txn.tx_class); self.pending.insert( txn_id, @@ -130,26 +106,25 @@ impl Scheduler { // no-determinism: dispatch_time is scheduler observability, not Calvin WAL data dispatch_time: Instant::now(), has_primary_write, + raises_write_mark, has_returning, - // The commit's resolved redo fills them. - change_sets: Vec::new(), - // The dependent-read active path STAGES (leader-verify OLLP + + // The dependent-read active path STAGES (OLLP verify + // buffer, no base apply); its response drives the same - // resolve → redo → flush as the static path, for - // WAL-only-restart durability. `resolve_staged_commit` reads the - // `read_set_valid: None` the active handler returns as "commit". - commit_state: Some(super::super::super::types::CommitState::Staged), + // resolve and redo proposal as the static path. + // `resolve_staged_commit` reads the `stage_vote` the active + // handler sets. + commit_state: super::super::super::types::CommitState::Staged, + // The dispatch below records its request. + awaiting: None, // Set only once the txn parks in `AwaitingVerdict`. verdict_deadline: None, stage_error: None, - // Set once a committed txn appends its redo record. - redo_records: None, - flush_scope, + scope, + // Set once a committed slice resolves its redo. + redo: None, superseded, gates, ungated: false, - // Taken when the flush dispatches. - install_permit: None, }, ); diff --git a/nodedb/src/control/cluster/calvin/scheduler/driver/core/dispatch/bind_identities.rs b/nodedb/src/control/cluster/calvin/scheduler/driver/core/dispatch/bind_identities.rs index 9481dc9f2..d3cbed167 100644 --- a/nodedb/src/control/cluster/calvin/scheduler/driver/core/dispatch/bind_identities.rs +++ b/nodedb/src/control/cluster/calvin/scheduler/driver/core/dispatch/bind_identities.rs @@ -3,40 +3,43 @@ //! Install the pk → surrogate identities a local Calvin write slice carries. //! //! The coordinator assigned each surrogate at plan time in its own catalog. -//! Every participant — group leader and follower alike — applies the slice -//! to its Data Plane, so every participant installs the same binding here, -//! or a later point read by primary key resolves nothing on this node. +//! The data-group leader binds them here before it stages the slice, and +//! carries them in the slice's redo entry. Every other replica binds the +//! same identities before the redo installs, so a later point read by +//! primary key resolves the row on every node. use nodedb_physical::physical_plan::PhysicalPlan; use super::super::halt::{HaltReason, HaltStep}; use super::super::scheduler::Scheduler; use crate::control::cluster::calvin::scheduler::lock_manager::TxnId; -use crate::control::surrogate::bind_plan_identities; +use crate::control::surrogate::{CarriedIdentity, bind_plan_identities, collect_plan_identities}; use crate::types::{DatabaseId, TenantId}; impl Scheduler { - /// Bind every identity in `plans` first-wins and rewrite each surrogate + /// Bind every identity in `plans` first-wins, rewrite each surrogate /// slot with the authoritative value, the same walk the replicated-write - /// decoder runs. + /// decoder runs, and return the identities the slice's redo carries. /// - /// A catalog error is local to this replica, and its peers apply the - /// slice. So the scheduler halts: the txn is not yet in `pending`, its - /// locks stay held under its lock owner, and its position stays - /// unapplied. Returns `false` after the halt; the caller returns at once. + /// A catalog error is local to this node, and its peers apply the slice. + /// So the scheduler halts: the txn is not yet in `pending`, its locks + /// stay held under its lock owner, and its position stays unapplied. + /// Returns `None` after the halt; the caller returns at once. pub(super) fn bind_local_identities( &mut self, plans: &mut [PhysicalPlan], database_id: DatabaseId, tenant_id: TenantId, txn_id: TxnId, - ) -> bool { + ) -> Option> { let assigner = &self.shared.surrogate_assigner; - let bound = plans + let rewritten = plans .iter_mut() .try_for_each(|plan| bind_plan_identities(assigner, database_id, tenant_id, plan)); + let bound = rewritten + .and_then(|()| collect_plan_identities(assigner, database_id, tenant_id, plans)); match bound { - Ok(()) => true, + Ok(identities) => Some(identities), Err(e) => { self.halt_apply( txn_id, @@ -44,7 +47,7 @@ impl Scheduler { HaltStep::IdentityBind, format!("surrogate binding failed: {e}"), ); - false + None } } } diff --git a/nodedb/src/control/cluster/calvin/scheduler/driver/core/dispatch/follow.rs b/nodedb/src/control/cluster/calvin/scheduler/driver/core/dispatch/follow.rs new file mode 100644 index 000000000..80094c351 --- /dev/null +++ b/nodedb/src/control/cluster/calvin/scheduler/driver/core/dispatch/follow.rs @@ -0,0 +1,98 @@ +// SPDX-License-Identifier: BUSL-1.1 + +//! A granted txn on a replica that does not lead the vShard's data group. +//! +//! The follower holds the txn's locks and stages nothing. It owes no vote. +//! The txn completes on an abort verdict, on a COMMIT verdict for a slice +//! with no local write, or once the slice's stamped redo applies from the +//! log. A promotion stages it. A dependent-read txn keeps its barrier log +//! meanwhile. + +use std::time::Instant; + +use nodedb_cluster::calvin::types::SequencedTxn; + +use super::super::scheduler::Scheduler; +use crate::control::cluster::calvin::scheduler::driver::types::{ + CommitState, PendingTxn, SliceScope, +}; +use crate::control::cluster::calvin::scheduler::lock_manager::TxnId; + +impl Scheduler { + /// Hold `txn` `Following`. A verdict stored already resolves it at once. + pub(in crate::control::cluster::calvin::scheduler::driver::core) fn follow( + &mut self, + txn: SequencedTxn, + txn_id: TxnId, + lock_owner: TxnId, + ) { + let scope = self.local_slice_scope(&txn, txn_id); + let dependent = txn.tx_class.dependent_reads.is_some(); + self.pending.insert( + txn_id, + PendingTxn { + txn, + lock_owner, + // no-determinism: dispatch time is observability only. + dispatch_time: Instant::now(), + has_primary_write: false, + raises_write_mark: false, + has_returning: false, + commit_state: CommitState::Following, + awaiting: None, + verdict_deadline: None, + stage_error: None, + scope, + redo: None, + superseded: false, + gates: Vec::new(), + ungated: false, + }, + ); + // A dependent txn keeps its barrier log from its grant on, so a + // promotion decides its barrier from every entry the log holds. + if dependent { + self.absorb_buffered_reads(txn_id); + } + if let Some(verdict) = self.registry.verdict(nodedb_cluster::calvin::TxnId::new( + txn_id.epoch, + txn_id.position, + )) { + self.resume_on_verdict(txn_id, verdict); + } + } + + /// The scope of `txn`'s slice on this vShard, from the txn's own plans. + /// A follower completes a COMMIT with no entry only when this scope + /// writes nothing, so it never comes from a leader's step in progress. + /// Plans that do not decode or route make the leader vote abort, so they + /// hold no write here. + pub(in crate::control::cluster::calvin::scheduler::driver::core) fn local_slice_scope( + &self, + txn: &SequencedTxn, + txn_id: TxnId, + ) -> SliceScope { + let local = + super::super::super::helpers::decode_plans(&txn.tx_class.plans).and_then(|plans| { + self.local_calvin_plans( + plans, + txn.tx_class.database_id, + txn_id.epoch, + txn_id.position, + ) + }); + match local { + Ok(local) => SliceScope::of_plans(&local), + Err(error) => { + tracing::debug!( + vshard_id = self.vshard_id, + epoch = txn_id.epoch, + position = txn_id.position, + %error, + "calvin: follower holds a txn whose plans the leader rejects" + ); + SliceScope::of_plans(&[]) + } + } + } +} diff --git a/nodedb/src/control/cluster/calvin/scheduler/driver/core/dispatch/incarnation.rs b/nodedb/src/control/cluster/calvin/scheduler/driver/core/dispatch/incarnation.rs index 8ad904fb2..0af735c75 100644 --- a/nodedb/src/control/cluster/calvin/scheduler/driver/core/dispatch/incarnation.rs +++ b/nodedb/src/control/cluster/calvin/scheduler/driver/core/dispatch/incarnation.rs @@ -1,18 +1,18 @@ // SPDX-License-Identifier: BUSL-1.1 -//! The incarnation check a Calvin transaction passes when a replica stages it. +//! The incarnation check a Calvin transaction passes when the leader stages +//! it. //! //! The coordinator stamps the incarnation of every collection the transaction -//! names. Each replica checks all of them, not only its own slice's, against +//! names. The leader checks all of them, not only its own slice's, against //! its committed catalog under the collections' gates, held shared until the //! transaction leaves `pending`. A mismatch makes the leader's vote an abort, //! and the sequencer's global verdict then aborts every participant alike: no -//! part of the transaction applies. A follower with a mismatch under a COMMIT -//! verdict halts rather than flush into a recreated collection. +//! part of the transaction applies. //! -//! A halt releases every gate no flush in flight needs, so a purge never waits -//! on a halted scheduler. A txn whose gates a halt released checks again, and -//! takes them back, before its flush. +//! A halt releases every gate, so a purge never waits on a halted +//! scheduler. A txn whose gates a halt released checks again, and takes them +//! back, before it proposes its redo. use std::collections::BTreeSet; @@ -20,7 +20,6 @@ use nodedb_cluster::calvin::types::TxClass; use nodedb_types::{CollectionKey, Hlc}; use tracing::error; -use super::super::super::types::CommitState; use super::super::scheduler::Scheduler; use crate::control::cluster::calvin::scheduler::lock_manager::TxnId; use crate::control::write_gate::{self, GateKey, SharedGate}; @@ -85,40 +84,26 @@ impl Scheduler { (superseded, gates) } - /// Release the gates a halt leaves nothing to protect. + /// Release every gate the pending txns hold, so a purge never waits on + /// this halted scheduler. /// - /// A halted scheduler never re-sends a parked request, and `stuck` never - /// applies before restart. Only a committed flush the Data Plane already - /// holds can still write, and it keeps its gates until it answers. Every - /// other txn gives its gates back, so a purge never waits on this - /// scheduler. + /// No write of a txn is in the Data Plane's hands under these gates: a + /// committed slice installs from the data-group log, and the apply loop + /// checks the incarnations its entry carries. pub(in crate::control::cluster::calvin::scheduler::driver::core) fn release_gates_on_halt( &mut self, - stuck: TxnId, ) { - let release: Vec = self - .pending - .iter() - .filter(|(txn_id, pending)| { - **txn_id == stuck - || !writes_in_flight(pending.commit_state) - || self.has_parked_dispatch(**txn_id) - }) - .map(|(txn_id, _)| *txn_id) - .collect(); - for txn_id in release { - if let Some(pending) = self.pending.get_mut(&txn_id) - && !pending.ungated - { + for pending in self.pending.values_mut() { + if !pending.ungated { pending.gates.clear(); pending.ungated = true; } } } - /// Take back the gates of `txn_id` before its flush, when a halt released - /// them. Fails when a collection the txn names no longer holds its planned - /// incarnation: the flush cannot apply here. + /// Take back the gates of `txn_id` before it proposes its redo, when a + /// halt released them. Fails when a collection the txn names no longer + /// holds its planned incarnation: the redo cannot commit into it. pub(in crate::control::cluster::calvin::scheduler::driver::core) fn regate( &mut self, txn_id: TxnId, @@ -149,18 +134,6 @@ impl Scheduler { } } -/// Whether a txn in `state` can have a write in the Data Plane's hands: a -/// committed flush, or a direct apply. -fn writes_in_flight(state: Option) -> bool { - matches!( - state, - None | Some(CommitState::AwaitingResolve { - committed: true, - .. - }) - ) -} - #[cfg(test)] mod tests { use nodedb_cluster::calvin::types::CalvinIncarnation; @@ -170,7 +143,7 @@ mod tests { make_sequenced_txn, scheduler_with_pending, staged_response, }; use super::*; - use crate::bridge::envelope::Status; + use crate::bridge::envelope::{StageVote, Status}; use crate::control::cluster::calvin::scheduler::driver::types::CommitState; use crate::control::cluster::calvin::scheduler::lock_manager::TxnId; use crate::control::security::catalog::StoredCollection; @@ -185,8 +158,8 @@ mod tests { /// A transaction staged against `orders` before a purge and a same-name /// create is superseded on this replica, though `ledger` is unchanged. The /// leader's vote is an abort, which the global verdict applies to every - /// participant, `ledger`'s included. A replica that staged it under a - /// COMMIT verdict halts instead of flushing. + /// participant, `ledger`'s included. A leader that staged it under a + /// COMMIT verdict halts instead of committing it. #[test] fn a_txn_staged_before_a_recreate_is_superseded_everywhere() { let txn_id = TxnId::new(3, 0); @@ -237,13 +210,16 @@ mod tests { ); // The staged slice votes abort, whatever its own read-set says, and - // cannot flush under a COMMIT verdict. + // cannot commit under a COMMIT verdict. if let Some(pending) = scheduler.pending.get_mut(&txn_id) { pending.superseded = true; } - scheduler.resolve_staged_commit(txn_id, &staged_response(Status::Ok, Some(true))); + scheduler.resolve_staged_commit( + txn_id, + &staged_response(Status::Ok, Some(StageVote::Commit)), + ); let pending = scheduler.pending.get(&txn_id).expect("still parked"); - assert_eq!(pending.commit_state, Some(CommitState::AwaitingVerdict)); + assert_eq!(pending.commit_state, CommitState::AwaitingVerdict); assert!(pending.stage_error.is_some(), "a COMMIT verdict halts it"); } @@ -267,11 +243,12 @@ mod tests { drop(reclaim); } - /// A replica that halts on a txn it never staged gives back the gates the - /// txn held: a purge of its collection proceeds. The txn checks again - /// before a flush and fails once the collection was recreated. + /// A leader that halts on a committed txn whose collection it found + /// superseded gives back the gates the txn held: a purge of its + /// collection proceeds. The txn checks again before a proposal and fails + /// once the collection was recreated. #[tokio::test] - async fn a_halt_releases_the_gates_and_a_later_flush_checks_again() { + async fn a_halt_releases_the_gates_and_a_later_proposal_checks_again() { let txn_id = TxnId::new(5, 0); let (mut scheduler, _dir) = scheduler_with_pending(txn_id, CommitState::AwaitingVerdict); let shared = std::sync::Arc::clone(&scheduler.shared); @@ -294,7 +271,8 @@ mod tests { if let Some(pending) = scheduler.pending.get_mut(&txn_id) { pending.txn.tx_class = tx_class; pending.gates = gates; - pending.stage_error = Some("stage refused on this replica".into()); + pending.superseded = true; + pending.stage_error = Some("a collection was superseded on this replica".into()); } scheduler.resume_on_verdict(txn_id, true); @@ -321,7 +299,7 @@ mod tests { assert_eq!(pending.gates.len(), 1, "the check took the gate back"); assert!(!pending.ungated); - scheduler.release_gates_on_halt(txn_id); + scheduler.release_gates_on_halt(); catalog .delete_collection(DatabaseId::DEFAULT, 1, "orders") .expect("purge"); @@ -333,7 +311,7 @@ mod tests { .expect("recreate"); assert!( scheduler.regate(txn_id).is_err(), - "a flush never lands in the recreated collection" + "a redo never commits into the recreated collection" ); } } diff --git a/nodedb/src/control/cluster/calvin/scheduler/driver/core/dispatch/mod.rs b/nodedb/src/control/cluster/calvin/scheduler/driver/core/dispatch/mod.rs index 303ce0094..1042783f6 100644 --- a/nodedb/src/control/cluster/calvin/scheduler/driver/core/dispatch/mod.rs +++ b/nodedb/src/control/cluster/calvin/scheduler/driver/core/dispatch/mod.rs @@ -1,9 +1,12 @@ // SPDX-License-Identifier: BUSL-1.1 -//! Static and active (dependent-read) txn dispatch to the Data Plane. +//! Granted txn dispatch: the leader stages static and active (dependent-read) +//! txns on the Data Plane, and a follower holds them. mod active_dispatch; mod bind_identities; +mod follow; mod incarnation; mod primary_write; mod static_dispatch; +mod unstaged; diff --git a/nodedb/src/control/cluster/calvin/scheduler/driver/core/dispatch/primary_write.rs b/nodedb/src/control/cluster/calvin/scheduler/driver/core/dispatch/primary_write.rs index 7e9cfca34..7211821b0 100644 --- a/nodedb/src/control/cluster/calvin/scheduler/driver/core/dispatch/primary_write.rs +++ b/nodedb/src/control/cluster/calvin/scheduler/driver/core/dispatch/primary_write.rs @@ -1,7 +1,7 @@ // SPDX-License-Identifier: BUSL-1.1 -//! Per-slice write classification: primary-write and RETURNING predicates -//! shared by the static and active dispatch paths. +//! Per-slice write classification: primary-write, write-mark and RETURNING +//! predicates shared by the static and active dispatch paths. use nodedb_physical::physical_plan::PhysicalPlan; @@ -21,7 +21,7 @@ pub(super) fn txn_has_non_derived_write(plans: &[PhysicalPlan]) -> bool { /// ONE deposited response, primary-write participants coalesce first-wins, and /// a derived participant's response describes a row the user's statement never /// named. A balance write that won that race handed an `INSERT` tag a count — -/// or, when its flush found the commit already resolved and answered with an +/// or, when its install found the commit already resolved and answered with an /// empty payload, no count at all — belonging to a different write entirely. /// /// `is_derived_side_effect` is the named predicate rather than an inline @@ -47,6 +47,17 @@ pub(super) fn plans_have_primary_write( .any(crate::control::planner::calvin::is_write_plan) } +/// Whether this vShard's slice raises its tenant's write mark when it +/// installs: a primary write that changes a tenant row. A slice that only +/// installs schema, such as a constraint set the leader's write gate +/// sequenced, changes no row, so a restore overwrites no write of it. +pub(super) fn plans_raise_write_mark(plans: &[PhysicalPlan], has_primary_write: bool) -> bool { + has_primary_write + && plans + .iter() + .any(crate::control::server::shared::write_admission::plan_writes_user_data) +} + /// Whether this vShard's slice carries a RETURNING-bearing write — a plan whose /// applied response is DATA-ROWs rather than a bare affected-count. Uses the /// SAME `describe_plan` classification the coordinator's response-shaping uses, @@ -57,3 +68,38 @@ pub(super) fn plans_have_returning(plans: &[PhysicalPlan]) -> bool { .iter() .any(|plan| matches!(describe_plan(plan), PlanKind::ReturningRows)) } + +#[cfg(test)] +mod tests { + use super::*; + use nodedb_physical::physical_plan::{CrdtOp, KvOp}; + use nodedb_types::{DatabaseId, QualifiedCollection, Surrogate}; + + fn collection() -> QualifiedCollection { + QualifiedCollection::new(DatabaseId::DEFAULT, "c") + } + + /// A routed constraint set is the slice's primary write, and it changes + /// no row. Its install raises no write mark. + #[test] + fn only_a_slice_that_changes_a_row_raises_the_write_mark() { + let set = PhysicalPlan::Crdt(CrdtOp::SetConstraints { + collection: collection(), + constraint_version: 1, + constraints: Vec::new(), + }); + let put = PhysicalPlan::Kv(KvOp::Put { + collection: collection(), + key: b"k".to_vec(), + value: b"v".to_vec(), + ttl_ms: 0, + surrogate: Surrogate::new(1), + returning: None, + rls_filters: Vec::new(), + provenance: None, + }); + assert!(!plans_raise_write_mark(std::slice::from_ref(&set), true)); + assert!(plans_raise_write_mark(std::slice::from_ref(&put), true)); + assert!(!plans_raise_write_mark(&[put], false)); + } +} diff --git a/nodedb/src/control/cluster/calvin/scheduler/driver/core/dispatch/static_dispatch.rs b/nodedb/src/control/cluster/calvin/scheduler/driver/core/dispatch/static_dispatch.rs index 22b531f89..b9c8b56e9 100644 --- a/nodedb/src/control/cluster/calvin/scheduler/driver/core/dispatch/static_dispatch.rs +++ b/nodedb/src/control/cluster/calvin/scheduler/driver/core/dispatch/static_dispatch.rs @@ -1,22 +1,20 @@ // SPDX-License-Identifier: BUSL-1.1 -//! Static-set ready-transaction dispatch: local-plan routing, group-leader -//! resolution, and the `CalvinExecuteStatic` submit/stage orchestration. +//! Static-set ready-transaction dispatch on the data-group leader: local-plan +//! routing and the `CalvinExecuteStatic` submit/stage orchestration. use std::time::Instant; -use tracing::error; - use nodedb_cluster::calvin::types::SequencedTxn; use nodedb_physical::physical_plan::PhysicalPlan; use nodedb_physical::physical_plan::meta::MetaOp; use super::super::deferred::{DispatchOutcome, DispatchStep}; -use super::super::owed::SchedulerProposal; use super::super::routing::PlanRouting; use super::super::scheduler::Scheduler; use super::primary_write::{ - plans_have_primary_write, plans_have_returning, txn_has_non_derived_write, + plans_have_primary_write, plans_have_returning, plans_raise_write_mark, + txn_has_non_derived_write, }; use crate::control::cluster::calvin::scheduler::lock_manager::TxnId; use crate::types::DatabaseId; @@ -28,52 +26,11 @@ struct LocalSlice { body_plans: Vec, /// Every local plan is a body's, and a client plan homes elsewhere. body_only: bool, + /// The identities the local plans carry, bound in this node's catalog. + identities: Vec, } impl Scheduler { - /// Whether THIS node is currently the leader of the data-group owning this - /// scheduler's vshard. - /// - /// Stamped into the `CalvinExecute{Static,Active}` MetaOp at dispatch time - /// so the Data Plane runs the OLLP optimistic-lock verification (and emits - /// `OllpRetryRequired`) ONLY on the leader, while every replica applies the - /// carried predicted write-set verbatim — preserving Calvin determinism. - /// - /// Resolved via the existing routing → group-role check (no new election). - /// On a poisoned lock the inner guard is recovered; a momentarily-unknown - /// leadership (e.g. mid-election) resolves to `false`, i.e. follower-style - /// apply against the predicted set, which is always determinism-safe. - pub(in crate::control::cluster::calvin::scheduler::driver::core) fn is_group_leader( - &self, - ) -> bool { - let mr = match self.multi_raft.lock() { - Ok(g) => g, - Err(poisoned) => poisoned.into_inner(), - }; - mr.vshard_role_is_leader(self.vshard_id) - } - - /// Broadcast a terminal, NON-retryable routing-failure signal via the - /// sequencer-group Raft so every replica's `CalvinCompletionRegistry` - /// fires `note_routing_failed`, waking the coordinator's completion - /// waiter immediately with the reason instead of leaving it to burn the - /// full deadline and report a generic timeout. Mirrors the OllpMismatch - /// broadcast in `handle_executor_response`. Shared by `dispatch_txn` and - /// `dispatch_active_txn`, and by a multi-part transaction whose part - /// fails to decode. - pub(in crate::control::cluster::calvin::scheduler::driver::core) fn propose_routing_failure( - &mut self, - txn_id: TxnId, - err: &crate::Error, - ) { - self.propose_sequencer_entry( - txn_id, - SchedulerProposal::RoutingFailed { - detail: err.to_string(), - }, - ); - } - /// Filter a transaction's write plans down to the slice that homes to this /// scheduler's vShard. /// @@ -147,14 +104,7 @@ impl Scheduler { let plans = match super::super::super::helpers::decode_plans(&txn.tx_class.plans) { Ok(p) => p, Err(e) => { - error!( - vshard_id = self.vshard_id, - epoch, - position, - error = %e, - "calvin scheduler: plan decode failed; releasing locks and skipping txn" - ); - self.on_unpending_txn_complete(txn_id, lock_owner); + self.reject_plan(txn, txn_id, lock_owner, &e); return; } }; @@ -176,27 +126,21 @@ impl Scheduler { match self.local_calvin_plans(plans, txn.tx_class.database_id, epoch, position) { Ok(p) => p, Err(e) => { - error!( - vshard_id = self.vshard_id, - epoch, - position, - error = %e, - "calvin scheduler: static txn routing failed; releasing locks" - ); - self.propose_routing_failure(txn_id, &e); - self.on_unpending_txn_complete(txn_id, lock_owner); + self.reject_plan(txn, txn_id, lock_owner, &e); return; } }; - if !self.bind_local_identities(&mut local, txn.tx_class.database_id, tenant_id, txn_id) { + let Some(identities) = + self.bind_local_identities(&mut local, txn.tx_class.database_id, tenant_id, txn_id) + else { return; - } + }; // A participant with no local WRITE slice is either a READ-ONLY // participant — writes home elsewhere, but a read homes HERE, so it must // still validate its slice of the read-set and cast a real commit/abort // vote — or a routing bug (neither writes nor reads home here). Only the - // latter is an error; the former stages a validate-only task below. + // latter rejects the plans; the former stages a validate-only task below. if local.is_empty() && !super::super::routing::homes_versioned_read( &txn.tx_class.versioned_reads, @@ -211,15 +155,7 @@ impl Scheduler { self.vshard_id ), }; - error!( - vshard_id = self.vshard_id, - epoch, - position, - error = %e, - "calvin scheduler: static txn homes no local work; releasing locks" - ); - self.propose_routing_failure(txn_id, &e); - self.on_unpending_txn_complete(txn_id, lock_owner); + self.reject_plan(txn, txn_id, lock_owner, &e); return; } @@ -238,6 +174,7 @@ impl Scheduler { plans: local, body_plans: local_body_plans, body_only, + identities, }, has_non_derived_write, ); @@ -281,7 +218,7 @@ impl Scheduler { /// the validate-only read path (`plans` empty). Both carry the txn's FULL /// `versioned_reads` to the apply core, which validates the LOCAL slice of /// the read-set — whether or not `plans` is empty — and returns the commit - /// vote on `read_set_valid`. A validate-only task has `has_primary_write == + /// vote on `stage_vote`. A validate-only task has `has_primary_write == /// false`, so it deposits no result sidecar entry, exactly as intended. /// /// The txn enters `pending` before the dispatch, so a stage refused at @@ -299,6 +236,7 @@ impl Scheduler { plans, body_plans, body_only, + identities, } = slice; // The apply-slot identity (used in the CalvinExecuteStatic task and // error logs) is exactly `txn_id`; deriving it here keeps the two in @@ -307,9 +245,11 @@ impl Scheduler { let position = txn_id.position; let request_id = self.next_request_id(); let has_primary_write = plans_have_primary_write(&plans, has_non_derived_write); + let raises_write_mark = plans_raise_write_mark(&plans, has_primary_write); let has_returning = plans_have_returning(&plans); - let mut flush_scope = super::super::super::types::FlushScope::of_plans(&plans); - flush_scope.body_only = body_only; + let mut scope = super::super::super::types::SliceScope::of_plans(&plans); + scope.body_only = body_only; + scope.identities = identities; let database_id = txn.tx_class.database_id; let plan = PhysicalPlan::Meta(MetaOp::CalvinExecuteStatic { epoch, @@ -317,7 +257,6 @@ impl Scheduler { tenant_id, plans, epoch_system_ms: txn.epoch_system_ms, - is_group_leader: self.is_group_leader(), // The replicated read-set travels to the apply core so each // participant can check, at apply, whether its slice of the reads was // still current. Empty for pure-write / autocommit transactions. @@ -325,15 +264,12 @@ impl Scheduler { body_plans, }); - // Calvin allocates the CalvinApplied WAL LSN post-apply (in the - // scheduler's response handler), so no committed LSN is known at - // dispatch time to stamp here. - let mut request = self.build_exempt_request(request_id, tenant_id, database_id, plan, None); + // A stage writes no WAL record, so no committed LSN rides on it. + let mut request = self.build_exempt_request(request_id, tenant_id, database_id, plan); // The stage tags the rows it stages by the slice's source. - request.event_source = - super::super::request::slice_event_source(&txn.tx_class, &flush_scope); + request.event_source = super::super::request::slice_event_source(&txn.tx_class, &scope); - // Every replica checks the transaction's collection incarnations. + // The leader checks the transaction's collection incarnations. let (superseded, gates) = self.check_incarnations(&txn.tx_class); self.pending.insert( txn_id, @@ -343,24 +279,22 @@ impl Scheduler { // no-determinism: dispatch_time is scheduler observability, not Calvin WAL data dispatch_time: Instant::now(), has_primary_write, + raises_write_mark, has_returning, - // The commit's resolved redo fills them. - change_sets: Vec::new(), // This dispatch STAGES the txn (validate + buffer, no apply); - // its response carries the local commit vote that drives the - // subsequent flush-or-drop. - commit_state: Some(super::super::super::types::CommitState::Staged), + // its response carries the local commit vote. + commit_state: super::super::super::types::CommitState::Staged, + // The dispatch below records its request. + awaiting: None, // Set only once the txn parks in `AwaitingVerdict`. verdict_deadline: None, stage_error: None, - // Set once a committed txn appends its redo record. - redo_records: None, - flush_scope, + scope, + // Set once a committed slice resolves its redo. + redo: None, superseded, gates, ungated: false, - // Taken when the flush dispatches. - install_permit: None, }, ); diff --git a/nodedb/src/control/cluster/calvin/scheduler/driver/core/dispatch/unstaged.rs b/nodedb/src/control/cluster/calvin/scheduler/driver/core/dispatch/unstaged.rs new file mode 100644 index 000000000..c02d6e001 --- /dev/null +++ b/nodedb/src/control/cluster/calvin/scheduler/driver/core/dispatch/unstaged.rs @@ -0,0 +1,286 @@ +// SPDX-License-Identifier: BUSL-1.1 + +//! A transaction the leader cannot stage: its plans are rejected, or its +//! dependent reads never arrived. +//! +//! The txn still takes the commit barrier. It enters `pending`, holds its +//! locks, and owes an abort vote. At the abort verdict it drops like any +//! staged txn: it proposes its `CompletionAck`, releases its locks, and only +//! then marks its position applied. A follower rejects the same plans and +//! holds the txn `Following` for the same verdict. + +use std::time::Instant; + +use tracing::error; + +use nodedb_cluster::calvin::AbortReason; +use nodedb_cluster::calvin::types::SequencedTxn; + +use super::super::owed::SchedulerProposal; +use super::super::scheduler::Scheduler; +use crate::control::cluster::calvin::scheduler::driver::types::{ + CommitState, PendingTxn, SliceScope, +}; +use crate::control::cluster::calvin::scheduler::lock_manager::TxnId; + +impl Scheduler { + /// Park `txn` with an abort vote for plans this vShard rejects. + /// + /// Plan decode, part assembly, and plan routing read only replicated + /// input. So every replica rejects the same plans, and the leader votes + /// `PlanRejected`. A follower owes no vote: it holds the txn for the + /// abort verdict. + pub(in crate::control::cluster::calvin::scheduler::driver::core) fn reject_plan( + &mut self, + txn: SequencedTxn, + txn_id: TxnId, + lock_owner: TxnId, + error: &crate::Error, + ) { + if !self.role.is_leader() { + self.follow(txn, txn_id, lock_owner); + return; + } + error!( + vshard_id = self.vshard_id, + epoch = txn_id.epoch, + position = txn_id.position, + error = %error, + "calvin scheduler: plans rejected; voting abort" + ); + self.park_unstaged( + txn, + txn_id, + lock_owner, + AbortReason::PlanRejected, + error.to_string(), + ); + } + + /// Park `txn`, which never staged on this replica, on the commit barrier + /// with an abort vote for `reason`. + /// + /// `stage_error` stays on the pending entry. This leader holds nothing + /// to resolve, so a COMMIT verdict restages the txn through the grant + /// path: the plans route again, or the barrier opens again. + pub(in crate::control::cluster::calvin::scheduler::driver::core) fn park_unstaged( + &mut self, + txn: SequencedTxn, + txn_id: TxnId, + lock_owner: TxnId, + reason: AbortReason, + stage_error: String, + ) { + // no-determinism: dispatch time and stall deadline are observability only; the replicated verdict decides. + let now = Instant::now(); + self.pending.insert( + txn_id, + PendingTxn { + txn, + lock_owner, + dispatch_time: now, + has_primary_write: false, + raises_write_mark: false, + has_returning: false, + commit_state: CommitState::AwaitingVerdict, + awaiting: None, + verdict_deadline: Some(now + self.config.verdict_stall_warn()), + stage_error: Some(stage_error), + scope: SliceScope::of_plans(&[]), + redo: None, + superseded: false, + gates: Vec::new(), + ungated: false, + }, + ); + self.propose_sequencer_entry( + txn_id, + SchedulerProposal::Vote { + abort: Some(reason), + }, + ); + // The verdict can be stored already: on replay, or when a peer's + // vote completed the tally first. + if let Some(verdict) = self.registry.verdict(nodedb_cluster::calvin::TxnId::new( + txn_id.epoch, + txn_id.position, + )) { + self.resume_on_verdict(txn_id, verdict); + } + } +} + +#[cfg(test)] +mod tests { + use std::sync::Arc; + + use nodedb_cluster::calvin::types::SchedulerInput; + use nodedb_cluster::calvin::{ + AttemptOutcome, CalvinCompletionRegistry, ParticipantVote, SequencerEntry, + }; + + use super::*; + use crate::bridge::dispatch::CoreChannelDataSide; + use crate::bridge::envelope::{StageVote, Status}; + use crate::control::cluster::calvin::scheduler::driver::core::test_proposer::{ + CapturingProposer, lead_data_group, + }; + use crate::control::cluster::calvin::scheduler::driver::core::test_support::{ + build_test_scheduler_with_data_side, make_sequenced_txn, staged_pending, staged_response, + }; + + const REJECTING: u32 = 7; + const PEER: u32 = 8; + + /// A scheduler on `vshard` that leads its data group and captures its + /// proposals. + fn leading_scheduler( + vshard: u32, + registry: &Arc, + ) -> ( + Scheduler, + tempfile::TempDir, + CoreChannelDataSide, + Arc, + ) { + let (mut scheduler, dir, data_side) = + build_test_scheduler_with_data_side(vshard, Arc::clone(registry)); + let proposer = CapturingProposer::accepting(); + scheduler.sequencer_proposer = proposer.clone(); + lead_data_group(&mut scheduler); + (scheduler, dir, data_side, proposer) + } + + /// Apply every vote `proposer` accepted to `registry`, as the sequencer + /// log applies them on every replica. + fn apply_votes(registry: &CalvinCompletionRegistry, proposer: &CapturingProposer) { + for entry in proposer.accepted() { + if let SequencerEntry::Vote { + epoch, + position, + vshard, + } = entry + { + registry.note_vote( + nodedb_cluster::calvin::TxnId::new(epoch, position), + vshard, + ParticipantVote::Commit, + ); + } else if let SequencerEntry::AbortVote { + epoch, + position, + vshard, + reason, + } = entry + { + registry.note_vote( + nodedb_cluster::calvin::TxnId::new(epoch, position), + vshard, + ParticipantVote::Abort(reason), + ); + } + } + } + + /// A txn whose plan bytes do not decode. + fn undecodable_txn(epoch: u64, position: u32) -> SequencedTxn { + let mut txn = make_sequenced_txn(epoch, position); + txn.tx_class.plans = vec![0xc1]; + txn + } + + /// A rejected plan votes `PlanRejected`. The tally completes with the + /// peer's commit vote, the verdict aborts, and both participants drop, + /// ack, release their locks, and mark the position applied. + #[tokio::test] + async fn a_rejected_plan_aborts_every_participant_through_the_verdict() { + let registry = CalvinCompletionRegistry::new_detached(); + let (mut rejecting, _dir_r, _data_r, proposer_r) = leading_scheduler(REJECTING, ®istry); + let (mut peer, _dir_p, _data_p, proposer_p) = leading_scheduler(PEER, ®istry); + let txn_id = TxnId::new(30, 0); + let cluster_txn = nodedb_cluster::calvin::TxnId::new(30, 0); + registry.seed_expected(cluster_txn, 2); + let outcome = registry.register_completion(cluster_txn, 2); + + rejecting.process_scheduler_input(SchedulerInput::Txn(Box::new(undecodable_txn(30, 0)))); + // A later txn on the same key waits behind the rejected txn's locks. + rejecting.process_scheduler_input(SchedulerInput::Txn(Box::new(undecodable_txn(31, 0)))); + assert!(rejecting.blocked.contains_key(&TxnId::new(31, 0))); + assert_eq!( + rejecting.pending.get(&txn_id).map(|p| p.commit_state), + Some(CommitState::AwaitingVerdict) + ); + assert!(!rejecting.applied.is_applied(30, 0)); + + peer.pending + .insert(txn_id, staged_pending(make_sequenced_txn(30, 0), txn_id)); + peer.resolve_staged_commit( + txn_id, + &staged_response(Status::Ok, Some(StageVote::Commit)), + ); + assert_eq!( + peer.pending.get(&txn_id).map(|p| p.commit_state), + Some(CommitState::AwaitingVerdict) + ); + + apply_votes(®istry, &proposer_r); + apply_votes(®istry, &proposer_p); + assert!(proposer_r.accepted().contains(&SequencerEntry::AbortVote { + epoch: 30, + position: 0, + vshard: REJECTING, + reason: AbortReason::PlanRejected, + })); + let verdicts = registry.drain_unproposed_verdicts(); + assert_eq!( + verdicts, + vec![( + cluster_txn, + nodedb_cluster::calvin::VerdictOutcome::Abort(AbortReason::PlanRejected) + )] + ); + registry.note_verdict(cluster_txn, verdicts[0].1); + + for (scheduler, vshard) in [(&mut rejecting, REJECTING), (&mut peer, PEER)] { + scheduler.resume_on_verdict(txn_id, false); + assert_eq!( + scheduler.pending.get(&txn_id).map(|p| p.commit_state), + Some(CommitState::AwaitingDrop), + "vShard {vshard} leaves AwaitingVerdict for its drop" + ); + assert!(!scheduler.applied.is_applied(30, 0)); + scheduler.finish_drop(txn_id, &staged_response(Status::Ok, None)); + assert!(!scheduler.pending.contains_key(&txn_id)); + assert!(scheduler.applied.is_applied(30, 0)); + registry.note_completion_ack(cluster_txn, vshard); + } + assert!( + !rejecting.blocked.contains_key(&TxnId::new(31, 0)), + "the released locks promote the waiting txn" + ); + assert_eq!( + outcome.await.expect("outcome fires"), + AttemptOutcome::Aborted { + reason: AbortReason::PlanRejected + } + ); + } + + /// A verdict stored before the rejection resumes the txn at once. + #[tokio::test] + async fn a_rejection_after_the_verdict_drops_at_once() { + let registry = CalvinCompletionRegistry::new_detached(); + let (mut scheduler, _dir, _data, _proposer) = leading_scheduler(REJECTING, ®istry); + let txn_id = TxnId::new(32, 0); + registry.note_verdict( + nodedb_cluster::calvin::TxnId::new(32, 0), + nodedb_cluster::calvin::VerdictOutcome::Abort(AbortReason::PlanRejected), + ); + + scheduler.dispatch_txn(undecodable_txn(32, 0), txn_id, txn_id); + assert_eq!( + scheduler.pending.get(&txn_id).map(|p| p.commit_state), + Some(CommitState::AwaitingDrop) + ); + } +} diff --git a/nodedb/src/control/cluster/calvin/scheduler/driver/core/drop_dispatch.rs b/nodedb/src/control/cluster/calvin/scheduler/driver/core/drop_dispatch.rs new file mode 100644 index 000000000..c0d1b8c44 --- /dev/null +++ b/nodedb/src/control/cluster/calvin/scheduler/driver/core/drop_dispatch.rs @@ -0,0 +1,40 @@ +// SPDX-License-Identifier: BUSL-1.1 + +//! Dispatch of a `CalvinDrop`: discard what a txn staged on its core. + +use nodedb_physical::physical_plan::PhysicalPlan; +use nodedb_physical::physical_plan::meta::MetaOp; + +use super::commit_redo::missing_pending_error; +use super::deferred::{DispatchOutcome, DispatchStep}; +use super::scheduler::Scheduler; +use crate::control::cluster::calvin::scheduler::lock_manager::TxnId; + +impl Scheduler { + /// Dispatch the drop of `txn_id`'s staged state as `step`: a + /// [`DispatchStep::Drop`] the txn completes on, or a + /// [`DispatchStep::Discard`] nothing waits for. + /// + /// A drop writes nothing, so no WAL LSN rides on it. A capacity refusal + /// returns [`DispatchOutcome::Deferred`]: the drop is parked for re-send + /// and the txn stays in flight. A txn with no `pending` entry returns + /// [`DispatchOutcome::Failed`]. + pub(in crate::control::cluster::calvin::scheduler::driver::core) fn dispatch_drop( + &mut self, + txn_id: TxnId, + step: DispatchStep, + ) -> DispatchOutcome { + let Some(pending) = self.pending.get(&txn_id) else { + return DispatchOutcome::Failed(missing_pending_error(txn_id)); + }; + let tenant_id = pending.txn.tx_class.tenant_id; + let database_id = pending.txn.tx_class.database_id; + let plan = PhysicalPlan::Meta(MetaOp::CalvinDrop { + epoch: txn_id.epoch, + position: txn_id.position, + }); + let request_id = self.next_request_id(); + let request = self.build_exempt_request(request_id, tenant_id, database_id, plan); + self.dispatch_sequenced(txn_id, step, request) + } +} diff --git a/nodedb/src/control/cluster/calvin/scheduler/driver/core/flush_turn.rs b/nodedb/src/control/cluster/calvin/scheduler/driver/core/flush_turn.rs deleted file mode 100644 index 82ed6d1ca..000000000 --- a/nodedb/src/control/cluster/calvin/scheduler/driver/core/flush_turn.rs +++ /dev/null @@ -1,231 +0,0 @@ -// SPDX-License-Identifier: BUSL-1.1 - -//! Sequencer-order flushes. -//! -//! A vShard's transactions stage, vote, and resolve in any order: their -//! responses and verdicts arrive in any order. Their flushes run one at a -//! time, in `(epoch, position)` order. A committed transaction's flush -//! dispatches once no lower transaction of this vShard is unfinished. -//! -//! One core applies every request of a vShard, in dispatch order. So the core -//! installs committed transactions, and emits their change events, in -//! sequencer order on every replica. The commit tail, which publishes the -//! transaction's Control-Plane changes, runs in the same order. A Calvin -//! change feed therefore never delivers a lower position after a higher one. -//! -//! Every wait points at a lower transaction: the scheduler takes input, and -//! grants locks, in sequencer order. So the turn order adds no wait cycle. - -use super::super::types::CommitState; -use super::commit_resolution_dispatch::CommitResolution; -use super::deferred::{DispatchOutcome, DispatchStep}; -use super::install_gate::FlushAdmission; -use super::scheduler::Scheduler; -use crate::control::cluster::calvin::scheduler::lock_manager::TxnId; - -impl Scheduler { - /// Queue the committed flush of `txn_id` for its turn, then dispatch it - /// when its turn came. - pub(in crate::control::cluster::calvin::scheduler::driver::core) fn queue_flush( - &mut self, - txn_id: TxnId, - redo_lsn: Option, - ) { - if let Some(pending) = self.pending.get_mut(&txn_id) { - pending.commit_state = Some(CommitState::AwaitingFlushTurn { redo_lsn }); - } - self.pump_flush_turn(); - } - - /// Dispatch the flush of the lowest unfinished transaction when it waits - /// for its turn. - pub(in crate::control::cluster::calvin::scheduler::driver::core) fn pump_flush_turn(&mut self) { - let Some(lowest) = self.lowest_unfinished() else { - return; - }; - let resolve_turn = self - .pending - .get(&lowest) - .is_some_and(|pending| pending.commit_state == Some(CommitState::AwaitingResolveTurn)); - if resolve_turn { - self.dispatch_resolve_turn(lowest); - return; - } - let Some(redo_lsn) = - self.pending - .get(&lowest) - .and_then(|pending| match pending.commit_state { - Some(CommitState::AwaitingFlushTurn { redo_lsn }) => Some(redo_lsn), - _ => None, - }) - else { - return; - }; - // The flush holds its data group's apply gate shared until its - // position is marked applied (see `super::install_gate`). - let permit = match self.admit_flush() { - FlushAdmission::Go(permit) => permit, - FlushAdmission::Wait | FlushAdmission::Retired => return, - }; - if let Some(pending) = self.pending.get_mut(&lowest) { - pending.install_permit = permit; - } - // A flush refused at capacity is parked for re-send. The txn awaits its - // flush response either way, so the state below is the same. - if let DispatchOutcome::Failed(error) = - self.dispatch_commit_resolution(lowest, CommitResolution::Flush { redo_lsn }) - { - self.fail_dispatch_step(lowest, DispatchStep::Flush, error); - return; - } - if let Some(pending) = self.pending.get_mut(&lowest) { - pending.commit_state = Some(CommitState::AwaitingResolve { - committed: true, - redo_lsn, - }); - } - } - - /// Dispatch the resolve of `txn_id`, whose turn came. Its flush then - /// follows at once: it stays the lowest unfinished txn. - fn dispatch_resolve_turn(&mut self, txn_id: TxnId) { - if let DispatchOutcome::Failed(error) = self.dispatch_calvin_resolve(txn_id) { - self.fail_dispatch_step(txn_id, DispatchStep::Resolve, error); - return; - } - if let Some(pending) = self.pending.get_mut(&txn_id) { - pending.commit_state = Some(CommitState::AwaitingRedoResolve); - } - } - - /// The lowest transaction of this vShard that has not finished: in - /// flight, blocked on a lock, or waiting at a dependent-read barrier. - fn lowest_unfinished(&self) -> Option { - let pending = self.pending.keys().next().copied(); - let barrier = self.dependent_barrier.keys().next().copied(); - let blocked = self - .blocked - .values() - .map(|blocked| TxnId::new(blocked.txn.epoch, blocked.txn.position)) - .min(); - // A multi-part txn holding its locks while its parts arrive is - // unfinished too: every later flush waits for it. - let awaiting_parts = self.parts.lowest_awaiting(); - [pending, barrier, blocked, awaiting_parts] - .into_iter() - .flatten() - .min() - } -} - -#[cfg(test)] -mod tests { - use super::*; - use crate::control::cluster::calvin::scheduler::driver::core::test_support::{ - build_test_scheduler_with_data_side, make_sequenced_txn, staged_pending, - }; - use nodedb_cluster::calvin::CalvinCompletionRegistry; - - /// A higher committed txn waits for the lower one; the lower one's - /// completion gives it its turn. - #[tokio::test] - async fn a_higher_flush_waits_for_every_lower_txn() { - let low = TxnId::new(5, 0); - let high = TxnId::new(5, 1); - let (mut scheduler, _dir, _data_side) = - build_test_scheduler_with_data_side(7, CalvinCompletionRegistry::new_detached()); - for txn_id in [low, high] { - scheduler.pending.insert( - txn_id, - staged_pending(make_sequenced_txn(txn_id.epoch, txn_id.position), txn_id), - ); - } - - scheduler.queue_flush(high, None); - assert_eq!( - scheduler.pending.get(&high).and_then(|p| p.commit_state), - Some(CommitState::AwaitingFlushTurn { redo_lsn: None }), - "the lower txn is unfinished, so the higher flush waits" - ); - - scheduler.pending.remove(&low); - scheduler.pump_flush_turn(); - assert_eq!( - scheduler.pending.get(&high).and_then(|p| p.commit_state), - Some(CommitState::AwaitingResolve { - committed: true, - redo_lsn: None, - }), - "with the lower txn finished, the higher flush dispatches" - ); - } - - /// A committed TRUNCATE of rows resolves only at its turn: while a lower - /// txn of this vShard is unfinished, its resolve waits, so it reads every - /// row sequenced before it. - #[tokio::test] - async fn a_truncate_resolves_only_at_its_turn() { - let low = TxnId::new(5, 0); - let truncate = TxnId::new(5, 1); - let (mut scheduler, _dir, _data_side) = - build_test_scheduler_with_data_side(7, CalvinCompletionRegistry::new_detached()); - for txn_id in [low, truncate] { - scheduler.pending.insert( - txn_id, - staged_pending(make_sequenced_txn(txn_id.epoch, txn_id.position), txn_id), - ); - } - if let Some(pending) = scheduler.pending.get_mut(&truncate) { - pending.flush_scope.resolve_at_turn = true; - pending.commit_state = Some(CommitState::AwaitingResolveTurn); - } - - scheduler.pump_flush_turn(); - assert_eq!( - scheduler - .pending - .get(&truncate) - .and_then(|p| p.commit_state), - Some(CommitState::AwaitingResolveTurn), - "the lower txn is unfinished, so the resolve waits" - ); - - scheduler.pending.remove(&low); - scheduler.pump_flush_turn(); - assert_eq!( - scheduler - .pending - .get(&truncate) - .and_then(|p| p.commit_state), - Some(CommitState::AwaitingRedoResolve), - "with the lower txn finished, the resolve dispatches" - ); - } - - /// Every truncate of rows reads a whole collection at resolve; a point - /// write does not. A TRUNCATE's edge share records a cut, which reads no - /// stored edge, so it resolves without waiting for its turn. - #[test] - fn a_truncate_slice_resolves_at_its_turn() { - use nodedb_physical::physical_plan::{DocumentOp, GraphOp, PhysicalPlan}; - let collection = nodedb_types::QualifiedCollection::from_stored("c".to_string()); - let truncate = PhysicalPlan::Document(DocumentOp::Truncate { - collection: collection.clone(), - restart_identity: false, - resolved_sum_targets: Vec::new(), - declared_primary_key: None, - }); - let share = PhysicalPlan::Graph(GraphOp::TruncateEdges { - collection, - vshard: 3, - }); - use crate::control::cluster::calvin::scheduler::driver::types::FlushScope; - assert!(FlushScope::of_plans(std::slice::from_ref(&truncate)).resolve_at_turn); - assert!(!FlushScope::of_plans(std::slice::from_ref(&share)).resolve_at_turn); - assert!( - FlushScope::of_plans(&[truncate, share]).resolve_at_turn, - "the rows' vShard holds both, and its rows' truncate waits" - ); - assert!(!FlushScope::of_plans(&[]).resolve_at_turn); - } -} diff --git a/nodedb/src/control/cluster/calvin/scheduler/driver/core/halt.rs b/nodedb/src/control/cluster/calvin/scheduler/driver/core/halt.rs index 33b24843c..47f3dfadd 100644 --- a/nodedb/src/control/cluster/calvin/scheduler/driver/core/halt.rs +++ b/nodedb/src/control/cluster/calvin/scheduler/driver/core/halt.rs @@ -10,17 +10,17 @@ //! //! A halted scheduler: //! - keeps the stuck txn unapplied, with its locks and its `pending` entry; -//! - releases the collection gates of every txn with no flush in flight, so a -//! purge never waits on it. Such a txn checks its incarnations again, and -//! takes its gates back, before a later flush; +//! - releases the collection gates of every txn, so a purge never waits on +//! it. A leader's txn checks its incarnations again, and takes its gates +//! back, before it proposes its redo; //! - closes intake with [`IntakeClosure::ApplyHalted`]. The fan-out then drops //! new input and arms catch-up, which keeps the sequencer log retained; //! - stops the deferred re-send and the catch-up drain; -//! - keeps routing responses, verdicts, and promotions for other in-flight -//! txns. Each one holds locks disjoint from the stuck txn, so its outcome -//! does not depend on it. Peer vShards wait on its votes, and the per-position -//! applied gate keeps restart replay exact. A later infrastructure error on -//! another txn holds that txn the same way. +//! - keeps routing responses, verdicts, promotions, and apply events for +//! other in-flight txns. Each one holds locks disjoint from the stuck txn, +//! so its outcome does not depend on it. Peer vShards wait on its votes, +//! and the per-position applied gate keeps restart replay exact. A later +//! infrastructure error on another txn holds that txn the same way. //! //! The first cause wins. It logs one `error!` line, sets the //! `nodedb_calvin_apply_halted` gauge, and records the node-wide @@ -51,17 +51,27 @@ pub(in crate::control::cluster::calvin::scheduler::driver::core) enum HaltReason ResponseDisconnected, /// The resolve of a committed txn failed or returned an undecodable record. ResolveFailed, - /// The flush of a committed txn returned an error. - FlushFailed, - /// This replica failed to stage a txn whose verdict is COMMIT. + /// This leader cannot stage a txn whose verdict is COMMIT: its + /// restages ran out, or a collection it names was superseded. LocalStageFailed, /// The surrogate catalog refused a coordinator-assigned identity. IdentityBindFailed, - /// A redo or `CalvinApplied` WAL append failed. - WalAppendFailed, /// The metadata group left this node before its apply reached a held /// txn's floor. MetadataGroupGone, + /// A stage response carries no vote the scheduler can read. + StageVoteInvalid, + /// A committed slice's redo entry cannot be built. The verdict is + /// COMMIT, so the txn cannot drop. + RedoProposeFailed, + /// The vShard's data group applied a committed slice's redo and refused + /// it for good. Every replica refuses the same entry. + RedoInstallRefused, + /// A committed slice's redo did not become durable on this replica. + RedoApplyFailed, + /// The vShard's core fail-stopped. It refuses every later request, the + /// committed install included. + CoreFailStopped, } impl HaltReason { @@ -72,11 +82,14 @@ impl HaltReason { Self::DispatchRefused => apply_halt_reason::DISPATCH_REFUSED, Self::ResponseDisconnected => apply_halt_reason::RESPONSE_DISCONNECTED, Self::ResolveFailed => apply_halt_reason::RESOLVE_FAILED, - Self::FlushFailed => apply_halt_reason::FLUSH_FAILED, Self::LocalStageFailed => apply_halt_reason::LOCAL_STAGE_FAILED, Self::IdentityBindFailed => apply_halt_reason::IDENTITY_BIND_FAILED, - Self::WalAppendFailed => apply_halt_reason::WAL_APPEND_FAILED, Self::MetadataGroupGone => apply_halt_reason::METADATA_GROUP_GONE, + Self::StageVoteInvalid => apply_halt_reason::STAGE_VOTE_INVALID, + Self::RedoProposeFailed => apply_halt_reason::REDO_PROPOSE_FAILED, + Self::RedoInstallRefused => apply_halt_reason::REDO_INSTALL_REFUSED, + Self::RedoApplyFailed => apply_halt_reason::REDO_APPLY_FAILED, + Self::CoreFailStopped => apply_halt_reason::CORE_FAIL_STOPPED, } } @@ -95,20 +108,18 @@ pub(in crate::control::cluster::calvin::scheduler::driver::core) enum HaltStep { Stage, /// The resolve request of a committed txn. Resolve, - /// The flush request of a committed txn. - Flush, - /// The drop request of an aborted txn. + /// The proposal of a committed slice's redo. + RedoPropose, + /// The install of a committed slice's redo, from the data-group log. + RedoApply, + /// The drop request of a txn that ends with no log entry. Drop, - /// The direct apply of a txn with no commit state. - Apply, - /// The `TransactionRedo` WAL append. - RedoAppend, - /// The `CalvinApplied` WAL append. - AppliedMarker, /// The surrogate identity binding before the stage dispatch. IdentityBind, /// The wait for this node's metadata apply to reach the txn's floor. MetadataHold, + /// The passive read of a dependent-read txn. + PassiveRead, } impl HaltStep { @@ -119,33 +130,27 @@ impl HaltStep { match self { Self::Stage => "stage", Self::Resolve => "resolve", - Self::Flush => "flush", + Self::RedoPropose => "redo_propose", + Self::RedoApply => "redo_apply", Self::Drop => "drop", - Self::Apply => "apply", - Self::RedoAppend => "redo_append", - Self::AppliedMarker => "applied_marker", Self::IdentityBind => "identity_bind", Self::MetadataHold => "metadata_hold", + Self::PassiveRead => "passive_read", } } /// The request a txn in `state` awaits a response for. pub(in crate::control::cluster::calvin::scheduler::driver::core) fn awaited_by( - state: Option, + state: CommitState, ) -> Self { match state { - Some(CommitState::Staged | CommitState::AwaitingVerdict) => Self::Stage, - Some(CommitState::AwaitingRedoResolve | CommitState::AwaitingResolveTurn) => { - Self::Resolve + CommitState::Staged | CommitState::AwaitingVerdict | CommitState::AwaitingRestage => { + Self::Stage } - Some(CommitState::AwaitingFlushTurn { .. }) => Self::Flush, - Some(CommitState::AwaitingResolve { - committed: true, .. - }) => Self::Flush, - Some(CommitState::AwaitingResolve { - committed: false, .. - }) => Self::Drop, - None => Self::Apply, + CommitState::AwaitingRedoResolve | CommitState::AwaitingResolveTurn => Self::Resolve, + CommitState::AwaitingRedoApply { .. } | CommitState::Following => Self::RedoApply, + CommitState::AwaitingDrop => Self::Drop, + CommitState::ReadingPassive => Self::PassiveRead, } } } @@ -205,9 +210,6 @@ impl Scheduler { step: HaltStep, error: String, ) { - // The txn stays unapplied here, so restart replay must reach its redo - // record. - self.hold_redo_records(txn_id); let reason = if self.node_shutting_down() { HaltReason::Draining } else { @@ -236,7 +238,7 @@ impl Scheduler { "calvin scheduler: apply halted; holding another txn unapplied" ); } - self.release_gates_on_halt(txn_id); + self.release_gates_on_halt(); return; } @@ -283,7 +285,7 @@ impl Scheduler { .record(report.clone()); } self.halt.first = Some(ApplyHalt { reason, report }); - self.release_gates_on_halt(txn_id); + self.release_gates_on_halt(); } /// Whether the node shuts down: the shutdown watch fired, or the diff --git a/nodedb/src/control/cluster/calvin/scheduler/driver/core/install_gate.rs b/nodedb/src/control/cluster/calvin/scheduler/driver/core/install_gate.rs index da4feba01..f195d9abe 100644 --- a/nodedb/src/control/cluster/calvin/scheduler/driver/core/install_gate.rs +++ b/nodedb/src/control/cluster/calvin/scheduler/driver/core/install_gate.rs @@ -1,46 +1,22 @@ // SPDX-License-Identifier: BUSL-1.1 -//! The scheduler's admission of a flush against its data group's snapshots. +//! The scheduler's view of its vShard's installed base. //! -//! A flush installs a committed transaction's slice on the vShard's core. -//! The data group's apply gate orders it against the group's snapshots: +//! A data-group snapshot install replaces the vShard's base and its applied +//! ledger. A scheduler started under an older base proposes nothing more: +//! the next scheduler reconcile starts it again from the installed state. //! -//! - A snapshot capture on this node holds the gate exclusive. A flush holds -//! it shared from its dispatch until its position is marked applied, so -//! the capture reads storage and applied positions that agree. -//! - A snapshot install on this node holds it exclusive too, and replaces -//! the vShard's base. A scheduler started under an older base installs -//! nothing more: the next scheduler reconcile starts it again from the -//! installed state. -//! -//! A flush the gate refuses waits for the gate's release and is tried again. - -use tokio::sync::watch; +//! The data group's apply gate orders every install against snapshot +//! captures and installs: a committed slice installs from the log, in the +//! apply loop, under that gate. use super::scheduler::Scheduler; use crate::control::security::auth_fence::cluster::group_of_vshard; use crate::control::state::SharedState; -/// What the gate says to one flush. -pub(in crate::control::cluster::calvin::scheduler::driver::core) enum FlushAdmission { - /// Dispatch, holding the permit until the position is marked applied. - /// `None` when no apply gate is wired here. - Go(Option), - /// A snapshot holds the gate: try again once it releases. - Wait, - /// A snapshot install replaced this vShard's base since this scheduler - /// started: install nothing. - Retired, -} - -/// The scheduler's view of its data group's apply gate. +/// The base generation a scheduler started under. pub(in crate::control::cluster::calvin::scheduler::driver::core) struct InstallGate { - /// The vShard's base generation this scheduler started under. generation: u64, - /// Changes each time a snapshot releases a group's gate. - released: Option>, - /// Whether a flush waits for a release. - waiting: bool, } impl InstallGate { @@ -51,72 +27,21 @@ impl InstallGate { ) -> Self { Self { generation: shared.calvin.bases.generation(vshard_id), - released: shared - .raft_apply_gates - .get() - .map(|gates| gates.subscribe_released()), - waiting: false, - } - } - - /// Admit one flush of `vshard_id`. - fn admit(&mut self, shared: &SharedState, vshard_id: u32) -> FlushAdmission { - if shared.calvin.bases.generation(vshard_id) != self.generation { - return FlushAdmission::Retired; - } - let Some(gates) = shared.raft_apply_gates.get() else { - return FlushAdmission::Go(None); - }; - let Ok(group_id) = group_of_vshard(shared, vshard_id) else { - // No routing table holds the vShard: no snapshot of it runs. - return FlushAdmission::Go(None); - }; - // Marked as seen before the try, so a release after it still wakes - // the wait. - if let Some(released) = self.released.as_mut() { - drop(released.borrow_and_update()); - } - match gates.try_apply(group_id) { - Some(permit) => { - self.waiting = false; - FlushAdmission::Go(Some(permit)) - } - None => { - self.waiting = true; - FlushAdmission::Wait - } } } - /// Whether a flush waits for the gate's release. - pub(in crate::control::cluster::calvin::scheduler::driver::core) fn is_waiting(&self) -> bool { - self.waiting - } - - /// Resolves once a snapshot released a group's gate since the last - /// admission. - pub(in crate::control::cluster::calvin::scheduler::driver::core) async fn released(&mut self) { - match self.released.as_mut() { - Some(released) => { - if released.changed().await.is_err() { - // The gates are gone: no snapshot holds one again. - self.waiting = false; - } - } - None => std::future::pending().await, - } + /// Whether a snapshot install replaced `vshard_id`'s base since this + /// scheduler started. + pub(in crate::control::cluster::calvin::scheduler::driver::core) fn is_retired( + &self, + shared: &SharedState, + vshard_id: u32, + ) -> bool { + shared.calvin.bases.generation(vshard_id) != self.generation } } impl Scheduler { - /// Admit the flush of the lowest unfinished transaction. - pub(in crate::control::cluster::calvin::scheduler::driver::core) fn admit_flush( - &mut self, - ) -> FlushAdmission { - let vshard_id = self.vshard_id; - self.install_gate.admit(&self.shared, vshard_id) - } - /// Record that this replica's Calvin state of the vShard has a hole, and /// make its data-group replica refuse log entries until a snapshot /// brings the state back. @@ -151,76 +76,30 @@ impl Scheduler { #[cfg(test)] mod tests { - use std::sync::{Arc, RwLock}; - use std::time::Duration; - - use nodedb_cluster::{GroupApplyGates, RoutingTable}; + use std::sync::Arc; use super::*; use crate::bridge::dispatch::Dispatcher; use crate::wal::WalManager; - /// A snapshot capture holds the group's gate exclusive. A flush that - /// starts meanwhile waits, and a capture that starts while a flush is - /// out waits until the flush marked its position and dropped its - /// permit. So every capture sees each vShard's flushes either whole or - /// not at all, on every core. A snapshot install that replaced the - /// vShard's base stops the scheduler started before it. - #[tokio::test] - async fn a_capture_never_sees_a_flush_half_applied() { + /// A snapshot install that replaced the vShard's base retires the + /// scheduler started before it. + #[test] + fn an_installed_base_retires_the_older_scheduler() { let dir = tempfile::tempdir().expect("tempdir"); let wal = Arc::new(WalManager::open_for_testing(&dir.path().join("gate.wal")).expect("wal")); let (dispatcher, _sides) = Dispatcher::new(1, 64); - let mut state = SharedState::new(dispatcher, wal).expect("shared state"); - Arc::get_mut(&mut state) - .expect("sole owner of fresh state") - .cluster_routing = Some(Arc::new(RwLock::new(RoutingTable::uniform(2, &[1], 1)))); - let gates = Arc::new(GroupApplyGates::new()); + let state = SharedState::new(dispatcher, wal).expect("shared state"); let vshard_id = 0; - let group_id = group_of_vshard(&state, vshard_id).expect("the vShard has a group"); - gates.mount(group_id); - assert!(state.raft_apply_gates.set(Arc::clone(&gates)).is_ok()); - - let mut gate = InstallGate::new(&state, vshard_id); - let capture = gates.install(group_id).await; - assert!(matches!( - gate.admit(&state, vshard_id), - FlushAdmission::Wait - )); - assert!(gate.is_waiting()); - drop(capture); - tokio::time::timeout(Duration::from_secs(5), gate.released()) - .await - .expect("the capture's release wakes the waiting flush"); - - let FlushAdmission::Go(Some(permit)) = gate.admit(&state, vshard_id) else { - panic!("an open gate admits the flush with a permit"); - }; - assert!(!gate.is_waiting()); - let next_capture = { - let gates = Arc::clone(&gates); - tokio::spawn(async move { drop(gates.install(group_id).await) }) - }; - tokio::time::sleep(Duration::from_millis(50)).await; - assert!( - !next_capture.is_finished(), - "a capture waits for the flush in flight" - ); - drop(permit); - tokio::time::timeout(Duration::from_secs(5), next_capture) - .await - .expect("the capture starts once the flush finished") - .expect("capture task"); + let gate = InstallGate::new(&state, vshard_id); + assert!(!gate.is_retired(&state, vshard_id)); state .calvin .bases .record_snapshot(state.credentials.catalog(), &[vshard_id], 9) .expect("record the installed base"); - assert!(matches!( - gate.admit(&state, vshard_id), - FlushAdmission::Retired - )); + assert!(gate.is_retired(&state, vshard_id)); } } diff --git a/nodedb/src/control/cluster/calvin/scheduler/driver/core/intake.rs b/nodedb/src/control/cluster/calvin/scheduler/driver/core/intake.rs index 06cbf77b9..4c45112e3 100644 --- a/nodedb/src/control/cluster/calvin/scheduler/driver/core/intake.rs +++ b/nodedb/src/control/cluster/calvin/scheduler/driver/core/intake.rs @@ -147,7 +147,7 @@ impl Scheduler { #[cfg(test)] mod tests { use super::*; - use std::collections::BTreeSet; + use std::collections::BTreeMap; use std::sync::Arc; use std::sync::atomic::Ordering; use std::time::{Duration, Instant}; @@ -157,6 +157,7 @@ mod tests { use nodedb_types::TenantId; use tokio::sync::mpsc; + use crate::control::cluster::calvin::scheduler::driver::core::test_proposer::lead_data_group; use crate::control::cluster::calvin::scheduler::driver::core::test_support::{ build_test_scheduler, build_test_scheduler_with_data_side, fill_tenant_inflight, make_sequenced_txn, make_validate_only_txn, release_filler, spawn_scheduler_loop, @@ -189,7 +190,7 @@ mod tests { fn blocked_fixture(epoch: u64) -> BlockedTxn { BlockedTxn { txn: make_sequenced_txn(epoch, 0), - keys: BTreeSet::new(), + keys: BTreeMap::new(), // no-determinism: test-only blocked_at timestamp for a fabricated BlockedTxn fixture. blocked_at: Instant::now(), } @@ -236,6 +237,7 @@ mod tests { let registry = CalvinCompletionRegistry::new_detached(); let (mut scheduler, _dir, mut data_side) = build_test_scheduler_with_data_side(test_coll_vshard(), registry); + lead_data_group(&mut scheduler); let shared = Arc::clone(&scheduler.shared); let fillers = fill_tenant_inflight(&shared, &mut data_side, TenantId::new(1)); scheduler @@ -317,12 +319,10 @@ mod tests { let (mut scheduler, _dir, _data_side) = build_test_scheduler_with_data_side(test_coll_vshard(), registry); let held = TxnId::new(3, 0); - scheduler - .pending - .insert(held, staged_pending(make_validate_only_txn(3, 0), held)); - scheduler - .handle_completion(held, RequestId::new(9), None) - .await; + let mut pending = staged_pending(make_validate_only_txn(3, 0), held); + pending.awaiting = Some(RequestId::new(9)); + scheduler.pending.insert(held, pending); + scheduler.handle_completion(held, RequestId::new(9), None); assert_eq!(scheduler.intake_closure(), Some(IntakeClosure::ApplyHalted)); let metrics = Arc::clone(&scheduler.metrics); diff --git a/nodedb/src/control/cluster/calvin/scheduler/driver/core/mod.rs b/nodedb/src/control/cluster/calvin/scheduler/driver/core/mod.rs index d4895393b..5583080ca 100644 --- a/nodedb/src/control/cluster/calvin/scheduler/driver/core/mod.rs +++ b/nodedb/src/control/cluster/calvin/scheduler/driver/core/mod.rs @@ -3,38 +3,49 @@ //! Calvin scheduler driver core. //! //! One [`Scheduler`] task runs per vshard hosted on this node. It receives -//! sequenced txns from the sequencer, acquires deterministic locks, dispatches -//! static / dependent-read transactions to the Data Plane, waits for executor -//! responses, and writes `CalvinApplied` WAL records. Each sub-module owns one -//! concern; see that file's own doc comment for what it does. +//! sequenced txns from the sequencer and acquires deterministic locks on +//! every replica. The vShard's data-group leader stages each granted txn on +//! the Data Plane, votes, and proposes a committed slice's stamped redo to +//! the data group. Every replica installs that redo from the log, and the +//! apply loop reports each install through the scheduler's inbox. Each +//! sub-module owns one concern; see that file's own doc comment for what it +//! does. //! //! All bookkeeping uses `BTreeMap`/`BTreeSet` — never `HashMap`/`HashSet` — //! and dispatch order is `(epoch, position)` order. `Instant::now()` is used -//! only for lock-wait latency metrics and the dependent-read barrier -//! `timeout_at`, both off the WAL-influencing path; every call site carries a -//! `// no-determinism:` marker. +//! only for lock-wait latency metrics, the instant a leader proposes a +//! dependent-read barrier's timeout entry, and the restage backoff, all off +//! the WAL-influencing path; every call site carries a `// no-determinism:` +//! marker. +mod apply_result; pub mod catch_up; pub mod commit_redo; -pub mod commit_resolution_dispatch; pub mod commit_resolve; pub mod completion_route; mod cut_marker; pub mod deferred; pub mod dispatch; -pub mod flush_turn; +mod drop_dispatch; pub mod halt; mod install_gate; pub mod intake; mod metadata_hold; pub mod owed; pub mod parts; +mod parts_error; pub mod parts_lane; +mod passive_read; pub mod process; pub mod propose; pub mod read_result; -mod redo_window; +mod read_timeout; +pub mod redo_applied; +pub mod redo_propose; pub mod request; +pub mod resolve_turn; +mod restage; +pub mod role; pub mod routing; pub mod run_loop; pub mod scheduler; @@ -44,7 +55,6 @@ pub mod staged_vote; mod test_proposer; #[cfg(test)] mod test_support; -pub mod write_version_record; pub use propose::{CalvinReadResultProposal, propose_calvin_read_result}; pub use scheduler::{Scheduler, SchedulerParams}; diff --git a/nodedb/src/control/cluster/calvin/scheduler/driver/core/owed/entry.rs b/nodedb/src/control/cluster/calvin/scheduler/driver/core/owed/entry.rs index 51f485d9a..6f3f43733 100644 --- a/nodedb/src/control/cluster/calvin/scheduler/driver/core/owed/entry.rs +++ b/nodedb/src/control/cluster/calvin/scheduler/driver/core/owed/entry.rs @@ -25,10 +25,6 @@ pub enum SchedulerProposal { /// the coordinator reads: the timeseries install counts of the apply. /// Empty when there are none. CompletionAck { result: Vec }, - /// The active executor saw its OLLP prediction drift. - OllpMismatch, - /// The txn's local plan routing failed for good. - RoutingFailed { detail: String }, } impl SchedulerProposal { @@ -37,8 +33,6 @@ impl SchedulerProposal { match self { Self::Vote { .. } => OwedKind::Vote, Self::CompletionAck { .. } => OwedKind::CompletionAck, - Self::OllpMismatch => OwedKind::OllpMismatch, - Self::RoutingFailed { .. } => OwedKind::RoutingFailed, } } @@ -67,12 +61,6 @@ impl SchedulerProposal { result: result.clone(), from_node: node_id, }, - Self::OllpMismatch => SequencerEntry::OllpMismatch { epoch, position }, - Self::RoutingFailed { detail } => SequencerEntry::TxnRoutingFailed { - epoch, - position, - detail: detail.clone(), - }, } } } @@ -82,8 +70,6 @@ impl SchedulerProposal { pub enum OwedKind { Vote, CompletionAck, - OllpMismatch, - RoutingFailed, } impl OwedKind { @@ -97,24 +83,17 @@ impl OwedKind { match self { Self::Vote => sequencer_propose_kind::VOTE, Self::CompletionAck => sequencer_propose_kind::COMPLETION_ACK, - Self::OllpMismatch => sequencer_propose_kind::OLLP_MISMATCH, - Self::RoutingFailed => sequencer_propose_kind::ROUTING_FAILED, } } - /// Whether only the vShard's data-group leader proposes this kind. - pub fn leader_only(self) -> bool { - matches!(self, Self::Vote | Self::CompletionAck) - } - /// Whether this node applied the entry of this kind. /// /// `progress` is the registry's view of the txn for the scheduler's /// vShard. `entry_seen` is whether the registry held an entry for the - /// txn at an earlier check. The registry removes an entry only once the - /// txn's outcome fired, and a txn with a fired outcome needs no entry of - /// any kind. A missing entry that was never seen means this node has not - /// seeded the txn, so the entry is still owed. + /// txn at an earlier check. The registry evicts only a terminal entry: + /// its verdict is stored and every participant acked, so the txn needs + /// no entry of any kind. A missing entry that was never seen means this + /// node has not seeded the txn, so the entry is still owed. /// /// A stored verdict also settles a vote: the verdict forms only once /// every participant's vote is in the tally. @@ -125,8 +104,6 @@ impl OwedKind { match self { Self::Vote => p.voted || p.has_verdict, Self::CompletionAck => p.acked, - Self::OllpMismatch => p.mismatched, - Self::RoutingFailed => p.routing_failed, } } } @@ -146,8 +123,8 @@ pub struct OwedEntry { /// Owed entries keyed by txn and kind, in deterministic order. /// /// Bounded: it holds at most one entry per (txn, kind), and a txn owes at -/// most two kinds (a vote and a completion ack, or a single terminal -/// signal). An entry leaves once this node applies it. Entries pile up only +/// most two kinds, a vote and a completion ack. An entry leaves once this +/// node applies it. Entries pile up only /// while no sequencer leader is reachable, and then the sequencer admits no /// new txns either. pub type OwedEntries = BTreeMap<(TxnId, OwedKind), OwedEntry>; @@ -161,17 +138,10 @@ mod tests { voted: false, acked: false, has_verdict: false, - mismatched: false, - routing_failed: false, } } - const ALL_KINDS: [OwedKind; 4] = [ - OwedKind::Vote, - OwedKind::CompletionAck, - OwedKind::OllpMismatch, - OwedKind::RoutingFailed, - ]; + const ALL_KINDS: [OwedKind; 2] = [OwedKind::Vote, OwedKind::CompletionAck]; #[test] fn nothing_is_applied_while_the_registry_shows_no_effect() { @@ -190,20 +160,10 @@ mod tests { acked: true, ..progress() }; - let mismatched = ParticipantProgress { - mismatched: true, - ..progress() - }; - let routing_failed = ParticipantProgress { - routing_failed: true, - ..progress() - }; assert!(OwedKind::Vote.is_applied(Some(voted), false)); assert!(!OwedKind::CompletionAck.is_applied(Some(voted), false)); assert!(OwedKind::CompletionAck.is_applied(Some(acked), false)); assert!(!OwedKind::Vote.is_applied(Some(acked), false)); - assert!(OwedKind::OllpMismatch.is_applied(Some(mismatched), false)); - assert!(OwedKind::RoutingFailed.is_applied(Some(routing_failed), false)); } #[test] @@ -247,5 +207,15 @@ mod tests { reason: AbortReason::SerializationConflict } )); + assert!(matches!( + SchedulerProposal::Vote { + abort: Some(AbortReason::PlanRejected) + } + .entry(txn, 9, 1), + SequencerEntry::AbortVote { + reason: AbortReason::PlanRejected, + .. + } + )); } } diff --git a/nodedb/src/control/cluster/calvin/scheduler/driver/core/owed/retry.rs b/nodedb/src/control/cluster/calvin/scheduler/driver/core/owed/retry.rs index 7ddf13a87..4dc9142fd 100644 --- a/nodedb/src/control/cluster/calvin/scheduler/driver/core/owed/retry.rs +++ b/nodedb/src/control/cluster/calvin/scheduler/driver/core/owed/retry.rs @@ -5,9 +5,7 @@ //! Re-proposing is safe because every entry kind applies idempotently in the //! completion registry. A vote is stored per vShard, so a repeat overwrites //! it with the same value, and the verdict it completes is emitted once. An -//! ack is a set insert per vShard, and the completion fires once. The -//! mismatch and routing-failure signals set a flag, and each fires its -//! waiter once. +//! ack is a set insert per vShard, and the completion fires once. use tracing::{debug, error}; @@ -24,6 +22,13 @@ impl Scheduler { /// The entry stays owed until this node's completion registry shows it /// applied. [`Self::retry_owed_sequencer_entries`] proposes it again on /// the stall tick until then. + /// + /// A vote and a `CompletionAck` answer for the vShard on every node: the + /// first one the sequencer log holds counts. So only the vShard's + /// data-group leader proposes them. A node that left the group leads none + /// of it, so an entry from its stale state never enters the log. Every + /// other replica keeps the entry owed and proposes it if it comes to lead + /// before one applies. Which entry counts then depends on the log alone. pub(in crate::control::cluster::calvin::scheduler::driver::core) fn propose_sequencer_entry( &mut self, txn_id: TxnId, @@ -46,7 +51,7 @@ impl Scheduler { } }; let entry_seen = self.participant_progress(txn_id).is_some(); - let in_flight = self.may_propose(kind) + let in_flight = self.is_group_leader() && propose_owed( self.sequencer_proposer.as_ref(), self.vshard_id, @@ -81,11 +86,12 @@ impl Scheduler { !kind.is_applied(progress, owed.entry_seen) }); + if !leads { + // Every entry stays owed: this replica proposes it if it comes + // to lead. + return; + } for ((txn_id, kind), owed) in self.owed.iter_mut() { - if kind.leader_only() && !leads { - // Stays owed: this replica proposes it if it comes to lead. - continue; - } if owed.in_flight { owed.in_flight = false; continue; @@ -102,18 +108,6 @@ impl Scheduler { } } - /// Whether this scheduler can propose an entry of `kind` now. - /// - /// A vote and a `CompletionAck` answer for the vShard on every node: the - /// first one the sequencer log holds counts. So only the vShard's - /// data-group leader proposes them. A node that left the group leads none - /// of it, so an entry from its stale state never enters the log. Every - /// other replica keeps the entry owed and proposes it if it comes to lead - /// before one applies. Which entry counts then depends on the log alone. - fn may_propose(&self, kind: OwedKind) -> bool { - !kind.leader_only() || self.is_group_leader() - } - /// The registry's view of `txn_id` for this scheduler's vShard. fn participant_progress(&self, txn_id: TxnId) -> Option { self.registry @@ -158,7 +152,7 @@ mod tests { use nodedb_cluster::calvin::{ParticipantVote, SequencerEntry}; use super::*; - use crate::bridge::envelope::Status; + use crate::bridge::envelope::{StageVote, Status}; use crate::control::cluster::calvin::scheduler::driver::core::test_proposer::{ CapturingProposer, elect_data_group_leader, }; @@ -189,7 +183,10 @@ mod tests { .pending .insert(txn_id, staged_pending(make_sequenced_txn(20, 1), txn_id)); - scheduler.resolve_staged_commit(txn_id, &staged_response(Status::Ok, Some(true))); + scheduler.resolve_staged_commit( + txn_id, + &staged_response(Status::Ok, Some(StageVote::Commit)), + ); assert_eq!(proposer.attempt_count(), 1, "the first proposal is refused"); scheduler.retry_owed_sequencer_entries(); @@ -238,22 +235,14 @@ mod tests { #[tokio::test] async fn completion_ack_is_reproposed_until_the_registry_records_it() { let txn_id = TxnId::new(21, 0); - let (mut scheduler, _dir) = scheduler_with_pending( - txn_id, - CommitState::AwaitingResolve { - committed: false, - redo_lsn: None, - }, - ); + let (mut scheduler, _dir) = scheduler_with_pending(txn_id, CommitState::AwaitingDrop); let proposer = CapturingProposer::failing_first(1); scheduler.sequencer_proposer = proposer.clone(); // Only the vShard's group leader proposes the ack. elect_data_group_leader(&scheduler); scheduler.registry.seed_expected(cluster_txn_id(txn_id), 2); - scheduler - .finish_resolved_commit(txn_id, staged_response(Status::Ok, None), false, None) - .await; + scheduler.finish_drop(txn_id, &staged_response(Status::Ok, None)); assert!(!scheduler.pending.contains_key(&txn_id)); assert_eq!(proposer.attempt_count(), 1, "the first proposal is refused"); @@ -289,14 +278,15 @@ mod tests { ); } - /// The registry removes a txn's entry once its outcome fired. An owed - /// entry for such a txn is settled, not proposed again. + /// The registry evicts a txn's entry once it is terminal. An owed entry + /// for such a txn is settled, not proposed again. #[tokio::test] - async fn entry_for_a_txn_whose_outcome_fired_is_settled() { + async fn entry_for_an_evicted_txn_is_settled() { let (mut scheduler, _dir) = build_test_scheduler(VSHARD); let proposer = CapturingProposer::failing_first(1); scheduler.sequencer_proposer = proposer.clone(); elect_data_group_leader(&scheduler); + scheduler.registry.set_waiterless_ttl(Duration::ZERO); let txn_id = TxnId::new(22, 0); let _outcome = scheduler .registry @@ -306,6 +296,10 @@ mod tests { txn_id, SchedulerProposal::CompletionAck { result: Vec::new() }, ); + scheduler.registry.note_verdict( + cluster_txn_id(txn_id), + nodedb_cluster::calvin::VerdictOutcome::Commit, + ); scheduler .registry .note_completion_ack(cluster_txn_id(txn_id), VSHARD); @@ -338,37 +332,6 @@ mod tests { assert_eq!(proposer.attempt_count(), 1); } - /// A follower that staged a txn owes an abort vote. It proposes the vote - /// only once it leads the vShard's group and no vote has applied, so the - /// txn ends in a retryable abort instead of waiting on a lost leader. - #[tokio::test] - async fn a_follower_that_comes_to_lead_votes_abort_for_its_staged_txn() { - let (mut scheduler, _dir) = build_test_scheduler(VSHARD); - let proposer = CapturingProposer::accepting(); - scheduler.sequencer_proposer = proposer.clone(); - let txn_id = TxnId::new(26, 0); - scheduler.registry.seed_expected(cluster_txn_id(txn_id), 2); - scheduler - .pending - .insert(txn_id, staged_pending(make_sequenced_txn(26, 0), txn_id)); - - scheduler.resolve_staged_commit(txn_id, &staged_response(Status::Ok, Some(true))); - scheduler.retry_owed_sequencer_entries(); - assert_eq!(proposer.attempt_count(), 0, "a follower casts no vote"); - - elect_data_group_leader(&scheduler); - scheduler.retry_owed_sequencer_entries(); - assert_eq!( - proposer.accepted(), - vec![SequencerEntry::AbortVote { - epoch: 26, - position: 0, - vshard: VSHARD, - reason: nodedb_cluster::calvin::AbortReason::SerializationConflict, - }] - ); - } - /// A txn this node never seeded keeps its entry owed: a missing registry /// entry that was never seen does not settle it. #[tokio::test] @@ -376,9 +339,15 @@ mod tests { let (mut scheduler, _dir) = build_test_scheduler(VSHARD); let proposer = CapturingProposer::accepting(); scheduler.sequencer_proposer = proposer.clone(); + elect_data_group_leader(&scheduler); let txn_id = TxnId::new(23, 0); - scheduler.propose_sequencer_entry(txn_id, SchedulerProposal::OllpMismatch); + scheduler.propose_sequencer_entry( + txn_id, + SchedulerProposal::Vote { + abort: Some(nodedb_cluster::calvin::AbortReason::PlanRejected), + }, + ); scheduler.retry_owed_sequencer_entries(); scheduler.retry_owed_sequencer_entries(); @@ -392,10 +361,11 @@ mod tests { let (mut scheduler, _dir) = build_test_scheduler(VSHARD); let proposer = CapturingProposer::failing_first(1); scheduler.sequencer_proposer = proposer.clone(); + elect_data_group_leader(&scheduler); scheduler.propose_sequencer_entry( TxnId::new(24, 0), - SchedulerProposal::RoutingFailed { - detail: "unroutable".to_string(), + SchedulerProposal::Vote { + abort: Some(nodedb_cluster::calvin::AbortReason::PlanRejected), }, ); assert!(proposer.accepted().is_empty()); diff --git a/nodedb/src/control/cluster/calvin/scheduler/driver/core/parts.rs b/nodedb/src/control/cluster/calvin/scheduler/driver/core/parts.rs index c0601d7f3..b82f19882 100644 --- a/nodedb/src/control/cluster/calvin/scheduler/driver/core/parts.rs +++ b/nodedb/src/control/cluster/calvin/scheduler/driver/core/parts.rs @@ -23,7 +23,7 @@ //! every replica alike. //! - A txn whose locks are granted before its parts arrived waits in //! `awaiting`, holding its locks. It counts as unfinished, so every later -//! txn's flush on this vShard waits for it. +//! whole-collection resolve on this vShard waits for it. //! - Once every part that targets this vShard arrived, the txn stages as a //! single-entry txn whose plans are its local tasks. Its manifest keeps //! the whole-transaction facts the dispatch reads. @@ -35,8 +35,9 @@ use std::sync::Arc; use nodedb_cluster::calvin::types::{SequencedTxn, TaskChunk, TxnIdWire}; use nodedb_physical::physical_plan::PhysicalPlan; -use tracing::{error, warn}; +use tracing::warn; +use super::parts_error::PartsError; use super::routing::{PlanRouting, plan_vshard_in_database}; use super::scheduler::Scheduler; use crate::control::cluster::calvin::scheduler::lock_manager::TxnId; @@ -84,12 +85,12 @@ struct Assembler { /// The task whose chunks are being joined. spanning: Option, /// Why a part cannot be read. The txn fails at dispatch. - failed: Option, + failed: Option, } impl Assembler { - fn fail(&mut self, detail: impl FnOnce() -> String) { - self.failed.get_or_insert_with(detail); + fn fail(&mut self, error: impl FnOnce() -> PartsError) { + self.failed.get_or_insert_with(error); } /// Keep `plan`, the txn's task `task` from part `index`, when it routes @@ -102,13 +103,17 @@ impl Assembler { } } PlanRouting::ControlPlaneOnly => { - self.fail(|| format!("part {index} task {task} is a control-plane-only plan")); + self.fail(|| PartsError::ControlPlaneOnly { part: index, task }); } PlanRouting::NotAWrite => { - self.fail(|| format!("part {index} task {task} is not a write")); + self.fail(|| PartsError::NotAWrite { part: index, task }); } PlanRouting::Unroutable(reason) => { - self.fail(|| format!("part {index} task {task} is unroutable: {reason}")); + self.fail(|| PartsError::Unroutable { + part: index, + task, + reason, + }); } } } @@ -117,7 +122,7 @@ impl Assembler { fn take_tasks(&mut self, index: u32, first_task: u32, plans: &[u8], vshard_id: u32) { if let Some(spanning) = &self.spanning { let task = spanning.task; - self.fail(|| format!("part {index} interrupts the chunks of task {task}")); + self.fail(|| PartsError::InterruptsChunks { part: index, task }); return; } match super::super::helpers::decode_plans(plans) { @@ -126,7 +131,10 @@ impl Assembler { self.keep_if_local(index, task, plan, vshard_id); } } - Err(error) => self.fail(|| format!("part {index} does not decode: {error}")), + Err(error) => self.fail(|| PartsError::PartUndecodable { + part: index, + source: Box::new(error), + }), } } @@ -148,7 +156,7 @@ impl Assembler { } }; if !continues { - self.fail(|| format!("part {index} does not continue the chunks of task {task}")); + self.fail(|| PartsError::BrokenChunks { part: index, task }); self.spanning = None; return; } @@ -166,12 +174,15 @@ impl Assembler { return; }; if len > whole.total_len { - self.fail(|| format!("task {task} chunks run past its {} bytes", whole.total_len)); + self.fail(|| PartsError::ChunksOverrun { + task, + total_len: whole.total_len, + }); return; } match nodedb_physical::physical_plan::wire::decode(&whole.bytes) { Ok(plan) => self.keep_if_local(index, task, plan, vshard_id), - Err(error) => self.fail(|| format!("task {task} does not decode: {error}")), + Err(source) => self.fail(|| PartsError::TaskUndecodable { task, source }), } } } @@ -224,6 +235,18 @@ impl PartsState { ) -> Option { self.awaiting.keys().next().copied() } + + /// Stop waiting for the parts of `txn_id`, which finished without them: + /// its redo installed from another replica's stage. Returns the owner of + /// the locks it holds. + pub(in crate::control::cluster::calvin::scheduler::driver::core) fn release_awaiting( + &mut self, + txn_id: TxnId, + ) -> Option { + let awaiting = self.awaiting.remove(&txn_id)?; + self.assemblies.remove(&txn_id); + Some(awaiting.lock_owner) + } } impl Scheduler { @@ -253,14 +276,15 @@ impl Scheduler { /// /// A multi-part txn with every part that targets this vShard is /// assembled here. One still missing parts waits with its locks. An - /// abandoned one, and one whose part cannot be read, completes here. + /// abandoned one completes here. One whose part cannot be read votes + /// abort on its rejected plans. /// /// Returns the txn to dispatch: a single-entry txn, or a multi-part one /// with every part that targets this vShard assembled. `None` means /// nothing dispatches: the txn waits for parts, or it completed. pub(in crate::control::cluster::calvin::scheduler::driver::core) fn gate_on_parts( &mut self, - txn: SequencedTxn, + mut txn: SequencedTxn, txn_id: TxnId, lock_owner: TxnId, ) -> Option { @@ -286,11 +310,11 @@ impl Scheduler { } let vshard_id = self.vshard_id; let assembled = match self.parts.assemblies.remove(&txn_id) { - Some(assembly) => assemble(txn, assembly, vshard_id), - None => Err("the txn reached dispatch with no assembly of its parts".to_owned()), + Some(assembly) => assemble(&mut txn, assembly, vshard_id), + None => Err(PartsError::NoAssembly), }; match assembled { - Ok(txn) => Some(txn), + Ok(()) => Some(txn), Err(detail) => { let error = crate::Error::Internal { detail: format!( @@ -298,9 +322,7 @@ impl Scheduler { txn_id.epoch, txn_id.position, self.vshard_id ), }; - error!(vshard_id = self.vshard_id, %error, "calvin scheduler: part unreadable"); - self.propose_routing_failure(txn_id, &error); - self.on_unpending_txn_complete(txn_id, lock_owner); + self.reject_plan(txn, txn_id, lock_owner, &error); None } } @@ -380,12 +402,8 @@ impl Scheduler { /// /// Reads the parts in index order, so the result depends only on their /// contents. Every replica that holds the same parts assembles the same -/// plans, or fails alike. -fn assemble( - mut txn: SequencedTxn, - assembly: Assembly, - vshard_id: u32, -) -> Result { +/// plans, or fails alike. A failure leaves `txn` unchanged. +fn assemble(txn: &mut SequencedTxn, assembly: Assembly, vshard_id: u32) -> Result<(), PartsError> { let mut assembler = Assembler { database_id: assembly.database_id, local: Vec::new(), @@ -404,12 +422,11 @@ fn assemble( return Err(failed); } if let Some(spanning) = assembler.spanning { - return Err(format!( - "task {} stopped at byte {} of {}", - spanning.task, - spanning.bytes.len(), - spanning.total_len - )); + return Err(PartsError::Unfinished { + task: spanning.task, + received: spanning.bytes.len(), + total_len: spanning.total_len, + }); } let mut local = assembler.local; local.sort_by_key(|(task, _)| *task); @@ -419,26 +436,35 @@ fn assemble( .map(|(local_index, _)| local_index) .collect(); let plans: Vec<&PhysicalPlan> = local.iter().map(|(_, plan)| plan).collect(); - txn.tx_class.plans = - zerompk::to_msgpack_vec(&plans).map_err(|e| format!("local plans do not encode: {e}"))?; + let bytes = zerompk::to_msgpack_vec(&plans) + .map_err(|source| PartsError::PlansDoNotEncode { source })?; + txn.tx_class.plans = bytes; txn.tx_class.body_plans = body_plans; - Ok(txn) + Ok(()) } #[cfg(test)] mod tests { - use nodedb_cluster::calvin::CalvinCompletionRegistry; use nodedb_cluster::calvin::types::{ MultiPartPlans, PartStreamId, SchedulerInput, VShardParts, }; + use nodedb_cluster::calvin::{AbortReason, CalvinCompletionRegistry, SequencerEntry}; use nodedb_physical::physical_plan::DocumentOp; use nodedb_physical::physical_plan::wire as plan_wire; use nodedb_types::QualifiedCollection; use super::*; + use crate::bridge::dispatch::CoreChannelDataSide; + use crate::bridge::envelope::Status; + use crate::control::cluster::calvin::scheduler::driver::core::owed::OwedKind; + use crate::control::cluster::calvin::scheduler::driver::core::test_proposer::{ + CapturingProposer, lead_data_group, + }; use crate::control::cluster::calvin::scheduler::driver::core::test_support::{ - build_test_scheduler_with_data_side, make_local_write_txn, test_coll_vshard, + build_test_scheduler_with_data_side, make_local_write_txn, staged_response, + test_coll_vshard, }; + use crate::control::cluster::calvin::scheduler::driver::types::CommitState; use crate::types::DatabaseId; fn truncate_plan() -> PhysicalPlan { @@ -514,6 +540,102 @@ mod tests { plan_wire::encode_batch(&vec![truncate_plan()]).expect("encode one truncate") } + /// A scheduler on the `test_coll` vShard that leads its data group and + /// captures its proposals. + fn leading_scheduler() -> ( + Scheduler, + tempfile::TempDir, + CoreChannelDataSide, + Arc, + ) { + let (mut scheduler, dir, data_side) = build_test_scheduler_with_data_side( + test_coll_vshard(), + CalvinCompletionRegistry::new_detached(), + ); + let proposer = CapturingProposer::accepting(); + scheduler.sequencer_proposer = proposer.clone(); + lead_data_group(&mut scheduler); + (scheduler, dir, data_side, proposer) + } + + /// Whether a truncate of `test_coll` reached the Data Plane. Drains the + /// request ring. + fn truncate_reached_data_plane(data_side: &mut CoreChannelDataSide) -> bool { + let mut reached = false; + while let Ok(request) = data_side.request_rx.try_pop() { + reached |= request.inner.plan == truncate_plan(); + } + reached + } + + /// `txn_id` is rejected before it stages. It waits in `pending` for its + /// verdict with a stage error, owes a `PlanRejected` vote, and holds its + /// locks: `next`, on the same key, waits behind it. At the abort verdict + /// it drops, marks its position applied, and frees its locks, so `next` + /// stages. + async fn rejected_then_dropped_at_its_verdict( + scheduler: &mut Scheduler, + data_side: &mut CoreChannelDataSide, + proposer: &CapturingProposer, + txn_id: TxnId, + next: SequencedTxn, + ) { + let pending = scheduler + .pending + .get(&txn_id) + .expect("waits for its verdict"); + assert_eq!(pending.commit_state, CommitState::AwaitingVerdict); + assert!(pending.stage_error.is_some(), "the rejection is recorded"); + assert!(!scheduler.parts.is_awaiting(txn_id)); + assert!(scheduler.owed.contains_key(&(txn_id, OwedKind::Vote))); + assert!(proposer.accepted().contains(&SequencerEntry::AbortVote { + epoch: txn_id.epoch, + position: txn_id.position, + vshard: test_coll_vshard(), + reason: AbortReason::PlanRejected, + })); + assert!(!scheduler.applied.is_applied(txn_id.epoch, txn_id.position)); + + let next_id = TxnId::new(next.epoch, next.position); + scheduler.process_scheduler_input(SchedulerInput::Txn(Box::new(next))); + assert!( + scheduler.blocked.contains_key(&next_id), + "the rejected txn holds its locks" + ); + + scheduler.registry.note_verdict( + nodedb_cluster::calvin::TxnId::new(txn_id.epoch, txn_id.position), + nodedb_cluster::calvin::VerdictOutcome::Abort(AbortReason::PlanRejected), + ); + scheduler.resume_on_verdict(txn_id, false); + assert_eq!( + scheduler.pending.get(&txn_id).map(|p| p.commit_state), + Some(CommitState::AwaitingDrop) + ); + assert!( + !truncate_reached_data_plane(data_side), + "no plan of the rejected txn stages" + ); + assert!(!scheduler.applied.is_applied(txn_id.epoch, txn_id.position)); + + scheduler.finish_drop(txn_id, &staged_response(Status::Ok, None)); + assert!(!scheduler.pending.contains_key(&txn_id)); + assert!(scheduler.applied.is_applied(txn_id.epoch, txn_id.position)); + assert!( + proposer.accepted().iter().any(|entry| matches!( + entry, + SequencerEntry::CompletionAck { epoch, position, .. } + if *epoch == txn_id.epoch && *position == txn_id.position + )), + "the dropped txn acks" + ); + assert!(!scheduler.blocked.contains_key(&next_id)); + assert!( + scheduler.pending.contains_key(&next_id), + "the freed locks let the next txn stage" + ); + } + /// A txn whose locks are granted before its parts holds them and waits. /// It stages once the last part that targets this vShard arrived, with /// every local task of every part. @@ -614,13 +736,11 @@ mod tests { assert_eq!(plans, vec![truncate_plan()]); } - /// A chunk that skips bytes fails the txn: nothing stages. + /// A chunk that skips bytes rejects the txn before it stages. It votes + /// abort and drops at the verdict. #[tokio::test] - async fn a_chunk_that_skips_bytes_stages_nothing() { - let (mut scheduler, _dir, _data_side) = build_test_scheduler_with_data_side( - test_coll_vshard(), - CalvinCompletionRegistry::new_detached(), - ); + async fn a_chunk_that_skips_bytes_is_rejected_before_it_stages() { + let (mut scheduler, _dir, mut data_side, proposer) = leading_scheduler(); let txn_id = TxnId::new(8, 0); let task = zerompk::to_msgpack_vec(&truncate_plan()).expect("encode one task"); let total = task.len() as u64; @@ -628,30 +748,35 @@ mod tests { scheduler.process_scheduler_input(chunk_part(8, 0, 0, &task[..2], 0, total)); scheduler.process_scheduler_input(chunk_part(8, 1, 0, &task[3..], 3, total)); - assert!(!scheduler.pending.contains_key(&txn_id), "nothing staged"); - assert!(scheduler.applied.is_applied(8, 0), "the txn completed"); + rejected_then_dropped_at_its_verdict( + &mut scheduler, + &mut data_side, + &proposer, + txn_id, + plain_write(8, 1), + ) + .await; } - /// A late part that does not decode fails the txn before it stages: the - /// earlier part's task never stages, and the txn's locks are released. + /// A late part that does not decode rejects the txn before it stages: + /// the earlier part's task never reaches the Data Plane. The txn votes + /// abort and drops at the verdict. #[tokio::test] - async fn a_late_part_that_fails_stages_nothing_of_the_earlier_parts() { - let (mut scheduler, _dir, _data_side) = build_test_scheduler_with_data_side( - test_coll_vshard(), - CalvinCompletionRegistry::new_detached(), - ); + async fn a_late_part_that_fails_rejects_the_txn_before_the_earlier_parts_stage() { + let (mut scheduler, _dir, mut data_side, proposer) = leading_scheduler(); let txn_id = TxnId::new(3, 0); scheduler.process_scheduler_input(SchedulerInput::Txn(Box::new(two_part_header(3, 0)))); scheduler.process_scheduler_input(part(3, 0, 0, one_truncate())); scheduler.process_scheduler_input(part(3, 0, 1, vec![0xc1, 0xff, 0x00])); - assert!(!scheduler.pending.contains_key(&txn_id), "nothing staged"); - assert!(!scheduler.parts.is_awaiting(txn_id)); - assert!(scheduler.applied.is_applied(3, 0), "the txn completed"); - - // Its locks are free: the next txn on the same key stages at once. - scheduler.process_scheduler_input(SchedulerInput::Txn(Box::new(plain_write(3, 1)))); - assert!(scheduler.pending.contains_key(&TxnId::new(3, 1))); + rejected_then_dropped_at_its_verdict( + &mut scheduler, + &mut data_side, + &proposer, + txn_id, + plain_write(3, 1), + ) + .await; } /// An abandoned txn holding its locks completes at once and frees them. diff --git a/nodedb/src/control/cluster/calvin/scheduler/driver/core/parts_error.rs b/nodedb/src/control/cluster/calvin/scheduler/driver/core/parts_error.rs new file mode 100644 index 000000000..9e3a39ef4 --- /dev/null +++ b/nodedb/src/control/cluster/calvin/scheduler/driver/core/parts_error.rs @@ -0,0 +1,44 @@ +// SPDX-License-Identifier: BUSL-1.1 + +//! Why a multi-part txn's parts do not assemble into local plans. + +use nodedb_physical::physical_plan::wire::WireError; + +/// A part set that does not assemble. Every replica holding the same parts +/// fails with the same error, so the txn is rejected alike everywhere. +#[derive(Debug, thiserror::Error)] +pub(in crate::control::cluster::calvin::scheduler::driver::core) enum PartsError { + #[error("the txn reached dispatch with no assembly of its parts")] + NoAssembly, + #[error("part {part} task {task} is a control-plane-only plan")] + ControlPlaneOnly { part: u32, task: u32 }, + #[error("part {part} task {task} is not a write")] + NotAWrite { part: u32, task: u32 }, + #[error("part {part} task {task} is unroutable: {reason}")] + Unroutable { + part: u32, + task: u32, + reason: &'static str, + }, + #[error("part {part} interrupts the chunks of task {task}")] + InterruptsChunks { part: u32, task: u32 }, + #[error("part {part} does not decode: {source}")] + PartUndecodable { + part: u32, + source: Box, + }, + #[error("part {part} does not continue the chunks of task {task}")] + BrokenChunks { part: u32, task: u32 }, + #[error("task {task} chunks run past its {total_len} bytes")] + ChunksOverrun { task: u32, total_len: u64 }, + #[error("task {task} does not decode: {source}")] + TaskUndecodable { task: u32, source: WireError }, + #[error("task {task} stopped at byte {received} of {total_len}")] + Unfinished { + task: u32, + received: usize, + total_len: u64, + }, + #[error("local plans do not encode: {source}")] + PlansDoNotEncode { source: zerompk::Error }, +} diff --git a/nodedb/src/control/cluster/calvin/scheduler/driver/core/parts_lane.rs b/nodedb/src/control/cluster/calvin/scheduler/driver/core/parts_lane.rs index 041c34aec..7720c83f0 100644 --- a/nodedb/src/control/cluster/calvin/scheduler/driver/core/parts_lane.rs +++ b/nodedb/src/control/cluster/calvin/scheduler/driver/core/parts_lane.rs @@ -7,7 +7,8 @@ //! of the backlog, though, finishes only on input: //! //! - A multi-part txn that holds its locks finishes only once its parts -//! arrive, and every later flush on this vShard waits for it. +//! arrive, and every later whole-collection resolve on this vShard waits +//! for it. //! - A blocked txn below it can wait on a reservation release. //! //! While the gate is closed for the backlog, the lane lets exactly those @@ -41,7 +42,7 @@ //! parts and any abandonment of it follow its header in the log. They sit //! in the channel or in the log past the armed start, and the lane reaches //! both. So does a release a blocked txn waits on. The txn then stages, -//! flushes, and the backlog behind it drains, which opens the gate. +//! applies, and the backlog behind it drains, which opens the gate. use std::collections::VecDeque; diff --git a/nodedb/src/control/cluster/calvin/scheduler/driver/core/passive_read.rs b/nodedb/src/control/cluster/calvin/scheduler/driver/core/passive_read.rs new file mode 100644 index 000000000..34d2638de --- /dev/null +++ b/nodedb/src/control/cluster/calvin/scheduler/driver/core/passive_read.rs @@ -0,0 +1,494 @@ +// SPDX-License-Identifier: BUSL-1.1 + +//! A dependent-read txn on the data-group leader of one of its vShards. +//! +//! The vShard's role in the txn decides the route: +//! +//! - passive (the txn reads rows here): the leader dispatches +//! `MetaOp::CalvinExecutePassive` under the txn's locks, and proposes the +//! values it returns to the data group of every active vShard; +//! - active (the write set names this vShard): the leader opens the +//! dependent-read barrier, once its own passive read answered when the +//! vShard is passive too; +//! - neither: a read-set participant, staged as a static slice. +//! +//! A passive-only vShard writes nothing. Its read answer is its vote: a +//! successful read votes commit, a failed one votes abort. + +use std::sync::Arc; +use std::time::Instant; + +use nodedb_cluster::calvin::AbortReason; +use nodedb_cluster::calvin::types::SequencedTxn; +use nodedb_physical::physical_plan::PhysicalPlan; +use nodedb_physical::physical_plan::meta::MetaOp; + +use super::deferred::{DispatchOutcome, DispatchStep}; +use super::halt::error_response_text; +use super::propose::{CalvinReadResultProposal, propose_calvin_read_result}; +use super::scheduler::Scheduler; +use crate::bridge::envelope::{Response, Status}; +use crate::control::cluster::calvin::scheduler::driver::types::{CommitState, PendingTxn}; +use crate::control::cluster::calvin::scheduler::lock_manager::TxnId; + +/// This vShard's role in a dependent-read txn. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +struct DependentRole { + /// The txn reads rows on this vShard. + passive: bool, + /// The txn's write set names this vShard. + active: bool, +} + +impl Scheduler { + /// Route a granted dependent-read txn on this leader by the vShard's + /// role in it. + pub(in crate::control::cluster::calvin::scheduler::driver::core) fn route_dependent( + &mut self, + txn: SequencedTxn, + txn_id: TxnId, + lock_owner: TxnId, + ) { + let role = match self.dependent_role(&txn) { + Ok(role) => role, + Err(error) => { + self.reject_plan(txn, txn_id, lock_owner, &error); + return; + } + }; + match role { + DependentRole { passive: true, .. } => { + self.dispatch_passive_read(txn, txn_id, lock_owner); + } + DependentRole { + passive: false, + active: true, + } => self.insert_dependent_barrier(txn, txn_id, lock_owner), + DependentRole { + passive: false, + active: false, + } => self.dispatch_txn(txn, txn_id, lock_owner), + } + } + + /// This vShard's role in dependent-read txn `txn`. + fn dependent_role(&self, txn: &SequencedTxn) -> crate::Result { + let passive = txn + .tx_class + .dependent_reads + .as_ref() + .is_some_and(|spec| spec.passive_reads.contains_key(&self.vshard_id)); + let active = txn + .tx_class + .active_vshards() + .map_err(|error| crate::Error::Internal { + detail: format!( + "calvin txn {}/{}: the write set names no vShard of its database: {error}", + txn.epoch, txn.position + ), + })? + .contains(&self.vshard_id); + Ok(DependentRole { passive, active }) + } + + /// Dispatch the passive read of `txn` on this vShard, and hold the txn + /// in [`CommitState::ReadingPassive`] until it answers. + fn dispatch_passive_read(&mut self, txn: SequencedTxn, txn_id: TxnId, lock_owner: TxnId) { + let keys_to_read = txn + .tx_class + .dependent_reads + .as_ref() + .and_then(|spec| spec.passive_reads.get(&self.vshard_id)) + .cloned() + .unwrap_or_default(); + let tenant_id = txn.tx_class.tenant_id; + let database_id = txn.tx_class.database_id; + let plan = PhysicalPlan::Meta(MetaOp::CalvinExecutePassive { + epoch: txn_id.epoch, + position: txn_id.position, + tenant_id, + keys_to_read, + }); + let request_id = self.next_request_id(); + let request = self.build_exempt_request(request_id, tenant_id, database_id, plan); + // A passive-only vShard votes on its read, so it checks the + // collections the txn names as a stage does. + let (superseded, gates) = self.check_incarnations(&txn.tx_class); + // The slice's own scope: a vShard that is passive and active writes + // here, whatever step the txn is at. + let scope = self.local_slice_scope(&txn, txn_id); + self.pending.insert( + txn_id, + PendingTxn { + txn, + lock_owner, + // no-determinism: dispatch time is observability only. + dispatch_time: Instant::now(), + has_primary_write: false, + raises_write_mark: false, + has_returning: false, + commit_state: CommitState::ReadingPassive, + // The dispatch below records its request. + awaiting: None, + verdict_deadline: None, + stage_error: None, + scope, + redo: None, + superseded, + gates, + ungated: false, + }, + ); + self.absorb_buffered_reads(txn_id); + if let DispatchOutcome::Failed(error) = + self.dispatch_sequenced(txn_id, DispatchStep::ReadPassive, request) + { + self.fail_dispatch_step(txn_id, DispatchStep::ReadPassive, error); + } + } + + /// Act on the answer of `txn_id`'s passive read: propose the values to + /// every active vShard, then vote on them on a passive-only vShard, or + /// open the barrier on an active one. + pub(in crate::control::cluster::calvin::scheduler::driver::core) fn finish_passive_read( + &mut self, + txn_id: TxnId, + response: &Response, + ) { + let Some(role) = self + .pending + .get(&txn_id) + .map(|pending| self.dependent_role(&pending.txn)) + else { + return; + }; + let role = match role { + Ok(role) => role, + Err(error) => { + if let Some(pending) = self.pending.remove(&txn_id) { + self.reject_plan(pending.txn, txn_id, pending.lock_owner, &error); + } + return; + } + }; + let read = response.status == Status::Ok; + if read { + self.propose_read_results(txn_id, response.payload.to_vec()); + } + if !role.active { + // A passive-only vShard writes nothing: its read answer carries + // its vote, and a COMMIT verdict drops the empty slice. + self.resolve_staged_commit(txn_id, response); + return; + } + let Some(pending) = self.pending.remove(&txn_id) else { + return; + }; + if read { + self.insert_dependent_barrier(pending.txn, txn_id, pending.lock_owner); + } else { + self.park_unstaged( + pending.txn, + txn_id, + pending.lock_owner, + AbortReason::ParticipantError, + error_response_text("passive read", response), + ); + } + } + + /// Propose `values`, this vShard's passive read of `txn_id`, to the data + /// group of every active vShard of the txn. Each proposal runs on its + /// own task until `config.passive_timeout()` passes. + fn propose_read_results(&self, txn_id: TxnId, values: Vec) { + let Some(pending) = self.pending.get(&txn_id) else { + return; + }; + let tx_class = &pending.txn.tx_class; + let targets = match tx_class.active_vshards() { + Ok(targets) => targets, + Err(error) => { + tracing::error!( + vshard_id = self.vshard_id, + epoch = txn_id.epoch, + position = txn_id.position, + %error, + "calvin: no active vShard to propose the passive read to" + ); + return; + } + }; + #[cfg(feature = "failpoints")] + let gates: Vec = tx_class + .dependent_reads + .as_ref() + .and_then(|spec| spec.passive_reads.get(&self.vshard_id)) + .map(|keys| { + keys.iter() + .map(|key| { + format!( + "calvin::before_read_result_propose::{}", + key.engine_key.collection() + ) + }) + .collect() + }) + .unwrap_or_default(); + let budget = self.config.passive_timeout(); + for target_vshard in targets { + let proposal = CalvinReadResultProposal { + target_vshard, + epoch: txn_id.epoch, + position: txn_id.position, + tenant_id: tx_class.tenant_id, + database_id: tx_class.database_id, + passive_vshard: self.vshard_id, + values: values.clone(), + }; + let state = Arc::downgrade(&self.shared); + #[cfg(feature = "failpoints")] + let gates = gates.clone(); + #[cfg(feature = "failpoints")] + let fail_scope = crate::fail_point::FailScope::Node(self.shared.node_id); + tokio::spawn(async move { + #[cfg(feature = "failpoints")] + for gate in &gates { + crate::control::fail_gate::wait(fail_scope, gate).await; + } + // no-determinism: the proposal's retry budget is liveness only; the log order decides the result. + let deadline = tokio::time::Instant::now() + budget; + if let Err(error) = propose_calvin_read_result(state, proposal, deadline).await { + tracing::warn!( + target_vshard, + epoch = txn_id.epoch, + position = txn_id.position, + %error, + "calvin: passive read result not proposed before its deadline; \ + the active barrier's timeout entry ends the txn" + ); + } + }); + } + } +} + +#[cfg(test)] +mod tests { + use std::collections::BTreeMap; + + use nodedb_cluster::calvin::types::{ + DependentReadSpec, EngineKeySet, PassiveReadKey, SchedulerInput, SortedVec, + }; + use nodedb_cluster::calvin::{CalvinCompletionRegistry, SequencerEntry}; + + use super::*; + use crate::bridge::dispatch::CoreChannelDataSide; + use crate::bridge::envelope::{ErrorCode, Payload, StageVote}; + use crate::control::cluster::calvin::scheduler::driver::core::test_proposer::{ + CapturingProposer, lead_data_group, + }; + use crate::control::cluster::calvin::scheduler::driver::core::test_support::{ + build_test_scheduler_with_data_side, make_local_write_txn, test_coll_vshard, + }; + + /// A dependent txn that writes vShard `active` only, and reads `items` + /// on the scheduler's own vShard. + fn passive_only_txn(epoch: u64) -> SequencedTxn { + let mut txn = make_local_write_txn(epoch, 0); + let other = (0u32..4096) + .map(|i| format!("passive_target_{i}")) + .find(|name| { + nodedb_types::CollectionKey::from_bare(nodedb_types::DatabaseId::DEFAULT, name) + .vshard() + .as_u32() + != test_coll_vshard() + }) + .expect("a collection on another vShard"); + txn.tx_class.write_set = + nodedb_cluster::calvin::types::ReadWriteSet::new(vec![EngineKeySet::Document { + collection: other, + surrogates: SortedVec::new(vec![1]), + }]); + txn.tx_class.dependent_reads = Some(DependentReadSpec { + passive_reads: BTreeMap::from([( + test_coll_vshard(), + vec![PassiveReadKey { + engine_key: EngineKeySet::Kv { + collection: "test_coll".to_owned(), + keys: SortedVec::new(vec![b"k".to_vec()]), + }, + }], + )]), + expected: BTreeMap::new(), + }); + txn + } + + fn leader() -> ( + Scheduler, + tempfile::TempDir, + CoreChannelDataSide, + Arc, + ) { + let registry = CalvinCompletionRegistry::new_detached(); + let (mut scheduler, dir, data_side) = + build_test_scheduler_with_data_side(test_coll_vshard(), registry); + let proposer = CapturingProposer::accepting(); + scheduler.sequencer_proposer = proposer.clone(); + lead_data_group(&mut scheduler); + (scheduler, dir, data_side, proposer) + } + + /// The request id of the passive read the Data Plane received. + fn passive_read_request( + data_side: &mut CoreChannelDataSide, + ) -> Option { + let mut found = None; + while let Ok(request) = data_side.request_rx.try_pop() { + if matches!( + request.inner.plan, + PhysicalPlan::Meta(MetaOp::CalvinExecutePassive { .. }) + ) { + found = Some(request.inner.request_id); + } + } + found + } + + fn answer(request_id: crate::types::RequestId, status: Status) -> Response { + Response { + request_id, + status, + attempt: 1, + partial: false, + payload: Payload::from_vec( + zerompk::to_msgpack_vec(&Vec::<( + nodedb_physical::physical_plan::meta::PassiveReadKeyId, + nodedb_types::Value, + )>::new()) + .expect("encode"), + ), + watermark_lsn: crate::types::Lsn::ZERO, + error_code: (status == Status::Error).then(|| { + Box::new(ErrorCode::Internal { + detail: "read failed".into(), + }) + }), + stage_vote: (status == Status::Ok).then_some(StageVote::Commit), + read_versions: crate::types::ReadVersions::new(), + write_set: Vec::new(), + } + } + + /// A passive-only vShard dispatches its read under the txn's locks and + /// votes commit on a successful answer. + #[tokio::test] + async fn a_passive_only_vshard_reads_then_votes_commit() { + let (mut scheduler, _dir, mut data_side, proposer) = leader(); + scheduler.process_scheduler_input(SchedulerInput::Txn(Box::new(passive_only_txn(70)))); + let txn_id = TxnId::new(70, 0); + assert_eq!( + scheduler.pending.get(&txn_id).map(|p| p.commit_state), + Some(CommitState::ReadingPassive) + ); + let request_id = passive_read_request(&mut data_side).expect("the passive read"); + + scheduler.handle_completion(txn_id, request_id, Some(answer(request_id, Status::Ok))); + + assert_eq!( + scheduler.pending.get(&txn_id).map(|p| p.commit_state), + Some(CommitState::AwaitingVerdict) + ); + assert!(proposer.accepted().iter().any(|entry| matches!( + entry, + SequencerEntry::Vote { + epoch: 70, + position: 0, + .. + } + ))); + } + + /// A failed passive read votes abort on a passive-only vShard. + #[tokio::test] + async fn a_failed_passive_read_votes_abort() { + let (mut scheduler, _dir, mut data_side, proposer) = leader(); + scheduler.process_scheduler_input(SchedulerInput::Txn(Box::new(passive_only_txn(71)))); + let txn_id = TxnId::new(71, 0); + let request_id = passive_read_request(&mut data_side).expect("the passive read"); + + scheduler.handle_completion(txn_id, request_id, Some(answer(request_id, Status::Error))); + + assert!(proposer.accepted().iter().any(|entry| matches!( + entry, + SequencerEntry::AbortVote { + epoch: 71, + reason: AbortReason::ParticipantError, + .. + } + ))); + } + + /// A vShard both passive and active opens its barrier once its own read + /// answered. The barrier waits for its own result through the log. + #[tokio::test] + async fn a_passive_and_active_vshard_opens_its_barrier_after_reading() { + let (mut scheduler, _dir, mut data_side, _proposer) = leader(); + let mut txn = make_local_write_txn(72, 0); + txn.tx_class.dependent_reads = Some(DependentReadSpec { + passive_reads: BTreeMap::from([(test_coll_vshard(), Vec::new())]), + expected: BTreeMap::new(), + }); + scheduler.process_scheduler_input(SchedulerInput::Txn(Box::new(txn))); + let txn_id = TxnId::new(72, 0); + let request_id = passive_read_request(&mut data_side).expect("the passive read"); + + scheduler.handle_completion(txn_id, request_id, Some(answer(request_id, Status::Ok))); + + assert!(!scheduler.pending.contains_key(&txn_id)); + assert!(scheduler.dependent_barrier.contains_key(&txn_id)); + } + + /// A leader that loses the vShard while its passive read runs follows + /// the txn with the slice's own write scope. A COMMIT verdict then waits + /// for the slice's redo, and the position stays unapplied until it + /// installs. With the passive read's empty scope, the follower ended the + /// COMMIT with no entry, marked the position applied, and skipped the + /// redo the next leader proposed. + #[tokio::test] + async fn a_demoted_passive_and_active_leader_waits_for_the_redo() { + let (mut scheduler, _dir, mut data_side, _proposer) = leader(); + let mut txn = make_local_write_txn(73, 0); + txn.tx_class.dependent_reads = Some(DependentReadSpec { + passive_reads: BTreeMap::from([(test_coll_vshard(), Vec::new())]), + expected: BTreeMap::new(), + }); + scheduler.process_scheduler_input(SchedulerInput::Txn(Box::new(txn))); + let txn_id = TxnId::new(73, 0); + assert!(passive_read_request(&mut data_side).is_some()); + assert!( + scheduler + .pending + .get(&txn_id) + .is_some_and(|pending| pending.scope.writes), + "the passive read holds the slice's write scope" + ); + + scheduler.demote("the test moves the vShard's leadership"); + assert_eq!( + scheduler.pending.get(&txn_id).map(|p| p.commit_state), + Some(CommitState::Following) + ); + scheduler.resume_on_verdict(txn_id, true); + + assert_eq!( + scheduler.pending.get(&txn_id).map(|p| p.commit_state), + Some(CommitState::Following), + "a COMMIT for a write slice waits for its redo" + ); + assert!( + !scheduler.ledger.is_applied(73, 0), + "the position stays unapplied, so the redo installs" + ); + } +} diff --git a/nodedb/src/control/cluster/calvin/scheduler/driver/core/process.rs b/nodedb/src/control/cluster/calvin/scheduler/driver/core/process.rs index d8b9e8818..7fd903831 100644 --- a/nodedb/src/control/cluster/calvin/scheduler/driver/core/process.rs +++ b/nodedb/src/control/cluster/calvin/scheduler/driver/core/process.rs @@ -3,7 +3,7 @@ //! New-txn processing, dependent-read barrier setup, and txn-completion //! bookkeeping for the Calvin scheduler. -use std::collections::BTreeMap; +use std::collections::BTreeSet; use std::sync::Arc; use std::time::Instant; @@ -13,6 +13,17 @@ use nodedb_cluster::calvin::types::{ use super::super::barrier::PendingDependentBarrier; use super::scheduler::Scheduler; + +/// Who records a finished position in the vShard's applied ledger. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub(in crate::control::cluster::calvin::scheduler::driver::core) enum LedgerMark { + /// The position ended with no log entry: an abort, a slice with no + /// write, or an abandoned txn. The scheduler marks it. + Terminal, + /// The position's stamped redo installed. The apply loop marked it + /// before the group's applied index passed the entry. + ByApply, +} use crate::control::cluster::calvin::scheduler::lock_manager::{AcquireOutcome, TxnId}; /// Epochs a read reservation can live before the scheduler reaps it as orphaned. @@ -53,7 +64,11 @@ impl Scheduler { SchedulerInput::Txn(txn) => self.process_or_hold_for_metadata(*txn), SchedulerInput::Reserve { owner, key } => self.install_reservation(owner, key), SchedulerInput::Release { owner, reason } => self.release_reservation(owner, reason), - SchedulerInput::CutMarker { hlc } => self.receive_cut_marker(hlc), + SchedulerInput::CutMarker { + hlc, + restore_point, + barrier, + } => self.receive_cut_marker(hlc, restore_point, barrier), SchedulerInput::TxnPart { txn, index, @@ -144,13 +159,12 @@ impl Scheduler { self.applied .note_expected(txn.epoch, txn.position, txn.epoch_vshard_txn_count); - // Exact per-position skip: never re-apply a position that already - // committed (its CalvinApplied marker is durable), and never re-run a - // whole epoch that has fully folded into the watermark. Re-running an - // applied position will re-fire its side effects — this gate IS the - // exactly-once mechanism. Skipping a whole epoch on its first completing - // position (the previous per-epoch gate) dropped every other position of - // that epoch across a restart: a torn transaction. + // Exact per-position skip: never re-run a position that already + // finished here (its stamped redo installed, or it ended with no log + // entry), and never re-run a whole epoch that has fully folded into + // the watermark. Re-running an applied position re-fires its side + // effects: this gate IS the exactly-once mechanism. A position whose + // redo applied before its input arrived is marked here already. if self.applied.is_applied(txn.epoch, txn.position) { // Learning the count for an already-applied position can complete a // historical epoch's applied set (during restart re-fan-out), folding @@ -221,61 +235,93 @@ impl Scheduler { } } - /// Route a ready txn to either a static dispatch or a dependent barrier. + /// Route a granted txn: a follower holds it, the leader stages it as a + /// static dispatch or a dependent barrier. pub(in crate::control::cluster::calvin::scheduler::driver::core) fn dispatch_or_barrier( &mut self, txn: SequencedTxn, txn_id: TxnId, lock_owner: TxnId, ) { - // A multi-part txn dispatches only once its parts arrived. + // A multi-part txn dispatches only once its parts arrived. Every + // replica assembles them: a promotion stages the assembled txn. let Some(txn) = self.gate_on_parts(txn, txn_id, lock_owner) else { return; }; - let is_dependent = txn.tx_class.dependent_reads.is_some(); - if is_dependent { - self.insert_dependent_barrier(txn, txn_id, lock_owner); + // Its redo applied while it waited for its locks. + if let Some(event) = self.early_applied.remove(&txn_id) { + self.complete_applied(txn_id, lock_owner, event); + return; + } + self.route_granted(txn, txn_id, lock_owner); + } + + /// Route a granted, assembled txn: a follower holds it, the leader + /// stages it as a static dispatch, or routes a dependent-read txn by its + /// role on this vShard. + pub(in crate::control::cluster::calvin::scheduler::driver::core) fn route_granted( + &mut self, + txn: SequencedTxn, + txn_id: TxnId, + lock_owner: TxnId, + ) { + if !self.role.is_leader() { + self.follow(txn, txn_id, lock_owner); + return; + } + if txn.tx_class.dependent_reads.is_some() { + self.route_dependent(txn, txn_id, lock_owner); } else { self.dispatch_txn(txn, txn_id, lock_owner); } } - /// Insert a dependent-read barrier for an active vshard. - fn insert_dependent_barrier(&mut self, txn: SequencedTxn, txn_id: TxnId, lock_owner: TxnId) { - let spec = match &txn.tx_class.dependent_reads { - Some(s) => s, + /// Open a dependent-read barrier for `txn` on this active vShard. + /// + /// The barrier waits for the read result of every passive vShard. This + /// leader proposes the barrier's timeout entry once it waited past + /// `config.passive_timeout()`. + pub(in crate::control::cluster::calvin::scheduler::driver::core) fn insert_dependent_barrier( + &mut self, + txn: SequencedTxn, + txn_id: TxnId, + lock_owner: TxnId, + ) { + let passive: BTreeSet = match &txn.tx_class.dependent_reads { + Some(spec) => spec.passive_vshards().collect(), None => { - // Shouldn't happen; fall through to static dispatch. - self.dispatch_txn(txn, txn_id, lock_owner); + let error = crate::Error::Internal { + detail: format!( + "calvin txn {}/{} reached a dependent-read barrier with no \ + dependent reads", + txn_id.epoch, txn_id.position + ), + }; + self.reject_plan(txn, txn_id, lock_owner, &error); return; } }; - - let waiting_for: std::collections::BTreeSet = - spec.passive_reads.keys().copied().collect(); - // no-determinism: passive barrier timeout is scheduler observability, not Calvin WAL data - let timeout_at = Instant::now() + self.config.passive_timeout(); - let barrier = PendingDependentBarrier { txn, lock_owner, - waiting_for, - received: BTreeMap::new(), - timeout_at, + passive, + // no-determinism: the leader's clock decides only when it proposes the timeout entry; the entry's place in the log decides the barrier. + timeout_due: Instant::now() + self.config.passive_timeout(), }; - - self.dependent_barrier.insert(txn_id, barrier); + self.open_barrier(txn_id, barrier); } - /// Complete an in-flight txn (success or infrastructure error). + /// Complete an in-flight txn. /// /// Releases the lock-table owner recorded in its `pending` entry. A txn /// with no `pending` entry has already completed, so this logs and - /// releases nothing. A txn that fails before it enters `pending` uses - /// [`Self::on_unpending_txn_complete`] instead. + /// releases nothing. A multi-part txn whose parts the sequencer abandoned + /// never enters `pending`, and uses [`Self::on_unpending_txn_complete`] + /// instead. `mark` says who records the position in the ledger. pub(in crate::control::cluster::calvin::scheduler::driver::core) fn on_txn_complete( &mut self, txn_id: TxnId, + mark: LedgerMark, ) { let Some(pending) = self.pending.remove(&txn_id) else { tracing::error!( @@ -286,26 +332,31 @@ impl Scheduler { ); return; }; - // The flush applied the txn's redo record. - if let Some(records) = pending.redo_records { - records.settle(); - } - self.release_and_mark_applied(txn_id, pending.lock_owner); + self.release_and_mark_applied(txn_id, pending.lock_owner, mark); } - /// Complete a txn that failed before it entered `pending`, releasing the - /// locks held under `lock_owner`. + /// Complete a txn that never entered `pending`, releasing the locks held + /// under `lock_owner`. Only a txn whose stored verdict is already + /// `Abort(PartsLost)` completes this way. pub(in crate::control::cluster::calvin::scheduler::driver::core) fn on_unpending_txn_complete( &mut self, txn_id: TxnId, lock_owner: TxnId, ) { - self.release_and_mark_applied(txn_id, lock_owner); + self.release_and_mark_applied(txn_id, lock_owner, LedgerMark::Terminal); } /// Release `lock_owner`'s locks, dispatch the promoted waiters, and mark /// `txn_id`'s position applied. - fn release_and_mark_applied(&mut self, txn_id: TxnId, lock_owner: TxnId) { + pub(in crate::control::cluster::calvin::scheduler::driver::core) fn release_and_mark_applied( + &mut self, + txn_id: TxnId, + lock_owner: TxnId, + mark: LedgerMark, + ) { + // The txn restages no more, and no barrier of it opens again. + let held_log = self.barrier_logs.contains_key(&txn_id); + self.forget_held_state(txn_id); // Release this txn's locks. `release` promotes any waiter queued behind // each freed key to holder (moving it pending -> held) and returns the // fully-promoted ids. Those ids are already holders in the table the @@ -324,12 +375,19 @@ impl Scheduler { // so any advertised watermark reflects a FULLY-applied epoch — the value // `BEGIN` needs for a torn-free cross-shard snapshot anchor. let folded = self.applied.mark_applied(txn_id.epoch, txn_id.position); - self.applied_mirror.mark(txn_id.epoch, txn_id.position); + match mark { + LedgerMark::Terminal => self.ledger.mark_terminal(txn_id.epoch, txn_id.position), + LedgerMark::ByApply => {} + } + // The ledger holds the position now, so the apply loop buffers and + // stores no later barrier entry of the txn. Each one it took before + // is spent. + self.forget_barrier_entries(txn_id, held_log); if let Some(watermark) = folded { self.publish_watermark(watermark); } - // A finished txn can give the next committed flush its turn. - self.pump_flush_turn(); + // A finished txn can give the next whole-collection resolve its turn. + self.pump_resolve_turn(); } /// Dispatch transactions that a `LockManager::release` promoted to holder. @@ -411,7 +469,7 @@ impl Scheduler { #[cfg(test)] mod tests { use super::*; - use std::collections::BTreeSet; + use std::collections::BTreeMap; use std::sync::atomic::Ordering; use nodedb_cluster::calvin::CalvinCompletionRegistry; @@ -419,6 +477,7 @@ mod tests { use nodedb_physical::physical_plan::meta::MetaOp; use nodedb_types::TenantId; + use crate::control::cluster::calvin::scheduler::driver::core::test_proposer::lead_data_group; use crate::control::cluster::calvin::scheduler::driver::core::test_support::{ await_data_plane_request, build_test_scheduler, build_test_scheduler_with_data_side, fill_tenant_inflight, make_sequenced_txn, make_validate_only_txn, release_filler, @@ -432,6 +491,7 @@ mod tests { let registry = CalvinCompletionRegistry::new_detached(); let (mut scheduler, _dir, mut data_side) = build_test_scheduler_with_data_side(test_coll_vshard(), registry); + lead_data_group(&mut scheduler); let shared = Arc::clone(&scheduler.shared); fill_tenant_inflight(&shared, &mut data_side, TenantId::new(1)); let watermark_before = shared.calvin.last_applied_epoch.load(Ordering::Acquire); @@ -457,6 +517,7 @@ mod tests { let registry = CalvinCompletionRegistry::new_detached(); let (mut scheduler, _dir, mut data_side) = build_test_scheduler_with_data_side(test_coll_vshard(), registry); + lead_data_group(&mut scheduler); let shared = Arc::clone(&scheduler.shared); fill_tenant_inflight(&shared, &mut data_side, TenantId::new(1)); @@ -477,6 +538,7 @@ mod tests { let registry = CalvinCompletionRegistry::new_detached(); let (mut scheduler, _dir, mut data_side) = build_test_scheduler_with_data_side(test_coll_vshard(), registry); + lead_data_group(&mut scheduler); let shared = Arc::clone(&scheduler.shared); fill_tenant_inflight(&shared, &mut data_side, TenantId::new(1)); let tracked_before = shared.tracker.in_flight(); @@ -498,6 +560,7 @@ mod tests { let registry = CalvinCompletionRegistry::new_detached(); let (mut scheduler, _dir, mut data_side) = build_test_scheduler_with_data_side(test_coll_vshard(), registry); + lead_data_group(&mut scheduler); let shared = Arc::clone(&scheduler.shared); let fillers = fill_tenant_inflight(&shared, &mut data_side, TenantId::new(1)); @@ -538,7 +601,7 @@ mod tests { txn_id, crate::control::cluster::calvin::scheduler::driver::types::BlockedTxn { txn: txn.clone(), - keys: BTreeSet::new(), + keys: BTreeMap::new(), // no-determinism: test-only blocked_at timestamp for a fabricated BlockedTxn fixture. blocked_at: Instant::now(), }, diff --git a/nodedb/src/control/cluster/calvin/scheduler/driver/core/propose.rs b/nodedb/src/control/cluster/calvin/scheduler/driver/core/propose.rs index 58c329dc5..be9176007 100644 --- a/nodedb/src/control/cluster/calvin/scheduler/driver/core/propose.rs +++ b/nodedb/src/control/cluster/calvin/scheduler/driver/core/propose.rs @@ -1,56 +1,127 @@ // SPDX-License-Identifier: BUSL-1.1 -//! Propose `ReplicatedWrite::CalvinReadResult` Raft entries from passive -//! participants. +//! Propose a passive participant's read values to an active vShard's data +//! group as a `ReplicatedWrite::CalvinReadResult` entry. +//! +//! The passive vShard's data-group leader proposes to the data group of each +//! active vShard. That group's leader is often another node, so the entry +//! travels through the node's async propose phase, which forwards it to the +//! group leader. The phase takes no vShard admission slot: the entry writes +//! no row, and the target leader's write gate exempts it. +//! +//! A refused proposal is retried until the deadline. A second copy of an +//! entry that already landed changes nothing: the first result of a passive +//! vShard is the one a barrier counts. A proposal that never lands leaves +//! the active barrier waiting, and its leader's timeout entry ends it on +//! every replica alike. +use std::sync::Weak; +use std::time::Duration; + +use crate::control::state::SharedState; use crate::control::wal_replication::{ReplicatedEntry, ReplicatedWrite}; use crate::types::{DatabaseId, TenantId}; -/// The passive read result to replicate: identity of the originating vshard -/// plus the sequencer coordinate, scope, and read values. +/// Pause between two attempts of a refused proposal. +const RETRY_PAUSE: Duration = Duration::from_millis(20); + +/// One passive vShard's read values, for one active vShard's data group. pub struct CalvinReadResultProposal { - pub vshard_id: u32, + /// The active vShard whose data group the entry goes to. + pub target_vshard: u32, pub epoch: u64, pub position: u32, pub tenant_id: TenantId, pub database_id: DatabaseId, + /// The vShard that read the values. pub passive_vshard: u32, - pub values_payload: Vec, + /// The zerompk-encoded `Vec<(PassiveReadKeyId, Value)>` the passive read + /// returned. + pub values: Vec, } -/// Propose a `ReplicatedWrite::CalvinReadResult` entry to the per-vshard -/// Raft group after a passive participant responds. -/// -/// This is a Control Plane function (Tokio async context). It constructs the -/// Raft entry and hands it to the WAL replication layer. +/// Propose `proposal` to the data group of `proposal.target_vshard`, and +/// wait for this node's apply of it when this node replicates that group. /// -/// `Instant::now()` is NOT used here (this goes to the WAL/Raft path). +/// Retries a refused proposal until `deadline`. Returns the last refusal +/// once the deadline passed. Returns `Ok` once an attempt landed in the +/// target leader's log, whatever this node's apply wait reports: a node that +/// replicates no copy of the group never applies it. pub async fn propose_calvin_read_result( - raft_proposer: &(dyn Fn(u32, Vec) -> crate::Result<(u64, u64)> + Send + Sync), + state: Weak, proposal: CalvinReadResultProposal, + deadline: tokio::time::Instant, ) -> crate::Result<()> { let CalvinReadResultProposal { - vshard_id, + target_vshard, epoch, position, tenant_id, database_id, passive_vshard, - values_payload, + values, } = proposal; let entry = ReplicatedEntry::new( tenant_id.as_u64(), database_id.as_u64(), - vshard_id, + target_vshard, ReplicatedWrite::CalvinReadResult { epoch, position, passive_vshard, tenant_id: tenant_id.as_u64(), - values: values_payload, + values, }, ); - let data = entry.encode()?; - raft_proposer(vshard_id, data)?; - Ok(()) + let bytes = entry.encode()?; + loop { + let submit = { + let Some(state) = state.upgrade() else { + return Err(crate::Error::Internal { + detail: "calvin read result not proposed: the node is shutting down".into(), + }); + }; + std::sync::Arc::clone(state.async_raft_submit()?) + }; + match submit( + target_vshard, + entry.idempotency_key, + bytes.clone(), + deadline, + ) + .await + { + Ok(proposed) => { + if let Err(error) = proposed.applied.await { + tracing::debug!( + target_vshard, + epoch, + position, + %error, + "calvin: read result proposed; this node's apply wait ended without it" + ); + } + return Ok(()); + } + Err(error) => { + // no-determinism: the retry budget is liveness only; the log order decides the result. + if tokio::time::Instant::now() >= deadline { + return Err(error); + } + tracing::debug!( + target_vshard, + epoch, + position, + %error, + "calvin: read result proposal refused; proposing it again" + ); + tokio::time::sleep( + RETRY_PAUSE + // no-determinism: a retry pause only. + .min(deadline.saturating_duration_since(tokio::time::Instant::now())), + ) + .await; + } + } + } } diff --git a/nodedb/src/control/cluster/calvin/scheduler/driver/core/read_result.rs b/nodedb/src/control/cluster/calvin/scheduler/driver/core/read_result.rs index 41a0c0093..53b0bc8bc 100644 --- a/nodedb/src/control/cluster/calvin/scheduler/driver/core/read_result.rs +++ b/nodedb/src/control/cluster/calvin/scheduler/driver/core/read_result.rs @@ -1,85 +1,665 @@ // SPDX-License-Identifier: BUSL-1.1 -//! `CalvinReadResult` handling and dependent-read barrier timeout sweeps. +//! Dependent-read barrier entries and the barrier's decision. +//! +//! The data-group apply loop folds every `CalvinReadResult` and +//! `CalvinReadTimeout` entry into the txn's stored barrier row and this +//! vShard's read-result buffer. The scheduler takes a txn's entries into its +//! `barrier_logs` once it holds the txn: at the grant, and on each push for a +//! txn it holds. A txn the buffer holds as stored reads its row. A barrier then +//! decides from the log alone (see +//! [`crate::control::cluster::calvin::scheduler::BarrierLog`]), so every +//! replica that leads the vShard decides it alike: +//! +//! - complete: the leader checks the values against the ones the coordinator +//! read, and stages the active slice or votes `PredictionDrift`; +//! - timed out: the leader votes abort; +//! - waiting: the barrier stays open, unless an abort verdict is stored. +//! +//! A follower keeps the log of each dependent txn it holds. A promotion +//! opens the barrier from it and reaches the decision the earlier leader +//! reached. -use tracing::warn; +use std::sync::Arc; -use super::super::barrier::ReadResultEvent; +use tracing::{error, info, warn}; + +use nodedb_cluster::calvin::AbortReason; + +use super::super::barrier::{PendingDependentBarrier, expected_reads_drift}; use super::scheduler::Scheduler; +use crate::control::cluster::calvin::scheduler::barrier_store; use crate::control::cluster::calvin::scheduler::lock_manager::TxnId; +use crate::control::cluster::calvin::scheduler::{BarrierLog, BarrierOutcome, BufferedEvents}; impl Scheduler { - /// Handle a `ReadResultEvent` from the per-vshard Raft apply loop. - pub(in crate::control::cluster::calvin::scheduler::driver::core) fn handle_read_result( + /// Take the buffered barrier entries of every dependent txn this + /// scheduler holds, and decide each open barrier they reach. + pub(in crate::control::cluster::calvin::scheduler::driver::core) fn drain_read_results( &mut self, - event: ReadResultEvent, ) { - let txn_id = TxnId::new(event.epoch, event.position); - - let barrier = match self.dependent_barrier.get_mut(&txn_id) { - Some(b) => b, - None => { - // No barrier for this txn — can have already timed out or - // been dispatched. Log and ignore. - warn!( - vshard_id = self.vshard_id, - epoch = event.epoch, - position = event.position, - passive_vshard = event.passive_vshard, - "calvin: received CalvinReadResult for unknown txn; ignoring" - ); - return; + let buffer = Arc::clone(&self.read_results); + let taken = buffer.take_held(|txn_id| self.holds_barrier_txn(txn_id)); + for (txn_id, events) in taken { + if self.absorb(txn_id, events) { + self.decide_barrier(txn_id); } - }; - - barrier.waiting_for.remove(&event.passive_vshard); - barrier.received.insert(event.passive_vshard, event.values); + } + } - if !barrier.is_complete() { - return; + /// Take the buffered barrier entries of `txn_id`, which this scheduler + /// holds from now on. + pub(in crate::control::cluster::calvin::scheduler::driver::core) fn absorb_buffered_reads( + &mut self, + txn_id: TxnId, + ) { + if let Some(events) = self.read_results.take(txn_id) { + self.absorb(txn_id, events); } + } - // All passive results in — remove barrier and dispatch active. - let Some(barrier) = self.dependent_barrier.remove(&txn_id) else { - return; + /// Fold `events` of `txn_id` into its barrier log. A stored txn's + /// events are its row's, extended with the ones memory held. Returns + /// whether the log took them. + /// + /// A row this node cannot read goes back to the buffer, still stored: + /// the barrier stays open, and the next pass reads the row again. + fn absorb(&mut self, txn_id: TxnId, events: BufferedEvents) -> bool { + let mut log = if events.stored { + match barrier_store::load_log(self.shared.credentials.catalog(), self.vshard_id, txn_id) + { + Ok(stored) => stored, + Err(e) => { + error!( + vshard_id = self.vshard_id, + epoch = txn_id.epoch, + position = txn_id.position, + error = %e, + "calvin: the barrier row of a held txn does not load; reading it again \ + on the next pass" + ); + crate::diag::calvin_barrier_log_store_failed( + self.vshard_id, + Some((txn_id.epoch, txn_id.position)), + "load", + &e, + ); + self.read_results.restore(txn_id, events); + return false; + } + } + } else { + BarrierLog::default() }; + log.extend(events.memory); + self.barrier_logs.entry(txn_id).or_default().extend(log); + true + } - let injected_reads = barrier.assemble_injected_reads(); - let lock_owner = barrier.lock_owner; - let txn = barrier.txn; - - self.dispatch_active_txn(txn, txn_id, lock_owner, injected_reads); + /// Whether `txn_id` is a dependent txn this scheduler holds: at a + /// barrier, or in `pending`. + fn holds_barrier_txn(&self, txn_id: TxnId) -> bool { + self.dependent_barrier.contains_key(&txn_id) + || self + .pending + .get(&txn_id) + .is_some_and(|pending| pending.txn.tx_class.dependent_reads.is_some()) } - /// Check all pending dependent barriers for timeout. - pub(in crate::control::cluster::calvin::scheduler::driver::core) fn check_dependent_barrier_timeouts( + /// Open `barrier` for `txn_id` and decide it from the entries the log + /// holds already. + pub(in crate::control::cluster::calvin::scheduler::driver::core) fn open_barrier( &mut self, + txn_id: TxnId, + barrier: PendingDependentBarrier, ) { - // Collect timed-out txn ids first (avoid borrowing issues). - let timed_out: Vec = self - .dependent_barrier - .iter() - .filter(|(_, b)| b.is_timed_out()) - .map(|(id, _)| *id) - .collect(); + self.dependent_barrier.insert(txn_id, barrier); + self.absorb_buffered_reads(txn_id); + self.decide_barrier(txn_id); + } - for txn_id in timed_out { - if let Some(barrier) = self.dependent_barrier.remove(&txn_id) { + /// Decide the open barrier of `txn_id` from its log. + fn decide_barrier(&mut self, txn_id: TxnId) { + let Some(barrier) = self.dependent_barrier.get(&txn_id) else { + return; + }; + let empty = BarrierLog::default(); + let log = self.barrier_logs.get(&txn_id).unwrap_or(&empty); + match log.outcome(&barrier.passive) { + BarrierOutcome::Complete => self.complete_barrier(txn_id), + BarrierOutcome::TimedOut => { + let missing = log.missing(&barrier.passive); warn!( vshard_id = self.vshard_id, epoch = txn_id.epoch, position = txn_id.position, - still_waiting = ?barrier.waiting_for, - "calvin: dependent-read barrier timed out; releasing locks" + ?missing, + "calvin: the log timed out the dependent-read barrier; voting abort" ); self.metrics.record_executor_error(); self.metrics.record_infra_abort( crate::control::cluster::calvin::scheduler::metrics::infra_abort_reason::PASSIVE_PARTICIPANT_TIMEOUT, ); - // A barrier txn never entered `pending`: release its locks - // under the owner the barrier recorded. - self.on_unpending_txn_complete(txn_id, barrier.lock_owner); + self.abort_open_barrier( + txn_id, + AbortReason::ParticipantError, + format!( + "the dependent-read barrier timed out in the log before the read \ + results of passive vShards {missing:?}" + ), + ); + } + BarrierOutcome::Waiting => { + // An abort verdict is stored already: a peer voted abort. The + // txn stages nothing here, so its locks free at once. + let verdict = self.registry.verdict(nodedb_cluster::calvin::TxnId::new( + txn_id.epoch, + txn_id.position, + )); + if verdict == Some(false) { + self.abort_open_barrier( + txn_id, + AbortReason::ParticipantError, + "an abort verdict reached the txn's open dependent-read barrier".to_owned(), + ); + } } } } + + /// Close the complete barrier of `txn_id`: stage the active slice with + /// the read values, or vote `PredictionDrift` when a value differs from + /// the one the coordinator read. + fn complete_barrier(&mut self, txn_id: TxnId) { + let Some(barrier) = self.dependent_barrier.remove(&txn_id) else { + return; + }; + let injected = self + .barrier_logs + .get(&txn_id) + .map(BarrierLog::injected_reads) + .unwrap_or_default(); + let drift = barrier + .txn + .tx_class + .dependent_reads + .as_ref() + .and_then(|spec| expected_reads_drift(spec, &injected)); + if let Some(detail) = drift { + info!( + vshard_id = self.vshard_id, + epoch = txn_id.epoch, + position = txn_id.position, + %detail, + "calvin: a passive read moved since the coordinator read it; voting prediction drift" + ); + self.park_unstaged( + barrier.txn, + txn_id, + barrier.lock_owner, + AbortReason::PredictionDrift, + detail, + ); + return; + } + self.dispatch_active_txn(barrier.txn, txn_id, barrier.lock_owner, injected); + } + + /// Close the open barrier of `txn_id` with an abort vote for `reason`. + pub(in crate::control::cluster::calvin::scheduler::driver::core) fn abort_open_barrier( + &mut self, + txn_id: TxnId, + reason: AbortReason, + detail: String, + ) { + let Some(barrier) = self.dependent_barrier.remove(&txn_id) else { + return; + }; + self.park_unstaged(barrier.txn, txn_id, barrier.lock_owner, reason, detail); + } + + /// Hand the barrier log of every txn this scheduler did not finish back + /// to the vShard's read-result buffer. A stopping scheduler calls it, so + /// the next scheduler of the vShard starts from the same entries. + pub(in crate::control::cluster::calvin::scheduler::driver::core) fn return_barrier_logs( + &mut self, + ) { + for (txn_id, log) in std::mem::take(&mut self.barrier_logs) { + if !self.ledger.is_applied(txn_id.epoch, txn_id.position) { + self.read_results.restore( + txn_id, + BufferedEvents { + memory: log, + stored: false, + }, + ); + } + } + } + + /// Remove every stored barrier row of this vShard whose txn finished. + /// A failed pass keeps the rows, and the next stall tick retries. + pub(in crate::control::cluster::calvin::scheduler::driver::core) fn sweep_finished_barrier_rows( + &self, + ) { + if let Err(e) = barrier_store::sweep_finished( + self.shared.credentials.catalog(), + self.vshard_id, + &self.ledger, + ) { + warn!( + vshard_id = self.vshard_id, + error = %e, + "calvin: the barrier rows of finished txns were not removed; retrying on \ + the next stall tick" + ); + crate::diag::calvin_barrier_log_store_failed(self.vshard_id, None, "remove", &e); + } + } + + /// Drop every barrier entry of `txn_id`, which finished here: the + /// buffered ones, and its stored row. + pub(in crate::control::cluster::calvin::scheduler::driver::core) fn forget_barrier_entries( + &mut self, + txn_id: TxnId, + held_log: bool, + ) { + let buffered = self.read_results.forget_txn(txn_id); + if !held_log && !buffered { + return; + } + // A failed remove leaves the row of a finished txn. No barrier reads + // it, and the stall tick's sweep removes it. + if let Err(e) = + barrier_store::remove_log(self.shared.credentials.catalog(), self.vshard_id, txn_id) + { + warn!( + vshard_id = self.vshard_id, + epoch = txn_id.epoch, + position = txn_id.position, + error = %e, + "calvin: the barrier row of a finished txn was not removed" + ); + crate::diag::calvin_barrier_log_store_failed( + self.vshard_id, + Some((txn_id.epoch, txn_id.position)), + "remove", + &e, + ); + } + } +} + +#[cfg(test)] +mod tests { + use std::collections::BTreeMap; + + use nodedb_cluster::calvin::types::{DependentReadSpec, SchedulerInput, SequencedTxn}; + use nodedb_cluster::calvin::{ + AbortReason, CalvinCompletionRegistry, SequencerEntry, VerdictOutcome, + }; + use nodedb_physical::physical_plan::PhysicalPlan; + use nodedb_physical::physical_plan::meta::{MetaOp, PassiveReadKeyId}; + use nodedb_types::{QualifiedCollection, Value}; + + use super::*; + use crate::bridge::dispatch::CoreChannelDataSide; + use crate::control::cluster::calvin::scheduler::BarrierEvent; + use crate::control::cluster::calvin::scheduler::driver::core::process::LedgerMark; + use crate::control::cluster::calvin::scheduler::driver::core::test_proposer::{ + CapturingProposer, lead_data_group, + }; + use crate::control::cluster::calvin::scheduler::driver::core::test_support::{ + build_test_scheduler_with_data_side, make_local_write_txn, test_coll_vshard, + }; + use crate::control::cluster::calvin::scheduler::driver::types::CommitState; + + /// The passive vShard of every test txn: not the scheduler's own. + const PASSIVE: u32 = 9; + + fn row() -> PassiveReadKeyId { + PassiveReadKeyId::kv( + QualifiedCollection::from_stored("items".to_owned()), + b"alice:sword".to_vec(), + ) + } + + /// A dependent txn that writes `test_coll` and reads `row()` on + /// [`PASSIVE`], whose coordinator read `expected` for it. + fn dependent_txn(epoch: u64, expected: Option<&[u8]>) -> SequencedTxn { + let mut txn = make_local_write_txn(epoch, 0); + txn.tx_class.dependent_reads = Some(DependentReadSpec { + passive_reads: BTreeMap::from([(PASSIVE, Vec::new())]), + expected: BTreeMap::from([(row(), expected.map(<[u8]>::to_vec))]), + }); + txn + } + + fn read(value: &[u8]) -> BarrierEvent { + BarrierEvent::Read { + passive_vshard: PASSIVE, + values: vec![(row(), Value::Bytes(value.to_vec()))], + } + } + + /// A scheduler on the `test_coll` vShard with a capturing proposer. Not + /// a leader until the test makes it one. + fn scheduler() -> ( + Scheduler, + tempfile::TempDir, + CoreChannelDataSide, + Arc, + Arc, + ) { + let registry = CalvinCompletionRegistry::new_detached(); + let (mut scheduler, dir, data_side) = + build_test_scheduler_with_data_side(test_coll_vshard(), Arc::clone(®istry)); + let proposer = CapturingProposer::accepting(); + scheduler.sequencer_proposer = proposer.clone(); + (scheduler, dir, data_side, proposer, registry) + } + + fn buffer(scheduler: &Scheduler, epoch: u64, event: BarrierEvent) { + scheduler + .read_results + .push(TxnId::new(epoch, 0), event, 64, true); + } + + fn grant(scheduler: &mut Scheduler, txn: SequencedTxn) { + scheduler.process_scheduler_input(SchedulerInput::Txn(Box::new(txn))); + } + + /// Whether the Data Plane received the active stage of `(epoch, 0)`. + fn staged_active(data_side: &mut CoreChannelDataSide, epoch: u64) -> bool { + let mut staged = false; + while let Ok(request) = data_side.request_rx.try_pop() { + staged |= matches!( + request.inner.plan, + PhysicalPlan::Meta(MetaOp::CalvinExecuteActive { epoch: e, position: 0, .. }) + if e == epoch + ); + } + staged + } + + fn abort_votes(proposer: &CapturingProposer) -> Vec { + proposer + .accepted() + .into_iter() + .filter_map(|entry| match entry { + SequencerEntry::AbortVote { reason, .. } => Some(reason), + _ => None, + }) + .collect() + } + + /// A read result the log applied before this node granted the txn waits + /// in the buffer. The grant opens the barrier from it, and the active + /// slice stages at once. + #[tokio::test] + async fn a_result_buffered_before_the_grant_completes_the_barrier() { + let (mut scheduler, _dir, mut data_side, proposer, _registry) = scheduler(); + lead_data_group(&mut scheduler); + buffer(&scheduler, 50, read(b"v1")); + + grant(&mut scheduler, dependent_txn(50, Some(b"v1"))); + + assert!(staged_active(&mut data_side, 50)); + assert!(scheduler.dependent_barrier.is_empty()); + assert_eq!(scheduler.read_results.waiting_txns(), 0); + assert_eq!( + scheduler + .pending + .get(&TxnId::new(50, 0)) + .map(|p| p.commit_state), + Some(CommitState::Staged) + ); + assert!(abort_votes(&proposer).is_empty()); + } + + /// A restarted node finds a txn's result in its stored row alone. The + /// grant opens the barrier from the row, the active slice stages, and + /// the txn's finish removes the row. + #[tokio::test] + async fn a_stored_result_completes_the_barrier_after_a_restart() { + let (mut scheduler, _dir, mut data_side, proposer, _registry) = scheduler(); + lead_data_group(&mut scheduler); + let txn_id = TxnId::new(51, 0); + let catalog = scheduler.shared.credentials.catalog(); + barrier_store::save_event(catalog, scheduler.vshard_id, txn_id, &read(b"v1")) + .expect("store the result"); + scheduler.read_results.mark_stored([txn_id]); + + grant(&mut scheduler, dependent_txn(51, Some(b"v1"))); + + assert!(staged_active(&mut data_side, 51)); + assert!(scheduler.dependent_barrier.is_empty()); + assert_eq!(scheduler.read_results.waiting_txns(), 0); + assert!(abort_votes(&proposer).is_empty()); + + let lock_owner = scheduler + .pending + .get(&txn_id) + .map(|pending| pending.lock_owner) + .expect("the staged txn is pending"); + scheduler.release_and_mark_applied(txn_id, lock_owner, LedgerMark::ByApply); + assert!( + scheduler + .shared + .credentials + .catalog() + .load_calvin_barrier_log(scheduler.vshard_id, 51, 0) + .expect("load") + .is_none(), + "the finished txn's row is removed" + ); + } + + /// A result pushed while the barrier is open completes it on the drain. + #[tokio::test] + async fn a_result_after_the_grant_completes_the_open_barrier() { + let (mut scheduler, _dir, mut data_side, _proposer, _registry) = scheduler(); + lead_data_group(&mut scheduler); + grant(&mut scheduler, dependent_txn(51, Some(b"v1"))); + assert!(scheduler.dependent_barrier.contains_key(&TxnId::new(51, 0))); + + buffer(&scheduler, 51, read(b"v1")); + scheduler.drain_read_results(); + + assert!(staged_active(&mut data_side, 51)); + assert!(scheduler.dependent_barrier.is_empty()); + } + + /// The timeout entry above every result changes nothing: the barrier + /// completed in the log first. + #[tokio::test] + async fn a_timeout_after_every_result_is_ignored() { + let (mut scheduler, _dir, mut data_side, proposer, _registry) = scheduler(); + lead_data_group(&mut scheduler); + buffer(&scheduler, 52, read(b"v1")); + buffer(&scheduler, 52, BarrierEvent::Timeout); + + grant(&mut scheduler, dependent_txn(52, Some(b"v1"))); + + assert!(staged_active(&mut data_side, 52)); + assert!(abort_votes(&proposer).is_empty()); + } + + /// A timeout entry below a result times the barrier out: the leader + /// votes abort, and the late result does not count. + #[tokio::test] + async fn a_timeout_before_the_result_votes_abort() { + let (mut scheduler, _dir, mut data_side, proposer, _registry) = scheduler(); + lead_data_group(&mut scheduler); + grant(&mut scheduler, dependent_txn(53, Some(b"v1"))); + + buffer(&scheduler, 53, BarrierEvent::Timeout); + buffer(&scheduler, 53, read(b"v1")); + scheduler.drain_read_results(); + + assert!(!staged_active(&mut data_side, 53)); + assert_eq!( + scheduler + .pending + .get(&TxnId::new(53, 0)) + .map(|p| p.commit_state), + Some(CommitState::AwaitingVerdict) + ); + assert_eq!(abort_votes(&proposer), vec![AbortReason::ParticipantError]); + assert!(!scheduler.applied.is_applied(53, 0)); + } + + /// A value that moved since the coordinator read it votes + /// `PredictionDrift` and stages nothing. + #[tokio::test] + async fn a_moved_value_votes_prediction_drift() { + let (mut scheduler, _dir, mut data_side, proposer, _registry) = scheduler(); + lead_data_group(&mut scheduler); + buffer(&scheduler, 54, read(b"v2")); + + grant(&mut scheduler, dependent_txn(54, Some(b"v1"))); + + assert!(!staged_active(&mut data_side, 54)); + assert_eq!(abort_votes(&proposer), vec![AbortReason::PredictionDrift]); + } + + /// A follower holds the log of a dependent txn its leader committed. The + /// result reached the follower before its grant, and the follower waited + /// past any timeout of its own. Promoted, it opens the barrier from the + /// log, stages the slice, and neither votes abort nor halts. + #[tokio::test] + async fn a_promoted_follower_stages_the_committed_txn_without_halting() { + let (mut scheduler, _dir, mut data_side, proposer, registry) = scheduler(); + scheduler.config.dependent_read_passive_timeout_ms = 20; + buffer(&scheduler, 55, read(b"v1")); + grant(&mut scheduler, dependent_txn(55, Some(b"v1"))); + let txn_id = TxnId::new(55, 0); + assert_eq!( + scheduler.pending.get(&txn_id).map(|p| p.commit_state), + Some(CommitState::Following) + ); + // The earlier leader voted commit and the verdict is stored. + registry.note_verdict( + nodedb_cluster::calvin::TxnId::new(55, 0), + VerdictOutcome::Commit, + ); + scheduler.resume_on_verdict(txn_id, true); + // The leader's barrier timeout passes on the follower's clock. + // no-determinism: test-only pause past the configured timeout. + tokio::time::sleep(scheduler.config.passive_timeout()).await; + + lead_data_group(&mut scheduler); + + assert!(staged_active(&mut data_side, 55)); + assert!(scheduler.apply_halt().is_none()); + assert!(abort_votes(&proposer).is_empty()); + assert_eq!( + scheduler.pending.get(&txn_id).map(|p| p.commit_state), + Some(CommitState::Staged) + ); + } + + /// A follower whose log timed the barrier out votes abort once + /// promoted: the decision the earlier leader reached from the same log. + #[tokio::test] + async fn a_promoted_follower_decides_a_logged_timeout_alike() { + let (mut scheduler, _dir, mut data_side, proposer, _registry) = scheduler(); + grant(&mut scheduler, dependent_txn(56, Some(b"v1"))); + buffer(&scheduler, 56, BarrierEvent::Timeout); + scheduler.drain_read_results(); + buffer(&scheduler, 56, read(b"v1")); + scheduler.drain_read_results(); + + lead_data_group(&mut scheduler); + + assert!(!staged_active(&mut data_side, 56)); + assert_eq!(abort_votes(&proposer), vec![AbortReason::ParticipantError]); + } + + /// An abort verdict ends an open barrier: the txn drops with no stage. + #[tokio::test] + async fn an_abort_verdict_ends_an_open_barrier() { + let (mut scheduler, _dir, mut data_side, _proposer, registry) = scheduler(); + lead_data_group(&mut scheduler); + grant(&mut scheduler, dependent_txn(57, Some(b"v1"))); + let txn_id = TxnId::new(57, 0); + registry.note_verdict( + nodedb_cluster::calvin::TxnId::new(57, 0), + VerdictOutcome::Abort(AbortReason::ParticipantError), + ); + + scheduler.resume_on_verdict(txn_id, false); + + assert!(scheduler.dependent_barrier.is_empty()); + assert!(!staged_active(&mut data_side, 57)); + assert_eq!( + scheduler.pending.get(&txn_id).map(|p| p.commit_state), + Some(CommitState::AwaitingDrop) + ); + } + + /// The leader proposes the barrier's timeout entry once its timeout + /// passes, and decides nothing until the entry applies. + #[tokio::test] + async fn the_leader_proposes_a_due_timeout_and_waits_for_the_log() { + let (mut scheduler, _dir, _data_side, proposer, _registry) = scheduler(); + lead_data_group(&mut scheduler); + grant(&mut scheduler, dependent_txn(58, Some(b"v1"))); + let txn_id = TxnId::new(58, 0); + let group_id = { + let mr = scheduler + .multi_raft + .lock() + .unwrap_or_else(|p| p.into_inner()); + mr.routing() + .read() + .unwrap_or_else(|p| p.into_inner()) + .group_for_vshard(scheduler.vshard_id) + .expect("routed vShard") + }; + let term_start = scheduler + .multi_raft + .lock() + .unwrap_or_else(|p| p.into_inner()) + .term_start_index(group_id) + .expect("term start"); + if let Some(barrier) = scheduler.dependent_barrier.get_mut(&txn_id) { + // no-determinism: test-only deadline already passed. + barrier.timeout_due = std::time::Instant::now() - std::time::Duration::from_millis(1); + } + + scheduler.propose_due_read_timeouts(); + + assert!( + scheduler + .multi_raft + .lock() + .unwrap_or_else(|p| p.into_inner()) + .log_term_at(group_id, term_start + 1) + .is_some(), + "the timeout entry is in the group's log" + ); + let barrier = scheduler + .dependent_barrier + .get(&txn_id) + .expect("the barrier stays open until the entry applies"); + assert!(barrier.timeout_due > std::time::Instant::now()); + assert!(abort_votes(&proposer).is_empty()); + } + + /// A stopping scheduler hands the log of an unfinished txn back to the + /// buffer for the next scheduler of the vShard. + #[tokio::test] + async fn a_stopping_scheduler_hands_its_logs_back() { + let (mut scheduler, _dir, _data_side, _proposer, _registry) = scheduler(); + grant(&mut scheduler, dependent_txn(59, Some(b"v1"))); + buffer(&scheduler, 59, read(b"v1")); + scheduler.drain_read_results(); + assert_eq!(scheduler.read_results.waiting_txns(), 0); + + scheduler.return_barrier_logs(); + + assert_eq!(scheduler.read_results.waiting_txns(), 1); + assert!(scheduler.barrier_logs.is_empty()); + } } diff --git a/nodedb/src/control/cluster/calvin/scheduler/driver/core/read_timeout.rs b/nodedb/src/control/cluster/calvin/scheduler/driver/core/read_timeout.rs new file mode 100644 index 000000000..388b06511 --- /dev/null +++ b/nodedb/src/control/cluster/calvin/scheduler/driver/core/read_timeout.rs @@ -0,0 +1,122 @@ +// SPDX-License-Identifier: BUSL-1.1 + +//! The data-group leader proposes the timeout entry of a dependent-read +//! barrier that waited too long. +//! +//! The leader's own clock decides only when it proposes +//! `ReplicatedWrite::CalvinReadTimeout`. The entry's place in the vShard's +//! log decides what it does (see [`super::read_result`]): every replica +//! applies it after the same read results. The leader proposes straight to +//! its group with `MultiRaft::propose`, as it proposes a redo: the write +//! gate exempts the entry, which writes no row. +//! +//! A proposal re-arms the barrier's timeout. An entry a leader change +//! lost is proposed again then, and a duplicate that lands after the first +//! changes nothing. + +use std::time::Instant; + +use nodedb_cluster::ClusterError; +use nodedb_raft::RaftError; + +use super::scheduler::Scheduler; +use crate::control::cluster::calvin::scheduler::lock_manager::TxnId; +use crate::control::wal_replication::{ReplicatedEntry, ReplicatedWrite}; + +/// Why a timeout entry did not reach the log. +enum TimeoutRefusal { + /// This node does not lead the vShard's data group. + NotLeader, + /// The entry cannot be built, or the group refused it now. + Failed(String), +} + +impl Scheduler { + /// Propose the timeout entry of every open barrier whose timeout passed. + /// Runs on the leader only. + pub(in crate::control::cluster::calvin::scheduler::driver::core) fn propose_due_read_timeouts( + &mut self, + ) { + if !self.role.is_leader() || self.dependent_barrier.is_empty() { + return; + } + // no-determinism: the leader's clock decides only when it proposes the timeout entry; the entry's place in the log decides the barrier. + let now = Instant::now(); + let due: Vec = self + .dependent_barrier + .iter() + .filter(|(_, barrier)| barrier.timeout_due <= now) + .map(|(txn_id, _)| *txn_id) + .collect(); + for txn_id in due { + match self.propose_read_timeout(txn_id) { + Ok(()) => {} + Err(TimeoutRefusal::NotLeader) => { + self.on_not_leader(); + return; + } + Err(TimeoutRefusal::Failed(error)) => tracing::debug!( + vshard_id = self.vshard_id, + epoch = txn_id.epoch, + position = txn_id.position, + %error, + "calvin: barrier timeout entry not proposed; proposing it again later" + ), + } + if let Some(barrier) = self.dependent_barrier.get_mut(&txn_id) { + barrier.timeout_due = now + self.config.passive_timeout(); + } + } + } + + /// The earliest instant this leader proposes a barrier's timeout entry, + /// when it holds a barrier open. + pub(in crate::control::cluster::calvin::scheduler::driver::core) fn next_read_timeout_due( + &self, + ) -> Option { + if !self.role.is_leader() { + return None; + } + self.dependent_barrier + .values() + .map(|barrier| barrier.timeout_due) + .min() + } + + /// Append the timeout entry of `txn_id`'s barrier to this vShard's + /// data-group log. + fn propose_read_timeout(&self, txn_id: TxnId) -> Result<(), TimeoutRefusal> { + let Some(barrier) = self.dependent_barrier.get(&txn_id) else { + return Ok(()); + }; + let tenant_id = barrier.txn.tx_class.tenant_id.as_u64(); + let entry = ReplicatedEntry::new( + tenant_id, + barrier.txn.tx_class.database_id.as_u64(), + self.vshard_id, + ReplicatedWrite::CalvinReadTimeout { + epoch: txn_id.epoch, + position: txn_id.position, + tenant_id, + }, + ); + let bytes = entry + .encode() + .map_err(|error| TimeoutRefusal::Failed(error.to_string()))?; + let mut mr = self.multi_raft.lock().unwrap_or_else(|p| p.into_inner()); + match mr.propose(self.vshard_id, bytes) { + Ok(_) => { + tracing::info!( + vshard_id = self.vshard_id, + epoch = txn_id.epoch, + position = txn_id.position, + "calvin: dependent-read barrier waited past its timeout; proposed the \ + timeout entry" + ); + Ok(()) + } + Err(ClusterError::Raft(RaftError::NotLeader { .. })) => Err(TimeoutRefusal::NotLeader), + Err(error) => Err(TimeoutRefusal::Failed(error.to_string())), + } + } +} diff --git a/nodedb/src/control/cluster/calvin/scheduler/driver/core/redo_applied.rs b/nodedb/src/control/cluster/calvin/scheduler/driver/core/redo_applied.rs new file mode 100644 index 000000000..e9efa74d0 --- /dev/null +++ b/nodedb/src/control/cluster/calvin/scheduler/driver/core/redo_applied.rs @@ -0,0 +1,314 @@ +// SPDX-License-Identifier: BUSL-1.1 + +//! What the scheduler does when the data-group apply loop concluded a +//! committed slice's stamped redo. Every replica runs it alike. +//! +//! - `RedoApplied` for a txn this scheduler holds: deposit the reply, +//! owe the `CompletionAck`, release the locks, mark the gate, complete. +//! The ack's bytes come from the reply, which every replica renders the +//! same, so the first ack the sequencer log holds is the true one. +//! - `RedoApplied` for a txn still waiting for its locks: kept until the +//! locks are granted, then completed the same way. +//! - `RedoApplied` for a txn whose input has not arrived: the log ran ahead +//! of the sequencer fan-out. Mark the gate and owe the ack. The input is +//! skipped when it arrives. +//! - `RedoRefused` and `RedoNotApplied` halt the scheduler. The txn holds +//! its locks. +//! +//! The apply loop marked the ledger before it pushed the event. A halted +//! scheduler still drains its inbox, so the apply loop never waits on it. + +use crate::bridge::envelope::Response; +use crate::control::cluster::calvin::scheduler::CalvinApplyEvent; +use crate::control::cluster::calvin::scheduler::lock_manager::TxnId; +use crate::control::state::AckSlice; + +use super::super::types::CommitState; +use super::halt::{HaltReason, HaltStep}; +use super::owed::SchedulerProposal; +use super::process::LedgerMark; +use super::scheduler::Scheduler; + +/// A committed slice's install that is durable on this replica. +#[derive(Debug, Clone)] +pub(in crate::control::cluster::calvin::scheduler::driver::core) struct AppliedRedo { + pub reply: Response, + pub primary_write: bool, + pub returning: bool, +} + +impl Scheduler { + /// Handle every event waiting in this scheduler's inbox, in + /// `(epoch, position)` order. + pub(in crate::control::cluster::calvin::scheduler::driver::core) fn drain_inbox(&mut self) { + for ((epoch, position), event) in self.inbox.inbox().take_all() { + self.handle_calvin_apply_event(TxnId::new(epoch, position), event); + } + } + + /// Handle how the apply loop concluded the stamped redo of `txn_id`. + pub(in crate::control::cluster::calvin::scheduler::driver::core) fn handle_calvin_apply_event( + &mut self, + txn_id: TxnId, + event: CalvinApplyEvent, + ) { + match event { + CalvinApplyEvent::RedoApplied { + reply, + primary_write, + returning, + } => self.on_redo_applied( + txn_id, + AppliedRedo { + reply, + primary_write, + returning, + }, + ), + CalvinApplyEvent::RedoRefused { error } => self.halt_apply( + txn_id, + HaltReason::RedoInstallRefused, + HaltStep::RedoApply, + format!("the data group refused the committed slice's redo for good: {error}"), + ), + CalvinApplyEvent::RedoNotApplied { error } => self.halt_apply( + txn_id, + HaltReason::RedoApplyFailed, + HaltStep::RedoApply, + format!("the committed slice's redo did not become durable here: {error}"), + ), + } + } + + /// The stamped redo of `txn_id` installed on this replica. + fn on_redo_applied(&mut self, txn_id: TxnId, applied: AppliedRedo) { + if let Some(pending) = self.pending.get(&txn_id) { + let state = pending.commit_state; + let lock_owner = pending.lock_owner; + if holds_staged_state(state) { + // Another leader's copy installed while this node staged the + // txn again: the staged state is spent. + self.discard_staged(txn_id); + } + self.pending.remove(&txn_id); + self.complete_applied(txn_id, lock_owner, applied); + return; + } + if let Some(barrier) = self.dependent_barrier.remove(&txn_id) { + self.complete_applied(txn_id, barrier.lock_owner, applied); + return; + } + if let Some(lock_owner) = self.parts.release_awaiting(txn_id) { + self.complete_applied(txn_id, lock_owner, applied); + return; + } + let blocked = self + .blocked + .values() + .any(|blocked| TxnId::new(blocked.txn.epoch, blocked.txn.position) == txn_id); + if blocked { + self.early_applied.insert(txn_id, applied); + return; + } + // The input has not arrived. Its position is applied: the input is + // skipped when it does, and takes no lock. + self.owe_applied_ack(txn_id, &applied); + if let Some(watermark) = self.applied.mark_applied(txn_id.epoch, txn_id.position) { + self.publish_watermark(watermark); + } + } + + /// Complete `txn_id`, whose redo installed, releasing the locks + /// `lock_owner` holds. + pub(in crate::control::cluster::calvin::scheduler::driver::core) fn complete_applied( + &mut self, + txn_id: TxnId, + lock_owner: TxnId, + applied: AppliedRedo, + ) { + self.owe_applied_ack(txn_id, &applied); + self.metrics.record_completed(); + self.release_and_mark_applied(txn_id, lock_owner, LedgerMark::ByApply); + } + + /// Deposit a primary slice's reply, then owe the ack that carries it. + /// The deposit lands first, so the coordinator finds it when the ack + /// fires its waiter. + fn owe_applied_ack(&mut self, txn_id: TxnId, applied: &AppliedRedo) { + if applied.primary_write { + self.deposit_reply(txn_id, &applied.reply, applied.returning); + } + let result = self.ack_result_of( + &applied.reply, + AckSlice { + primary_write: applied.primary_write, + returning: applied.returning, + }, + ); + self.propose_sequencer_entry(txn_id, SchedulerProposal::CompletionAck { result }); + } +} + +/// Whether a txn in `state` holds staged state on this node's core. +fn holds_staged_state(state: CommitState) -> bool { + match state { + CommitState::Staged + | CommitState::AwaitingVerdict + | CommitState::AwaitingResolveTurn + | CommitState::AwaitingRedoResolve + | CommitState::AwaitingDrop => true, + // The install consumed the staged entry of a proposed slice, a + // follower staged nothing, a txn awaiting its restage discarded what + // it staged, and a passive read stages nothing. + CommitState::AwaitingRedoApply { .. } + | CommitState::Following + | CommitState::AwaitingRestage + | CommitState::ReadingPassive => false, + } +} + +#[cfg(test)] +mod tests { + use std::time::Duration; + + use nodedb_cluster::calvin::CalvinCompletionRegistry; + use nodedb_cluster::calvin::types::SchedulerInput; + + use super::*; + use crate::bridge::envelope::Status; + use crate::control::cluster::calvin::scheduler::driver::core::halt::HaltReason; + use crate::control::cluster::calvin::scheduler::driver::core::owed::OwedKind; + use crate::control::cluster::calvin::scheduler::driver::core::test_support::{ + build_test_scheduler_with_data_side, make_sequenced_txn, redo_applied, + scheduler_with_pending, spawn_scheduler_loop, staged_response, test_coll_vshard, + }; + + /// The installed slice's reply. + fn reply() -> Response { + staged_response(Status::Ok, None) + } + + /// A follower's held txn whose redo installed completes: it owes its + /// ack, releases its locks, and marks its position applied. The txn + /// waiting on its locks runs. + #[tokio::test] + async fn redo_applied_releases_locks_and_owes_the_ack() { + let (mut scheduler, _dir, _data_side) = build_test_scheduler_with_data_side( + test_coll_vshard(), + CalvinCompletionRegistry::new_detached(), + ); + let txn_id = TxnId::new(3, 0); + let next = TxnId::new(4, 0); + scheduler.process_scheduler_input(SchedulerInput::Txn(Box::new(make_sequenced_txn(3, 0)))); + scheduler.process_scheduler_input(SchedulerInput::Txn(Box::new(make_sequenced_txn(4, 0)))); + assert!(scheduler.blocked.contains_key(&next), "the same key waits"); + + scheduler.handle_calvin_apply_event(txn_id, redo_applied(reply())); + + assert!(!scheduler.pending.contains_key(&txn_id)); + assert!(scheduler.applied.is_applied(3, 0)); + assert!( + scheduler + .owed + .contains_key(&(txn_id, OwedKind::CompletionAck)) + ); + assert!(!scheduler.blocked.contains_key(&next)); + assert!( + scheduler.pending.contains_key(&next), + "the freed locks grant the next txn" + ); + assert!(!scheduler.is_apply_halted()); + } + + /// The log ran ahead of the sequencer fan-out: the redo installed before + /// this scheduler saw the txn. Its position is applied and its ack owed, + /// and the input is skipped when it arrives. + #[tokio::test] + async fn redo_applied_before_the_input_skips_the_input() { + let (mut scheduler, _dir, _data_side) = build_test_scheduler_with_data_side( + test_coll_vshard(), + CalvinCompletionRegistry::new_detached(), + ); + let txn_id = TxnId::new(3, 0); + + scheduler.handle_calvin_apply_event(txn_id, redo_applied(reply())); + assert!(scheduler.applied.is_applied(3, 0)); + assert!( + scheduler + .owed + .contains_key(&(txn_id, OwedKind::CompletionAck)) + ); + + scheduler.process_scheduler_input(SchedulerInput::Txn(Box::new(make_sequenced_txn(3, 0)))); + assert!( + !scheduler.pending.contains_key(&txn_id), + "the input is skipped" + ); + assert!(!scheduler.blocked.contains_key(&txn_id)); + scheduler.process_scheduler_input(SchedulerInput::Txn(Box::new(make_sequenced_txn(4, 0)))); + assert!( + scheduler.pending.contains_key(&TxnId::new(4, 0)), + "the skipped input took no lock" + ); + } + + /// A redo every replica refused for good halts the scheduler. The txn + /// keeps its locks and its position stays unapplied. + #[tokio::test] + async fn redo_refused_halts() { + let txn_id = TxnId::new(5, 1); + let (mut scheduler, _dir) = + scheduler_with_pending(txn_id, CommitState::AwaitingRedoApply { proposed: None }); + + scheduler.handle_calvin_apply_event( + txn_id, + CalvinApplyEvent::RedoRefused { + error: "constraint".to_string(), + }, + ); + + assert_eq!( + scheduler.apply_halt().map(|h| h.reason), + Some(HaltReason::RedoInstallRefused) + ); + assert!(scheduler.pending.contains_key(&txn_id)); + assert!(!scheduler.applied.is_applied(5, 1)); + } + + /// A halted scheduler still drains its inbox, so the apply loop never + /// waits on it. + #[tokio::test] + async fn a_halted_scheduler_still_drains_the_inbox() { + let txn_id = TxnId::new(5, 1); + let (mut scheduler, _dir) = + scheduler_with_pending(txn_id, CommitState::AwaitingRedoApply { proposed: None }); + scheduler.handle_calvin_apply_event( + txn_id, + CalvinApplyEvent::RedoNotApplied { + error: "fsync".to_string(), + }, + ); + assert!(scheduler.is_apply_halted()); + let inbox = std::sync::Arc::clone(scheduler.inbox.inbox()); + + let running = spawn_scheduler_loop(scheduler); + inbox + .push( + 6, + 0, + CalvinApplyEvent::RedoNotApplied { + error: "fsync".to_string(), + }, + ) + .await; + let drained = tokio::time::timeout(Duration::from_secs(5), async { + while inbox.waiting() > 0 { + tokio::time::sleep(Duration::from_millis(10)).await; + } + }) + .await; + running.stop().await; + + assert!(drained.is_ok(), "the halted scheduler drains its inbox"); + } +} diff --git a/nodedb/src/control/cluster/calvin/scheduler/driver/core/redo_propose.rs b/nodedb/src/control/cluster/calvin/scheduler/driver/core/redo_propose.rs new file mode 100644 index 000000000..91f142284 --- /dev/null +++ b/nodedb/src/control/cluster/calvin/scheduler/driver/core/redo_propose.rs @@ -0,0 +1,519 @@ +// SPDX-License-Identifier: BUSL-1.1 + +//! The data-group leader proposes a committed Calvin slice's stamped redo. +//! +//! The leader proposes right after the resolve, straight to its group with +//! `MultiRaft::propose`: no vShard admission slot and no forward. The +//! leader's write gate exempts a stamped redo, and the scheduler's lock +//! table already orders it against every conflicting write. Entries of +//! non-conflicting txns can enter the log in any order. +//! +//! A redo whose entry passes `tuning.calvin.max_redo_entry_bytes` travels +//! as chunk entries under a stream that names its position and attempt, +//! then a final entry naming the stream. +//! +//! The txn completes once its redo applies. Until then: +//! +//! | Outcome | Action | +//! |---|---| +//! | `NotLeader` | Demote. The next leader stages the txn again. | +//! | Proposed, then the group applied past the entry with no install | Propose again. | +//! | A transient refusal | Propose again on the stall tick. | +//! | The entry cannot be built | Halt: the verdict is COMMIT, so the txn cannot drop. | +//! | Sequenced after an ordered cut's marker whose barrier this node has not applied | Hold. Propose once the barrier applied. | +//! +//! Every copy that commits installs at most once: the apply loop claims the +//! position in the vShard's applied ledger. + +use nodedb_cluster::ClusterError; +use nodedb_raft::RaftError; + +use super::super::types::CommitState; +use super::halt::{HaltReason, HaltStep}; +use super::scheduler::Scheduler; +use crate::control::cluster::calvin::scheduler::lock_manager::TxnId; +use crate::control::wal_replication::ReplicatedWrite; +use crate::control::wal_replication::encode::{ + CalvinEntryStamps, RedoEntryTarget, RedoProposal, calvin_redo_proposal, +}; +use crate::wal::RedoStreamId; + +/// A committed slice's redo, kept while it is proposed. +#[derive(Debug, Clone)] +pub(in crate::control::cluster::calvin::scheduler::driver) struct OwedRedo { + pub payload: crate::control::wal_replication::transaction_redo::TransactionRedoPayload, + pub stamps: CalvinEntryStamps, + pub target: RedoEntryTarget, + /// Proposals made so far. Each names its own chunk stream. + pub attempts: u32, +} + +/// Why a proposal did not reach the log. +enum ProposeRefusal { + /// This node does not lead the vShard's data group. + NotLeader, + /// The group cannot take it now. The stall tick proposes again. + Transient(String), + /// The entries cannot be built. + Unbuildable(String), +} + +impl Scheduler { + /// Propose the redo `txn_id` owes, and record where it landed. + pub(in crate::control::cluster::calvin::scheduler::driver::core) fn propose_calvin_redo( + &mut self, + txn_id: TxnId, + ) { + if self + .pending + .get(&txn_id) + .and_then(|pending| pending.redo.as_ref()) + .is_none() + { + return; + } + #[cfg(feature = "failpoints")] + if self.redo_propose_held(txn_id) { + return; + } + // A txn sequenced after an ordered cut's marker proposes its redo + // only once this node applied the cut's barrier in the vShard's + // group, so the redo lands after the barrier. The pass that ends the + // hold proposes it again. + if self.held_for_cut(txn_id.epoch) { + return; + } + // A snapshot install replaced this vShard's base since this + // scheduler started. The next scheduler starts from the installed + // state and stages the txn again. + if self.install_gate.is_retired(&self.shared, self.vshard_id) { + return; + } + // A halt released the txn's gates: take them back, or refuse a redo + // into a recreated collection. + if let Err(error) = self.regate(txn_id) { + self.halt_apply( + txn_id, + HaltReason::RedoProposeFailed, + HaltStep::RedoPropose, + error.to_string(), + ); + return; + } + let proposed = self + .attempt_entries(txn_id) + .and_then(|entries| self.propose_entries(entries)); + match proposed { + Ok(proposed) => { + if let Some(pending) = self.pending.get_mut(&txn_id) { + pending.commit_state = CommitState::AwaitingRedoApply { + proposed: Some(proposed), + }; + } + } + Err(ProposeRefusal::NotLeader) => self.on_not_leader(), + Err(ProposeRefusal::Transient(error)) => { + tracing::debug!( + vshard_id = self.vshard_id, + epoch = txn_id.epoch, + position = txn_id.position, + %error, + "calvin: redo not proposed; the stall tick proposes it again" + ); + if let Some(pending) = self.pending.get_mut(&txn_id) { + pending.commit_state = CommitState::AwaitingRedoApply { proposed: None }; + } + } + Err(ProposeRefusal::Unbuildable(error)) => self.halt_apply( + txn_id, + HaltReason::RedoProposeFailed, + HaltStep::RedoPropose, + format!("the committed slice's redo entry cannot be built: {error}"), + ), + } + } + + /// Whether a crash test holds `txn_id`'s redo: it resolved and no copy + /// entered the log. The stall tick asks again. + #[cfg(feature = "failpoints")] + fn redo_propose_held(&self, txn_id: TxnId) -> bool { + let held = self + .pending + .get(&txn_id) + .and_then(|pending| pending.redo.as_ref()) + .is_some_and(|owed| { + crate::control::fail_gate::holds_redo_propose( + self.shared.node_id, + &owed.payload.collections, + ) + }); + if held { + tracing::info!( + node_id = self.shared.node_id, + vshard_id = self.vshard_id, + epoch = txn_id.epoch, + position = txn_id.position, + "calvin: redo proposal held at a fail point" + ); + } + held + } + + /// The encoded entries of the next proposal attempt of `txn_id`. + fn attempt_entries(&mut self, txn_id: TxnId) -> Result>, ProposeRefusal> { + let max_entry_bytes = self.shared.redo_chunks.limits().max_entry_bytes; + let vshard = self.vshard_id; + let until = self.request_deadline(); + let Some(owed) = self + .pending + .get_mut(&txn_id) + .and_then(|pending| pending.redo.as_mut()) + else { + return Err(ProposeRefusal::Unbuildable( + "the txn owes no redo".to_owned(), + )); + }; + let stream = RedoStreamId::Calvin { + vshard, + epoch: txn_id.epoch, + position: txn_id.position, + attempt: owed.attempts, + }; + owed.attempts = owed.attempts.saturating_add(1); + let unbuildable = |error: crate::Error| ProposeRefusal::Unbuildable(error.to_string()); + let proposal = calvin_redo_proposal( + owed.target, + &owed.payload, + &owed.stamps, + stream, + max_entry_bytes, + ) + .map_err(unbuildable)?; + let (chunks, last) = match proposal { + RedoProposal::Inline(last) => (Vec::new(), last), + RedoProposal::Chunked { + stream, + chunks, + last, + } => { + let len = if let Some(ReplicatedWrite::RedoChunk { len, .. }) = + chunks.first().map(|chunk| &chunk.write) + { + *len + } else { + 0 + }; + // The leader bounds the node's open stream bytes: a stream + // past the cap waits for the stall tick. + self.shared + .redo_chunks + .admit_stream(stream, len, until) + .map_err(|refusal| ProposeRefusal::Transient(refusal.to_string()))?; + (chunks, last) + } + }; + let mut entries = Vec::with_capacity(chunks.len() + 1); + for entry in chunks.iter().chain(std::iter::once(&last)) { + entries.push(entry.encode().map_err(unbuildable)?); + } + Ok(entries) + } + + /// Append `entries` to this vShard's data-group log, in order. Returns + /// `(group_id, log_index)` of the last. + fn propose_entries(&self, entries: Vec>) -> Result<(u64, u64), ProposeRefusal> { + let mut mr = self.multi_raft.lock().unwrap_or_else(|p| p.into_inner()); + let mut last = None; + for bytes in entries { + match mr.propose(self.vshard_id, bytes) { + Ok(at) => last = Some(at), + Err(ClusterError::Raft(RaftError::NotLeader { .. })) => { + return Err(ProposeRefusal::NotLeader); + } + Err(error) => return Err(ProposeRefusal::Transient(error.to_string())), + } + } + last.ok_or_else(|| ProposeRefusal::Unbuildable("a redo with no entry".to_owned())) + } + + /// Propose again every owed redo that is not in the log: never + /// proposed, refused, or proposed and passed by the group's applied + /// index with no install, which a new term's entries overwrote. Runs + /// on the stall tick, after the inbox drained, on the leader only. + pub(in crate::control::cluster::calvin::scheduler::driver::core) fn retry_redo_proposals( + &mut self, + ) { + if !self.role.is_leader() { + return; + } + let owed: Vec<(TxnId, Option<(u64, u64)>)> = self + .pending + .iter() + .filter_map(|(txn_id, pending)| match pending.commit_state { + CommitState::AwaitingRedoApply { proposed } => Some((*txn_id, proposed)), + CommitState::Following + | CommitState::Staged + | CommitState::AwaitingVerdict + | CommitState::AwaitingResolveTurn + | CommitState::AwaitingRedoResolve + | CommitState::AwaitingDrop + | CommitState::AwaitingRestage + | CommitState::ReadingPassive => None, + }) + .collect(); + for (txn_id, proposed) in owed { + let lost = match proposed { + None => true, + Some((group_id, log_index)) => { + self.shared.applied_index_watcher(group_id).current() >= log_index + } + }; + if lost { + if proposed.is_some() { + tracing::warn!( + vshard_id = self.vshard_id, + epoch = txn_id.epoch, + position = txn_id.position, + "calvin: the group applied past the slice's redo with no install; \ + proposing it again" + ); + } + self.propose_calvin_redo(txn_id); + } + if !self.role.is_leader() { + return; + } + } + } +} + +#[cfg(test)] +mod tests { + use nodedb_cluster::calvin::CalvinCompletionRegistry; + + use super::*; + use crate::bridge::dispatch::CoreChannelDataSide; + use crate::bridge::envelope::{Payload, Response, Status}; + use crate::control::cluster::calvin::scheduler::driver::core::test_proposer::{ + lead_data_group, step_down_from_data_group, + }; + use crate::control::cluster::calvin::scheduler::driver::core::test_support::{ + build_test_scheduler_with_data_side, make_sequenced_txn, staged_pending, staged_response, + }; + use crate::control::wal_replication::ReplicatedEntry; + use crate::wal::RedoRecord; + + const VSHARD: u32 = 7; + + /// A leader on [`VSHARD`] with `txn_id` committed and its resolve + /// awaited. + fn resolving_leader(txn_id: TxnId) -> (Scheduler, tempfile::TempDir, CoreChannelDataSide) { + let (mut scheduler, dir, data_side) = + build_test_scheduler_with_data_side(VSHARD, CalvinCompletionRegistry::new_detached()); + lead_data_group(&mut scheduler); + let mut pending = staged_pending(make_sequenced_txn(txn_id.epoch, txn_id.position), txn_id); + pending.commit_state = CommitState::AwaitingRedoResolve; + scheduler.pending.insert(txn_id, pending); + (scheduler, dir, data_side) + } + + /// A resolve answer whose redo record carries no ops. + fn resolved_answer() -> Response { + let redo = RedoRecord { + version: 1, + ops: Vec::new(), + calvin_stamp: None, + cross_shard_applied: None, + row_sources: Vec::new(), + publishes: Vec::new(), + row_changes: Vec::new(), + }; + let resolved = nodedb_physical::physical_plan::CalvinResolved { + redo: redo.to_bytes().expect("encode redo"), + reply: nodedb_physical::physical_plan::CalvinReplySpec::Count(Vec::new()), + }; + let mut response = staged_response(Status::Ok, None); + response.payload = + Payload::from_vec(zerompk::to_msgpack_vec(&resolved).expect("encode resolved")); + response + } + + /// Where `txn_id`'s redo entry landed, if it is in the log. + fn proposed_at(scheduler: &Scheduler, txn_id: TxnId) -> Option<(u64, u64)> { + match scheduler.pending.get(&txn_id).map(|p| p.commit_state) { + Some(CommitState::AwaitingRedoApply { proposed }) => proposed, + other => panic!("the txn awaits its redo apply, found {other:?}"), + } + } + + /// The committed entry of `group_id` at `index`. Ticks the single-voter + /// group until the entry commits. + fn committed_entry(scheduler: &Scheduler, group_id: u64, index: u64) -> Vec { + let mut mr = scheduler + .multi_raft + .lock() + .unwrap_or_else(|p| p.into_inner()); + for _ in 0..20 { + let entries = mr + .read_committed_entries(group_id, index, index) + .expect("read the group's log"); + if let Some(entry) = entries.into_iter().find(|entry| entry.index == index) { + return entry.data; + } + mr.wait_all_durable_blocking(); + mr.tick().expect("tick"); + } + panic!("the entry at {index} never committed"); + } + + /// A committed write slice's resolve proposes exactly one entry: the + /// stamped redo of the slice, on the vShard's data group. + #[tokio::test] + async fn a_committed_write_slice_proposes_one_stamped_entry() { + let txn_id = TxnId::new(14, 2); + let (mut scheduler, _dir, _data_side) = resolving_leader(txn_id); + let group_id = { + let mr = scheduler + .multi_raft + .lock() + .unwrap_or_else(|p| p.into_inner()); + mr.routing() + .read() + .unwrap_or_else(|p| p.into_inner()) + .group_for_vshard(VSHARD) + .expect("the fixture routes every vShard") + }; + let last_before = scheduler + .multi_raft + .lock() + .unwrap_or_else(|p| p.into_inner()) + .last_log_index(group_id) + .expect("the group is mounted"); + + scheduler.finish_redo_resolve(txn_id, resolved_answer()); + + let (proposed_group, index) = + proposed_at(&scheduler, txn_id).expect("the redo is in the log"); + assert_eq!(proposed_group, group_id); + assert_eq!(index, last_before + 1, "one entry, no chunks"); + let entry = ReplicatedEntry::from_bytes(&committed_entry(&scheduler, group_id, index)) + .expect("a replicated entry"); + assert_eq!(entry.vshard_id, VSHARD); + assert!( + matches!( + entry.write, + ReplicatedWrite::TransactionRedo { + calvin: Some(_), + .. + } + ), + "the entry is the slice's stamped redo" + ); + assert!(!scheduler.applied.is_applied(14, 2)); + assert!(!scheduler.is_apply_halted()); + } + + /// The group applied past the proposed entry with no install: a new + /// term overwrote it. The stall tick proposes the redo again. + #[tokio::test] + async fn an_overwritten_proposal_is_proposed_again() { + let txn_id = TxnId::new(14, 2); + let (mut scheduler, _dir, _data_side) = resolving_leader(txn_id); + scheduler.finish_redo_resolve(txn_id, resolved_answer()); + let (group_id, first) = proposed_at(&scheduler, txn_id).expect("proposed"); + + scheduler.retry_redo_proposals(); + assert_eq!( + proposed_at(&scheduler, txn_id), + Some((group_id, first)), + "an entry the group has not applied past stays" + ); + + scheduler.shared.applied_index_watcher(group_id).bump(first); + scheduler.retry_redo_proposals(); + + let (_, second) = proposed_at(&scheduler, txn_id).expect("proposed again"); + assert!(second > first, "a new entry carries the redo"); + if let Some(pending) = scheduler.pending.get(&txn_id) { + assert_eq!( + pending.redo.as_ref().map(|owed| owed.attempts), + Some(2), + "each proposal names its own attempt" + ); + } + } + + /// A slice sequenced after an ordered cut's marker proposes nothing until + /// this node applied the cut's barrier in the vShard's group. Then the + /// pass that ends the hold proposes it. + #[tokio::test] + async fn a_slice_after_a_cut_marker_waits_for_the_barrier() { + use crate::control::backup::cut_order::OrderedCut; + use nodedb_cluster::calvin::types::SchedulerInput; + + let txn_id = TxnId::new(14, 2); + let (mut scheduler, _dir, _data_side) = resolving_leader(txn_id); + let cut = OrderedCut { + hlc: scheduler.shared.hlc_clock.now().wall_ns, + restore_point: 0, + capture: None, + }; + scheduler.process_scheduler_input(SchedulerInput::CutMarker { + hlc: cut.hlc, + restore_point: 0, + barrier: Some(std::sync::Arc::new(cut.to_wire())), + }); + + scheduler.finish_redo_resolve(txn_id, resolved_answer()); + assert_eq!(proposed_at(&scheduler, txn_id), None, "the redo is held"); + scheduler.retry_redo_proposals(); + assert_eq!( + proposed_at(&scheduler, txn_id), + None, + "the stall tick keeps it held" + ); + + let group_id = { + let mr = scheduler + .multi_raft + .lock() + .unwrap_or_else(|p| p.into_inner()); + mr.routing() + .read() + .unwrap_or_else(|p| p.into_inner()) + .group_for_vshard(VSHARD) + .expect("the fixture routes every vShard") + }; + scheduler + .shared + .calvin + .cut_barriers + .note_applied(&cut, group_id, 1); + assert!(scheduler.release_cut_holds()); + scheduler.retry_redo_proposals(); + assert!( + proposed_at(&scheduler, txn_id).is_some(), + "the barrier applied, so the redo is proposed" + ); + assert!(!scheduler.is_apply_halted()); + } + + /// A proposal refused because this node no longer leads demotes the + /// scheduler: the txn is held `Following` for the next leader. + #[tokio::test] + async fn not_leader_demotes() { + let txn_id = TxnId::new(14, 2); + let (mut scheduler, _dir, _data_side) = resolving_leader(txn_id); + step_down_from_data_group(&scheduler); + + scheduler.finish_redo_resolve(txn_id, resolved_answer()); + + assert!(!scheduler.role.is_leader()); + assert_eq!( + scheduler.pending.get(&txn_id).map(|p| p.commit_state), + Some(CommitState::Following) + ); + assert!(!scheduler.is_apply_halted()); + assert!(!scheduler.applied.is_applied(14, 2)); + } +} diff --git a/nodedb/src/control/cluster/calvin/scheduler/driver/core/redo_window.rs b/nodedb/src/control/cluster/calvin/scheduler/driver/core/redo_window.rs deleted file mode 100644 index 04dab5e8b..000000000 --- a/nodedb/src/control/cluster/calvin/scheduler/driver/core/redo_window.rs +++ /dev/null @@ -1,124 +0,0 @@ -// SPDX-License-Identifier: BUSL-1.1 - -//! Close the outcome-floor window of a committed txn's `TransactionRedo` -//! record. -//! -//! The record joins the pending txn when it is appended. The window settles -//! when the txn completes: the flush applied the record. It holds when the -//! txn stays unapplied here, because restart replay must reach the record. - -use super::scheduler::Scheduler; -use crate::control::cluster::calvin::scheduler::lock_manager::TxnId; - -impl Scheduler { - /// Hold the redo record of `txn_id`: the scheduler halted with the txn - /// unapplied. - pub(in crate::control::cluster::calvin::scheduler::driver::core) fn hold_redo_records( - &mut self, - txn_id: TxnId, - ) { - if let Some(records) = self - .pending - .get_mut(&txn_id) - .and_then(|pending| pending.redo_records.take()) - { - records.hold(); - } - } - - /// Hold the redo record of every pending txn: the scheduler stops with - /// them unapplied. - pub(in crate::control::cluster::calvin::scheduler::driver::core) fn hold_all_redo_records( - &mut self, - ) { - for pending in self.pending.values_mut() { - if let Some(records) = pending.redo_records.take() { - records.hold(); - } - } - } -} - -#[cfg(test)] -mod tests { - use super::*; - use crate::control::cluster::calvin::scheduler::driver::core::halt::{HaltReason, HaltStep}; - use crate::control::cluster::calvin::scheduler::driver::core::test_support::scheduler_with_pending; - use crate::control::cluster::calvin::scheduler::driver::types::CommitState; - use crate::control::server::dispatch_utils::MintedRecords; - use crate::types::{DatabaseId, Lsn, TenantId, VShardId}; - - /// Attach an appended redo record to the pending txn, as a committed - /// resolve does. - fn attach_redo_record(scheduler: &mut Scheduler, txn_id: TxnId) -> Lsn { - let records = MintedRecords::open(&scheduler.shared.outcome_floor); - let lsn = records - .appender(&scheduler.shared.wal, crate::wal::manager::NO_APPLY_KEY) - .with_event_source(crate::event::EventSource::User) - .append_put( - TenantId::new(1), - VShardId::new(0), - DatabaseId::DEFAULT, - b"redo", - ) - .expect("append"); - if let Some(pending) = scheduler.pending.get_mut(&txn_id) { - pending.redo_records = Some(records); - } - lsn - } - - fn awaiting_flush() -> CommitState { - CommitState::AwaitingResolve { - committed: true, - redo_lsn: None, - } - } - - #[tokio::test] - async fn a_halted_txn_holds_its_redo_record() { - let txn_id = TxnId::new(3, 1); - let (mut scheduler, _dir) = scheduler_with_pending(txn_id, awaiting_flush()); - let lsn = attach_redo_record(&mut scheduler, txn_id); - - scheduler.halt_apply( - txn_id, - HaltReason::FlushFailed, - HaltStep::Flush, - "flush refused".to_string(), - ); - - let floor = &scheduler.shared.outcome_floor; - assert!( - floor.floor() < lsn, - "the held record keeps the floor below it" - ); - assert_eq!(floor.leaked_windows(), 0); - } - - #[tokio::test] - async fn a_completed_txn_settles_its_redo_record() { - let txn_id = TxnId::new(3, 2); - let (mut scheduler, _dir) = scheduler_with_pending(txn_id, awaiting_flush()); - let lsn = attach_redo_record(&mut scheduler, txn_id); - - scheduler.on_txn_complete(txn_id); - - let floor = &scheduler.shared.outcome_floor; - assert_eq!(floor.floor(), lsn); - assert_eq!(floor.leaked_windows(), 0); - } - - #[tokio::test] - async fn a_stopping_scheduler_holds_every_pending_redo_record() { - let txn_id = TxnId::new(3, 3); - let (mut scheduler, _dir) = scheduler_with_pending(txn_id, awaiting_flush()); - let lsn = attach_redo_record(&mut scheduler, txn_id); - - scheduler.hold_all_redo_records(); - - let floor = &scheduler.shared.outcome_floor; - assert!(floor.floor() < lsn); - assert_eq!(floor.leaked_windows(), 0); - } -} diff --git a/nodedb/src/control/cluster/calvin/scheduler/driver/core/request.rs b/nodedb/src/control/cluster/calvin/scheduler/driver/core/request.rs index 95d79d73d..11cf941ab 100644 --- a/nodedb/src/control/cluster/calvin/scheduler/driver/core/request.rs +++ b/nodedb/src/control/cluster/calvin/scheduler/driver/core/request.rs @@ -7,7 +7,7 @@ use std::time::{Duration, Instant}; use super::scheduler::Scheduler; use crate::bridge::envelope::{Admission, ExemptReason, Priority, Request}; use crate::control::cluster::calvin::scheduler::lock_manager::TxnId; -use crate::types::{DatabaseId, Lsn, ReadConsistency, RequestId, TenantId, VShardId}; +use crate::types::{DatabaseId, ReadConsistency, RequestId, TenantId, VShardId}; use nodedb_physical::physical_plan::PhysicalPlan; /// The event source a Calvin sub-operation runs with when its transaction @@ -15,9 +15,9 @@ use nodedb_physical::physical_plan::PhysicalPlan; pub(in crate::control::cluster::calvin::scheduler::driver::core) const CALVIN_EVENT_SOURCE: crate::event::EventSource = crate::event::EventSource::User; -/// The event source a sequenced transaction's writes carry. Its redo record -/// and its flush both carry it, so WAL replay rebuilds the events the flush -/// emits, and a server-run body's writes fire no triggers. +/// The event source a sequenced transaction's writes carry. Its redo entry +/// carries it, so every replica's install and WAL replay emit the same +/// events, and a server-run body's writes fire no triggers. pub(in crate::control::cluster::calvin::scheduler::driver::core) fn txn_event_source( tx_class: &nodedb_cluster::calvin::types::TxClass, ) -> crate::event::EventSource { @@ -27,14 +27,14 @@ pub(in crate::control::cluster::calvin::scheduler::driver::core) fn txn_event_so /// The event source this vShard's slice of a sequenced transaction commits /// under. A slice only trigger bodies wrote, in a transaction a client also /// wrote, takes the source a body's row takes in one overlay. Every other -/// slice takes the transaction's source. The stage, the resolve, the redo -/// record and the flush all carry it. +/// slice takes the transaction's source. The stage and the redo entry both +/// carry it. pub(in crate::control::cluster::calvin::scheduler::driver::core) fn slice_event_source( tx_class: &nodedb_cluster::calvin::types::TxClass, - flush_scope: &super::super::types::FlushScope, + scope: &super::super::types::SliceScope, ) -> crate::event::EventSource { let source = txn_event_source(tx_class); - if flush_scope.body_only { + if scope.body_only { source.committed_row_override(crate::event::EventSource::Trigger) } else { source @@ -46,17 +46,16 @@ impl Scheduler { /// /// Calvin has already globally ordered the transaction, so the request is /// admitted `Exempt(AlreadyOrdered)` — it bypasses admission control. Only - /// `request_id`, `tenant_id`, `plan`, and `wal_lsn` vary between call sites; - /// everything else (vshard, deadline, and the fixed metadata constants) is - /// derived from `&self` or constant. `wal_lsn` is `Some` only when the caller - /// has already durably appended the record this request applies. + /// `request_id`, `tenant_id`, `database_id` and `plan` vary between call + /// sites; everything else (vshard, deadline, and the fixed metadata + /// constants) is derived from `&self` or constant. No scheduler request + /// writes a WAL record, so none carries a WAL LSN. pub(in crate::control::cluster::calvin::scheduler::driver::core) fn build_exempt_request( &self, request_id: RequestId, tenant_id: TenantId, database_id: DatabaseId, plan: PhysicalPlan, - wal_lsn: Option, ) -> Request { Request { request_id, @@ -74,9 +73,10 @@ impl Scheduler { user_id: None, statement_digest: None, txn_id: None, - wal_lsn, + wal_lsn: None, resolved_now_ms: None, commit_hlc: None, + entry_version: None, admission: Admission::Exempt(ExemptReason::AlreadyOrdered), } } diff --git a/nodedb/src/control/cluster/calvin/scheduler/driver/core/resolve_turn.rs b/nodedb/src/control/cluster/calvin/scheduler/driver/core/resolve_turn.rs new file mode 100644 index 000000000..291b93e62 --- /dev/null +++ b/nodedb/src/control/cluster/calvin/scheduler/driver/core/resolve_turn.rs @@ -0,0 +1,126 @@ +// SPDX-License-Identifier: BUSL-1.1 + +//! The resolve turn of a slice that reads a whole collection at resolve. +//! +//! A committed slice resolves as soon as its verdict arrives, and its redo +//! is proposed at once: the log orders it against every other entry of the +//! group, and conflicting txns never overlap under the lock table. A +//! TRUNCATE of rows is the exception. Its resolve reads every row of the +//! collection, so it waits until every lower txn of this vShard finished: +//! their rows are then installed, and no higher txn's rows are. +//! +//! Every wait points at a lower transaction: the scheduler takes input, and +//! grants locks, in sequencer order. So the turn order adds no wait cycle. + +use super::super::types::CommitState; +use super::scheduler::Scheduler; +use crate::control::cluster::calvin::scheduler::lock_manager::TxnId; + +impl Scheduler { + /// Resolve the lowest unfinished transaction when it waits for its turn. + pub(in crate::control::cluster::calvin::scheduler::driver::core) fn pump_resolve_turn( + &mut self, + ) { + let Some(lowest) = self.lowest_unfinished() else { + return; + }; + let at_turn = self + .pending + .get(&lowest) + .is_some_and(|pending| pending.commit_state == CommitState::AwaitingResolveTurn); + if at_turn { + self.start_resolve(lowest); + } + } + + /// The lowest transaction of this vShard that has not finished: in + /// flight, blocked on a lock, or waiting at a dependent-read barrier. + fn lowest_unfinished(&self) -> Option { + let pending = self.pending.keys().next().copied(); + let barrier = self.dependent_barrier.keys().next().copied(); + let blocked = self + .blocked + .values() + .map(|blocked| TxnId::new(blocked.txn.epoch, blocked.txn.position)) + .min(); + // A multi-part txn holding its locks while its parts arrive is + // unfinished too: every later whole-collection resolve waits for it. + let awaiting_parts = self.parts.lowest_awaiting(); + [pending, barrier, blocked, awaiting_parts] + .into_iter() + .flatten() + .min() + } +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::control::cluster::calvin::scheduler::driver::core::test_support::{ + build_test_scheduler_with_data_side, make_sequenced_txn, staged_pending, + }; + use crate::control::cluster::calvin::scheduler::driver::types::SliceScope; + use nodedb_cluster::calvin::CalvinCompletionRegistry; + + /// A committed TRUNCATE of rows resolves only at its turn: while a lower + /// txn of this vShard is unfinished, its resolve waits, so it reads every + /// row sequenced before it. + #[tokio::test] + async fn a_truncate_resolves_only_at_its_turn() { + let low = TxnId::new(5, 0); + let truncate = TxnId::new(5, 1); + let (mut scheduler, _dir, _data_side) = + build_test_scheduler_with_data_side(7, CalvinCompletionRegistry::new_detached()); + for txn_id in [low, truncate] { + scheduler.pending.insert( + txn_id, + staged_pending(make_sequenced_txn(txn_id.epoch, txn_id.position), txn_id), + ); + } + if let Some(pending) = scheduler.pending.get_mut(&truncate) { + pending.scope.resolve_at_turn = true; + pending.commit_state = CommitState::AwaitingResolveTurn; + } + + scheduler.pump_resolve_turn(); + assert_eq!( + scheduler.pending.get(&truncate).map(|p| p.commit_state), + Some(CommitState::AwaitingResolveTurn), + "the lower txn is unfinished, so the resolve waits" + ); + + scheduler.pending.remove(&low); + scheduler.pump_resolve_turn(); + assert_eq!( + scheduler.pending.get(&truncate).map(|p| p.commit_state), + Some(CommitState::AwaitingRedoResolve), + "with the lower txn finished, the resolve dispatches" + ); + } + + /// Every truncate of rows reads a whole collection at resolve; a point + /// write does not. A TRUNCATE's edge share records a cut, which reads no + /// stored edge, so it resolves without waiting for its turn. + #[test] + fn a_truncate_slice_resolves_at_its_turn() { + use nodedb_physical::physical_plan::{DocumentOp, GraphOp, PhysicalPlan}; + let collection = nodedb_types::QualifiedCollection::from_stored("c".to_string()); + let truncate = PhysicalPlan::Document(DocumentOp::Truncate { + collection: collection.clone(), + restart_identity: false, + resolved_sum_targets: Vec::new(), + declared_primary_key: None, + }); + let share = PhysicalPlan::Graph(GraphOp::TruncateEdges { + collection, + vshard: 3, + }); + assert!(SliceScope::of_plans(std::slice::from_ref(&truncate)).resolve_at_turn); + assert!(!SliceScope::of_plans(std::slice::from_ref(&share)).resolve_at_turn); + assert!( + SliceScope::of_plans(&[truncate, share]).resolve_at_turn, + "the rows' vShard holds both, and its rows' truncate waits" + ); + assert!(!SliceScope::of_plans(&[]).resolve_at_turn); + } +} diff --git a/nodedb/src/control/cluster/calvin/scheduler/driver/core/restage.rs b/nodedb/src/control/cluster/calvin/scheduler/driver/core/restage.rs new file mode 100644 index 000000000..aad369e37 --- /dev/null +++ b/nodedb/src/control/cluster/calvin/scheduler/driver/core/restage.rs @@ -0,0 +1,548 @@ +// SPDX-License-Identifier: BUSL-1.1 + +//! Restage of a committed txn whose stage failed on this leader. +//! +//! A COMMIT verdict reaches a leader whose own stage failed only when an +//! earlier leader of the vShard voted COMMIT first. The txn cannot drop: +//! its peers apply it. So the leader discards what the failed stage left, +//! holds the txn with its locks, and stages it again once a backoff passes. +//! +//! - A restage takes the path a promotion takes, +//! [`Scheduler::route_granted`]: a static stage, a dependent barrier, or a +//! plan rejection. A rejected plan or a timed-out barrier parks the txn +//! unstaged again. That counts as one more failed stage. +//! - The verdict is known, so the restage's vote changes nothing, and its +//! park probe resumes the txn at once. A restage that stages resolves and +//! proposes its redo like any committed slice. +//! - Due restages run in sequencer order. A txn waits for every lower txn +//! that also waits for its restage. +//! - The leader halts with `LocalStageFailed` once the restages run out. A +//! txn whose collection is superseded cannot commit here, and halts at +//! once. +//! +//! The backoff runs on this node's clock. It only delays a retry of a +//! decided verdict, so no replica applies anything different. + +use std::sync::atomic::Ordering; +use std::time::Instant; + +use tracing::{info, warn}; + +use super::super::types::{CommitState, PendingTxn}; +use super::halt::{HaltReason, HaltStep}; +use super::scheduler::Scheduler; +use crate::control::cluster::calvin::scheduler::lock_manager::TxnId; + +/// How far a committed txn is through its restages. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub(in crate::control::cluster::calvin::scheduler::driver::core) struct Restage { + /// Restages dispatched so far. + pub attempts: u32, + /// When the next restage dispatches. `None` while one is in flight. + pub due: Option, +} + +impl Scheduler { + /// Act on a COMMIT verdict for `txn_id`, whose stage failed on this + /// leader: hold it for a restage, or halt once it cannot commit here. + pub(in crate::control::cluster::calvin::scheduler::driver::core) fn restage_or_halt( + &mut self, + txn_id: TxnId, + ) { + let Some((superseded, stage_error)) = self.pending.get_mut(&txn_id).map(|pending| { + pending.verdict_deadline = None; + ( + pending.superseded, + pending.stage_error.clone().unwrap_or_default(), + ) + }) else { + return; + }; + if superseded { + self.halt_apply( + txn_id, + HaltReason::LocalStageFailed, + HaltStep::Stage, + format!( + "COMMIT verdict for a txn whose collection this leader found superseded: \ + {stage_error}" + ), + ); + return; + } + let done = self + .restages + .get(&txn_id) + .map_or(0, |restage| restage.attempts); + if done >= self.config.restage_attempts { + self.restages.remove(&txn_id); + self.halt_apply( + txn_id, + HaltReason::LocalStageFailed, + HaltStep::Stage, + format!( + "COMMIT verdict for a txn this leader failed to stage in {done} restages: \ + {stage_error}" + ), + ); + return; + } + if let Some(pending) = self.pending.get_mut(&txn_id) { + pending.commit_state = CommitState::AwaitingRestage; + pending.awaiting = None; + pending.stage_error = None; + } + let backoff = self.config.restage_backoff(done); + // no-determinism: the restage backoff only delays a retry of a decided COMMIT verdict; it never changes what a replica applies. + let due = Instant::now() + backoff; + self.restages.insert( + txn_id, + Restage { + attempts: done, + due: Some(due), + }, + ); + warn!( + vshard_id = self.vshard_id, + epoch = txn_id.epoch, + position = txn_id.position, + restages_done = done, + backoff_ms = u64::try_from(backoff.as_millis()).unwrap_or(u64::MAX), + error = %stage_error, + "calvin: COMMIT verdict for a txn this leader failed to stage; staging it again \ + after the backoff" + ); + // Clear what the failed stage left on the core. The restage reaches + // the same core after it, in dispatch order. + self.discard_staged(txn_id); + } + + /// Stage again, in sequencer order, every txn whose restage backoff + /// passed. Runs on the leader only, and not once the scheduler halted. + pub(in crate::control::cluster::calvin::scheduler::driver::core) fn restage_due(&mut self) { + if self.restages.is_empty() || !self.role.is_leader() || self.is_apply_halted() { + return; + } + // no-determinism: the restage backoff only delays a retry of a decided COMMIT verdict. + let now = Instant::now(); + let mut due = Vec::new(); + for (txn_id, restage) in &self.restages { + let Some(at) = restage.due else { + continue; + }; + // A lower txn's restage goes first. A discard that waits for + // capacity holds its restage back, so the stage never overtakes it. + if at > now || self.has_deferred_for(*txn_id) { + break; + } + due.push(*txn_id); + } + for txn_id in due { + self.restage(txn_id); + if self.is_apply_halted() || !self.role.is_leader() { + return; + } + } + } + + /// When the run loop next wakes for a restage: the backoff end of the + /// lowest txn that waits for its restage. `None` when no restage can + /// run, or when one waits for capacity, which wakes the loop itself. + pub(in crate::control::cluster::calvin::scheduler::driver::core) fn next_restage_at( + &self, + ) -> Option { + if !self.role.is_leader() || self.is_apply_halted() { + return None; + } + let (txn_id, at) = self + .restages + .iter() + .find_map(|(txn_id, restage)| restage.due.map(|at| (*txn_id, at)))?; + (!self.has_deferred_for(txn_id)).then_some(at) + } + + /// Forget the restage and the barrier log of `txn_id`, which finished. + pub(in crate::control::cluster::calvin::scheduler::driver::core) fn forget_held_state( + &mut self, + txn_id: TxnId, + ) { + self.restages.remove(&txn_id); + self.barrier_logs.remove(&txn_id); + } + + /// Stage `txn_id` again through the grant path. + fn restage(&mut self, txn_id: TxnId) { + let awaits = self + .pending + .get(&txn_id) + .is_some_and(|pending| pending.commit_state == CommitState::AwaitingRestage); + let Some(restage) = self.restages.get_mut(&txn_id) else { + return; + }; + if !awaits { + restage.due = None; + return; + } + restage.attempts = restage.attempts.saturating_add(1); + restage.due = None; + let attempt = restage.attempts; + let Some(pending) = self.pending.remove(&txn_id) else { + return; + }; + self.metrics.record_restage(); + self.shared + .calvin + .counters + .stage_restages + .fetch_add(1, Ordering::Relaxed); + info!( + vshard_id = self.vshard_id, + epoch = txn_id.epoch, + position = txn_id.position, + attempt, + "calvin: staging a committed txn again" + ); + let PendingTxn { + txn, + lock_owner, + gates, + .. + } = pending; + // The old gates stay held until the restage holds its own, so a + // purge of a named collection never runs between the two. + self.route_granted(txn, txn_id, lock_owner); + drop(gates); + } +} + +#[cfg(test)] +mod tests { + use std::sync::Arc; + use std::time::Duration; + + use nodedb_cluster::calvin::types::SchedulerInput; + use nodedb_cluster::calvin::{CalvinCompletionRegistry, VerdictOutcome}; + use nodedb_physical::physical_plan::PhysicalPlan; + use nodedb_physical::physical_plan::meta::MetaOp; + + use super::*; + use crate::bridge::dispatch::CoreChannelDataSide; + use crate::bridge::envelope::{ErrorCode, StageVote, Status}; + use crate::control::cluster::calvin::scheduler::driver::core::test_proposer::{ + lead_data_group, step_down_from_data_group, + }; + use crate::control::cluster::calvin::scheduler::driver::core::test_support::{ + build_test_scheduler_with_data_side, error_response, make_local_write_txn, staged_response, + test_coll_vshard, + }; + + /// A data-group leader on the `test_coll` vShard, sharing `registry`. + fn leader( + registry: &Arc, + ) -> (Scheduler, tempfile::TempDir, CoreChannelDataSide) { + let (mut scheduler, dir, data_side) = + build_test_scheduler_with_data_side(test_coll_vshard(), Arc::clone(registry)); + lead_data_group(&mut scheduler); + (scheduler, dir, data_side) + } + + /// The Calvin plans on the Data Plane side, in arrival order. Drains + /// the request ring. + fn requests(data_side: &mut CoreChannelDataSide) -> Vec { + let mut plans = Vec::new(); + while let Ok(request) = data_side.request_rx.try_pop() { + plans.push(request.inner.plan); + } + plans + } + + fn is_stage(plan: &PhysicalPlan, txn_id: TxnId) -> bool { + matches!( + plan, + PhysicalPlan::Meta(MetaOp::CalvinExecuteStatic { epoch, position, .. }) + if *epoch == txn_id.epoch && *position == txn_id.position + ) + } + + fn is_drop(plan: &PhysicalPlan, txn_id: TxnId) -> bool { + *plan + == PhysicalPlan::Meta(MetaOp::CalvinDrop { + epoch: txn_id.epoch, + position: txn_id.position, + }) + } + + fn stage_failed() -> crate::bridge::envelope::Response { + error_response(ErrorCode::Internal { + detail: "stage failed on this core".to_string(), + }) + } + + /// Store a COMMIT verdict for `txn_id`, as the sequencer log applies it. + fn commit_verdict(registry: &CalvinCompletionRegistry, txn_id: TxnId) { + registry.note_verdict( + nodedb_cluster::calvin::TxnId::new(txn_id.epoch, txn_id.position), + VerdictOutcome::Commit, + ); + } + + /// End the backoff of `txn_id`'s next restage now. + fn end_backoff(scheduler: &mut Scheduler, txn_id: TxnId) { + let restage = scheduler + .restages + .get_mut(&txn_id) + .expect("the txn waits for a restage"); + // no-determinism: test-only backoff end in the past. + restage.due = Some(Instant::now() - Duration::from_millis(1)); + } + + fn restages(scheduler: &Scheduler) -> u64 { + scheduler + .shared + .calvin + .counters + .stage_restages + .load(Ordering::Relaxed) + } + + /// A leader whose stage of a committed write slice failed: it stages the + /// slice again after the backoff, fails once more, then stages it and + /// commits. The resolve follows the normal commit path. + #[tokio::test] + async fn a_restage_that_succeeds_on_the_second_attempt_commits() { + let registry = CalvinCompletionRegistry::new_detached(); + let (mut scheduler, _dir, mut data_side) = leader(®istry); + scheduler.config.restage_backoff_ms = 60_000; + let txn_id = TxnId::new(20, 0); + scheduler + .process_scheduler_input(SchedulerInput::Txn(Box::new(make_local_write_txn(20, 0)))); + assert!(requests(&mut data_side).iter().any(|p| is_stage(p, txn_id))); + + scheduler.resolve_staged_commit(txn_id, &stage_failed()); + commit_verdict(®istry, txn_id); + scheduler.resume_on_verdict(txn_id, true); + + assert_eq!( + scheduler.pending.get(&txn_id).map(|p| p.commit_state), + Some(CommitState::AwaitingRestage) + ); + assert!( + requests(&mut data_side).iter().any(|p| is_drop(p, txn_id)), + "the failed stage's state is discarded" + ); + assert!(!scheduler.is_apply_halted()); + + scheduler.restage_due(); + assert!( + requests(&mut data_side).is_empty(), + "nothing stages before the backoff ends" + ); + assert!(scheduler.next_restage_at().is_some()); + + end_backoff(&mut scheduler, txn_id); + scheduler.restage_due(); + assert!(requests(&mut data_side).iter().any(|p| is_stage(p, txn_id))); + assert_eq!( + scheduler.pending.get(&txn_id).map(|p| p.commit_state), + Some(CommitState::Staged) + ); + assert_eq!(restages(&scheduler), 1); + + // The second stage fails too. The known verdict holds it for the + // next restage at once. + scheduler.resolve_staged_commit(txn_id, &stage_failed()); + assert_eq!( + scheduler.pending.get(&txn_id).map(|p| p.commit_state), + Some(CommitState::AwaitingRestage) + ); + assert_eq!(scheduler.restages.get(&txn_id).map(|r| r.attempts), Some(1)); + requests(&mut data_side); + + end_backoff(&mut scheduler, txn_id); + scheduler.restage_due(); + assert!(requests(&mut data_side).iter().any(|p| is_stage(p, txn_id))); + assert_eq!(restages(&scheduler), 2); + + scheduler.resolve_staged_commit( + txn_id, + &staged_response(Status::Ok, Some(StageVote::Commit)), + ); + assert_eq!( + scheduler.pending.get(&txn_id).map(|p| p.commit_state), + Some(CommitState::AwaitingRedoResolve) + ); + assert!( + requests(&mut data_side).iter().any(|p| matches!( + p, + PhysicalPlan::Meta(MetaOp::CalvinResolve { + epoch: 20, + position: 0 + }) + )), + "the committed slice resolves its redo" + ); + assert!(!scheduler.is_apply_halted()); + assert!(!scheduler.applied.is_applied(20, 0)); + } + + /// Once its restages run out, a leader that still cannot stage a + /// committed txn halts. The txn stays parked with its locks, unapplied. + #[tokio::test] + async fn a_spent_restage_bound_halts_unapplied() { + let registry = CalvinCompletionRegistry::new_detached(); + let (mut scheduler, _dir, mut data_side) = leader(®istry); + scheduler.config.restage_attempts = 1; + let txn_id = TxnId::new(21, 0); + scheduler + .process_scheduler_input(SchedulerInput::Txn(Box::new(make_local_write_txn(21, 0)))); + commit_verdict(®istry, txn_id); + scheduler.resolve_staged_commit(txn_id, &stage_failed()); + assert_eq!( + scheduler.pending.get(&txn_id).map(|p| p.commit_state), + Some(CommitState::AwaitingRestage) + ); + + end_backoff(&mut scheduler, txn_id); + scheduler.restage_due(); + requests(&mut data_side); + scheduler.resolve_staged_commit(txn_id, &stage_failed()); + + assert_eq!( + scheduler.apply_halt().map(|h| h.reason), + Some(HaltReason::LocalStageFailed) + ); + let pending = scheduler + .pending + .get(&txn_id) + .expect("the txn stays pending"); + assert_eq!(pending.commit_state, CommitState::AwaitingVerdict); + assert_eq!(pending.verdict_deadline, None); + assert!(scheduler.restages.is_empty()); + assert_eq!(restages(&scheduler), 1); + assert!(!scheduler.applied.is_applied(21, 0)); + assert!( + !requests(&mut data_side) + .iter() + .any(|p| matches!(p, PhysicalPlan::Meta(MetaOp::CalvinResolve { .. }))), + "no resolve reaches the Data Plane" + ); + } + + /// A committed txn whose collection this leader found superseded cannot + /// commit here. It halts at once, with no restage. + #[tokio::test] + async fn a_superseded_collection_halts_at_once() { + let registry = CalvinCompletionRegistry::new_detached(); + let (mut scheduler, _dir, mut data_side) = leader(®istry); + let txn_id = TxnId::new(22, 0); + scheduler + .process_scheduler_input(SchedulerInput::Txn(Box::new(make_local_write_txn(22, 0)))); + requests(&mut data_side); + if let Some(pending) = scheduler.pending.get_mut(&txn_id) { + pending.superseded = true; + } + commit_verdict(®istry, txn_id); + + scheduler.resolve_staged_commit( + txn_id, + &staged_response(Status::Ok, Some(StageVote::Commit)), + ); + + assert_eq!( + scheduler.apply_halt().map(|h| h.reason), + Some(HaltReason::LocalStageFailed) + ); + assert!(scheduler.restages.is_empty()); + assert_eq!(restages(&scheduler), 0); + assert!( + requests(&mut data_side).is_empty(), + "nothing is discarded or staged" + ); + } + + /// An abort verdict for a txn this leader failed to stage drops it. No + /// restage follows. + #[tokio::test] + async fn an_abort_verdict_drops_without_restaging() { + let registry = CalvinCompletionRegistry::new_detached(); + let (mut scheduler, _dir, mut data_side) = leader(®istry); + let txn_id = TxnId::new(23, 0); + scheduler + .process_scheduler_input(SchedulerInput::Txn(Box::new(make_local_write_txn(23, 0)))); + requests(&mut data_side); + scheduler.resolve_staged_commit(txn_id, &stage_failed()); + + scheduler.resume_on_verdict(txn_id, false); + + assert_eq!( + scheduler.pending.get(&txn_id).map(|p| p.commit_state), + Some(CommitState::AwaitingDrop) + ); + assert!(requests(&mut data_side).iter().any(|p| is_drop(p, txn_id))); + assert!(scheduler.restages.is_empty()); + assert_eq!(scheduler.next_restage_at(), None); + assert_eq!(restages(&scheduler), 0); + assert!(!scheduler.is_apply_halted()); + } + + /// A txn whose plans this leader rejected restages through the grant + /// path. The plans reject again, which spends the bound and halts. + #[tokio::test] + async fn a_rejected_plan_restages_through_the_grant_path() { + let registry = CalvinCompletionRegistry::new_detached(); + let (mut scheduler, _dir, _data_side) = leader(®istry); + scheduler.config.restage_attempts = 1; + let txn_id = TxnId::new(24, 0); + let mut txn = make_local_write_txn(24, 0); + txn.tx_class.plans = vec![0xc1]; + scheduler.process_scheduler_input(SchedulerInput::Txn(Box::new(txn))); + assert_eq!( + scheduler.pending.get(&txn_id).map(|p| p.commit_state), + Some(CommitState::AwaitingVerdict) + ); + + commit_verdict(®istry, txn_id); + scheduler.resume_on_verdict(txn_id, true); + assert_eq!( + scheduler.pending.get(&txn_id).map(|p| p.commit_state), + Some(CommitState::AwaitingRestage) + ); + + end_backoff(&mut scheduler, txn_id); + scheduler.restage_due(); + + assert_eq!(restages(&scheduler), 1); + assert_eq!( + scheduler.apply_halt().map(|h| h.reason), + Some(HaltReason::LocalStageFailed) + ); + assert!( + scheduler.pending.contains_key(&txn_id), + "the rejected restage parks the txn again" + ); + } + + /// A demoted leader holds a txn awaiting its restage `Following`, and + /// forgets the restage. + #[tokio::test] + async fn demotion_forgets_a_waiting_restage() { + let registry = CalvinCompletionRegistry::new_detached(); + let (mut scheduler, _dir, _data_side) = leader(®istry); + let txn_id = TxnId::new(25, 0); + scheduler + .process_scheduler_input(SchedulerInput::Txn(Box::new(make_local_write_txn(25, 0)))); + commit_verdict(®istry, txn_id); + scheduler.resolve_staged_commit(txn_id, &stage_failed()); + assert!(scheduler.restages.contains_key(&txn_id)); + + step_down_from_data_group(&scheduler); + scheduler.refresh_role(); + + assert_eq!( + scheduler.pending.get(&txn_id).map(|p| p.commit_state), + Some(CommitState::Following) + ); + assert!(scheduler.restages.is_empty()); + assert_eq!(scheduler.next_restage_at(), None); + } +} diff --git a/nodedb/src/control/cluster/calvin/scheduler/driver/core/role.rs b/nodedb/src/control/cluster/calvin/scheduler/driver/core/role.rs new file mode 100644 index 000000000..ae1f2a614 --- /dev/null +++ b/nodedb/src/control/cluster/calvin/scheduler/driver/core/role.rs @@ -0,0 +1,472 @@ +// SPDX-License-Identifier: BUSL-1.1 + +//! This scheduler's role in its vShard's data group. +//! +//! Only the data-group leader stages, votes, resolves, and proposes a +//! committed slice's redo. Every other replica holds each granted txn as +//! [`CommitState::Following`] and installs its slice from the log. +//! +//! A node that wins a term leads only once the term's election no-op +//! applied here: the stage gate. Every entry of an earlier term in its log +//! sits below the no-op and commits with it. So a redo an earlier leader +//! proposed installs before this leader stages anything, and this leader +//! finds that position applied. +//! +//! - Promotion stages every `Following` txn as a new grant, in sequencer +//! order. A txn whose verdict is already COMMIT proposes its vote again, +//! which changes nothing: the first vote of the vShard counts. +//! - Demotion discards the staged state of every txn the leader drove, +//! drops its owed votes and its restages, and holds the txn `Following`. +//! A redo this node proposed can still commit in the new term: the +//! applied ledger installs one copy per position. +//! +//! [`CommitState::Following`]: super::super::types::CommitState::Following + +use super::super::types::CommitState; +use super::deferred::{DispatchOutcome, DispatchStep}; +use super::owed::OwedKind; +use super::scheduler::Scheduler; +use crate::control::cluster::calvin::scheduler::lock_manager::TxnId; + +/// What this node does for its vShard's Calvin txns. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub(in crate::control::cluster::calvin::scheduler::driver::core) enum Role { + /// Holds granted txns and installs committed slices from the log. + Follower, + /// Stages, votes, resolves, and proposes. + Leader, +} + +/// The stage gate: the role, and the leadership term it opened in. +#[derive(Debug, Clone, Copy)] +pub(in crate::control::cluster::calvin::scheduler::driver::core) struct StageGate { + role: Role, + /// The data-group term this node leads, as last seen. + term: Option, + /// The index of the term's election no-op. The gate opens once this + /// node applied it. + term_start: Option, +} + +impl Default for StageGate { + fn default() -> Self { + Self { + role: Role::Follower, + term: None, + term_start: None, + } + } +} + +impl StageGate { + /// Whether this node stages the vShard's txns. + pub(in crate::control::cluster::calvin::scheduler::driver::core) fn is_leader(&self) -> bool { + self.role == Role::Leader + } + + /// Whether this node leads the data group and waits for its term's + /// no-op to apply. + pub(in crate::control::cluster::calvin::scheduler::driver::core) fn awaits_term_start( + &self, + ) -> bool { + self.role == Role::Follower && self.term.is_some() + } +} + +/// What the data group says about this node now. +struct Leadership { + group_id: u64, + term: u64, + term_start: Option, +} + +impl Scheduler { + /// Read this node's place in the vShard's data group, and promote or + /// demote on a change. + pub(in crate::control::cluster::calvin::scheduler::driver::core) fn refresh_role(&mut self) { + let Some(leadership) = self.leadership() else { + self.role.term = None; + self.role.term_start = None; + if self.role.is_leader() { + self.demote("this node no longer leads the vShard's data group"); + } + return; + }; + if self.role.term != Some(leadership.term) { + if self.role.is_leader() { + self.demote("this node leads a later term of the vShard's data group"); + } + self.role.term = Some(leadership.term); + self.role.term_start = leadership.term_start; + tracing::debug!( + node_id = self.shared.node_id, + vshard_id = self.vshard_id, + group_id = leadership.group_id, + term = leadership.term, + term_start = ?leadership.term_start, + applied = self + .shared + .applied_index_watcher(leadership.group_id) + .current(), + "calvin scheduler: this node won the data group; staging waits for the term's \ + first entry to apply" + ); + } + if self.role.is_leader() { + return; + } + let applied = self + .shared + .applied_index_watcher(leadership.group_id) + .current(); + if self.role.term_start.is_some_and(|start| applied >= start) { + self.promote(); + } + } + + /// This node's term and term start in the vShard's data group, while it + /// leads the group. + fn leadership(&self) -> Option { + let mr = self.multi_raft.lock().unwrap_or_else(|p| p.into_inner()); + let group_id = mr + .routing() + .read() + .unwrap_or_else(|p| p.into_inner()) + .group_for_vshard(self.vshard_id) + .ok()?; + let term = mr.leader_term(group_id)?; + Some(Leadership { + group_id, + term, + term_start: mr.term_start_index(group_id), + }) + } + + /// Whether this node leads the vShard's data group in Raft right now, + /// whatever its stage gate says. Owed sequencer entries are proposed + /// only then. + pub(in crate::control::cluster::calvin::scheduler::driver::core) fn is_group_leader( + &self, + ) -> bool { + let mr = self.multi_raft.lock().unwrap_or_else(|p| p.into_inner()); + mr.vshard_role_is_leader(self.vshard_id) + } + + /// Open the stage gate: stage every `Following` txn, in sequencer order. + fn promote(&mut self) { + self.role.role = Role::Leader; + tracing::info!( + node_id = self.shared.node_id, + vshard_id = self.vshard_id, + term = self.role.term, + "calvin scheduler: this node leads the vShard; staging its held txns" + ); + let following: Vec = self + .pending + .iter() + .filter(|(_, pending)| pending.commit_state == CommitState::Following) + .map(|(txn_id, _)| *txn_id) + .collect(); + for txn_id in following { + let Some(pending) = self.pending.remove(&txn_id) else { + continue; + }; + self.route_granted(pending.txn, txn_id, pending.lock_owner); + } + } + + /// Close the stage gate: discard the staged state of every txn this + /// node drove, drop its owed votes, and hold each txn `Following`. + pub(in crate::control::cluster::calvin::scheduler::driver::core) fn demote( + &mut self, + why: &str, + ) { + self.role.role = Role::Follower; + tracing::info!( + node_id = self.shared.node_id, + vshard_id = self.vshard_id, + reason = why, + "calvin scheduler: this node stops leading the vShard; holding its txns" + ); + let driven: Vec = self + .pending + .iter() + .filter(|(_, pending)| leader_drives(pending.commit_state)) + .map(|(txn_id, _)| *txn_id) + .collect(); + for txn_id in driven { + self.release_to_following(txn_id); + } + // Only a leader restages. A later term counts its own restages. + self.restages.clear(); + // A barrier waits only on the leader. Its txn holds its locks and + // follows the log like any other. Its barrier log stays, so a later + // barrier of it decides from the same entries. + let barriers: Vec = self.dependent_barrier.keys().copied().collect(); + for txn_id in barriers { + if let Some(barrier) = self.dependent_barrier.remove(&txn_id) { + self.follow(barrier.txn, txn_id, barrier.lock_owner); + } + } + } + + /// Hold `txn_id` `Following`, and discard what this node staged for it. + /// + /// The txn's scope becomes its slice's own, from its plans. A leader's + /// step can hold a narrower one: a passive read or a parked abort vote + /// holds a scope that writes nothing. A follower that kept it would end a + /// COMMIT with no entry, mark the position applied, and then skip the + /// slice's redo when it applies. + fn release_to_following(&mut self, txn_id: TxnId) { + let Some(scope) = self + .pending + .get(&txn_id) + .map(|pending| self.local_slice_scope(&pending.txn, txn_id)) + else { + return; + }; + let Some(pending) = self.pending.get_mut(&txn_id) else { + return; + }; + pending.commit_state = CommitState::Following; + pending.scope = scope; + pending.awaiting = None; + pending.verdict_deadline = None; + pending.stage_error = None; + pending.superseded = false; + pending.redo = None; + pending.gates.clear(); + pending.ungated = false; + self.owed.remove(&(txn_id, OwedKind::Vote)); + self.discard_staged(txn_id); + } + + /// Dispatch a `CalvinDrop` that clears what this node staged for + /// `txn_id`. An absent staged entry makes it a no-op on the core. + pub(in crate::control::cluster::calvin::scheduler::driver::core) fn discard_staged( + &mut self, + txn_id: TxnId, + ) { + if let DispatchOutcome::Failed(error) = self.dispatch_drop(txn_id, DispatchStep::Discard) { + self.fail_dispatch_step(txn_id, DispatchStep::Discard, error); + } + } + + /// Note that a redo proposal found this node no longer leads. + pub(in crate::control::cluster::calvin::scheduler::driver::core) fn on_not_leader(&mut self) { + if self.role.is_leader() { + self.demote("a redo proposal found this node no longer leads"); + } + self.role.term = None; + self.role.term_start = None; + } +} + +/// Whether a txn in `state` is one the leader drives: it holds staged state +/// or waits on a step only the leader takes. +fn leader_drives(state: CommitState) -> bool { + match state { + CommitState::Staged + | CommitState::AwaitingVerdict + | CommitState::AwaitingResolveTurn + | CommitState::AwaitingRedoResolve + | CommitState::AwaitingRedoApply { .. } + | CommitState::AwaitingRestage + | CommitState::ReadingPassive => true, + // A drop ends the txn with no log entry whatever the role. + CommitState::AwaitingDrop | CommitState::Following => false, + } +} + +#[cfg(test)] +mod tests { + use nodedb_cluster::calvin::CalvinCompletionRegistry; + use nodedb_cluster::calvin::types::{ + EngineKeySet, ReadWriteSet, SchedulerInput, SequencedTxn, SortedVec, + }; + use nodedb_physical::physical_plan::PhysicalPlan; + use nodedb_physical::physical_plan::meta::MetaOp; + + use super::*; + use crate::bridge::dispatch::CoreChannelDataSide; + use crate::bridge::envelope::{StageVote, Status}; + use crate::control::cluster::calvin::scheduler::driver::core::test_proposer::{ + elect_data_group_leader, lead_data_group, step_down_from_data_group, + }; + use crate::control::cluster::calvin::scheduler::driver::core::test_support::{ + build_test_scheduler_with_data_side, make_validate_only_txn, staged_response, + test_coll_vshard, + }; + + /// A scheduler on the `test_coll` vShard with its Data Plane side. + fn scheduler() -> (Scheduler, tempfile::TempDir, CoreChannelDataSide) { + build_test_scheduler_with_data_side( + test_coll_vshard(), + CalvinCompletionRegistry::new_detached(), + ) + } + + /// A validate-only txn at `(epoch, 0)` that locks `test_coll` row + /// `surrogate`, so txns on distinct rows hold their locks together. + fn validate_only_on(epoch: u64, surrogate: u32) -> SequencedTxn { + let mut txn = make_validate_only_txn(epoch, 0); + txn.tx_class.write_set = ReadWriteSet::new(vec![EngineKeySet::Document { + collection: "test_coll".to_string(), + surrogates: SortedVec::new(vec![surrogate]), + }]); + txn + } + + /// The `(epoch, position)` of every Calvin stage request on the Data + /// Plane side, in arrival order. Drains the request ring. + fn staged_requests(data_side: &mut CoreChannelDataSide) -> Vec<(u64, u32)> { + let mut staged = Vec::new(); + while let Ok(request) = data_side.request_rx.try_pop() { + if let PhysicalPlan::Meta(MetaOp::CalvinExecuteStatic { + epoch, position, .. + }) = request.inner.plan + { + staged.push((epoch, position)); + } + } + staged + } + + /// Whether a `CalvinDrop` of `txn_id` reached the Data Plane side. + /// Drains the request ring. + fn dropped(data_side: &mut CoreChannelDataSide, txn_id: TxnId) -> bool { + let mut dropped = false; + while let Ok(request) = data_side.request_rx.try_pop() { + dropped |= request.inner.plan + == PhysicalPlan::Meta(MetaOp::CalvinDrop { + epoch: txn_id.epoch, + position: txn_id.position, + }); + } + dropped + } + + /// A replica that does not lead the data group holds a granted txn + /// `Following`. It stages nothing and owes no vote. + #[tokio::test] + async fn a_follower_grant_dispatches_nothing() { + let (mut scheduler, _dir, mut data_side) = scheduler(); + let txn_id = TxnId::new(3, 0); + + scheduler.process_scheduler_input(SchedulerInput::Txn(Box::new(validate_only_on(3, 1)))); + + assert_eq!( + scheduler.pending.get(&txn_id).map(|p| p.commit_state), + Some(CommitState::Following) + ); + assert!(data_side.request_rx.try_pop().is_err(), "nothing staged"); + assert!(scheduler.owed.is_empty(), "a follower owes no vote"); + assert!(!scheduler.applied.is_applied(3, 0)); + } + + /// A node that won a term stages nothing until the term's no-op applied + /// here. Once it applied, the held txn stages. + #[tokio::test] + async fn a_new_leader_stages_nothing_before_the_commit_index_applies() { + let (mut scheduler, _dir, mut data_side) = scheduler(); + let group_id = elect_data_group_leader(&scheduler); + scheduler.refresh_role(); + assert!(!scheduler.role.is_leader(), "the gate stays closed"); + assert!(scheduler.role.awaits_term_start()); + let txn_id = TxnId::new(3, 0); + + scheduler.process_scheduler_input(SchedulerInput::Txn(Box::new(validate_only_on(3, 1)))); + assert_eq!( + scheduler.pending.get(&txn_id).map(|p| p.commit_state), + Some(CommitState::Following) + ); + assert!(staged_requests(&mut data_side).is_empty()); + + let term_start = scheduler + .multi_raft + .lock() + .unwrap_or_else(|p| p.into_inner()) + .term_start_index(group_id) + .expect("the elected node names its term start"); + scheduler + .shared + .applied_index_watcher(group_id) + .bump(term_start); + scheduler.refresh_role(); + + assert!(scheduler.role.is_leader(), "the gate opens"); + assert_eq!( + scheduler.pending.get(&txn_id).map(|p| p.commit_state), + Some(CommitState::Staged) + ); + assert_eq!(staged_requests(&mut data_side), vec![(3, 0)]); + } + + /// Promotion stages every `Following` txn, in sequencer order. + #[tokio::test] + async fn promotion_stages_following_txns_in_sequencer_order() { + let (mut scheduler, _dir, mut data_side) = scheduler(); + for (epoch, surrogate) in [(3, 1), (4, 2), (5, 3)] { + scheduler.process_scheduler_input(SchedulerInput::Txn(Box::new(validate_only_on( + epoch, surrogate, + )))); + } + assert!(staged_requests(&mut data_side).is_empty()); + + lead_data_group(&mut scheduler); + + assert_eq!( + staged_requests(&mut data_side), + vec![(3, 0), (4, 0), (5, 0)] + ); + for epoch in [3, 4, 5] { + assert_eq!( + scheduler + .pending + .get(&TxnId::new(epoch, 0)) + .map(|p| p.commit_state), + Some(CommitState::Staged) + ); + } + } + + /// Demotion discards what the leader staged, drops its owed vote, and + /// holds the txn `Following` with its locks. + #[tokio::test] + async fn demotion_drops_staged_state_and_owed_votes() { + let (mut scheduler, _dir, mut data_side) = scheduler(); + lead_data_group(&mut scheduler); + let txn_id = TxnId::new(3, 0); + scheduler.process_scheduler_input(SchedulerInput::Txn(Box::new(validate_only_on(3, 1)))); + assert_eq!(staged_requests(&mut data_side), vec![(3, 0)]); + scheduler.resolve_staged_commit( + txn_id, + &staged_response(Status::Ok, Some(StageVote::Commit)), + ); + assert!(scheduler.owed.contains_key(&(txn_id, OwedKind::Vote))); + + step_down_from_data_group(&scheduler); + scheduler.refresh_role(); + + assert!(!scheduler.role.is_leader()); + assert_eq!( + scheduler.pending.get(&txn_id).map(|p| p.commit_state), + Some(CommitState::Following) + ); + assert!( + !scheduler.owed.contains_key(&(txn_id, OwedKind::Vote)), + "a demoted node keeps no vote owed" + ); + assert!( + dropped(&mut data_side, txn_id), + "the staged state is dropped" + ); + assert!(!scheduler.applied.is_applied(3, 0)); + let blocked = validate_only_on(4, 1); + scheduler.process_scheduler_input(SchedulerInput::Txn(Box::new(blocked))); + assert!( + scheduler.blocked.contains_key(&TxnId::new(4, 0)), + "the held txn keeps its locks" + ); + } +} diff --git a/nodedb/src/control/cluster/calvin/scheduler/driver/core/run_loop.rs b/nodedb/src/control/cluster/calvin/scheduler/driver/core/run_loop.rs index 60892742b..8f4142d2b 100644 --- a/nodedb/src/control/cluster/calvin/scheduler/driver/core/run_loop.rs +++ b/nodedb/src/control/cluster/calvin/scheduler/driver/core/run_loop.rs @@ -10,6 +10,10 @@ use super::catch_up::CatchUpDrain; use super::scheduler::Scheduler; use crate::control::shutdown::ShutdownReceiver; +/// How often a new data-group leader checks whether its term's no-op +/// applied, while its stage gate is closed. +const ROLE_POLL: std::time::Duration = std::time::Duration::from_millis(5); + impl Scheduler { /// Run the scheduler event loop until shutdown is signaled. pub async fn run(mut self, mut shutdown: ShutdownReceiver) { @@ -30,6 +34,12 @@ impl Scheduler { // Woken when a routed Data-Plane response frees dispatcher capacity. let capacity_freed = Arc::clone(&self.capacity_freed); + // Woken when the data-group apply loop concludes a stamped redo. + let inbox = Arc::clone(self.inbox.inbox()); + // Woken when the data-group apply loop buffers a barrier entry. + let read_results = Arc::clone(&self.read_results); + // Woken when a cut's barrier is proposed or applied on this node. + let shared = Arc::clone(&self.shared); // Set when a tick left armed catch-up unreplayed. The next open-gate // pass fires the tick at once to resume it. let mut catch_up_resume = false; @@ -44,9 +54,23 @@ impl Scheduler { if self.resends_deferred() { self.redispatch_deferred(); } + // Registered before the hold check below, for the same reason. + let cut_changed = shared.calvin.cut_barriers.notified(); + tokio::pin!(cut_changed); + cut_changed.as_mut().enable(); - self.check_dependent_barrier_timeouts(); + // Every pass reads the role first: a promotion stages held txns, + // a demotion stops staging. + self.refresh_role(); + self.drain_inbox(); + // A cut's barrier applied here: propose the redo it held. + if self.release_cut_holds() { + self.retry_redo_proposals(); + } + self.drain_read_results(); + self.propose_due_read_timeouts(); self.check_awaiting_verdict_stalls(); + self.restage_due(); self.resume_metadata_hold(); // Open only once the inputs a closed gate held are processed. A @@ -57,6 +81,12 @@ impl Scheduler { catch_up_resume = false; stall_tick.reset_immediately(); } + // The backoff end of the next restage, when one waits. + let restage_at = self.next_restage_at().map(tokio::time::Instant::from_std); + // When this leader next proposes a barrier's timeout entry. + let read_timeout_at = self + .next_read_timeout_due() + .map(tokio::time::Instant::from_std); tokio::select! { biased; @@ -74,10 +104,14 @@ impl Scheduler { self.log_superseded("completion"); break; }; - // Awaited in the arm: the loop takes no other input - // until this completion, its durability wait included, - // is fully handled. - self.handle_completion(txn_id, request_id, resp_opt).await; + self.handle_completion(txn_id, request_id, resp_opt); + } + + // The apply loop concluded a stamped redo. Drained in every + // state, halted included, so the apply loop never waits on + // this scheduler. + () = inbox.ready() => { + self.drain_inbox(); } maybe_verdict = self.verdict_rx.recv() => { @@ -86,16 +120,14 @@ impl Scheduler { break; }; // A durable global verdict landed: resume the matching - // parked txn into its flush (commit) or drop (abort). + // txn. self.handle_verdict_signal(signal); } - maybe_event = self.read_result_rx.recv() => { - let Some(event) = maybe_event else { - self.log_superseded("read result"); - break; - }; - self.handle_read_result(event); + // The apply loop buffered a barrier entry. Drained in every + // state, so a held txn keeps every entry of its log. + () = read_results.pushed() => { + self.drain_read_results(); } maybe_promoted = self.promotion_rx.recv() => { @@ -116,17 +148,37 @@ impl Scheduler { // The next loop pass re-checks the held txn's floor. } + _ = tokio::time::sleep_until( + // no-determinism: a wake timer only; the restage reads replicated state. + restage_at.unwrap_or_else(tokio::time::Instant::now) + ), if restage_at.is_some() => { + // The next loop pass stages again every txn whose + // restage backoff ended. + } + + _ = tokio::time::sleep_until( + // no-determinism: a wake timer only; the timeout entry's log position decides. + read_timeout_at.unwrap_or_else(tokio::time::Instant::now) + ), if read_timeout_at.is_some() => { + // The next loop pass proposes the timeout entry of every + // barrier whose timeout passed. + } + + _ = tokio::time::sleep(ROLE_POLL), if self.role.awaits_term_start() => { + // The next loop pass checks whether the term's no-op + // applied, and stages the held txns once it has. + } + + _ = &mut cut_changed, if !self.cut_holds.is_empty() => { + // The next loop pass ends the hold of every cut whose + // barrier applied here. + } + _ = &mut capacity_notified, if self.resends_deferred() => { // Capacity freed: the next loop pass re-sends deferred // requests in FIFO order. } - _ = self.install_gate.released(), if self.install_gate.is_waiting() => { - // A snapshot released a group's gate: the waiting flush - // tries again. - self.pump_flush_turn(); - } - maybe_txn = self.receiver.recv(), if intake_open => { match maybe_txn { Some(input) => self.process_scheduler_input(input), @@ -150,15 +202,30 @@ impl Scheduler { !intake_open || self.drain_catch_up() == CatchUpDrain::Remaining; // Propose again every owed sequencer entry not yet applied. self.retry_owed_sequencer_entries(); + // Propose again every committed slice's redo that is not + // in the log. Apply events are drained first, so an + // install already reported is never proposed again. + self.drain_inbox(); + self.retry_redo_proposals(); + // A barrier entry the apply loop buffered after the + // scheduler finished its txn is spent. + let ledger = Arc::clone(&self.ledger); + self.read_results.discard_finished(|txn| { + ledger.is_applied(txn.epoch, txn.position) + }); + // So is a stored row of a finished txn its finish did + // not remove. + self.sweep_finished_barrier_rows(); // The top-of-loop check_awaiting_verdict_stalls / - // check_dependent_barrier_timeouts and the deferred re-send + // propose_due_read_timeouts and the deferred re-send // pass run on every wake; this arm guarantees the loop wakes // to run them (and the drain) when no other event arrives. } } } - // Every txn still pending stays unapplied on this replica. - self.hold_all_redo_records(); + // The next scheduler of the vShard starts from the barrier entries + // this one took. + self.return_barrier_logs(); } /// Log that a scheduler channel closed and the loop exits. diff --git a/nodedb/src/control/cluster/calvin/scheduler/driver/core/scheduler.rs b/nodedb/src/control/cluster/calvin/scheduler/driver/core/scheduler.rs index 002090c4a..089354880 100644 --- a/nodedb/src/control/cluster/calvin/scheduler/driver/core/scheduler.rs +++ b/nodedb/src/control/cluster/calvin/scheduler/driver/core/scheduler.rs @@ -11,7 +11,7 @@ use nodedb_cluster::MultiRaft; use nodedb_cluster::calvin::types::SchedulerInput; use nodedb_cluster::calvin::{CalvinCompletionRegistry, SequencerStateMachine, VerdictSignal}; -use super::super::barrier::{PendingDependentBarrier, ReadResultEvent}; +use super::super::barrier::PendingDependentBarrier; use super::super::config::SchedulerConfig; use super::super::types::{BlockedTxn, PendingTxn}; use super::deferred::DeferredQueue; @@ -49,10 +49,12 @@ pub struct Scheduler { /// Shared control-plane state used for dispatch, response tracking, WAL, /// and request-id allocation. pub(in crate::control::cluster::calvin::scheduler::driver::core) shared: Arc, - /// Handle to MultiRaft for the data-group leader check and the catch-up - /// read of the sequencer log. + /// Handle to MultiRaft for the data-group role, the redo proposals, and + /// the catch-up read of the sequencer log. pub(in crate::control::cluster::calvin::scheduler::driver::core) multi_raft: Arc>, + /// This node's role in the vShard's data group. See [`super::role`]. + pub(in crate::control::cluster::calvin::scheduler::driver::core) role: super::role::StageGate, /// Hands sequencer entries to the sequencer group, locally on its leader /// and by forward from any other node. pub(in crate::control::cluster::calvin::scheduler::driver::core) sequencer_proposer: @@ -79,29 +81,47 @@ pub struct Scheduler { /// for the brief probe the gate takes. pub(in crate::control::cluster::calvin::scheduler::driver::core) lock_manager: Arc>, - /// In-flight static/active transactions awaiting executor response, - /// including those whose request waits in `deferred` for capacity. - /// `BTreeMap` ensures deterministic iteration order. + /// Granted transactions that have not finished: staged ones on the + /// leader, held ones on a follower, including those whose request waits + /// in `deferred` for capacity. `BTreeMap` ensures deterministic + /// iteration order. pub(in crate::control::cluster::calvin::scheduler::driver::core) pending: BTreeMap, + /// Installs the apply loop reported for txns still waiting for their + /// locks. Each completes when its locks are granted. + pub(in crate::control::cluster::calvin::scheduler::driver::core) early_applied: + BTreeMap, + /// This scheduler's registered inbox: how the data-group apply loop + /// concluded each stamped redo of the vShard. + pub(in crate::control::cluster::calvin::scheduler::driver::core) inbox: + crate::control::cluster::calvin::scheduler::InboxHandle, /// Blocked transactions awaiting lock release. pub(in crate::control::cluster::calvin::scheduler::driver::core) blocked: BTreeMap, - /// Dependent-read barriers awaiting passive read results. - /// `BTreeMap` for determinism. + /// Dependent-read barriers this leader holds open. `BTreeMap` for + /// determinism. pub(in crate::control::cluster::calvin::scheduler::driver::core) dependent_barrier: BTreeMap, - /// Channel receiving `CalvinReadResult` Raft apply events from the - /// per-vshard data Raft apply loop. Bounded. - pub(in crate::control::cluster::calvin::scheduler::driver::core) read_result_rx: - mpsc::Receiver, + /// The barrier entries of the data-group log, folded per dependent txn + /// this scheduler holds: at a barrier, or in `pending`. A barrier of the + /// txn decides from them. Each entry leaves when its txn completes. + pub(in crate::control::cluster::calvin::scheduler::driver::core) barrier_logs: + BTreeMap, + /// This vShard's read-result buffer: the barrier entries the apply loop + /// applied for txns this scheduler did not hold yet. + pub(in crate::control::cluster::calvin::scheduler::driver::core) read_results: + Arc, + /// Committed txns whose stage failed on this leader, with their + /// restage count and backoff. See [`super::restage`]. + pub(in crate::control::cluster::calvin::scheduler::driver::core) restages: + BTreeMap, /// Exactly-once applied gate: the fully-applied watermark plus the set of /// applied `(epoch, position)` pairs above it. Replaces a bare per-epoch /// counter so a multi-position epoch is never marked applied on the strength /// of its first completing position. pub(in crate::control::cluster::calvin::scheduler::driver::core) applied: AppliedGate, - /// Rebuild target epoch (highest applied epoch from the initial recovery - /// scan). + /// Rebuild target epoch: the highest applied epoch of the ledger when + /// the scheduler started. pub(in crate::control::cluster::calvin::scheduler::driver::core) rebuild_target_epoch: u64, /// Highest replicated epoch observed across all scheduler inputs so far. /// Advances monotonically as `process_scheduler_input` sees new inputs; the @@ -113,9 +133,17 @@ pub struct Scheduler { /// set and the markers not yet reported. pub(in crate::control::cluster::calvin::scheduler::driver::core) cut_floors: crate::control::cluster::calvin::scheduler::cut_floor::CutFloors, - /// Shared mirror of `applied`, read by authorization coverage. - pub(in crate::control::cluster::calvin::scheduler::driver::core) applied_mirror: - Arc, + /// Ordered cuts whose barrier this vShard's later redo waits for. + pub(in crate::control::cluster::calvin::scheduler::driver::core) cut_holds: + crate::control::cluster::calvin::scheduler::cut_hold::CutHolds, + /// This vShard's applied ledger: the gate's seed, and the shared record + /// of each position this scheduler finishes. + pub(in crate::control::cluster::calvin::scheduler::driver::core) ledger: + Arc, + /// This scheduler's caught-up entry, read by the startup readiness gate. + /// Set once the watermark reaches `rebuild_target_epoch`. + pub(in crate::control::cluster::calvin::scheduler::driver::core) caught_up: + crate::control::cluster::calvin::scheduler::CaughtUpHandle, /// Scheduler configuration. pub(in crate::control::cluster::calvin::scheduler::driver::core) config: SchedulerConfig, /// Metrics. @@ -178,7 +206,7 @@ pub struct Scheduler { /// staged yet. See [`super::parts`]. pub(in crate::control::cluster::calvin::scheduler::driver::core) parts: super::parts::PartsState, - /// Orders flushes against the data group's snapshots. See + /// The vShard base this scheduler started under. See /// [`super::install_gate`]. pub(in crate::control::cluster::calvin::scheduler::driver::core) install_gate: super::install_gate::InstallGate, @@ -196,14 +224,14 @@ pub struct SchedulerParams { /// Shared sequencer state machine, source of the per-vShard catch-up index /// the drain replays from. Same `Arc` the Raft apply loop drives. pub sequencer_state_machine: Arc>, - /// Fully-applied watermark seed from the recovery scan. - pub fully_applied_epoch: u64, - /// Applied `(epoch, position)` pairs seed from the recovery scan. - pub applied_tail: std::collections::BTreeSet<(u64, u32)>, + /// This vShard's applied ledger. The applied gate starts from its + /// state. + pub ledger: Arc, + /// The epoch the scheduler must fully apply before it reports caught + /// up. [`NOT_YET_APPLIED_EPOCH`] means nothing to rebuild. pub rebuild_target_epoch: u64, pub config: SchedulerConfig, pub metrics: Arc, - pub read_result_rx: mpsc::Receiver, /// The shared lock table for this vShard. Constructed by /// `reconcile_vshard_schedulers` and registered in /// `SharedState.calvin.lock_managers` under the SAME `Arc` passed here. @@ -231,12 +259,10 @@ impl Scheduler { multi_raft, sequencer_proposer, sequencer_state_machine, - fully_applied_epoch, - applied_tail, + ledger, rebuild_target_epoch, config, metrics, - read_result_rx, lock_manager, promotion_rx, registry, @@ -248,15 +274,17 @@ impl Scheduler { let completion_cap = config.channel_capacity; let (completion_tx, completion_rx) = mpsc::channel(completion_cap); - let applied_mirror = shared.authorization_fence.calvin_mirrors().register( - vshard_id, - fully_applied_epoch, - &applied_tail, - ); + let (fully_applied_epoch, applied_tail) = ledger.snapshot(); + let caught_up = shared.calvin.caught_up.register(vshard_id); // A backup's cut waits on every scheduler this node runs. shared.calvin.cuts.register(vshard_id); let install_gate = super::install_gate::InstallGate::new(&shared, vshard_id); + let inbox = shared + .calvin + .inboxes + .register(vshard_id, config.max_inflight_backlog); + let read_results = shared.calvin.read_results.vshard(vshard_id); let capacity_freed = shared .dispatcher @@ -264,22 +292,29 @@ impl Scheduler { .unwrap_or_else(|p| p.into_inner()) .capacity_freed(); - Self { + let scheduler = Self { vshard_id, receiver, shared, multi_raft, + role: super::role::StageGate::default(), sequencer_proposer, owed: OwedEntries::new(), sequencer_state_machine, lock_manager, pending: BTreeMap::new(), + early_applied: BTreeMap::new(), + inbox, blocked: BTreeMap::new(), dependent_barrier: BTreeMap::new(), - read_result_rx, + barrier_logs: BTreeMap::new(), + read_results, + restages: BTreeMap::new(), applied: AppliedGate::new(fully_applied_epoch, applied_tail), cut_floors: Default::default(), - applied_mirror, + cut_holds: Default::default(), + ledger, + caught_up, rebuild_target_epoch, max_input_epoch: 0, config, @@ -296,17 +331,20 @@ impl Scheduler { metadata_hold: None, parts: Default::default(), install_gate, - } + }; + // A scheduler with nothing to rebuild reports caught up at once. + scheduler.publish_caught_up(); + scheduler } /// Whether the scheduler has caught up to the rebuild target epoch. /// - /// `rebuild_target_epoch` is seeded from `AppliedRecovery::max_applied_epoch` - /// (see `recovery.rs`): [`NOT_YET_APPLIED_EPOCH`] means the WAL scan found NO - /// `CalvinApplied` marker for this vShard at all — a greenfield node with no - /// Calvin history — never a real epoch (epoch 0 with markers reports - /// `max_applied_epoch == 0`, distinct from the sentinel). With nothing to - /// rebuild, such a node is trivially caught up. + /// `rebuild_target_epoch` is seeded from the ledger's highest applied + /// epoch (see `CalvinAppliedLedger::max_applied_epoch`): + /// [`NOT_YET_APPLIED_EPOCH`] means the ledger holds NO applied position + /// for this vShard — a greenfield node with no Calvin history — never a + /// real epoch (epoch 0 with markers reports `0`, distinct from the + /// sentinel). With nothing to rebuild, such a node is trivially caught up. /// /// `fully_applied_epoch()` is conservatively seeded to the same sentinel by /// recovery (the watermark only advances once the sequencer's re-fan-out @@ -329,6 +367,21 @@ impl Scheduler { fully_applied >= self.rebuild_target_epoch } + /// Report this scheduler caught up to the readiness gate, once + /// [`Self::is_caught_up`] holds. + pub(in crate::control::cluster::calvin::scheduler::driver::core) fn publish_caught_up(&self) { + if self.caught_up.is_caught_up() || !self.is_caught_up() { + return; + } + self.caught_up.mark_caught_up(); + tracing::info!( + vshard_id = self.vshard_id, + rebuild_target_epoch = self.rebuild_target_epoch, + fully_applied_epoch = self.applied.fully_applied_epoch(), + "calvin scheduler caught up to its rebuild target" + ); + } + /// Publish an advanced fully-applied watermark to the metrics gauge and the /// shared cross-shard snapshot anchor. /// @@ -342,13 +395,14 @@ impl Scheduler { watermark: u64, ) { self.metrics.update_last_applied_epoch(watermark); - self.applied_mirror.fold(watermark); + self.ledger.fold(watermark); self.shared .calvin .last_applied_epoch .fetch_max(watermark, std::sync::atomic::Ordering::Release); // A marker passes once every epoch delivered before it folded. self.report_passed_cuts(); + self.publish_caught_up(); } } @@ -399,7 +453,7 @@ mod tests { ); } - /// A greenfield node with NO Calvin history at all: `read_applied_recovery` + /// A greenfield node with NO Calvin history at all: `recover_all_applied` /// seeds `max_applied_epoch` (hence `rebuild_target_epoch`) to /// `NOT_YET_APPLIED_EPOCH` too (see `recovery.rs`'s /// `greenfield_returns_sentinel_and_empty_tail` test) — this is distinct from @@ -423,4 +477,66 @@ mod tests { "no rebuild target (greenfield node) must report caught-up" ); } + + fn lagging(scheduler: &Scheduler) -> Vec { + scheduler.shared.calvin.caught_up.lagging() + } + + /// A scheduler with a real rebuild target holds the readiness gate until + /// its published watermark reaches that target. + #[tokio::test] + async fn a_scheduler_lags_until_its_watermark_reaches_the_rebuild_target() { + let (mut scheduler, _dir) = build_test_scheduler(4); + scheduler.rebuild_target_epoch = 3; + assert_eq!(lagging(&scheduler), vec![4]); + + scheduler.applied = AppliedGate::new(2, BTreeSet::new()); + scheduler.publish_watermark(2); + assert_eq!(lagging(&scheduler), vec![4], "epoch 2 is below the target"); + + scheduler.applied = AppliedGate::new(3, BTreeSet::new()); + scheduler.publish_watermark(3); + assert!(lagging(&scheduler).is_empty()); + } + + /// A published watermark folds the vShard's applied ledger, which + /// coverage, checkpoints and snapshot capture read. + #[tokio::test] + async fn a_published_watermark_folds_the_ledger() { + let (mut scheduler, _dir) = build_test_scheduler(4); + let ledger = scheduler + .shared + .calvin + .applied + .get(4) + .expect("the scheduler's ledger"); + assert!(Arc::ptr_eq(&ledger, &scheduler.ledger)); + assert!(!ledger.is_applied(3, 9)); + + scheduler.applied = AppliedGate::new(3, BTreeSet::new()); + scheduler.publish_watermark(3); + + assert!(ledger.is_applied(3, 9)); + assert!(!ledger.is_applied(4, 0)); + } + + /// With no Calvin history there is nothing to rebuild, so the scheduler + /// reports caught up without a watermark. + #[tokio::test] + async fn a_scheduler_with_nothing_to_rebuild_reports_caught_up() { + let (mut scheduler, _dir) = build_test_scheduler(4); + scheduler.rebuild_target_epoch = NOT_YET_APPLIED_EPOCH; + scheduler.publish_caught_up(); + assert!(lagging(&scheduler).is_empty()); + } + + /// A stopped scheduler no longer holds the readiness gate. + #[tokio::test] + async fn a_dropped_scheduler_leaves_the_readiness_registry() { + let (scheduler, _dir) = build_test_scheduler(4); + let shared = Arc::clone(&scheduler.shared); + assert_eq!(lagging(&scheduler), vec![4]); + drop(scheduler); + assert!(shared.calvin.caught_up.lagging().is_empty()); + } } diff --git a/nodedb/src/control/cluster/calvin/scheduler/driver/core/staged_vote.rs b/nodedb/src/control/cluster/calvin/scheduler/driver/core/staged_vote.rs index b0c7f92db..165864f9f 100644 --- a/nodedb/src/control/cluster/calvin/scheduler/driver/core/staged_vote.rs +++ b/nodedb/src/control/cluster/calvin/scheduler/driver/core/staged_vote.rs @@ -1,11 +1,11 @@ // SPDX-License-Identifier: BUSL-1.1 -//! This participant's local commit vote on a staged static Calvin transaction, -//! derived from the staged executor response. +//! This participant's local commit vote on a staged Calvin transaction, +//! read from the stage vote its executor response carries. use nodedb_cluster::calvin::AbortReason; -use crate::bridge::envelope::{ErrorCode, Response, Status}; +use crate::bridge::envelope::{ErrorCode, Response, StageVote, Status}; /// A participant's local verdict on its staged slice. #[derive(Clone, Copy, Debug, PartialEq, Eq)] @@ -19,11 +19,22 @@ pub(super) enum StagedVote { /// A collection the transaction names no longer holds its planned /// incarnation in this replica's catalog. CollectionSuperseded, - /// The stage refused with `OllpRetryRequired`: the state the coordinator - /// predicted moved. The coordinator reads again and retries. + /// The stage refused because the state the coordinator predicted moved. + /// The coordinator reads again and retries. PredictionDrift, } +impl From for StagedVote { + fn from(vote: StageVote) -> Self { + match vote { + StageVote::Commit => Self::Commit, + StageVote::SerializationConflict => Self::SerializationConflict, + StageVote::PredictionDrift => Self::PredictionDrift, + StageVote::ParticipantError => Self::ParticipantError, + } + } +} + impl StagedVote { /// `None` for a commit, the abort cause otherwise. pub(super) fn abort_reason(self) -> Option { @@ -37,21 +48,52 @@ impl StagedVote { } } -/// Derive a local staged vote without ever treating an executor error as a -/// commit, and keep the abort causes apart. A `None` read-set stays -/// affirmative only for a successful dependent-read or active staged response, -/// which has no versioned read-set. -pub(super) fn staged_commit_vote(response: &Response) -> StagedVote { - if response.status != Status::Ok { - if response.error_code.as_deref() == Some(&ErrorCode::OllpRetryRequired) { - return StagedVote::PredictionDrift; - } - return StagedVote::ParticipantError; +/// A stage response the scheduler cannot read a vote from. +#[derive(Clone, Copy, Debug, PartialEq, Eq, thiserror::Error)] +pub(super) enum StageVoteError { + /// A stage response that did not fail carries no stage vote. + #[error("stage returned {status:?} without a stage vote")] + Missing { status: Status }, + /// A stage response that did not succeed carries a commit vote. + #[error("stage returned {status:?} with a commit stage vote")] + CommitWithoutSuccess { status: Status }, + /// The vShard's core fail-stopped and refused the stage. It refuses + /// every later request too, the committed install included, so no vote + /// this replica casts can be kept. + #[error("stage refused by fail-stopped core {core_id}")] + CoreFailStopped { core_id: usize }, +} + +/// Read the local vote of a staged slice from its stage response. +/// +/// The response's stage vote maps 1:1. An error response with no vote never +/// reached a stage path: the request expired, or its core is gone or +/// drained. That slice never staged, so it votes `ParticipantError`. A +/// fail-stopped core's refusal is the exception: it is refused. +/// +/// A response that did not fail and carries no vote is refused, and so is a +/// commit vote on a response that did not succeed. Neither counts as a commit. +pub(super) fn staged_commit_vote(response: &Response) -> Result { + let status = response.status; + if let Some(ErrorCode::CoreFailStopped { core_id, .. }) = response.error_code.as_deref() { + return Err(StageVoteError::CoreFailStopped { core_id: *core_id }); } - if response.read_set_valid == Some(false) { - return StagedVote::SerializationConflict; + match (response.stage_vote, status) { + (Some(StageVote::Commit), Status::Ok) => Ok(StagedVote::Commit), + (Some(StageVote::Commit), Status::Partial | Status::Error) => { + Err(StageVoteError::CommitWithoutSuccess { status }) + } + ( + Some( + vote @ (StageVote::SerializationConflict + | StageVote::PredictionDrift + | StageVote::ParticipantError), + ), + _, + ) => Ok(vote.into()), + (None, Status::Error) => Ok(StagedVote::ParticipantError), + (None, Status::Ok | Status::Partial) => Err(StageVoteError::Missing { status }), } - StagedVote::Commit } #[cfg(test)] @@ -60,7 +102,7 @@ mod tests { use crate::bridge::envelope::Payload; use crate::types::{Lsn, RequestId}; - fn staged_response(status: Status, read_set_valid: Option) -> Response { + fn staged_response(status: Status, stage_vote: Option) -> Response { Response { request_id: RequestId::new(1), status, @@ -69,58 +111,100 @@ mod tests { payload: Payload::empty(), watermark_lsn: Lsn::ZERO, error_code: None, - read_set_valid, - read_version_lsn: Lsn::ZERO, + stage_vote, + read_versions: crate::types::ReadVersions::new(), write_set: Vec::new(), } } #[test] - fn executor_error_votes_participant_error_whatever_the_read_set_field_says() { - // The participant never staged, so no read-set was ever validated — - // reporting a serialization conflict here will be a lie. + fn every_stage_vote_maps_one_to_one() { + let cases = [ + (Status::Ok, StageVote::Commit, StagedVote::Commit), + ( + Status::Ok, + StageVote::SerializationConflict, + StagedVote::SerializationConflict, + ), + ( + Status::Error, + StageVote::PredictionDrift, + StagedVote::PredictionDrift, + ), + ( + Status::Error, + StageVote::ParticipantError, + StagedVote::ParticipantError, + ), + ]; + for (status, vote, expected) in cases { + assert_eq!( + staged_commit_vote(&staged_response(status, Some(vote))), + Ok(expected), + "{vote:?} on {status:?}" + ); + } + } + + #[test] + fn a_prediction_drift_vote_aborts_with_prediction_drift() { + let mut response = staged_response(Status::Error, Some(StageVote::PredictionDrift)); + response.error_code = Some(Box::new(ErrorCode::OllpRetryRequired)); + let vote = staged_commit_vote(&response); + assert_eq!(vote, Ok(StagedVote::PredictionDrift)); assert_eq!( - staged_commit_vote(&staged_response(Status::Error, None)), - StagedVote::ParticipantError + vote.map(StagedVote::abort_reason), + Ok(Some(AbortReason::PredictionDrift)) ); + } + + #[test] + fn a_successful_stage_response_without_a_vote_is_refused() { assert_eq!( - staged_commit_vote(&staged_response(Status::Error, Some(true))), - StagedVote::ParticipantError + staged_commit_vote(&staged_response(Status::Ok, None)), + Err(StageVoteError::Missing { status: Status::Ok }) ); assert_eq!( - staged_commit_vote(&staged_response(Status::Error, Some(false))), - StagedVote::ParticipantError + staged_commit_vote(&staged_response(Status::Partial, None)), + Err(StageVoteError::Missing { + status: Status::Partial + }) ); } #[test] - fn an_ollp_retry_stage_error_votes_prediction_drift() { - let mut response = staged_response(Status::Error, None); - response.error_code = Some(Box::new(ErrorCode::OllpRetryRequired)); - assert_eq!(staged_commit_vote(&response), StagedVote::PredictionDrift); + fn a_commit_vote_on_an_unsuccessful_response_is_refused() { assert_eq!( - StagedVote::PredictionDrift.abort_reason(), - Some(AbortReason::PredictionDrift) + staged_commit_vote(&staged_response(Status::Error, Some(StageVote::Commit))), + Err(StageVoteError::CommitWithoutSuccess { + status: Status::Error + }) ); } #[test] - fn stale_read_set_on_a_successful_response_votes_serialization_conflict() { + fn an_error_that_never_reached_a_stage_path_votes_participant_error() { + // An expired request answers before any stage handler runs. + let mut response = staged_response(Status::Error, None); + response.error_code = Some(Box::new(ErrorCode::ExpiredBeforeExecution)); assert_eq!( - staged_commit_vote(&staged_response(Status::Ok, Some(false))), - StagedVote::SerializationConflict + staged_commit_vote(&response), + Ok(StagedVote::ParticipantError) ); } + /// A fail-stopped core's refusal is no vote: the core refuses the + /// committed install too. #[test] - fn successful_response_with_a_current_or_absent_read_set_votes_commit() { - assert_eq!( - staged_commit_vote(&staged_response(Status::Ok, Some(true))), - StagedVote::Commit - ); + fn a_fail_stopped_core_refusal_is_refused() { + let mut response = staged_response(Status::Error, None); + response.error_code = Some(Box::new(ErrorCode::CoreFailStopped { + core_id: 2, + detail: "rollback failed".into(), + })); assert_eq!( - staged_commit_vote(&staged_response(Status::Ok, None)), - StagedVote::Commit + staged_commit_vote(&response), + Err(StageVoteError::CoreFailStopped { core_id: 2 }) ); } @@ -139,5 +223,9 @@ mod tests { StagedVote::CollectionSuperseded.abort_reason(), Some(AbortReason::CollectionSuperseded) ); + assert_eq!( + StagedVote::PredictionDrift.abort_reason(), + Some(AbortReason::PredictionDrift) + ); } } diff --git a/nodedb/src/control/cluster/calvin/scheduler/driver/core/test_proposer.rs b/nodedb/src/control/cluster/calvin/scheduler/driver/core/test_proposer.rs index 6602ec96e..b08ede769 100644 --- a/nodedb/src/control/cluster/calvin/scheduler/driver/core/test_proposer.rs +++ b/nodedb/src/control/cluster/calvin/scheduler/driver/core/test_proposer.rs @@ -72,9 +72,28 @@ impl SequencerProposer for CapturingProposer { } /// Make the fixture node the elected leader of the data group that owns -/// `scheduler`'s vShard, so leader-only proposals such as the commit vote -/// run. -pub(super) fn elect_data_group_leader(scheduler: &Scheduler) { +/// `scheduler`'s vShard, apply the term's no-op, and open the scheduler's +/// stage gate, so it stages its grants and proposes its entries. +pub(super) fn lead_data_group(scheduler: &mut Scheduler) { + let group_id = elect_data_group_leader(scheduler); + let term_start = scheduler + .multi_raft + .lock() + .unwrap_or_else(|p| p.into_inner()) + .term_start_index(group_id) + .expect("the elected node names its term start"); + scheduler + .shared + .applied_index_watcher(group_id) + .bump(term_start); + scheduler.refresh_role(); + assert!(scheduler.role.is_leader(), "the stage gate opens"); +} + +/// Make the fixture node the elected leader of the data group that owns +/// `scheduler`'s vShard, with its term's no-op not yet applied. Returns +/// the group. +pub(super) fn elect_data_group_leader(scheduler: &Scheduler) -> u64 { let mut mr = scheduler .multi_raft .lock() @@ -95,8 +114,38 @@ pub(super) fn elect_data_group_leader(scheduler: &Scheduler) { for _ in 0..20 { mr.tick().expect("tick"); if mr.vshard_role_is_leader(scheduler.vshard_id) { - return; + return group_id; } } panic!("data group did not elect the single fixture node"); } + +/// Step the fixture node down from its data group: a new term starts on +/// another node. +pub(super) fn step_down_from_data_group(scheduler: &Scheduler) { + let mut mr = scheduler + .multi_raft + .lock() + .unwrap_or_else(|p| p.into_inner()); + let group_id = mr + .routing() + .read() + .unwrap_or_else(|p| p.into_inner()) + .group_for_vshard(scheduler.vshard_id) + .expect("the fixture routing maps every vShard"); + let term = mr.leader_term(group_id).expect("the fixture node leads"); + // A heartbeat of a later term from another node deposes this one. + mr.handle_append_entries(&nodedb_raft::message::AppendEntriesRequest { + term: term + 1, + leader_id: scheduler.shared.node_id + 1, + prev_log_index: 0, + prev_log_term: 0, + entries: Vec::new(), + leader_commit: 0, + group_id, + round: 0, + replicated_floor: 0, + }) + .expect("the group takes the heartbeat"); + assert!(!mr.vshard_role_is_leader(scheduler.vshard_id)); +} diff --git a/nodedb/src/control/cluster/calvin/scheduler/driver/core/test_support.rs b/nodedb/src/control/cluster/calvin/scheduler/driver/core/test_support.rs index a8ca94fc2..29102a0a4 100644 --- a/nodedb/src/control/cluster/calvin/scheduler/driver/core/test_support.rs +++ b/nodedb/src/control/cluster/calvin/scheduler/driver/core/test_support.rs @@ -2,7 +2,7 @@ //! Shared test fixtures for the Calvin scheduler driver's `core` unit tests. -use std::collections::{BTreeSet, HashMap}; +use std::collections::HashMap; use std::sync::{Arc, Mutex}; use std::time::{Duration, Instant}; @@ -20,17 +20,18 @@ use tokio::sync::mpsc; use crate::bridge::dispatch::{BridgeResponse, CoreChannelDataSide, Dispatcher}; use crate::bridge::envelope::{ - Admission, ErrorCode, ExemptReason, Payload, Priority, Request, Response, Status, + Admission, ErrorCode, ExemptReason, Payload, Priority, Request, Response, StageVote, Status, }; -use crate::control::cluster::calvin::scheduler::driver::barrier::ReadResultEvent; +use crate::control::cluster::calvin::scheduler::SchedulerConfig; use crate::control::cluster::calvin::scheduler::driver::core::scheduler::{ Scheduler, SchedulerParams, }; use crate::control::cluster::calvin::scheduler::driver::core::test_proposer::CapturingProposer; -use crate::control::cluster::calvin::scheduler::driver::types::{CommitState, PendingTxn}; +use crate::control::cluster::calvin::scheduler::driver::types::{ + CommitState, PendingTxn, SliceScope, +}; use crate::control::cluster::calvin::scheduler::lock_manager::{LockManager, TxnId}; use crate::control::cluster::calvin::scheduler::metrics::SchedulerMetrics; -use crate::control::cluster::calvin::scheduler::{NOT_YET_APPLIED_EPOCH, SchedulerConfig}; use crate::control::shutdown::ShutdownWatch; use crate::control::state::SharedState; use crate::types::{DatabaseId, Lsn, ReadConsistency, RequestId, VShardId}; @@ -50,6 +51,7 @@ pub(super) fn build_test_scheduler(vshard_id: u32) -> (Scheduler, tempfile::Temp .pop() .expect("one configured core has one data side"); let shared = SharedState::new(dispatcher, wal).unwrap(); + let ledger = shared.calvin.applied.get_or_create(vshard_id); let rt = RoutingTable::uniform(1, &[1], 1); let multi_raft = Arc::new(Mutex::new(MultiRaft::new(1, rt, dir.path().to_path_buf()))); @@ -60,7 +62,6 @@ pub(super) fn build_test_scheduler(vshard_id: u32) -> (Scheduler, tempfile::Temp ))); let (_tx, receiver) = tokio::sync::mpsc::channel(16); - let (_rr_tx, read_result_rx) = tokio::sync::mpsc::channel(16); let (_prom_tx, promotion_rx) = tokio::sync::mpsc::unbounded_channel(); let (verdict_tx, verdict_rx) = tokio::sync::mpsc::channel(16); registry.register_verdict_signal_sender(vshard_id, verdict_tx); @@ -74,18 +75,16 @@ pub(super) fn build_test_scheduler(vshard_id: u32) -> (Scheduler, tempfile::Temp multi_raft, sequencer_proposer: CapturingProposer::accepting(), sequencer_state_machine, - // A freshly-built scheduler has applied nothing, so its watermark is the - // not-yet-applied sentinel (matching `read_applied_recovery` for a clean - // node). Hardcoding `0` here will instead claim epoch 0 is fully applied, + // A freshly-built scheduler has applied nothing, so its ledger holds the + // not-yet-applied sentinel (matching `recover_all_applied` for a clean + // node). A watermark of `0` will instead claim epoch 0 is fully applied, // making the exactly-once gate (`AppliedGate::is_applied`) short-circuit // every epoch-0 replay before it reaches the lock table — silently // defeating the end-to-end drain tests below. - fully_applied_epoch: NOT_YET_APPLIED_EPOCH, - applied_tail: BTreeSet::new(), + ledger, rebuild_target_epoch: 0, config: SchedulerConfig::default(), metrics: SchedulerMetrics::new(), - read_result_rx, lock_manager, promotion_rx, registry, @@ -109,6 +108,7 @@ pub(super) fn build_test_scheduler_with_data_side( .pop() .expect("one configured core has one data side"); let shared = SharedState::new(dispatcher, wal).unwrap(); + let ledger = shared.calvin.applied.get_or_create(vshard_id); let rt = RoutingTable::uniform(1, &[1], 1); let multi_raft = Arc::new(Mutex::new(MultiRaft::new(1, rt, dir.path().to_path_buf()))); @@ -119,7 +119,6 @@ pub(super) fn build_test_scheduler_with_data_side( ))); let (_tx, receiver) = tokio::sync::mpsc::channel(16); - let (_rr_tx, read_result_rx) = tokio::sync::mpsc::channel(16); let (_prom_tx, promotion_rx) = tokio::sync::mpsc::unbounded_channel(); let (verdict_tx, verdict_rx) = tokio::sync::mpsc::channel(16); registry.register_verdict_signal_sender(vshard_id, verdict_tx); @@ -133,12 +132,10 @@ pub(super) fn build_test_scheduler_with_data_side( multi_raft, sequencer_proposer: CapturingProposer::accepting(), sequencer_state_machine, - fully_applied_epoch: NOT_YET_APPLIED_EPOCH, - applied_tail: BTreeSet::new(), + ledger, rebuild_target_epoch: 0, config: SchedulerConfig::default(), metrics: SchedulerMetrics::new(), - read_result_rx, lock_manager, promotion_rx, registry, @@ -197,9 +194,8 @@ pub(super) fn make_validate_only_txn(epoch: u64, position: u32) -> SequencedTxn engine: EngineTag::Document, collection: "test_coll".to_string(), key: ReadKeyIdent::Point(KeyRepr::Surrogate(1)), - read_lsn: Lsn::ZERO, + read_version: nodedb_types::WriteVersion::ZERO, home_vshard: None, - served_by: 0, }]); let tx_class = TxClass::new_single_vshard( ReadWriteSet::new(vec![]), @@ -295,6 +291,7 @@ fn filler_request(request_id: RequestId, tenant_id: TenantId) -> Request { wal_lsn: None, resolved_now_ms: None, commit_hlc: None, + entry_version: None, admission: Admission::Exempt(ExemptReason::Read), } } @@ -377,7 +374,6 @@ pub(super) struct RunningScheduler { shutdown: ShutdownWatch, handle: tokio::task::JoinHandle<()>, input_tx: mpsc::Sender, - _read_result_tx: mpsc::Sender, _promotion_tx: mpsc::UnboundedSender>, } @@ -409,10 +405,8 @@ impl RunningScheduler { /// liveness tick. pub(super) fn spawn_scheduler_loop(mut scheduler: Scheduler) -> RunningScheduler { let (input_tx, input_rx) = mpsc::channel(16); - let (read_result_tx, read_result_rx) = mpsc::channel(16); let (promotion_tx, promotion_rx) = mpsc::unbounded_channel(); scheduler.receiver = input_rx; - scheduler.read_result_rx = read_result_rx; scheduler.promotion_rx = promotion_rx; // The loop's liveness tick fires every quarter of this interval. scheduler.config.verdict_stall_warn_ms = 200; @@ -423,36 +417,58 @@ pub(super) fn spawn_scheduler_loop(mut scheduler: Scheduler) -> RunningScheduler shutdown, handle, input_tx, - _read_result_tx: read_result_tx, _promotion_tx: promotion_tx, } } -/// A `PendingTxn` staged and parked awaiting the cross-shard commit verdict. +/// A `PendingTxn` staged on the leader, its stage answer awaited. Its slice +/// writes `test_coll`. pub(super) fn staged_pending(txn: SequencedTxn, txn_id: TxnId) -> PendingTxn { + let mut scope = SliceScope::of_plans(&[]); + scope.writes = true; + scope.collections = vec!["test_coll".to_string()]; PendingTxn { txn, lock_owner: txn_id, // no-determinism: test-only dispatch timestamp for a fabricated PendingTxn fixture. dispatch_time: Instant::now(), has_primary_write: true, + raises_write_mark: true, has_returning: false, - change_sets: Vec::new(), - commit_state: Some(CommitState::Staged), + commit_state: CommitState::Staged, + awaiting: None, verdict_deadline: None, stage_error: None, - redo_records: None, - flush_scope: crate::control::cluster::calvin::scheduler::driver::types::FlushScope::default( - ), + scope, + redo: None, superseded: false, gates: Vec::new(), ungated: false, - install_permit: None, } } -/// A staged executor `Response` carrying the given status and read-set vote. -pub(super) fn staged_response(status: Status, read_set_valid: Option) -> Response { +/// A `PendingTxn` a follower holds for its verdict and its redo. +pub(super) fn following_pending(txn: SequencedTxn, txn_id: TxnId) -> PendingTxn { + let mut pending = staged_pending(txn, txn_id); + pending.commit_state = CommitState::Following; + pending.has_primary_write = false; + pending.raises_write_mark = false; + pending +} + +/// The answer the apply loop reports for an installed slice. +pub(super) fn redo_applied( + reply: Response, +) -> crate::control::cluster::calvin::scheduler::CalvinApplyEvent { + crate::control::cluster::calvin::scheduler::CalvinApplyEvent::RedoApplied { + reply, + primary_write: true, + returning: false, + } +} + +/// An executor `Response` carrying the given status and stage vote. +pub(super) fn staged_response(status: Status, stage_vote: Option) -> Response { Response { request_id: RequestId::new(1), status, @@ -461,8 +477,8 @@ pub(super) fn staged_response(status: Status, read_set_valid: Option) -> R payload: Payload::empty(), watermark_lsn: Lsn::ZERO, error_code: None, - read_set_valid, - read_version_lsn: Lsn::ZERO, + stage_vote, + read_versions: crate::types::ReadVersions::new(), write_set: Vec::new(), } } @@ -484,14 +500,16 @@ pub(super) fn begin_data_plane_drain(shared: &SharedState) { .begin_data_plane_drain(); } -/// A scheduler on vShard 7 with `txn_id` pending in `state`. +/// A scheduler on vShard 7 with `txn_id` pending in `state`, its answer to +/// request 9 awaited. pub(super) fn scheduler_with_pending( txn_id: TxnId, state: CommitState, ) -> (Scheduler, tempfile::TempDir) { let (mut scheduler, dir) = build_test_scheduler(7); let mut pending = staged_pending(make_sequenced_txn(txn_id.epoch, txn_id.position), txn_id); - pending.commit_state = Some(state); + pending.commit_state = state; + pending.awaiting = Some(RequestId::new(9)); scheduler.pending.insert(txn_id, pending); (scheduler, dir) } diff --git a/nodedb/src/control/cluster/calvin/scheduler/driver/core/write_version_record.rs b/nodedb/src/control/cluster/calvin/scheduler/driver/core/write_version_record.rs deleted file mode 100644 index 0d27cb9e8..000000000 --- a/nodedb/src/control/cluster/calvin/scheduler/driver/core/write_version_record.rs +++ /dev/null @@ -1,153 +0,0 @@ -// SPDX-License-Identifier: BUSL-1.1 - -//! Post-apply write-version recording for committed Calvin transactions. -//! -//! A Calvin apply's committed WAL LSN is the LSN of its `TransactionRedo` -//! record, or of the `CalvinApplied` marker a transaction that wrote nothing -//! here appends. The install records the collection floors and index-value -//! versions of the record at its LSN. The per-key versions of the local write -//! plans are recorded here: the scheduler dispatches a one-way, record-only op -//! back to the same core, which funnels the plans through the shared -//! write-version recorder at that LSN — the same shard-local WAL-LSN space -//! the single-shard fast path and read watermarks use. - -use super::deferred::{DispatchOutcome, DispatchStep}; -use super::scheduler::Scheduler; -use crate::control::cluster::calvin::scheduler::lock_manager::TxnId; -use crate::types::Lsn; -use nodedb_physical::physical_plan::PhysicalPlan; -use nodedb_physical::physical_plan::meta::MetaOp; - -impl Scheduler { - /// Record the per-key write versions of a just-committed Calvin - /// transaction's locally-applied write plans at its committed WAL - /// `applied_lsn`. - /// - /// Dispatches a record-only [`MetaOp::RecordCalvinWriteVersions`] op back to - /// this vShard's core with `applied_lsn` stamped on the request envelope's - /// `wal_lsn`; the core funnels the plans through the shared write-version - /// recorder at that LSN. The recorded version therefore lands in the same - /// WAL-LSN space as fast-path writes, so a later read-set validation against - /// these keys is not a false-Valid serializability hole. - /// - /// Fire-and-forget: the recorded version is not needed to complete the - /// transaction, so the response is drained and discarded. A brief index-lag - /// window before the record op lands is harmless — nothing enforces read-set - /// validation against these versions yet. A record refused at capacity is - /// parked and re-sent once capacity frees, never dropped. A decode or - /// routing failure, or a terminal dispatch refusal, leaves the version - /// unrecorded and never blocks the commit. - pub(in crate::control::cluster::calvin::scheduler::driver::core) fn record_calvin_write_versions( - &mut self, - txn_id: TxnId, - applied_lsn: Lsn, - ) { - let epoch = txn_id.epoch; - let position = txn_id.position; - - let Some(pending) = self.pending.get(&txn_id) else { - return; - }; - let tenant_id = pending.txn.tx_class.tenant_id; - let database_id = pending.txn.tx_class.database_id; - let plans = match super::super::helpers::decode_plans(&pending.txn.tx_class.plans) { - Ok(p) => p, - Err(e) => { - tracing::warn!( - vshard_id = self.vshard_id, - epoch, - position, - error = %e, - "calvin: write-version recording skipped — plan decode failed" - ); - return; - } - }; - // The locally-applied slice this vShard committed. Empty for a - // validate-only READ participant (no local writes) — the recorder then - // has nothing to record, which is correct. The recorder no-ops any plan - // without a per-key or collection version, so no gate on plan kind is - // applied here — gating on a narrower write predicate would silently - // skip recordable writes (e.g. a CRDT apply) and reopen the version gap. - let local = match self.local_calvin_plans(plans, database_id, epoch, position) { - Ok(p) => p, - Err(e) => { - tracing::warn!( - vshard_id = self.vshard_id, - epoch, - position, - error = %e, - "calvin: write-version recording skipped — local plan routing failed" - ); - return; - } - }; - - let request_id = self.next_request_id(); - let plan = PhysicalPlan::Meta(MetaOp::RecordCalvinWriteVersions { - tenant_id, - plans: local, - }); - // The committed write-LSN for this Calvin apply — recorded against - // every key the plans wrote, in the same WAL-LSN space as fast-path. - let request = - self.build_exempt_request(request_id, tenant_id, database_id, plan, Some(applied_lsn)); - - // The request carries everything a re-send needs, so a parked record - // outlives the txn's `pending` entry. - if let DispatchOutcome::Failed(error) = - self.dispatch_sequenced(txn_id, DispatchStep::WriteVersionRecord, request) - { - self.fail_dispatch_step(txn_id, DispatchStep::WriteVersionRecord, error); - } - } -} - -#[cfg(test)] -mod tests { - use std::sync::Arc; - - use nodedb_cluster::calvin::CalvinCompletionRegistry; - use nodedb_types::TenantId; - - use super::*; - use crate::control::cluster::calvin::scheduler::driver::core::test_support::{ - await_data_plane_request, build_test_scheduler_with_data_side, fill_tenant_inflight, - make_validate_only_txn, release_filler, spawn_scheduler_loop, staged_pending, - test_coll_vshard, - }; - - /// Once a Data Plane response frees tenant capacity, a write-version - /// record refused at capacity reaches the Data Plane. - #[tokio::test] - async fn refused_write_version_record_reaches_data_plane_after_capacity_frees() { - let registry = CalvinCompletionRegistry::new_detached(); - let (mut scheduler, _dir, mut data_side) = - build_test_scheduler_with_data_side(test_coll_vshard(), registry); - let txn_id = TxnId::new(21, 0); - scheduler.pending.insert( - txn_id, - staged_pending(make_validate_only_txn(21, 0), txn_id), - ); - let shared = Arc::clone(&scheduler.shared); - let fillers = fill_tenant_inflight(&shared, &mut data_side, TenantId::new(1)); - - scheduler.record_calvin_write_versions(txn_id, Lsn::new(42)); - let running = spawn_scheduler_loop(scheduler); - release_filler(&shared, &mut data_side, fillers[0]); - - let arrived = await_data_plane_request(&mut data_side, |plan| { - matches!( - plan, - PhysicalPlan::Meta(MetaOp::RecordCalvinWriteVersions { .. }) - ) - }) - .await; - running.stop().await; - - assert!( - arrived, - "the refused write-version record must reach the Data Plane once capacity frees" - ); - } -} diff --git a/nodedb/src/control/cluster/calvin/scheduler/driver/helpers.rs b/nodedb/src/control/cluster/calvin/scheduler/driver/helpers.rs index fb5816af2..7eaaf45e7 100644 --- a/nodedb/src/control/cluster/calvin/scheduler/driver/helpers.rs +++ b/nodedb/src/control/cluster/calvin/scheduler/driver/helpers.rs @@ -2,12 +2,12 @@ //! Expansion and decoding helpers for the Calvin scheduler driver. -use std::collections::BTreeSet; +use std::collections::BTreeMap; use std::sync::Arc; use nodedb_cluster::calvin::types::{EngineKeySet, LockKeyWire, SequencedTxn, TxnIdWire}; -use crate::control::cluster::calvin::scheduler::lock_manager::{LockKey, TxnId}; +use crate::control::cluster::calvin::scheduler::lock_manager::{LockKey, LockMode, TxnId}; use nodedb_physical::physical_plan::PhysicalPlan; use nodedb_physical::physical_plan::wire as plan_wire; @@ -42,75 +42,121 @@ pub(super) fn decode_lock_key(wire: &LockKeyWire) -> LockKey { src: *src, dst: *dst, }, + LockKeyWire::Collection { collection } => LockKey::Collection { + collection: Arc::from(collection.as_str()), + }, + LockKeyWire::Unique { + collection, + index, + value, + } => LockKey::Unique { + collection: Arc::from(collection.as_str()), + index: Arc::from(index.as_str()), + value: Arc::from(value.as_slice()), + }, } } -/// Expand the read_set ∪ write_set of a sequenced transaction into a -/// `BTreeSet`. +/// Expand the read set and write set of a sequenced transaction into the +/// moded lock request the scheduler acquires. /// -/// All keys in both the read set and write set are acquired as exclusive -/// (write) locks. -pub(crate) fn expand_rw_set(txn: &SequencedTxn) -> BTreeSet { - let mut keys = BTreeSet::new(); - let add_key_set = |keys: &mut BTreeSet, ks: &EngineKeySet| match ks { - EngineKeySet::Document { - collection, - surrogates, - } - | EngineKeySet::Vector { - collection, - surrogates, - } => { - let coll: Arc = Arc::from(collection.as_str()); +/// - A write row key (surrogate, KV key, edge pair, UNIQUE value) is +/// `Exclusive`, and its collection key is `Intent`. +/// - A write [`EngineKeySet::Collection`] and a write array are `Exclusive` +/// on the collection key. +/// - A read row key is `Shared`, with no collection key. +/// - A read [`EngineKeySet::Collection`] is `Shared` on the collection key. +/// +/// A key named twice takes the merged mode (see [`LockMode::merge`]). +pub(crate) fn expand_rw_set(txn: &SequencedTxn) -> BTreeMap { + let mut keys = BTreeMap::new(); + for ks in &txn.tx_class.read_set.0 { + add_key_set(&mut keys, ks, LockMode::Shared); + } + for ks in &txn.tx_class.write_set.0 { + add_key_set(&mut keys, ks, LockMode::Exclusive); + } + keys +} + +/// Expand write key sets alone into the moded lock request a transaction of +/// those writes takes, by the rules of [`expand_rw_set`]. The +/// write-admission gate locks an autocommit write with it, so the write and +/// a Calvin transaction of the same plan take the same keys. +pub(crate) fn expand_write_key_sets(sets: &[EngineKeySet]) -> BTreeMap { + let mut keys = BTreeMap::new(); + for ks in sets { + add_key_set(&mut keys, ks, LockMode::Exclusive); + } + keys +} + +/// Add the keys of `ks` to `keys`. `row_mode` is `Shared` for a read set and +/// `Exclusive` for a write set. +fn add_key_set(keys: &mut BTreeMap, ks: &EngineKeySet, row_mode: LockMode) { + let coll: Arc = Arc::from(ks.collection()); + let whole = LockKey::Collection { + collection: Arc::clone(&coll), + }; + match ks { + EngineKeySet::Document { surrogates, .. } | EngineKeySet::Vector { surrogates, .. } => { for &surrogate in surrogates.iter() { - keys.insert(LockKey::Surrogate { + let key = LockKey::Surrogate { collection: Arc::clone(&coll), surrogate, - }); + }; + request(keys, key, row_mode); } } - EngineKeySet::Kv { - collection, - keys: kv_keys, - } => { - let coll: Arc = Arc::from(collection.as_str()); + EngineKeySet::Kv { keys: kv_keys, .. } => { for k in kv_keys.iter() { - keys.insert(LockKey::Kv { + let key = LockKey::Kv { collection: Arc::clone(&coll), key: Arc::from(k.as_slice()), - }); + }; + request(keys, key, row_mode); } } - EngineKeySet::Edge { - collection, edges, .. - } => { - let coll: Arc = Arc::from(collection.as_str()); + EngineKeySet::Edge { edges, .. } => { for &(src, dst) in edges.iter() { - keys.insert(LockKey::Edge { + let key = LockKey::Edge { collection: Arc::clone(&coll), src, dst, - }); + }; + request(keys, key, row_mode); + } + } + EngineKeySet::Unique { index, values, .. } => { + let index: Arc = Arc::from(index.as_str()); + for value in values.iter() { + let key = LockKey::Unique { + collection: Arc::clone(&coll), + index: Arc::clone(&index), + value: Arc::from(value.as_slice()), + }; + request(keys, key, row_mode); } } // Each participating vShard's lock table holds the whole array on - // that vShard: the collection key every writer of the array takes. - EngineKeySet::Array { collection, .. } => { - keys.insert(LockKey::Surrogate { - collection: Arc::from(collection.as_str()), - surrogate: crate::control::planner::calvin::tx_class::write_keys::COLLECTION_KEY, - }); + // that vShard. A whole-collection key set covers the collection. + EngineKeySet::Array { .. } | EngineKeySet::Collection { .. } => { + request(keys, whole, row_mode); + return; } - }; - - for ks in &txn.tx_class.read_set.0 { - add_key_set(&mut keys, ks); } - for ks in &txn.tx_class.write_set.0 { - add_key_set(&mut keys, ks); + // A row writer announces itself on its collection, so a collection-wide + // writer or a predicate reader of that collection orders against it. + if row_mode == LockMode::Exclusive { + request(keys, whole, LockMode::Intent); } +} - keys +/// Request `key` in `mode`, merged with any mode already requested for it. +fn request(keys: &mut BTreeMap, key: LockKey, mode: LockMode) { + keys.entry(key) + .and_modify(|held| *held = held.merge(mode)) + .or_insert(mode); } /// Decode a `Vec` from the opaque plan bytes stored in a diff --git a/nodedb/src/control/cluster/calvin/scheduler/driver/mod.rs b/nodedb/src/control/cluster/calvin/scheduler/driver/mod.rs index 5baf6021c..6ac5f56ca 100644 --- a/nodedb/src/control/cluster/calvin/scheduler/driver/mod.rs +++ b/nodedb/src/control/cluster/calvin/scheduler/driver/mod.rs @@ -6,7 +6,6 @@ pub mod core; pub mod helpers; pub mod types; -pub use barrier::ReadResultEvent; pub use config::SchedulerConfig; pub use core::{ CalvinReadResultProposal, RaftSequencerProposer, Scheduler, SchedulerParams, SequencerProposer, diff --git a/nodedb/src/control/cluster/calvin/scheduler/driver/types.rs b/nodedb/src/control/cluster/calvin/scheduler/driver/types.rs index f3cda4a1a..8de4414e7 100644 --- a/nodedb/src/control/cluster/calvin/scheduler/driver/types.rs +++ b/nodedb/src/control/cluster/calvin/scheduler/driver/types.rs @@ -2,22 +2,25 @@ //! In-flight transaction types for the Calvin scheduler driver. -use std::collections::BTreeSet; +use std::collections::BTreeMap; use std::time::Instant; use nodedb_cluster::calvin::types::SequencedTxn; -use super::super::lock_manager::{LockKey, TxnId}; +use super::super::lock_manager::{LockKey, LockMode, TxnId}; -/// An in-flight transaction that has been dispatched and is awaiting a -/// Data Plane response. +/// A transaction this vShard granted and has not finished. +/// +/// On the data-group leader it stages, votes, resolves and proposes its +/// redo. On a follower it waits in [`CommitState::Following`] for its +/// verdict and its stamped redo. /// /// The executor response channel is held by a bridge task (see /// `Scheduler::spawn_response_bridge`) that forwards completions to the /// scheduler's fan-in `completion_rx`. This avoids polling and ensures the /// main `select!` loop wakes the moment a response arrives. pub(super) struct PendingTxn { - /// Original sequenced transaction (for WAL record on completion). + /// Original sequenced transaction. pub txn: SequencedTxn, /// The lock-table owner id for this txn (equals the apply-slot id unless a /// reservation owns the lock). Used by `on_txn_complete` to `release` the @@ -36,84 +39,69 @@ pub(super) struct PendingTxn { /// cleanup participants that dual-home alongside it carry no primary write and /// so never clobber the entry the coordinator drains. pub has_primary_write: bool, + /// Whether the slice's install raises its tenant's write mark: a + /// primary write that changes a tenant row. A schema install raises none. + pub raises_write_mark: bool, /// Whether this vShard's slice carries a RETURNING-bearing write (a plan - /// whose applied response is DATA-ROWs, not a bare affected-count). Used at - /// the commit tail to detect a genuine cross-shard RETURNING union: two - /// returning-bearing participants for one txn are unsupported, whereas - /// multiple plain-write participants (a multi-collection cross-shard COMMIT) - /// coalesce without conflict. + /// whose applied response is DATA-ROWs, not a bare affected-count). Two + /// returning-bearing participants for one txn are an unsupported + /// cross-shard RETURNING union. pub has_returning: bool, - /// Participant-local Control-Plane change manifests. Consumed once after - /// durable COMMIT apply; graph dual-home replicas are excluded at capture. - pub change_sets: Vec, - /// Commit-resolution state for a static-set Calvin txn. - /// - /// `Some(CommitState::Staged)` for a txn dispatched, or parked for re-send, - /// via the validate-and-stage path: its first executor response carries the local - /// commit vote and drives a flush-or-drop before the commit tail runs. - /// `None` for dependent/active txns, which apply directly. - pub commit_state: Option, + /// Commit-resolution state of the txn. + pub commit_state: CommitState, + /// The request whose response the txn waits for. A response to any + /// other request of the txn is from a step it left, and is ignored. + pub awaiting: Option, /// Stall deadline for a txn parked in [`CommitState::AwaitingVerdict`]. /// /// `Some(instant)` only while parked: if the deadline passes with the /// durable global verdict still unknown, the scheduler emits a stall /// warning and re-arms this deadline — it NEVER releases locks and NEVER - /// aborts (a unilateral abort while a peer can have already flushed a commit + /// aborts (a unilateral abort while a peer can have already committed /// will tear the transaction). `None` in every other state. /// /// `Instant::now()` is used for this deadline (observability / liveness /// only; never influences WAL bytes). pub verdict_deadline: Option, - /// Error text of a stage response that was not `Ok` on this replica. + /// Error text of a stage response that was not `Ok` on this leader. /// - /// `Some` when this replica never staged the txn. An abort verdict drops it - /// as usual. A COMMIT verdict halts the scheduler, because the txn cannot - /// apply here while its peers apply it. + /// `Some` when this leader never staged the txn. An abort verdict drops + /// it as usual. A COMMIT verdict restages it, a bounded number of times. + /// A superseded collection or a spent bound halts the scheduler, because + /// the txn cannot apply here while its peers apply it. pub stage_error: Option, - /// The `TransactionRedo` record appended for a committed txn's flush, - /// under its outcome-floor window. The window settles when the txn - /// completes, and holds when the scheduler halts or stops with the txn - /// still pending. - pub redo_records: Option, - /// What a committed flush names beside its redo record, derived once - /// from this vShard's slice when it stages. - pub flush_scope: FlushScope, + /// What this vShard's slice writes, derived once from its local plans. + pub scope: SliceScope, + /// The committed slice's redo, kept while it is proposed so a stall or + /// a new term can propose it again. + pub redo: Option, /// A collection the transaction names no longer held its planned - /// incarnation when this replica staged it. The vote is an abort. + /// incarnation when this leader staged it. The vote is an abort. pub superseded: bool, /// The gates of the collections the transaction names, held shared from /// the incarnation check until the txn leaves `pending`. A purge of one - /// waits, so a staged slice never flushes into a recreated collection. + /// waits, so a staged slice never commits into a recreated collection. pub gates: Vec, - /// A halt released `gates` while the txn had no write in flight. Its - /// flush checks the incarnations again and takes the gates back first. + /// A halt released `gates` while the txn waited. Its proposal checks the + /// incarnations again and takes the gates back first. pub ungated: bool, - /// The shared hold of the vShard's data-group apply gate, taken when the - /// flush dispatches and dropped once the txn left `pending` and its - /// position is marked applied. A data-group snapshot capture or install - /// holds the gate exclusive, so it never sees a flush half applied. - pub install_permit: Option, } -/// What a vShard's committed flush carries: the collections its slice -/// writes, their materialized-sum targets, and the redo record. The Data -/// Plane installs the record the way every committed transaction installs. -/// -/// The collections and sum targets are derived when the slice stages, from -/// the plans the stage sends. A flush that follows a COMMIT verdict -/// therefore never decodes or routes a plan, so it cannot fail after the -/// verdict is durable. -#[derive(Debug, Clone, Default)] -pub(super) struct FlushScope { +/// What this vShard's slice of a transaction writes: derived once from its +/// local plans, on every replica alike. +#[derive(Debug, Clone)] +pub(super) struct SliceScope { + /// Whether the slice holds any local write plan. A slice with none only + /// validates reads: it commits with no log entry. + pub writes: bool, pub collections: Vec, pub sum_targets: Vec, - /// The encoded redo record the resolve appended. Empty until the resolve - /// answers, and empty when the slice wrote nothing on this vShard. A - /// resent flush carries the same bytes. - pub redo: Vec, - /// Flushes sent for this txn. A refused install resends the flush until - /// the count reaches its bound. - pub sends: u32, + /// The identities the local plans carry, bound in this node's catalog. + /// Every replica binds them before the redo installs. + pub identities: Vec, + /// Which commit-boundary checks the install runs: a RESTORE's, or a + /// commit's. + pub origin: nodedb_physical::physical_plan::RedoOrigin, /// Every plan of this vShard's slice is one a trigger body buffered, and /// a client plan of the transaction homes on another vShard. The slice /// then commits under `Trigger`: a single overlay holding the whole @@ -121,16 +109,16 @@ pub(super) struct FlushScope { pub body_only: bool, /// The slice reads a whole collection at resolve: a TRUNCATE of rows. A /// TRUNCATE's edge share reads none. A resolve otherwise runs as soon as - /// its verdict arrives, - /// beside lower txns that have not flushed. A TRUNCATE's resolve then - /// misses their rows, so it waits for its flush turn. + /// its verdict arrives, beside lower txns that have not applied. A + /// TRUNCATE's resolve then misses their rows, so it waits for its turn. pub resolve_at_turn: bool, } -impl FlushScope { - /// The flush scope of the local plans `plans`. +impl SliceScope { + /// The scope of the local plans `plans`. pub(super) fn of_plans(plans: &[nodedb_physical::physical_plan::PhysicalPlan]) -> Self { Self { + writes: !plans.is_empty(), collections: crate::control::wal_replication::transaction_redo::collections::written_collections( plans, @@ -139,20 +127,38 @@ impl FlushScope { crate::control::wal_replication::transaction_redo::sum_targets::redo_sum_targets( plans, ), - // The resolve fills these once it answers. - redo: Vec::new(), - sends: 0, + // The leader fills them once it binds the plans. + identities: Vec::new(), + origin: redo_origin(plans), body_only: false, resolve_at_turn: plans.iter().any(reads_whole_collection_at_resolve), } } } +/// The origin of a Calvin slice's redo record: a RESTORE when its local +/// plans install a RESTORE batch, a commit otherwise. A RESTORE's rows +/// passed their collection's rules when they were first written, so the +/// install runs only the checks a RESTORE runs. +fn redo_origin( + plans: &[nodedb_physical::physical_plan::PhysicalPlan], +) -> nodedb_physical::physical_plan::RedoOrigin { + use nodedb_physical::physical_plan::{MetaOp, PhysicalPlan, RedoOrigin}; + if plans + .iter() + .any(|plan| matches!(plan, PhysicalPlan::Meta(MetaOp::RestoreRedo(_)))) + { + RedoOrigin::Restore + } else { + RedoOrigin::Commit + } +} + /// Whether the resolve of `plan` reads a whole collection's stored state. /// /// A TRUNCATE's edge share is not among them. Its resolve records a cut at /// the transaction's ordinal and reads no stored edge, and the cut hides by -/// ordinal whatever this vShard flushes before or after it. The rows' +/// ordinal whatever this vShard installs before or after it. The rows' /// truncate on the collection's vShard still reads every row at resolve. fn reads_whole_collection_at_resolve(plan: &nodedb_physical::physical_plan::PhysicalPlan) -> bool { use nodedb_physical::physical_plan::{ @@ -168,57 +174,60 @@ fn reads_whole_collection_at_resolve(plan: &nodedb_physical::physical_plan::Phys ) } -/// Commit-resolution state of a staged static Calvin transaction. +/// Commit-resolution state of a granted Calvin transaction. #[derive(Debug, Clone, Copy, PartialEq, Eq)] pub(in crate::control::cluster::calvin::scheduler::driver) enum CommitState { - /// Awaiting the validate-and-stage response, whose `read_set_valid` carries - /// the local commit vote that drives the flush-or-drop decision. + /// A follower holds the txn's locks and stages nothing. It completes on + /// an abort verdict, on a COMMIT verdict for a slice with no write, or + /// once the slice's stamped redo applies. A promotion stages it. + Following, + /// Awaiting the validate-and-stage response, whose `stage_vote` carries + /// the local commit vote. Staged, + /// A dependent-read txn whose passive read this leader dispatched. The + /// read stages nothing. Its answer is proposed to every active vShard's + /// data group. A passive-only vShard then votes on it, and an active + /// vShard opens its barrier. + ReadingPassive, /// The staged txn has cast its local vote and PARKED, awaiting the durable /// authoritative GLOBAL verdict aggregated across all participant vShards. /// /// The scheduler holds this txn's locks and its staged Data-Plane buffer - /// intact while parked; it resumes into a flush (verdict = commit) or a drop - /// (verdict = abort) only once `registry.verdict(txn)` is known — via the - /// verdict push, the probe on park, or the stall re-probe sweep. This is the - /// cross-shard commit barrier: no participant self-decides on its local - /// vote, so a torn commit (one shard flushes while a peer drops) is + /// intact while parked; it resumes into a resolve (verdict = commit) or a + /// drop (verdict = abort) only once `registry.verdict(txn)` is known — via + /// the verdict push, the probe on park, or the stall re-probe sweep. No + /// participant self-decides on its local vote, so a torn commit is /// impossible. AwaitingVerdict, - /// The txn committed and a `MetaOp::CalvinResolve` has been dispatched, or - /// parked for re-send at dispatcher capacity, to resolve its staged - /// post-images into a replayable `RedoRecord`; awaiting that response - /// before the redo is WAL-appended and the flush dispatched. - AwaitingRedoResolve, /// The txn committed, and its slice reads a whole collection at resolve /// (a TRUNCATE). Its resolve waits for its turn: every lower txn of this /// vShard finished, so the resolve reads every write sequenced before it - /// and none after. See [`FlushScope::resolve_at_turn`]. + /// and none after. See [`SliceScope::resolve_at_turn`]. AwaitingResolveTurn, - /// The txn committed and its redo record is appended. Its flush waits for - /// its turn: every lower txn of this vShard finished. See - /// [`super::core::flush_turn`]. - AwaitingFlushTurn { redo_lsn: Option }, - /// A flush (`committed = true`) or drop (`committed = false`) has been - /// dispatched, or parked for re-send at dispatcher capacity; awaiting its - /// response before the commit tail runs. - /// - /// `redo_lsn` is `Some(lsn)` when a `TransactionRedo` record was appended - /// for this commit's non-empty write set — `commit_apply_tail` then only - /// records write versions at it, since the redo record already IS the - /// applied marker. `None` for a drop, or a committed txn whose resolved - /// redo carried no ops (pure read / CRDT) — `commit_apply_tail` falls back - /// to appending a `CalvinApplied` marker in that case. - AwaitingResolve { - committed: bool, - redo_lsn: Option, - }, + /// The txn committed and a `MetaOp::CalvinResolve` has been dispatched, or + /// parked for re-send at dispatcher capacity, to resolve its staged + /// post-images into a replayable `RedoRecord`. + AwaitingRedoResolve, + /// The txn committed and its stamped redo is proposed to the vShard's + /// data group. `proposed` names the `(group_id, log_index)` of the entry + /// that installs it, `None` while no proposal is in the log. The txn + /// completes once its redo applies. + AwaitingRedoApply { proposed: Option<(u64, u64)> }, + /// A `CalvinDrop` has been dispatched, or parked for re-send: the abort + /// verdict, or a COMMIT verdict for a slice with no write. The txn + /// completes once the drop answers. + AwaitingDrop, + /// The verdict is COMMIT, and this leader's stage failed. Its staged + /// state is discarded. The txn holds its locks and stages again once + /// its backoff passes. + AwaitingRestage, } /// A transaction that is blocked on lock acquisition. pub(super) struct BlockedTxn { pub txn: SequencedTxn, - pub keys: BTreeSet, + /// The moded lock request the txn waits on. + pub keys: BTreeMap, /// Wall-clock time at first block (for latency metrics). /// /// `Instant::now()` used for observability only. diff --git a/nodedb/src/control/cluster/calvin/scheduler/inbox.rs b/nodedb/src/control/cluster/calvin/scheduler/inbox.rs new file mode 100644 index 000000000..63356b027 --- /dev/null +++ b/nodedb/src/control/cluster/calvin/scheduler/inbox.rs @@ -0,0 +1,290 @@ +// SPDX-License-Identifier: BUSL-1.1 + +//! What the data-group apply loop tells a vShard's scheduler about each +//! stamped Calvin redo it concluded. +//! +//! The apply loop pushes one [`CalvinApplyEvent`] per position, keyed by +//! `(epoch, position)`. The scheduler drains every event in key order. The +//! inbox is bounded: a push into a full inbox waits for room, and its entry +//! holds its own place in the group's lane meanwhile. Nothing is dropped. +//! +//! A vShard with no running scheduler has no registered inbox. The apply +//! loop then pushes nothing: the ledger it marks seeds the next scheduler. + +use std::collections::{BTreeMap, HashMap}; +use std::sync::{Arc, Mutex}; + +use tokio::sync::Notify; + +use crate::bridge::envelope::Response; + +/// How the apply loop concluded one stamped Calvin redo. +#[derive(Debug, Clone)] +pub enum CalvinApplyEvent { + /// The install is durable and the ledger holds the position. `reply` is + /// the slice's rendered answer. Every replica renders the same one. + RedoApplied { + reply: Response, + /// The slice writes a row the statement names. + primary_write: bool, + /// The slice answers rows, not a count. + returning: bool, + }, + /// The install refused for good. Every replica refuses the same entry. + /// The position stays claimed, so no later copy installs. + RedoRefused { error: String }, + /// The install did not become durable on this replica. The claim is + /// released, and a restart applies the entry again. + RedoNotApplied { error: String }, +} + +#[derive(Debug, Default)] +struct InboxState { + events: BTreeMap<(u64, u32), CalvinApplyEvent>, + /// The scheduler stopped: a push returns at once. + closed: bool, +} + +/// One vShard's inbox of apply events. +#[derive(Debug)] +pub struct CalvinVShardInbox { + state: Mutex, + /// Wakes the scheduler: an event arrived. + ready: Notify, + /// Wakes a waiting push: the scheduler took events. + room: Notify, + capacity: usize, +} + +impl CalvinVShardInbox { + /// An empty inbox holding at most `capacity` events. A zero capacity + /// holds one. + pub fn new(capacity: usize) -> Self { + Self { + state: Mutex::new(InboxState::default()), + ready: Notify::new(), + room: Notify::new(), + capacity: capacity.max(1), + } + } + + /// Hand `event` for `(epoch, position)` to the scheduler. Waits while + /// the inbox is full. Returns once the event is in, or at once when the + /// scheduler stopped. + pub async fn push(&self, epoch: u64, position: u32, event: CalvinApplyEvent) { + let mut event = Some(event); + loop { + let room = self.room.notified(); + tokio::pin!(room); + room.as_mut().enable(); + { + let mut state = self.state.lock().unwrap_or_else(|p| p.into_inner()); + if state.closed { + return; + } + let key = (epoch, position); + if state.events.len() < self.capacity || state.events.contains_key(&key) { + if let Some(event) = event.take() { + state.events.insert(key, event); + } + drop(state); + self.ready.notify_one(); + return; + } + } + room.await; + } + } + + /// Take every waiting event, in `(epoch, position)` order. + pub fn take_all(&self) -> Vec<((u64, u32), CalvinApplyEvent)> { + let taken = { + let mut state = self.state.lock().unwrap_or_else(|p| p.into_inner()); + std::mem::take(&mut state.events) + }; + if !taken.is_empty() { + self.room.notify_waiters(); + } + taken.into_iter().collect() + } + + /// Resolves once an event waits. + pub async fn ready(&self) { + loop { + let ready = self.ready.notified(); + tokio::pin!(ready); + ready.as_mut().enable(); + if !self + .state + .lock() + .unwrap_or_else(|p| p.into_inner()) + .events + .is_empty() + { + return; + } + ready.await; + } + } + + /// The scheduler stopped: every waiting push and every later push + /// returns at once. + fn close(&self) { + self.state.lock().unwrap_or_else(|p| p.into_inner()).closed = true; + self.room.notify_waiters(); + } + + /// How many events wait. + #[cfg(test)] + pub fn waiting(&self) -> usize { + self.state + .lock() + .unwrap_or_else(|p| p.into_inner()) + .events + .len() + } +} + +type Inboxes = Arc>>>; + +/// The inbox of every vShard whose scheduler runs on this node. +#[derive(Debug, Default)] +pub struct CalvinInboxes { + by_vshard: Inboxes, +} + +impl CalvinInboxes { + /// Register the inbox of the scheduler starting for `vshard_id`. It + /// replaces a predecessor's. + pub fn register(&self, vshard_id: u32, capacity: usize) -> InboxHandle { + let inbox = Arc::new(CalvinVShardInbox::new(capacity)); + let replaced = self + .by_vshard + .lock() + .unwrap_or_else(|p| p.into_inner()) + .insert(vshard_id, Arc::clone(&inbox)); + if let Some(previous) = replaced { + previous.close(); + } + InboxHandle { + vshard_id, + inbox, + by_vshard: Arc::clone(&self.by_vshard), + } + } + + /// The inbox of `vshard_id`'s running scheduler. + pub fn get(&self, vshard_id: u32) -> Option> { + self.by_vshard + .lock() + .unwrap_or_else(|p| p.into_inner()) + .get(&vshard_id) + .cloned() + } +} + +/// A scheduler's registered inbox. Dropping it closes the inbox and +/// removes it, unless a successor replaced it. +#[derive(Debug)] +pub struct InboxHandle { + vshard_id: u32, + inbox: Arc, + by_vshard: Inboxes, +} + +impl InboxHandle { + /// The inbox this handle registered. + pub fn inbox(&self) -> &Arc { + &self.inbox + } +} + +impl Drop for InboxHandle { + fn drop(&mut self) { + self.inbox.close(); + let mut by_vshard = self.by_vshard.lock().unwrap_or_else(|p| p.into_inner()); + if by_vshard + .get(&self.vshard_id) + .is_some_and(|inbox| Arc::ptr_eq(inbox, &self.inbox)) + { + by_vshard.remove(&self.vshard_id); + } + } +} + +#[cfg(test)] +mod tests { + use std::time::Duration; + + use super::*; + + fn refused(text: &str) -> CalvinApplyEvent { + CalvinApplyEvent::RedoRefused { + error: text.to_owned(), + } + } + + /// Events leave the inbox in `(epoch, position)` order, whatever order + /// they arrived in. + #[tokio::test] + async fn events_drain_in_position_order() { + let inbox = CalvinVShardInbox::new(8); + inbox.push(4, 1, refused("b")).await; + inbox.push(3, 7, refused("a")).await; + inbox.push(4, 0, refused("c")).await; + let keys: Vec<(u64, u32)> = inbox.take_all().into_iter().map(|(k, _)| k).collect(); + assert_eq!(keys, vec![(3, 7), (4, 0), (4, 1)]); + assert_eq!(inbox.waiting(), 0); + } + + /// A push into a full inbox waits until the scheduler takes events. + #[tokio::test] + async fn a_full_inbox_holds_the_push_until_events_leave() { + let inbox = Arc::new(CalvinVShardInbox::new(1)); + inbox.push(1, 0, refused("first")).await; + let pushing = { + let inbox = Arc::clone(&inbox); + tokio::spawn(async move { inbox.push(1, 1, refused("second")).await }) + }; + tokio::time::sleep(Duration::from_millis(50)).await; + assert!(!pushing.is_finished(), "the push waits for room"); + assert_eq!(inbox.take_all().len(), 1); + tokio::time::timeout(Duration::from_secs(5), pushing) + .await + .expect("the push finishes once room frees") + .expect("push task"); + assert_eq!(inbox.waiting(), 1); + } + + /// A stopped scheduler's inbox takes nothing, and a waiting push + /// returns. + #[tokio::test] + async fn a_dropped_handle_releases_a_waiting_push() { + let inboxes = CalvinInboxes::default(); + let handle = inboxes.register(5, 1); + let inbox = Arc::clone(handle.inbox()); + inbox.push(1, 0, refused("first")).await; + let pushing = { + let inbox = Arc::clone(&inbox); + tokio::spawn(async move { inbox.push(1, 1, refused("second")).await }) + }; + drop(handle); + tokio::time::timeout(Duration::from_secs(5), pushing) + .await + .expect("a closed inbox releases the push") + .expect("push task"); + assert!(inboxes.get(5).is_none()); + } + + /// A successor's registration replaces the inbox, and the predecessor's + /// drop leaves it in place. + #[test] + fn a_successor_keeps_its_inbox_when_the_predecessor_drops() { + let inboxes = CalvinInboxes::default(); + let first = inboxes.register(2, 4); + let second = inboxes.register(2, 4); + drop(first); + let current = inboxes.get(2).expect("the successor's inbox"); + assert!(Arc::ptr_eq(¤t, second.inbox())); + } +} diff --git a/nodedb/src/control/cluster/calvin/scheduler/lock/lock_entry.rs b/nodedb/src/control/cluster/calvin/scheduler/lock/lock_entry.rs index 6b8b53082..314ae76aa 100644 --- a/nodedb/src/control/cluster/calvin/scheduler/lock/lock_entry.rs +++ b/nodedb/src/control/cluster/calvin/scheduler/lock/lock_entry.rs @@ -2,9 +2,8 @@ //! Per-key lock state for the Calvin lock table. //! -//! A [`LockEntry`] holds either a single exclusive holder or a set of shared -//! holders, plus a FIFO waiter queue that carries each waiter's requested -//! [`LockMode`]. +//! A [`LockEntry`] holds one group of compatible holders, plus a FIFO waiter +//! queue that carries each waiter's requested [`LockMode`]. use std::collections::VecDeque; @@ -16,16 +15,53 @@ use super::lock_key::TxnId; /// The mode under which a lock is held or requested. /// -/// `Shared` locks are mutually compatible (many readers); `Exclusive` locks -/// conflict with everything else. +/// Compatibility: +/// +/// | held \ requested | Intent | Shared | Exclusive | +/// |------------------|--------|--------|-----------| +/// | Intent | yes | no | no | +/// | Shared | no | yes | no | +/// | Exclusive | no | no | no | #[derive(Debug, Clone, Copy, PartialEq, Eq)] pub enum LockMode { + /// A row writer's hold on its collection key. Compatible only with other + /// `Intent` holders, so row writers run together and exclude a predicate + /// reader or a collection-wide writer. + Intent, /// Compatible with other shared holders on the same key. Shared, /// Held by exactly one transaction; excludes all others. Exclusive, } +impl LockMode { + /// Whether a holder in `self` and a holder in `other` can hold one key + /// together. + pub fn compatible(self, other: LockMode) -> bool { + match (self, other) { + (LockMode::Intent, LockMode::Intent) | (LockMode::Shared, LockMode::Shared) => true, + (LockMode::Intent, LockMode::Shared | LockMode::Exclusive) + | (LockMode::Shared, LockMode::Intent | LockMode::Exclusive) + | (LockMode::Exclusive, LockMode::Intent | LockMode::Shared | LockMode::Exclusive) => { + false + } + } + } + + /// The one mode that covers both `self` and `other` for one transaction. + /// + /// Equal modes merge to themselves. Any other pair merges to `Exclusive`: + /// `Shared` plus `Intent` means the transaction both reads the whole + /// collection and writes rows of it. + pub fn merge(self, other: LockMode) -> LockMode { + if self == other { + self + } else { + LockMode::Exclusive + } + } +} + // ── AcquireOutcome ──────────────────────────────────────────────────────────── /// Result of a lock-acquire call. @@ -33,8 +69,9 @@ pub enum LockMode { pub enum AcquireOutcome { /// All requested locks were granted; the transaction is ready to dispatch. Ready, - /// At least one key conflicted with a current holder. The transaction has - /// been enqueued as a waiter on every unavailable key. + /// At least one key conflicted with a current holder or an earlier + /// waiter. The transaction holds every key it could take and waits in + /// FIFO order on every other key. Blocked, } @@ -43,17 +80,54 @@ pub enum AcquireOutcome { /// Per-key lock state. /// /// Invariants maintained by [`LockManager`](super::manager::LockManager): +/// - Every holder holds the key in `mode`. Two modes never mix, because no +/// two distinct modes are compatible. /// - `Exclusive` entries have exactly one holder. -/// - `Shared` entries have one or more holders, all compatible. /// - `waiters` is FIFO; each waiter carries the mode it requested so that /// promotion on release is mode-aware. pub(super) struct LockEntry { /// The mode the current holders hold this lock under. pub(super) mode: LockMode, /// The transactions currently holding this lock. Exclusive: exactly one; - /// Shared: one or more. + /// Shared or Intent: one or more. pub(super) holders: SmallVec<[TxnId; 2]>, /// Transactions waiting for this lock, in FIFO order, each tagged with the /// mode it requested. pub(super) waiters: VecDeque<(TxnId, LockMode)>, } + +#[cfg(test)] +mod tests { + use super::*; + + const ALL: [LockMode; 3] = [LockMode::Intent, LockMode::Shared, LockMode::Exclusive]; + + #[test] + fn compatibility_matrix() { + assert!(LockMode::Intent.compatible(LockMode::Intent)); + assert!(LockMode::Shared.compatible(LockMode::Shared)); + assert!(!LockMode::Intent.compatible(LockMode::Shared)); + assert!(!LockMode::Shared.compatible(LockMode::Intent)); + for mode in ALL { + assert!(!mode.compatible(LockMode::Exclusive)); + assert!(!LockMode::Exclusive.compatible(mode)); + } + } + + #[test] + fn merge_keeps_the_strongest_mode() { + for mode in ALL { + assert_eq!(mode.merge(mode), mode); + assert_eq!(mode.merge(LockMode::Exclusive), LockMode::Exclusive); + assert_eq!(LockMode::Exclusive.merge(mode), LockMode::Exclusive); + } + assert_eq!( + LockMode::Shared.merge(LockMode::Intent), + LockMode::Exclusive + ); + assert_eq!( + LockMode::Intent.merge(LockMode::Shared), + LockMode::Exclusive + ); + } +} diff --git a/nodedb/src/control/cluster/calvin/scheduler/lock/lock_key.rs b/nodedb/src/control/cluster/calvin/scheduler/lock/lock_key.rs index cd1fe5224..c8c5883ab 100644 --- a/nodedb/src/control/cluster/calvin/scheduler/lock/lock_key.rs +++ b/nodedb/src/control/cluster/calvin/scheduler/lock/lock_key.rs @@ -36,6 +36,17 @@ pub enum LockKey { src: u32, dst: u32, }, + /// A whole collection. Writers of single rows hold it `Intent`. Writers + /// of the whole collection hold it `Exclusive`. Predicate readers hold it + /// `Shared`. + Collection { collection: Arc }, + /// One claimed value of one UNIQUE index. Writers that claim the value + /// hold it `Exclusive`, so two claims of one value serialize. + Unique { + collection: Arc, + index: Arc, + value: Arc<[u8]>, + }, } // ── TxnId ───────────────────────────────────────────────────────────────────── diff --git a/nodedb/src/control/cluster/calvin/scheduler/lock/manager/acquire.rs b/nodedb/src/control/cluster/calvin/scheduler/lock/manager/acquire.rs index f896c15c9..0d13629bc 100644 --- a/nodedb/src/control/cluster/calvin/scheduler/lock/manager/acquire.rs +++ b/nodedb/src/control/cluster/calvin/scheduler/lock/manager/acquire.rs @@ -1,160 +1,103 @@ // SPDX-License-Identifier: BUSL-1.1 -//! Exclusive lock acquisition and waiter queueing. +//! Moded lock acquisition and FIFO waiter queueing. -use std::collections::{BTreeSet, VecDeque}; +use std::collections::BTreeMap; -use smallvec::smallvec; - -use crate::control::cluster::calvin::scheduler::lock::lock_entry::{ - AcquireOutcome, LockEntry, LockMode, -}; +use crate::control::cluster::calvin::scheduler::lock::lock_entry::{AcquireOutcome, LockMode}; use crate::control::cluster::calvin::scheduler::lock::lock_key::{LockKey, TxnId}; -use super::types::{ExclusiveWait, LockManager}; +use super::classify::KeyState; +use super::types::{ConflictResolution, LockManager}; impl LockManager { - /// Attempt to acquire **exclusive** locks on all keys for `txn`. + /// Acquire every key of `keys` for `txn`, each in its own mode. /// - /// If every key is free, already held exclusively by `txn` (promoted from - /// waiter), or held **shared solely by `txn`** (a reservation this txn placed - /// earlier, now upgraded in place to exclusive), records `txn` as sole holder - /// of each key and returns [`AcquireOutcome::Ready`]. + /// The request merges with any request `txn` already waits on. A key the + /// txn names twice takes the merged mode (see [`LockMode::merge`]). /// - /// Otherwise the whole key set is classified under the WOUND-WAIT discipline - /// (see [`Self::wound_or_block`]). `TxnId` order (`(epoch, position)`) is - /// the replicated total order, so "older" means a smaller id and the - /// decision is a pure function of the lock table plus the replicated ids — - /// every replica computes it identically: - /// - Any conflicting holder is **exclusive** → `txn` waits (an exclusive - /// holder is executing/applied work and is never wounded). - /// - All conflicting holders are **shared** reservations and `txn` is older - /// than every one of them → **wound** them all (revoke to plain OCC, no - /// notification) and take every key. Wounding is silent. - /// - Otherwise (`txn` younger than some conflicting shared holder) → wait. + /// When no key conflicts, `txn` takes every key and the call returns + /// [`AcquireOutcome::Ready`]. A key conflicts when an incompatible holder + /// holds it, or when an earlier waiter queues on it. /// - /// On the wait path `txn` is enqueued as an exclusive waiter on every - /// conflicting key and holds none; its key set is stored in `pending_keys` - /// so `release` can promote it atomically when all keys become available. - /// The whole key set is evaluated before any mutation, so acquisition stays - /// all-keys-or-none — the manager never partially wounds and then blocks. - pub fn acquire(&mut self, txn: TxnId, keys: BTreeSet) -> AcquireOutcome { - // First pass: determine whether any key is held by a *different* txn. - // A key already held exclusively by `txn` (promoted via release) or held - // shared solely by `txn` (an earlier reservation) counts as available — - // the former is a no-op re-acquire, the latter a self-upgrade to exclusive. - let all_available = keys.iter().all(|k| { - self.table.get(k).is_none_or(|entry| { - entry.held_exclusively_by(txn) || entry.held_shared_solely_by(txn) - }) - }); - - if all_available { - // Acquire all keys. For keys not yet in the table (free), insert a - // new exclusive entry. For keys already held by this txn (promoted - // waiter), leave the entry unchanged — the waiter queue is intact. - for key in &keys { - match self.table.get_mut(key) { - None => { - self.table.insert( - key.clone(), - LockEntry { - mode: LockMode::Exclusive, - holders: smallvec![txn], - waiters: VecDeque::new(), - }, - ); - } - Some(entry) => { - // A key this txn already holds shared-solely is upgraded - // to exclusive in place (holders is exactly `[txn]`, so no - // holder change and any waiter queue stays intact). A key - // already held exclusively by `txn` is left unchanged. - if entry.held_shared_solely_by(txn) { - entry.mode = LockMode::Exclusive; - } - } + /// A conflicting request is resolved under WOUND-WAIT (see + /// [`Self::resolve_conflict`]). `TxnId` order is the replicated total + /// order, so the decision is identical on every replica: + /// - Every conflicting holder is a younger read reservation in `Shared` + /// mode, and nobody waits on a conflicting key: `txn` revokes those + /// reservations and takes every key. + /// - Otherwise `txn` takes every key it can take now and waits FIFO on the + /// rest. Calls arrive in sequencer order, so every key grants its + /// requesters in sequencer order, whatever order earlier holders + /// release in. + pub fn acquire(&mut self, txn: TxnId, keys: BTreeMap) -> AcquireOutcome { + let mut request = self.pending_keys.remove(&txn).unwrap_or_default(); + for (key, mode) in keys { + request + .entry(key) + .and_modify(|held| *held = held.merge(mode)) + .or_insert(mode); + } + let states: Vec = request + .iter() + .map(|(key, mode)| self.classify(txn, key, *mode)) + .collect(); + + let resolution = if states.contains(&KeyState::Conflict) { + Some(self.resolve_conflict(txn, &request, &states)) + } else { + None + }; + + match resolution { + None => { + for (key, mode) in &request { + self.grant(txn, key, *mode); } + self.finish_ready(txn, request); + AcquireOutcome::Ready } - // Move out of pending (if the txn was previously blocked on this - // same key set) and into held_locks. - self.pending_keys.remove(&txn); - self.held_locks.insert(txn, keys); - return AcquireOutcome::Ready; - } - - // A conflict exists. Classify the WHOLE key set before mutating so the - // wound / block decision is atomic (never partially wound then block). - match self.wound_or_block(txn, &keys) { - ExclusiveWait::Wound => { - // Take every key exclusively. A conflicting shared entry has its - // holders revoked (the wounded readers, all younger than `txn`, - // degrade to plain OCC); `txn` becomes the sole holder while any - // existing waiters remain queued behind it. - for key in &keys { - match self.table.get_mut(key) { - Some(entry) => { - entry.mode = LockMode::Exclusive; - entry.holders.clear(); - entry.holders.push(txn); - } - None => { - self.table.insert( - key.clone(), - LockEntry { - mode: LockMode::Exclusive, - holders: smallvec![txn], - waiters: VecDeque::new(), - }, - ); + Some(ConflictResolution::Wound) => { + for ((key, mode), state) in request.iter().zip(&states) { + match state { + KeyState::Conflict => self.wound(txn, key, *mode), + KeyState::Free | KeyState::Held | KeyState::Join => { + self.grant(txn, key, *mode) } } } - self.pending_keys.remove(&txn); - self.held_locks.insert(txn, keys); + self.finish_ready(txn, request); AcquireOutcome::Ready } - ExclusiveWait::Block => { - // Enqueue as an exclusive waiter on every key held by a - // different txn. - for key in &keys { - if let Some(entry) = self.table.get_mut(key) { - // No conflict on this key means it is held solely by `txn` - // (a shared reservation to be upgraded, or an exclusive - // re-acquire) — leave it untouched; `txn` keeps the key and - // upgrades it once its conflicting keys are free. Same - // predicate as the `all_available` check above. - if entry.held_exclusively_by(txn) || entry.held_shared_solely_by(txn) { - continue; - } - // Real conflict on this key. If `txn` also holds it shared - // (an upgrade that must wait behind an OLDER shared holder), - // drop its own shared hold — degrading that read to plain - // OCC, never worse than today — so the key can drain to - // empty and normal promotion can grant `txn` the exclusive - // lock later. Without this, `txn` would occupy the key - // forever and its own exclusive request could never fire. - entry.holders.retain(|h| *h != txn); - if !entry.has_waiter(txn) { - entry.waiters.push_back((txn, LockMode::Exclusive)); + Some(ConflictResolution::Block) => { + for ((key, mode), state) in request.iter().zip(&states) { + match state { + KeyState::Conflict => self.enqueue(txn, key, *mode), + KeyState::Free | KeyState::Held | KeyState::Join => { + self.grant(txn, key, *mode) } } - // Free keys: no entry exists; the txn acquires them on the - // re-acquire path after all conflicting keys are released. } - // Store the full key set so that release can promote this txn - // atomically once all its keys become available. - self.pending_keys.insert(txn, keys); + self.pending_keys.insert(txn, request); AcquireOutcome::Blocked } } } + + /// Record `txn` as holder of every key of `request`. + pub(super) fn finish_ready(&mut self, txn: TxnId, request: BTreeMap) { + self.held_locks + .entry(txn) + .or_default() + .extend(request.into_keys()); + } } // ── Tests ───────────────────────────────────────────────────────────────────── #[cfg(test)] mod tests { + use std::collections::BTreeSet; use std::sync::Arc; use super::*; @@ -166,8 +109,21 @@ mod tests { } } - fn keyset(names: &[&str]) -> BTreeSet { - names.iter().map(|n| key(n)).collect() + fn coll(name: &str) -> LockKey { + LockKey::Collection { + collection: Arc::from(name), + } + } + + fn exclusive(names: &[&str]) -> BTreeMap { + names + .iter() + .map(|n| (key(n), LockMode::Exclusive)) + .collect() + } + + fn one(key: LockKey, mode: LockMode) -> BTreeMap { + BTreeMap::from([(key, mode)]) } fn txn(epoch: u64, pos: u32) -> TxnId { @@ -177,8 +133,7 @@ mod tests { #[test] fn acquire_free_keys_returns_ready() { let mut lm = LockManager::new(); - let t = txn(1, 0); - let outcome = lm.acquire(t, keyset(&["a", "b"])); + let outcome = lm.acquire(txn(1, 0), exclusive(&["a", "b"])); assert_eq!(outcome, AcquireOutcome::Ready); assert_eq!(lm.lock_count(), 2); } @@ -186,98 +141,191 @@ mod tests { #[test] fn acquire_held_key_returns_blocked_and_enqueues_waiter() { let mut lm = LockManager::new(); - let t1 = txn(1, 0); - let t2 = txn(1, 1); - lm.acquire(t1, keyset(&["x"])); - - let outcome = lm.acquire(t2, keyset(&["x"])); - assert_eq!(outcome, AcquireOutcome::Blocked); + let (t1, t2) = (txn(1, 0), txn(1, 1)); + lm.acquire(t1, exclusive(&["x"])); - // t2 should be in the waiter queue for "x". + assert_eq!(lm.acquire(t2, exclusive(&["x"])), AcquireOutcome::Blocked); assert!(lm.table.get(&key("x")).unwrap().has_waiter(t2)); } #[test] - fn exclusive_waits_on_exclusive() { + fn intent_intent_compatible() { let mut lm = LockManager::new(); - let t1 = txn(1, 0); - let t2 = txn(1, 1); + let (t1, t2) = (txn(1, 0), txn(1, 1)); + assert_eq!( + lm.acquire(t1, one(coll("c"), LockMode::Intent)), + AcquireOutcome::Ready + ); + assert_eq!( + lm.acquire(t2, one(coll("c"), LockMode::Intent)), + AcquireOutcome::Ready, + "two row writers of one collection hold its key together" + ); + let entry = lm.table.get(&coll("c")).unwrap(); + assert_eq!(entry.mode, LockMode::Intent); + assert_eq!(entry.holders.as_slice(), &[t1, t2]); + } - assert_eq!(lm.acquire(t1, keyset(&["k"])), AcquireOutcome::Ready); - assert_eq!(lm.acquire(t2, keyset(&["k"])), AcquireOutcome::Blocked); - assert!(lm.table.get(&key("k")).unwrap().has_waiter(t2)); + #[test] + fn intent_blocks_against_exclusive_and_shared() { + for other in [LockMode::Exclusive, LockMode::Shared] { + // Intent held, the other mode requested. + let mut lm = LockManager::new(); + let (t1, t2) = (txn(1, 0), txn(1, 1)); + assert_eq!( + lm.acquire(t1, one(coll("c"), LockMode::Intent)), + AcquireOutcome::Ready + ); + assert_eq!( + lm.acquire(t2, one(coll("c"), other)), + AcquireOutcome::Blocked, + "{other:?} waits behind an Intent holder" + ); + assert_eq!(lm.release(t1), vec![t2]); + + // The other mode held, Intent requested. + let mut lm = LockManager::new(); + assert_eq!(lm.acquire(t1, one(coll("c"), other)), AcquireOutcome::Ready); + assert_eq!( + lm.acquire(t2, one(coll("c"), LockMode::Intent)), + AcquireOutcome::Blocked, + "Intent waits behind a {other:?} holder" + ); + assert_eq!(lm.release(t1), vec![t2]); + } } #[test] - fn shared_reservation_self_upgrades_to_exclusive() { + fn a_compatible_request_queues_behind_an_earlier_waiter() { let mut lm = LockManager::new(); - let t = txn(1, 0); - - assert_eq!(lm.acquire_shared(t, key("k")), AcquireOutcome::Ready); - // The txn re-acquires its own shared reservation exclusively — this must - // NOT self-deadlock by blocking on its own held key. + let (reader, truncate, writer) = (txn(1, 0), txn(1, 1), txn(1, 2)); assert_eq!( - lm.acquire(t, keyset(&["k"])), - AcquireOutcome::Ready, - "self-upgrade from shared to exclusive must not block" + lm.acquire(reader, one(coll("c"), LockMode::Shared)), + AcquireOutcome::Ready ); - - let entry = lm.table.get(&key("k")).unwrap(); - assert_eq!(entry.mode, LockMode::Exclusive); - assert_eq!(entry.holders.len(), 1); - assert_eq!(entry.holders[0], t); + assert_eq!( + lm.acquire(truncate, one(coll("c"), LockMode::Exclusive)), + AcquireOutcome::Blocked + ); + // A later reader is compatible with the holder, yet it must not pass + // the truncate sequenced before it. + let late_reader = txn(1, 3); + assert_eq!( + lm.acquire(late_reader, one(coll("c"), LockMode::Shared)), + AcquireOutcome::Blocked + ); + assert_eq!( + lm.acquire(writer, one(coll("c"), LockMode::Intent)), + AcquireOutcome::Blocked + ); + assert_eq!(lm.release(reader), vec![truncate]); + assert_eq!(lm.release(truncate), vec![late_reader]); + assert_eq!(lm.release(late_reader), vec![writer]); } #[test] - fn self_upgrade_with_other_shared_holder_blocks_or_wounds() { - // T_old is older than T_young: T_old's self-upgrade must wound T_young. + fn a_blocked_txn_takes_its_free_keys_so_a_later_txn_queues_behind_it() { let mut lm = LockManager::new(); - let t_old = txn(1, 0); - let t_young = txn(1, 1); + let (first, second, third) = (txn(1, 0), txn(1, 1), txn(1, 2)); + assert_eq!(lm.acquire(first, exclusive(&["a"])), AcquireOutcome::Ready); + assert_eq!( + lm.acquire(second, exclusive(&["a", "b"])), + AcquireOutcome::Blocked + ); + // `b` is free at `second`'s request, so `second` holds it now. A + // later txn on `b` waits for `second`, whatever order releases run in. + assert_eq!( + lm.acquire(third, exclusive(&["b"])), + AcquireOutcome::Blocked + ); + assert_eq!(lm.release(first), vec![second]); + assert_eq!(lm.release(second), vec![third]); + } - assert_eq!(lm.acquire_shared(t_old, key("k")), AcquireOutcome::Ready); - assert_eq!(lm.acquire_shared(t_young, key("k")), AcquireOutcome::Ready); + #[test] + fn the_strongest_mode_wins_within_one_request() { + let mut lm = LockManager::new(); + let t = txn(1, 0); + assert_eq!( + lm.acquire(t, one(coll("c"), LockMode::Shared)), + AcquireOutcome::Ready + ); + // The same txn now writes a row: Shared plus Intent merges to + // Exclusive, held in place by the sole holder. + assert_eq!( + lm.acquire(t, one(coll("c"), LockMode::Intent)), + AcquireOutcome::Ready + ); + assert_eq!(lm.table.get(&coll("c")).unwrap().mode, LockMode::Exclusive); + let other = txn(1, 1); + assert_eq!( + lm.acquire(other, one(coll("c"), LockMode::Shared)), + AcquireOutcome::Blocked + ); + } + #[test] + fn grant_order_is_identical_for_the_same_input_sequence() { + // Every input of the sequence, applied in order: an acquire or a release. + enum Step { + Acquire(TxnId, BTreeMap), + Release(TxnId), + } + let steps = || { + let row = |n: &str| (key(n), LockMode::Exclusive); + let intent = (coll("c"), LockMode::Intent); + vec![ + Step::Acquire(txn(1, 0), BTreeMap::from([intent.clone(), row("a")])), + Step::Acquire(txn(1, 1), one(coll("c"), LockMode::Exclusive)), + Step::Acquire(txn(1, 2), BTreeMap::from([intent.clone(), row("b")])), + Step::Acquire(txn(1, 3), one(coll("c"), LockMode::Shared)), + Step::Acquire(txn(2, 0), BTreeMap::from([intent, row("a")])), + Step::Release(txn(1, 0)), + Step::Release(txn(1, 1)), + Step::Release(txn(1, 2)), + Step::Release(txn(1, 3)), + Step::Release(txn(2, 0)), + ] + }; + let run = || { + let mut lm = LockManager::new(); + let mut granted = Vec::new(); + for step in steps() { + match step { + Step::Acquire(t, keys) => { + if lm.acquire(t, keys) == AcquireOutcome::Ready { + granted.push(t); + } + } + Step::Release(t) => granted.extend(lm.release(t)), + } + } + granted + }; + let first = run(); + assert_eq!(first, run()); assert_eq!( - lm.acquire(t_old, keyset(&["k"])), - AcquireOutcome::Ready, - "the older self-upgrader wounds the younger shared holder" + first, + vec![txn(1, 0), txn(1, 1), txn(1, 2), txn(1, 3), txn(2, 0),], + "every txn is granted in sequencer order" ); - let entry = lm.table.get(&key("k")).unwrap(); - assert_eq!(entry.mode, LockMode::Exclusive); - assert_eq!(entry.holders.len(), 1); - assert_eq!(entry.holders[0], t_old); + } - // Symmetric case: the YOUNGER of the two self-upgrades and must block. + #[test] + fn shared_reservation_self_upgrades_to_exclusive() { let mut lm = LockManager::new(); - let t_old = txn(1, 0); - let t_young = txn(1, 1); - - assert_eq!(lm.acquire_shared(t_old, key("k")), AcquireOutcome::Ready); - assert_eq!(lm.acquire_shared(t_young, key("k")), AcquireOutcome::Ready); + let t = txn(1, 0); + assert_eq!(lm.acquire_shared(t, key("k")), AcquireOutcome::Ready); assert_eq!( - lm.acquire(t_young, keyset(&["k"])), - AcquireOutcome::Blocked, - "the younger self-upgrader must wait behind the older shared holder" + lm.acquire(t, exclusive(&["k"])), + AcquireOutcome::Ready, + "self-upgrade from shared to exclusive must not block" ); - // t_young drops its own shared hold (degrading to plain OCC) so the key - // can drain to empty and its exclusive request can later be promoted; - // t_old remains the sole shared holder, and t_young is enqueued as an - // exclusive waiter rather than left stuck as a non-waiting holder. - let entry = lm.table.get(&key("k")).unwrap(); - assert_eq!(entry.mode, LockMode::Shared); - assert!(entry.holders.contains(&t_old)); - assert!(!entry.holders.contains(&t_young)); - assert!(entry.has_waiter(t_young)); - - // Once t_old releases, t_young is promoted to sole exclusive holder. - let unblocked = lm.release(t_old); - assert!(unblocked.contains(&t_young)); + let entry = lm.table.get(&key("k")).unwrap(); assert_eq!(entry.mode, LockMode::Exclusive); - assert_eq!(entry.holders.len(), 1); - assert_eq!(entry.holders[0], t_young); + assert_eq!(entry.holders.as_slice(), &[t]); } #[test] @@ -286,62 +334,26 @@ mod tests { let t = txn(1, 0); let u = txn(1, 1); - // T reserves K1 shared; U holds K2 exclusively. assert_eq!(lm.acquire_shared(t, key("k1")), AcquireOutcome::Ready); - assert_eq!(lm.acquire(u, keyset(&["k2"])), AcquireOutcome::Ready); + assert_eq!(lm.acquire(u, exclusive(&["k2"])), AcquireOutcome::Ready); - // T tries to take both keys exclusively: K2 conflicts with U, so T must - // block on the whole set — and critically must NOT self-deadlock on K1. assert_eq!( - lm.acquire(t, keyset(&["k1", "k2"])), + lm.acquire(t, exclusive(&["k1", "k2"])), AcquireOutcome::Blocked, "conflict on k2 blocks the whole set" ); - // After U releases K2, T's re-acquire succeeds and upgrades K1 in place. - lm.release(u); + assert_eq!(lm.release(u), vec![t]); assert_eq!( - lm.acquire(t, keyset(&["k1", "k2"])), + lm.acquire(t, exclusive(&["k1", "k2"])), AcquireOutcome::Ready, - "once k2 frees up, t acquires both keys exclusively" - ); - let k1 = lm.table.get(&key("k1")).unwrap(); - assert_eq!(k1.mode, LockMode::Exclusive); - assert_eq!(k1.holders.len(), 1); - assert_eq!(k1.holders[0], t); - let k2 = lm.table.get(&key("k2")).unwrap(); - assert_eq!(k2.mode, LockMode::Exclusive); - assert_eq!(k2.holders.len(), 1); - assert_eq!(k2.holders[0], t); - } - - #[test] - fn two_non_conflicting_both_dispatch_immediately() { - let mut lm = LockManager::new(); - - let txn1 = TxnId::new(1, 0); - let txn2 = TxnId::new(1, 1); - - let keys1: BTreeSet = [LockKey::Surrogate { - collection: Arc::from("coll"), - surrogate: 1, - }] - .into(); - let keys2: BTreeSet = [LockKey::Surrogate { - collection: Arc::from("coll"), - surrogate: 2, - }] - .into(); - - let o1 = lm.acquire(txn1, keys1); - let o2 = lm.acquire(txn2, keys2); - - assert_eq!(o1, AcquireOutcome::Ready, "txn1 should be ready"); - assert_eq!( - o2, - AcquireOutcome::Ready, - "txn2 should be ready (disjoint keys)" + "the promoted txn re-acquires its keys as a no-op" ); + for k in ["k1", "k2"] { + let entry = lm.table.get(&key(k)).unwrap(); + assert_eq!(entry.mode, LockMode::Exclusive); + assert_eq!(entry.holders.as_slice(), &[t]); + } } #[test] @@ -352,27 +364,20 @@ mod tests { let pairs = [(2, 0), (1, 1), (3, 0), (1, 0), (2, 1)]; for (epoch, pos) in pairs { let tid = TxnId::new(epoch, pos); - let keys: BTreeSet = [LockKey::Surrogate { - collection: Arc::from(format!("c_{epoch}_{pos}")), - surrogate: epoch as u32 * 10 + pos, - }] - .into(); - let outcome = lm.acquire(tid, keys); - if outcome == AcquireOutcome::Ready { + let keys = one( + LockKey::Surrogate { + collection: Arc::from(format!("c_{epoch}_{pos}")), + surrogate: epoch as u32 * 10 + pos, + }, + LockMode::Exclusive, + ); + if lm.acquire(tid, keys) == AcquireOutcome::Ready { dispatched.push(tid); } } - assert_eq!( - dispatched.len(), - 5, - "all non-conflicting txns should be ready" - ); - - let mut expected = pairs.map(|(e, p)| TxnId::new(e, p)).to_vec(); - expected.sort(); - let mut sorted_dispatched = dispatched.clone(); - sorted_dispatched.sort(); - assert_eq!(sorted_dispatched, expected); + let expected: BTreeSet = pairs.map(|(e, p)| TxnId::new(e, p)).into(); + let got: BTreeSet = dispatched.into_iter().collect(); + assert_eq!(got, expected, "all non-conflicting txns are ready"); } } diff --git a/nodedb/src/control/cluster/calvin/scheduler/lock/manager/classify.rs b/nodedb/src/control/cluster/calvin/scheduler/lock/manager/classify.rs new file mode 100644 index 000000000..42492d644 --- /dev/null +++ b/nodedb/src/control/cluster/calvin/scheduler/lock/manager/classify.rs @@ -0,0 +1,100 @@ +// SPDX-License-Identifier: BUSL-1.1 + +//! Per-key classification of a lock request, and the grant that installs a +//! requester on a key it can take now. + +use std::collections::VecDeque; + +use smallvec::smallvec; + +use crate::control::cluster::calvin::scheduler::lock::lock_entry::{LockEntry, LockMode}; +use crate::control::cluster::calvin::scheduler::lock::lock_key::{LockKey, TxnId}; + +use super::types::LockManager; + +/// Where one requested key stands for one requester. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub(super) enum KeyState { + /// No entry: the requester can take the key. + Free, + /// The requester already holds the key in a mode that covers the request, + /// or holds it alone and can raise its mode in place. + Held, + /// Other holders hold the key in a compatible mode and nobody waits: the + /// requester can join them. + Join, + /// The requester must wait: an incompatible holder, an earlier waiter, or + /// its own earlier wait on this key. + Conflict, +} + +impl LockManager { + /// Classify `key` requested in `mode` by `txn`. + pub(super) fn classify(&self, txn: TxnId, key: &LockKey, mode: LockMode) -> KeyState { + let Some(entry) = self.table.get(key) else { + return KeyState::Free; + }; + if entry.holders.contains(&txn) { + if entry.mode.merge(mode) == entry.mode || entry.held_solely_by(txn) { + return KeyState::Held; + } + return KeyState::Conflict; + } + if entry.has_waiter(txn) { + return KeyState::Conflict; + } + if entry.mode.compatible(mode) && entry.waiters.is_empty() { + return KeyState::Join; + } + KeyState::Conflict + } + + /// Install `txn` on `key` in `mode`. The caller classified the key as + /// [`KeyState::Free`], [`KeyState::Held`], or [`KeyState::Join`]. + pub(super) fn grant(&mut self, txn: TxnId, key: &LockKey, mode: LockMode) { + match self.table.get_mut(key) { + None => { + self.table.insert( + key.clone(), + LockEntry { + mode, + holders: smallvec![txn], + waiters: VecDeque::new(), + }, + ); + } + Some(entry) => { + if entry.holders.contains(&txn) { + // A sole holder raises its mode in place. A co-holder only + // reaches here when its mode already covers the request. + if entry.held_solely_by(txn) { + entry.mode = entry.mode.merge(mode); + } + } else { + entry.holders.push(txn); + } + } + } + } + + /// Enqueue `txn` as a waiter on a [`KeyState::Conflict`] key. + /// + /// A txn that holds the key and needs a stronger mode drops its own hold + /// and waits for the merged mode. Its read degrades to plain OCC until the + /// grant, and the key can drain so the grant can fire. + pub(super) fn enqueue(&mut self, txn: TxnId, key: &LockKey, mode: LockMode) { + let Some(entry) = self.table.get_mut(key) else { + return; + }; + if entry.has_waiter(txn) { + return; + } + let wanted = if entry.holders.contains(&txn) { + entry.holders.retain(|h| *h != txn); + entry.mode.merge(mode) + } else { + mode + }; + entry.waiters.push_back((txn, wanted)); + } +} diff --git a/nodedb/src/control/cluster/calvin/scheduler/lock/manager/introspection.rs b/nodedb/src/control/cluster/calvin/scheduler/lock/manager/introspection.rs index ef9f4e13d..d4b0018a5 100644 --- a/nodedb/src/control/cluster/calvin/scheduler/lock/manager/introspection.rs +++ b/nodedb/src/control/cluster/calvin/scheduler/lock/manager/introspection.rs @@ -1,28 +1,60 @@ // SPDX-License-Identifier: BUSL-1.1 -//! Read-only inspection of lock manager state — readiness checks and the -//! test-only counters used to assert on table/holder sizes. +//! Read-only inspection of lock manager state: readiness checks, the keys +//! that block a request, and the test-only counters used to assert on +//! table and holder sizes. -use std::collections::BTreeSet; +use std::collections::BTreeMap; +use crate::control::cluster::calvin::scheduler::lock::lock_entry::LockMode; use crate::control::cluster::calvin::scheduler::lock::lock_key::{LockKey, TxnId}; +use super::classify::KeyState; use super::types::LockManager; +/// One key of a lock request that another transaction blocks: the mode it +/// asks, the mode its holders hold, the holders, and the waiters ahead. +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct KeyContention { + pub key: LockKey, + pub requested: LockMode, + pub held: LockMode, + pub holders: Vec, + pub waiters: Vec<(TxnId, LockMode)>, +} + impl LockManager { + /// Every key of `keys` that `txn` cannot take now, with what blocks it. + /// A request [`Self::try_acquire`] refuses names at least one. + pub fn contention(&self, txn: TxnId, keys: &BTreeMap) -> Vec { + keys.iter() + .filter(|(key, mode)| self.classify(txn, key, **mode) == KeyState::Conflict) + .filter_map(|(key, mode)| { + let entry = self.table.get(key)?; + Some(KeyContention { + key: key.clone(), + requested: *mode, + held: entry.mode, + holders: entry.holders.to_vec(), + waiters: entry.waiters.iter().copied().collect(), + }) + }) + .collect() + } + /// Check whether a previously-blocked transaction is now ready. /// /// A transaction is ready when for every key in its key set, the key is /// either: /// - Not present in the lock table (free), or - /// - Present in the lock table with `txn` among the current holders - /// (shared or exclusive). + /// - Present in the lock table with `txn` among the current holders, in + /// any mode. /// /// This is called after `release` returns `txn_id` in the unblocked set. - /// If `is_ready` returns `true`, the caller calls `acquire` again which - /// will succeed on the all-available path (because the waiter was promoted). - pub fn is_ready(&self, txn: TxnId, keys: &BTreeSet) -> bool { - keys.iter().all(|key| { + /// If `is_ready` returns `true`, the caller calls `acquire` again, which + /// grants nothing new because the waiter was promoted. + pub fn is_ready(&self, txn: TxnId, keys: &BTreeMap) -> bool { + keys.keys().all(|key| { match self.table.get(key) { None => true, // key is free Some(entry) => entry.holders.contains(&txn), // txn is a current holder @@ -69,14 +101,39 @@ mod tests { } } - fn keyset(names: &[&str]) -> BTreeSet { - names.iter().map(|n| key(n)).collect() + fn keyset(names: &[&str]) -> BTreeMap { + names + .iter() + .map(|n| (key(n), LockMode::Exclusive)) + .collect() } fn txn(epoch: u64, pos: u32) -> TxnId { TxnId::new(epoch, pos) } + /// A refused request names each key that blocks it, with the holders + /// and the waiters ahead. A free key names nothing. + #[test] + fn contention_names_the_keys_that_block_a_request() { + let mut lm = LockManager::new(); + let (holder, waiter, probe) = (txn(1, 0), txn(1, 1), txn(1, 2)); + lm.acquire(holder, keyset(&["x"])); + lm.acquire(waiter, keyset(&["x"])); + let request = keyset(&["x", "free"]); + assert!(!lm.try_acquire(probe, request.clone())); + assert_eq!( + lm.contention(probe, &request), + vec![KeyContention { + key: key("x"), + requested: LockMode::Exclusive, + held: LockMode::Exclusive, + holders: vec![holder], + waiters: vec![(waiter, LockMode::Exclusive)], + }] + ); + } + #[test] fn is_ready_returns_true_when_all_keys_free_or_self_at_front() { let mut lm = LockManager::new(); diff --git a/nodedb/src/control/cluster/calvin/scheduler/lock/manager/mod.rs b/nodedb/src/control/cluster/calvin/scheduler/lock/manager/mod.rs index e59e665b9..619b57359 100644 --- a/nodedb/src/control/cluster/calvin/scheduler/lock/manager/mod.rs +++ b/nodedb/src/control/cluster/calvin/scheduler/lock/manager/mod.rs @@ -7,11 +7,11 @@ //! The lock manager provides a deterministic, totally-ordered lock table over //! per-key entries keyed by //! [`LockKey`](crate::control::cluster::calvin::scheduler::lock::LockKey). -//! Locks come in two modes: `Exclusive` (one holder, excludes all others) and -//! `Shared` (many compatible holders). The Calvin batch acquire path takes -//! every key in a transaction's `read_set ∪ write_set` as an `Exclusive` -//! lock; single-key `Shared` locks are available via -//! [`LockManager::acquire_shared`]. +//! Locks come in three modes (see +//! [`LockMode`](crate::control::cluster::calvin::scheduler::lock::LockMode)): +//! `Exclusive` (one holder), `Shared` (many readers), and `Intent` (many row +//! writers of one collection). A transaction acquires one map of key to mode. +//! Read reservations take one `Shared` key via [`LockManager::acquire_shared`]. //! //! # Determinism //! @@ -20,19 +20,23 @@ //! requirement, not a style preference. //! //! Split by concern: -//! - [`types`]: the lock table struct and its internal decision enums. -//! - [`acquire`]: exclusive lock acquisition and waiter queueing. +//! - [`types`]: the lock table struct and its internal decision enum. +//! - [`classify`]: per-key request classification and grants. +//! - [`acquire`]: moded lock acquisition and FIFO waiter queueing. //! - [`wound_wait`]: shared-lock reservations and wound-wait conflict //! resolution. -//! - [`release`]: lock release and FIFO/shared waiter promotion. -//! - [`try_acquire`]: the non-blocking exclusive fast path. -//! - [`introspection`]: readiness checks and test counters. +//! - [`release`]: lock release and mode-aware FIFO waiter promotion. +//! - [`try_acquire`]: the non-blocking fast path. +//! - [`introspection`]: readiness checks, request contention and test +//! counters. mod acquire; +mod classify; mod introspection; mod release; mod try_acquire; mod types; mod wound_wait; +pub use introspection::KeyContention; pub use types::LockManager; diff --git a/nodedb/src/control/cluster/calvin/scheduler/lock/manager/release.rs b/nodedb/src/control/cluster/calvin/scheduler/lock/manager/release.rs index 1ba0550d5..f7f1ca556 100644 --- a/nodedb/src/control/cluster/calvin/scheduler/lock/manager/release.rs +++ b/nodedb/src/control/cluster/calvin/scheduler/lock/manager/release.rs @@ -1,113 +1,85 @@ // SPDX-License-Identifier: BUSL-1.1 -//! Lock release and FIFO/shared waiter promotion. +//! Lock release and FIFO, mode-aware waiter promotion. use std::collections::BTreeSet; -use crate::control::cluster::calvin::scheduler::lock::lock_entry::LockMode; use crate::control::cluster::calvin::scheduler::lock::lock_key::{LockKey, TxnId}; -use super::types::{LockManager, Promotion}; +use super::types::LockManager; impl LockManager { - /// Release all locks held by `txn`. + /// Release every lock `txn` holds and every wait it queues. /// - /// `txn` is removed from every entry's holder set. When an entry's holders - /// drain to empty, its FIFO waiters are promoted mode-aware: a leading run - /// of shared waiters is promoted together, or a single leading exclusive - /// waiter is promoted alone. A waiter that becomes holder on ALL its - /// pending keys is moved from `pending_keys` to `held_locks` immediately. + /// `txn` leaves the holders and the waiters of every key it held or + /// requested. Each such key then promotes its waiters (see + /// [`Self::promote_waiters`]). A waiter that now holds ALL its requested + /// keys moves from `pending_keys` to `held_locks`. /// - /// Returns the set of `TxnId`s that have been fully promoted (i.e. moved - /// into `held_locks`). The caller may use this list to dispatch those - /// transactions. + /// Returns the fully-promoted `TxnId`s in ascending order. The caller + /// dispatches them. pub fn release(&mut self, txn: TxnId) -> Vec { - let held = match self.held_locks.remove(&txn) { - Some(h) => h, - None => return Vec::new(), - }; + let mut keys: BTreeSet = self.held_locks.remove(&txn).unwrap_or_default(); + if let Some(pending) = self.pending_keys.remove(&txn) { + keys.extend(pending.into_keys()); + } let mut newly_promoted: BTreeSet = BTreeSet::new(); - - for key in &held { - // Drop `txn` from this key's holders. If other (shared) holders - // remain, the key stays held and there is nothing to promote. - let now_empty = match self.table.get_mut(key) { - Some(entry) => { - entry.holders.retain(|h| *h != txn); - entry.holders.is_empty() - } - None => continue, + for key in &keys { + let Some(entry) = self.table.get_mut(key) else { + continue; }; - if now_empty { - self.promote_waiters(key, &mut newly_promoted); - } + entry.holders.retain(|h| *h != txn); + entry.waiters.retain(|(w, _)| *w != txn); + self.promote_waiters(key, &mut newly_promoted); + } + if !keys.is_empty() { + self.released.notify_waiters(); } - newly_promoted.into_iter().collect() } - /// Promote the front of `key`'s waiter queue after its holders drained. + /// Grant the front of `key`'s waiter queue while it is compatible with the + /// current holders, in FIFO order. /// - /// A leading run of shared waiters is granted together; a single leading - /// exclusive waiter is granted alone; an empty queue frees the entry. Any - /// promoted txn that is now holder on all of its pending keys is moved into - /// `held_locks` and recorded in `newly_promoted`. + /// A key with no holders grants its first waiter in that waiter's mode, + /// then every following waiter compatible with it. A key with neither + /// holders nor waiters is removed. Any granted txn that now holds all of + /// its requested keys moves into `held_locks` and joins `newly_promoted`. fn promote_waiters(&mut self, key: &LockKey, newly_promoted: &mut BTreeSet) { - // Decide the promotion inside a scoped borrow so the readiness sweep - // below can re-borrow the table. - let decision = match self.table.get_mut(key) { - Some(entry) => match entry.waiters.front().map(|(_, mode)| *mode) { - None => Promotion::Freed, - Some(LockMode::Exclusive) => match entry.waiters.pop_front() { - Some((next, _)) => { - entry.mode = LockMode::Exclusive; - entry.holders.clear(); - entry.holders.push(next); - Promotion::Promoted(vec![next]) - } - None => Promotion::Freed, - }, - Some(LockMode::Shared) => { - entry.mode = LockMode::Shared; - entry.holders.clear(); - let mut promoted = Vec::new(); - while matches!(entry.waiters.front(), Some((_, LockMode::Shared))) { - if let Some((next, _)) = entry.waiters.pop_front() { - entry.holders.push(next); - promoted.push(next); - } - } - Promotion::Promoted(promoted) - } - }, - None => return, + let mut granted: Vec = Vec::new(); + let Some(entry) = self.table.get_mut(key) else { + return; }; - - let promoted = match decision { - Promotion::Freed => { - self.table.remove(key); - return; + while let Some(&(next, mode)) = entry.waiters.front() { + if !entry.holders.is_empty() && !entry.mode.compatible(mode) { + break; } - Promotion::Promoted(promoted) => promoted, - }; + entry.waiters.pop_front(); + if entry.holders.is_empty() { + entry.mode = mode; + } + entry.holders.push(next); + granted.push(next); + } + if entry.holders.is_empty() && entry.waiters.is_empty() { + self.table.remove(key); + } - // For each promoted txn, check whether it is now holder on ALL of its - // pending keys. If so, it is fully ready — move to held_locks. Remove - // first (rather than `get` + a follow-up `remove`) so there is no - // unwrap/expect on a "just confirmed Some" invariant: the owned - // `pending` set is reinserted on the not-yet-ready path. - for next in promoted { - if let Some(pending) = self.pending_keys.remove(&next) { - let all_held = pending - .iter() - .all(|k| self.table.get(k).is_none_or(|e| e.holders.contains(&next))); - if all_held { - self.held_locks.insert(next, pending); - newly_promoted.insert(next); - } else { - self.pending_keys.insert(next, pending); - } + for next in granted { + let Some(pending) = self.pending_keys.remove(&next) else { + continue; + }; + let all_held = pending.keys().all(|k| { + self.table + .get(k) + .is_some_and(|entry| entry.holders.contains(&next)) + }); + if all_held { + self.finish_ready(next, pending); + newly_promoted.insert(next); + } else { + self.pending_keys.insert(next, pending); } } } @@ -119,8 +91,10 @@ impl LockManager { mod tests { use std::sync::Arc; + use std::collections::BTreeMap; + use super::*; - use crate::control::cluster::calvin::scheduler::lock::lock_entry::AcquireOutcome; + use crate::control::cluster::calvin::scheduler::lock::lock_entry::{AcquireOutcome, LockMode}; fn key(name: &str) -> LockKey { LockKey::Surrogate { @@ -129,8 +103,11 @@ mod tests { } } - fn keyset(names: &[&str]) -> BTreeSet { - names.iter().map(|n| key(n)).collect() + fn keyset(names: &[&str]) -> BTreeMap { + names + .iter() + .map(|n| (key(n), LockMode::Exclusive)) + .collect() } fn txn(epoch: u64, pos: u32) -> TxnId { @@ -182,6 +159,22 @@ mod tests { ); } + /// A release wakes a waiter that enabled the release signal before it. + #[tokio::test] + async fn release_wakes_the_release_signal() { + let mut lm = LockManager::new(); + let holder = txn(1, 0); + assert_eq!(lm.acquire(holder, keyset(&["k"])), AcquireOutcome::Ready); + let signal = lm.release_signal(); + let notified = signal.notified(); + tokio::pin!(notified); + notified.as_mut().enable(); + lm.release(holder); + tokio::time::timeout(std::time::Duration::from_secs(5), notified) + .await + .expect("a release wakes the release signal"); + } + #[test] fn release_preserves_fifo_waiter_order() { let mut lm = LockManager::new(); @@ -290,10 +283,13 @@ mod tests { let txn1 = TxnId::new(1, 0); let txn2 = TxnId::new(1, 1); - let shared_key: BTreeSet = [LockKey::Surrogate { - collection: Arc::from("coll"), - surrogate: 42, - }] + let shared_key: BTreeMap = [( + LockKey::Surrogate { + collection: Arc::from("coll"), + surrogate: 42, + }, + LockMode::Exclusive, + )] .into(); let o1 = lm.acquire(txn1, shared_key.clone()); @@ -315,10 +311,13 @@ mod tests { let txn_n = TxnId::new(1, 0); let txn_n1 = TxnId::new(2, 0); - let key_k: BTreeSet = [LockKey::Surrogate { - collection: Arc::from("orders"), - surrogate: 100, - }] + let key_k: BTreeMap = [( + LockKey::Surrogate { + collection: Arc::from("orders"), + surrogate: 100, + }, + LockMode::Exclusive, + )] .into(); let o1 = lm.acquire(txn_n, key_k.clone()); diff --git a/nodedb/src/control/cluster/calvin/scheduler/lock/manager/try_acquire.rs b/nodedb/src/control/cluster/calvin/scheduler/lock/manager/try_acquire.rs index 0dee86f99..1455a67a9 100644 --- a/nodedb/src/control/cluster/calvin/scheduler/lock/manager/try_acquire.rs +++ b/nodedb/src/control/cluster/calvin/scheduler/lock/manager/try_acquire.rs @@ -1,18 +1,22 @@ // SPDX-License-Identifier: BUSL-1.1 -//! Non-blocking exclusive acquire fast path. +//! Non-blocking acquire fast path. -use std::collections::BTreeSet; +use std::collections::BTreeMap; -use crate::control::cluster::calvin::scheduler::lock::lock_entry::AcquireOutcome; +use crate::control::cluster::calvin::scheduler::lock::lock_entry::LockMode; use crate::control::cluster::calvin::scheduler::lock::lock_key::{LockKey, TxnId}; +use super::classify::KeyState; use super::types::LockManager; impl LockManager { - /// Non-blocking exclusive acquire: take all `keys` for `txn` iff every one is - /// free (or already held by `txn`), returning `true`; otherwise return - /// `false` WITHOUT enqueuing a waiter or recording any pending state. + /// Non-blocking acquire: take all `keys` for `txn` iff none conflicts, + /// returning `true`; otherwise return `false` WITHOUT enqueuing a waiter or + /// recording any pending state. + /// + /// A key is taken when it is free, already held by `txn`, or held in a + /// compatible mode with nobody waiting (see [`LockMode::compatible`]). /// /// This is the fast path's probe. Unlike [`acquire`](Self::acquire), the /// contended (`false`) path touches NOTHING — no holder, no `pending_keys`, @@ -21,20 +25,42 @@ impl LockManager { /// leaves an orphaned waiter that a later `release` would promote to an /// unowned holder. It also never perturbs the FIFO ordering that Calvin /// transactions depend on. - pub fn try_acquire(&mut self, txn: TxnId, keys: BTreeSet) -> bool { - if !self.is_ready(txn, &keys) { - // Contended: leave the table, waiter queues, and pending_keys - // completely untouched. + pub fn try_acquire(&mut self, txn: TxnId, keys: BTreeMap) -> bool { + let contended = keys + .iter() + .any(|(key, mode)| self.classify(txn, key, *mode) == KeyState::Conflict); + if contended { return false; } - // Every key is free or already held by `txn`, so `acquire` takes its - // all-available path — it inserts the holder and never enqueues. - let outcome = self.acquire(txn, keys); - debug_assert_eq!( - outcome, - AcquireOutcome::Ready, - "try_acquire: is_ready was true but acquire returned Blocked" - ); + for (key, mode) in &keys { + self.grant(txn, key, *mode); + } + self.finish_ready(txn, keys); true } } + +#[cfg(test)] +mod tests { + use std::sync::Arc; + + use super::*; + + fn coll() -> LockKey { + LockKey::Collection { + collection: Arc::from("c"), + } + } + + #[test] + fn try_acquire_joins_compatible_holders_and_refuses_conflicts() { + let mut lm = LockManager::new(); + let (a, b, c) = (TxnId::new(1, 0), TxnId::new(1, 1), TxnId::new(1, 2)); + assert!(lm.try_acquire(a, BTreeMap::from([(coll(), LockMode::Intent)]))); + assert!(lm.try_acquire(b, BTreeMap::from([(coll(), LockMode::Intent)]))); + assert!(!lm.try_acquire(c, BTreeMap::from([(coll(), LockMode::Exclusive)]))); + let entry = lm.table.get(&coll()).unwrap(); + assert_eq!(entry.holders.as_slice(), &[a, b]); + assert!(entry.waiters.is_empty(), "a refused probe leaves no waiter"); + } +} diff --git a/nodedb/src/control/cluster/calvin/scheduler/lock/manager/types.rs b/nodedb/src/control/cluster/calvin/scheduler/lock/manager/types.rs index 93e6862bd..0ec4039d6 100644 --- a/nodedb/src/control/cluster/calvin/scheduler/lock/manager/types.rs +++ b/nodedb/src/control/cluster/calvin/scheduler/lock/manager/types.rs @@ -1,9 +1,12 @@ // SPDX-License-Identifier: BUSL-1.1 -//! The lock table struct, its internal decision enums, and the small -//! per-entry predicates the acquire/release paths share. +//! The lock table struct and the small per-entry predicates the +//! acquire/release paths share. use std::collections::{BTreeMap, BTreeSet}; +use std::sync::Arc; + +use tokio::sync::Notify; use crate::control::cluster::calvin::scheduler::lock::lock_entry::{LockEntry, LockMode}; use crate::control::cluster::calvin::scheduler::lock::lock_key::{LockKey, TxnId}; @@ -15,11 +18,12 @@ use crate::control::cluster::calvin::scheduler::lock::lock_key::{LockKey, TxnId} /// /// # Key sets tracked per transaction /// -/// - `held_locks`: key sets for transactions that are a current holder on ALL -/// their keys and are actively executing (i.e. dispatched to the Data Plane). -/// - `pending_keys`: key sets for transactions that are blocked waiting for at -/// least one key. When `release` promotes a blocked txn to holder on every -/// one of its keys, the entry moves from `pending_keys` to `held_locks`. +/// - `held_locks`: keys of transactions that hold ALL their requested keys and +/// are actively executing (i.e. dispatched to the Data Plane). +/// - `pending_keys`: the full request of a transaction that waits for at least +/// one key. It already holds every other key of the request. When `release` +/// grants its last waited key, the entry moves from `pending_keys` to +/// `held_locks`. pub struct LockManager { /// Per-key lock entries. Uses `BTreeMap` for deterministic iteration. /// Visible to the whole `lock` module so the sibling `reap` module can @@ -30,39 +34,27 @@ pub struct LockManager { /// Visible to the whole `lock` module — see `table`. pub(in crate::control::cluster::calvin::scheduler::lock) held_locks: BTreeMap>, - /// Key sets for **blocked** (not-yet-dispatched) txns. Populated when - /// `acquire` returns `Blocked`; cleared (moved to `held_locks`) when all - /// keys have been acquired on the promotion path inside `release`. - pub(super) pending_keys: BTreeMap>, -} - -/// Outcome of inspecting a single key during [`LockManager::acquire_shared`]. -pub(super) enum SharedGrant { - /// The shared lock was granted (key was free or already held shared). - Granted, - /// The key is held exclusively by another txn; the request was enqueued. - Blocked, + /// Full requests of **blocked** (not-yet-dispatched) txns. Populated when + /// `acquire` returns `Blocked`; moved to `held_locks` when the last waited + /// key is granted on the promotion path inside `release`. + pub(in crate::control::cluster::calvin::scheduler::lock) pending_keys: + BTreeMap>, + /// Woken on every release. A write-admission waiter that holds no key + /// probes the table again after each wake. + pub(in crate::control::cluster::calvin::scheduler::lock) released: Arc, } -/// The wound-wait decision for an exclusive requester that meets a conflict. -pub(super) enum ExclusiveWait { - /// Every conflicting holder is a shared reservation and the requester is - /// older than all of them: wound (revoke) those shared holders and proceed. +/// How a requester resolves a request that conflicts on at least one key. +pub(super) enum ConflictResolution { + /// Every conflicting holder is a younger read reservation and no other + /// transaction waits on a conflicting key: revoke those reservations and + /// take every key. Wound, - /// The requester must block: a conflicting holder is exclusive, or the - /// requester is younger than some conflicting shared holder. + /// The requester waits: a conflicting holder is a transaction, an older + /// reservation, or an earlier waiter queues ahead of it. Block, } -/// The waiters promoted off one key when its holders drained, together with the -/// action to take on the now-empty entry. -pub(super) enum Promotion { - /// No waiters remained; the entry should be removed entirely. - Freed, - /// These waiters were installed as the new holders. - Promoted(Vec), -} - impl LockManager { /// Create an empty lock manager. pub fn new() -> Self { @@ -70,8 +62,16 @@ impl LockManager { table: BTreeMap::new(), held_locks: BTreeMap::new(), pending_keys: BTreeMap::new(), + released: Arc::new(Notify::new()), } } + + /// The signal every release wakes. A waiter enables its `notified()` + /// future before it probes the table, so no release between the probe + /// and the wait goes unseen. + pub fn release_signal(&self) -> Arc { + Arc::clone(&self.released) + } } impl Default for LockManager { @@ -81,17 +81,9 @@ impl Default for LockManager { } impl LockEntry { - /// Whether this entry is held exclusively by exactly `txn` (the self - /// re-acquire case on the exclusive path). - pub(super) fn held_exclusively_by(&self, txn: TxnId) -> bool { - self.mode == LockMode::Exclusive && self.holders.len() == 1 && self.holders[0] == txn - } - - /// Whether this entry is held **shared** by exactly `txn` and no one else — - /// the self-upgrade case: `txn` may take the key exclusively because it is - /// the sole current holder. - pub(super) fn held_shared_solely_by(&self, txn: TxnId) -> bool { - self.mode == LockMode::Shared && self.holders.len() == 1 && self.holders[0] == txn + /// Whether `txn` is the only holder of this entry. + pub(super) fn held_solely_by(&self, txn: TxnId) -> bool { + self.holders.len() == 1 && self.holders[0] == txn } /// Whether `txn` is already enqueued as a waiter on this entry. diff --git a/nodedb/src/control/cluster/calvin/scheduler/lock/manager/wound_wait.rs b/nodedb/src/control/cluster/calvin/scheduler/lock/manager/wound_wait.rs index 2a77b1a45..00d2579c3 100644 --- a/nodedb/src/control/cluster/calvin/scheduler/lock/manager/wound_wait.rs +++ b/nodedb/src/control/cluster/calvin/scheduler/lock/manager/wound_wait.rs @@ -1,119 +1,78 @@ // SPDX-License-Identifier: BUSL-1.1 //! Shared-lock reservations and the wound-wait conflict resolution used by -//! exclusive acquisition. +//! acquisition. -use std::collections::btree_map::Entry; -use std::collections::{BTreeSet, VecDeque}; +use std::collections::BTreeMap; -use smallvec::smallvec; - -use crate::control::cluster::calvin::scheduler::lock::lock_entry::{ - AcquireOutcome, LockEntry, LockMode, -}; +use crate::control::cluster::calvin::scheduler::lock::lock_entry::{AcquireOutcome, LockMode}; use crate::control::cluster::calvin::scheduler::lock::lock_key::{LockKey, TxnId}; -use super::types::{ExclusiveWait, LockManager, SharedGrant}; +use super::classify::KeyState; +use super::types::{ConflictResolution, LockManager}; impl LockManager { - /// Classify the wound-wait decision for an exclusive requester `txn` over - /// `keys`, given that at least one key already conflicts. - /// - /// Pure read over the lock table: any exclusive conflict forces - /// [`ExclusiveWait::Block`] (an exclusive holder is never wounded, so a mix - /// of exclusive and shared conflicts blocks too). Otherwise all conflicting - /// holders are shared reservations, and `txn` wounds them only when it is - /// older than every one (`txn < h` for each conflicting shared holder `h`); - /// if it is younger than any, it blocks. A key held only by `txn` itself is - /// not a conflict. - pub(super) fn wound_or_block(&self, txn: TxnId, keys: &BTreeSet) -> ExclusiveWait { - let mut shared_conflicts: Vec = Vec::new(); - for key in keys { - if let Some(entry) = self.table.get(key) { - match entry.mode { - LockMode::Exclusive => { - // Exclusive entries have exactly one holder; a holder - // other than `txn` is an exclusive conflict. - if !entry.holders.contains(&txn) { - return ExclusiveWait::Block; - } - } - LockMode::Shared => { - for holder in &entry.holders { - if *holder != txn { - shared_conflicts.push(*holder); - } - } - } - } - } - } - // Wound only when there is a shared conflict AND `txn` is older than - // every conflicting shared holder; otherwise block. `shared_conflicts` - // only ever holds *other* txns' shared holders (a key held shared solely - // by `txn` never reaches here — it takes the self-upgrade path in - // `acquire`), so an empty set here means every conflict was exclusive. - if !shared_conflicts.is_empty() && shared_conflicts.iter().all(|holder| txn < *holder) { - ExclusiveWait::Wound - } else { - ExclusiveWait::Block - } - } - - /// Attempt to acquire a **shared** lock on a single `key` for `txn`. + /// Decide how `txn` resolves a request with at least one conflicting key. /// - /// - Key free → create a shared entry holding `txn`, return - /// [`AcquireOutcome::Ready`]. - /// - Key held shared → add `txn` to the holders, return - /// [`AcquireOutcome::Ready`]. - /// - Key held exclusively by another txn → enqueue `txn` as a shared waiter - /// (FIFO) and return [`AcquireOutcome::Blocked`]. + /// Pure read over the lock table. `txn` wounds only when every conflicting + /// key meets all of these: + /// - nobody waits on the key, so no earlier request is passed; + /// - every other holder holds the key `Shared`; + /// - every other holder is a read reservation (`TxnId::is_reservation`) + /// younger than `txn`. /// - /// A shared request that meets an exclusive holder blocks FIFO for now; - /// wound-wait priority resolution lands in a following change. - pub fn acquire_shared(&mut self, txn: TxnId, key: LockKey) -> AcquireOutcome { - // Inspect / mutate the entry via the `Entry` API (which takes the key by - // value, sidestepping a get-then-insert borrow conflict) inside a scoped - // borrow so the map-level bookkeeping below can re-borrow `self`. - let grant = match self.table.entry(key.clone()) { - Entry::Vacant(slot) => { - slot.insert(LockEntry { - mode: LockMode::Shared, - holders: smallvec![txn], - waiters: VecDeque::new(), - }); - SharedGrant::Granted + /// A transaction holder is executing work and is never wounded, whatever + /// its mode. Otherwise `txn` blocks. + pub(super) fn resolve_conflict( + &self, + txn: TxnId, + request: &BTreeMap, + states: &[KeyState], + ) -> ConflictResolution { + for (key, state) in request.keys().zip(states) { + if *state != KeyState::Conflict { + continue; } - Entry::Occupied(mut slot) => { - let entry = slot.get_mut(); - if entry.mode == LockMode::Shared { - if !entry.holders.contains(&txn) { - entry.holders.push(txn); - } - SharedGrant::Granted - } else { - // Held exclusively by another txn: block FIFO. - if !entry.has_waiter(txn) { - entry.waiters.push_back((txn, LockMode::Shared)); - } - SharedGrant::Blocked - } + let Some(entry) = self.table.get(key) else { + return ConflictResolution::Block; + }; + if !entry.waiters.is_empty() || entry.mode != LockMode::Shared { + return ConflictResolution::Block; } - }; - - match grant { - SharedGrant::Granted => { - self.pending_keys.remove(&txn); - self.held_locks.entry(txn).or_default().insert(key); - AcquireOutcome::Ready + let mut others = entry.holders.iter().filter(|h| **h != txn).peekable(); + if others.peek().is_none() { + return ConflictResolution::Block; } - SharedGrant::Blocked => { - let mut pending = BTreeSet::new(); - pending.insert(key); - self.pending_keys.insert(txn, pending); - AcquireOutcome::Blocked + if others.any(|holder| !holder.is_reservation() || txn >= *holder) { + return ConflictResolution::Block; } } + ConflictResolution::Wound + } + + /// Revoke every other holder of `key` and make `txn` its sole holder in + /// `mode`, merged with any mode `txn` already held it in. The revoked + /// reservations degrade to plain OCC, with no notification. Waiters stay + /// queued. + pub(super) fn wound(&mut self, txn: TxnId, key: &LockKey, mode: LockMode) { + let Some(entry) = self.table.get_mut(key) else { + return; + }; + let wanted = if entry.holders.contains(&txn) { + entry.mode.merge(mode) + } else { + mode + }; + entry.holders.clear(); + entry.holders.push(txn); + entry.mode = wanted; + } + + /// Acquire a **shared** lock on a single `key` for `txn`: a read + /// reservation. Same rules as [`Self::acquire`]. A reservation that + /// meets an incompatible holder or an earlier waiter waits FIFO. + pub fn acquire_shared(&mut self, txn: TxnId, key: LockKey) -> AcquireOutcome { + self.acquire(txn, BTreeMap::from([(key, LockMode::Shared)])) } } @@ -132,182 +91,226 @@ mod tests { } } - fn keyset(names: &[&str]) -> BTreeSet { - names.iter().map(|n| key(n)).collect() + fn exclusive(names: &[&str]) -> BTreeMap { + names + .iter() + .map(|n| (key(n), LockMode::Exclusive)) + .collect() } fn txn(epoch: u64, pos: u32) -> TxnId { TxnId::new(epoch, pos) } + /// A read-reservation owner: position `n` of the reservation band. + fn reservation(epoch: u64, n: u32) -> TxnId { + TxnId::new(epoch, TxnId::RESERVATION_POSITION_BAND + n) + } + #[test] fn shared_shared_compatible() { let mut lm = LockManager::new(); - let t1 = txn(1, 0); - let t2 = txn(1, 1); + let (r1, r2) = (reservation(1, 0), reservation(1, 1)); - assert_eq!(lm.acquire_shared(t1, key("s")), AcquireOutcome::Ready); - assert_eq!(lm.acquire_shared(t2, key("s")), AcquireOutcome::Ready); + assert_eq!(lm.acquire_shared(r1, key("s")), AcquireOutcome::Ready); + assert_eq!(lm.acquire_shared(r2, key("s")), AcquireOutcome::Ready); let entry = lm.table.get(&key("s")).unwrap(); assert_eq!(entry.mode, LockMode::Shared); - assert!(entry.holders.contains(&t1)); - assert!(entry.holders.contains(&t2)); + assert!(entry.holders.contains(&r1)); + assert!(entry.holders.contains(&r2)); } #[test] - fn shared_blocks_exclusive() { + fn a_shared_transaction_holder_is_never_wounded() { let mut lm = LockManager::new(); - let t1 = txn(1, 0); - let t2 = txn(1, 1); + let reader = txn(1, 2); + let writer = txn(1, 1); // older than the reader - assert_eq!(lm.acquire_shared(t1, key("k")), AcquireOutcome::Ready); + assert_eq!(lm.acquire_shared(reader, key("k")), AcquireOutcome::Ready); assert_eq!( - lm.acquire(t2, keyset(&["k"])), + lm.acquire(writer, exclusive(&["k"])), AcquireOutcome::Blocked, - "an exclusive request must block behind a shared holder" + "a transaction's read lock is executing work, not a reservation" ); - assert!(lm.table.get(&key("k")).unwrap().has_waiter(t2)); + assert!(lm.table.get(&key("k")).unwrap().has_waiter(writer)); } #[test] fn exclusive_blocks_shared() { let mut lm = LockManager::new(); let t1 = txn(1, 0); - let t2 = txn(1, 1); + let r = reservation(1, 0); - assert_eq!(lm.acquire(t1, keyset(&["k"])), AcquireOutcome::Ready); + assert_eq!(lm.acquire(t1, exclusive(&["k"])), AcquireOutcome::Ready); assert_eq!( - lm.acquire_shared(t2, key("k")), + lm.acquire_shared(r, key("k")), AcquireOutcome::Blocked, "a shared request must block behind an exclusive holder" ); - assert!(lm.table.get(&key("k")).unwrap().has_waiter(t2)); + assert!(lm.table.get(&key("k")).unwrap().has_waiter(r)); } #[test] fn older_writer_wounds_shared() { let mut lm = LockManager::new(); - let t2 = txn(1, 2); // shared holder - let t1 = txn(1, 1); // exclusive requester, older than t2 + let r = reservation(1, 0); + let writer = txn(1, 1); // older than every reservation of epoch 1 - assert_eq!(lm.acquire_shared(t2, key("k")), AcquireOutcome::Ready); - // The older writer wounds the younger shared holder and proceeds. - assert_eq!(lm.acquire(t1, keyset(&["k"])), AcquireOutcome::Ready); + assert_eq!(lm.acquire_shared(r, key("k")), AcquireOutcome::Ready); + assert_eq!(lm.acquire(writer, exclusive(&["k"])), AcquireOutcome::Ready); let entry = lm.table.get(&key("k")).unwrap(); assert_eq!(entry.mode, LockMode::Exclusive); - assert!(entry.holders.contains(&t1), "R is now the exclusive holder"); - assert!( - !entry.holders.contains(&t2), - "the wounded shared holder is gone" + assert_eq!(entry.holders.as_slice(), &[writer]); + } + + #[test] + fn older_intent_writer_wounds_a_collection_reservation() { + let mut lm = LockManager::new(); + let r = reservation(1, 0); + let writer = txn(1, 1); + let coll = LockKey::Collection { + collection: Arc::from("c"), + }; + + assert_eq!(lm.acquire_shared(r, coll.clone()), AcquireOutcome::Ready); + assert_eq!( + lm.acquire(writer, BTreeMap::from([(coll.clone(), LockMode::Intent)])), + AcquireOutcome::Ready ); + let entry = lm.table.get(&coll).unwrap(); + assert_eq!(entry.mode, LockMode::Intent); + assert_eq!(entry.holders.as_slice(), &[writer]); } #[test] fn younger_writer_waits() { let mut lm = LockManager::new(); - let t1 = txn(1, 1); // shared holder - let t2 = txn(1, 2); // exclusive requester, younger than t1 + let r = reservation(1, 1); + let writer = txn(2, 0); // younger than the reservation - assert_eq!(lm.acquire_shared(t1, key("k")), AcquireOutcome::Ready); - // The younger writer must not wound; it waits behind the shared holder. - assert_eq!(lm.acquire(t2, keyset(&["k"])), AcquireOutcome::Blocked); + assert_eq!(lm.acquire_shared(r, key("k")), AcquireOutcome::Ready); + assert_eq!( + lm.acquire(writer, exclusive(&["k"])), + AcquireOutcome::Blocked + ); let entry = lm.table.get(&key("k")).unwrap(); - assert!(entry.holders.contains(&t1), "the shared holder still holds"); - assert!(!entry.holders.contains(&t2), "R holds nothing"); - assert!(entry.has_waiter(t2), "R is enqueued as an exclusive waiter"); + assert!(entry.holders.contains(&r), "the shared holder still holds"); + assert!(!entry.holders.contains(&writer)); + assert!(entry.has_waiter(writer)); } #[test] fn exclusive_waits_on_exclusive_regardless_of_age() { let mut lm = LockManager::new(); - let t2 = txn(1, 2); // exclusive holder (younger) - let t1 = txn(1, 1); // exclusive requester (older) + let t2 = txn(1, 2); + let t1 = txn(1, 1); - assert_eq!(lm.acquire(t2, keyset(&["k"])), AcquireOutcome::Ready); - // An exclusive holder is NEVER wounded, even by an older writer. - assert_eq!(lm.acquire(t1, keyset(&["k"])), AcquireOutcome::Blocked); + assert_eq!(lm.acquire(t2, exclusive(&["k"])), AcquireOutcome::Ready); + assert_eq!(lm.acquire(t1, exclusive(&["k"])), AcquireOutcome::Blocked); let entry = lm.table.get(&key("k")).unwrap(); - assert!( - entry.holders.contains(&t2), - "the exclusive holder is intact" - ); - assert!(!entry.holders.contains(&t1)); + assert_eq!(entry.holders.as_slice(), &[t2]); assert!(entry.has_waiter(t1)); } #[test] fn multi_key_atomic_wound_takes_both() { let mut lm = LockManager::new(); - let s1 = txn(1, 5); // shared holder on k1, younger than R - let s2 = txn(1, 6); // shared holder on k2, younger than R - let r = txn(1, 1); // exclusive requester, older than both + let s1 = reservation(1, 5); + let s2 = reservation(1, 6); + let r = txn(1, 1); assert_eq!(lm.acquire_shared(s1, key("k1")), AcquireOutcome::Ready); assert_eq!(lm.acquire_shared(s2, key("k2")), AcquireOutcome::Ready); - assert_eq!(lm.acquire(r, keyset(&["k1", "k2"])), AcquireOutcome::Ready); + assert_eq!( + lm.acquire(r, exclusive(&["k1", "k2"])), + AcquireOutcome::Ready + ); for k in ["k1", "k2"] { let entry = lm.table.get(&key(k)).unwrap(); assert_eq!(entry.mode, LockMode::Exclusive); - assert!(entry.holders.contains(&r), "R holds {k}"); + assert_eq!(entry.holders.as_slice(), &[r]); } - assert!(!lm.table.get(&key("k1")).unwrap().holders.contains(&s1)); - assert!(!lm.table.get(&key("k2")).unwrap().holders.contains(&s2)); } #[test] - fn multi_key_atomic_wait_holds_none() { + fn multi_key_wait_takes_no_conflicting_key() { let mut lm = LockManager::new(); - let s1 = txn(1, 5); // shared holder on k1, younger than R - let s2 = txn(1, 0); // shared holder on k2, OLDER than R - let r = txn(1, 1); // exclusive requester + let s1 = reservation(1, 5); // younger than R + let s2 = reservation(0, 0); // older than R + let r = txn(1, 1); assert_eq!(lm.acquire_shared(s1, key("k1")), AcquireOutcome::Ready); assert_eq!(lm.acquire_shared(s2, key("k2")), AcquireOutcome::Ready); - // R is younger than the holder on k2, so it must wait on BOTH keys and - // hold neither (all-or-nothing). + // R is younger than the holder on k2, so it wounds nobody and waits on + // both keys behind their shared holders. assert_eq!( - lm.acquire(r, keyset(&["k1", "k2"])), + lm.acquire(r, exclusive(&["k1", "k2"])), AcquireOutcome::Blocked ); + for (k, holder) in [("k1", s1), ("k2", s2)] { + let entry = lm.table.get(&key(k)).unwrap(); + assert_eq!(entry.holders.as_slice(), &[holder]); + assert!(entry.has_waiter(r)); + } + } + + #[test] + fn self_upgrade_with_other_shared_holder_wounds_or_blocks() { + // The older reservation's self-upgrade wounds the younger one. + let mut lm = LockManager::new(); + let (r_old, r_young) = (reservation(1, 0), reservation(1, 1)); + + assert_eq!(lm.acquire_shared(r_old, key("k")), AcquireOutcome::Ready); + assert_eq!(lm.acquire_shared(r_young, key("k")), AcquireOutcome::Ready); + assert_eq!(lm.acquire(r_old, exclusive(&["k"])), AcquireOutcome::Ready); + let entry = lm.table.get(&key("k")).unwrap(); + assert_eq!(entry.mode, LockMode::Exclusive); + assert_eq!(entry.holders.as_slice(), &[r_old]); - assert!( - !lm.table.get(&key("k1")).unwrap().holders.contains(&r), - "R holds no key" + // The younger one's self-upgrade drops its own hold and waits. + let mut lm = LockManager::new(); + assert_eq!(lm.acquire_shared(r_old, key("k")), AcquireOutcome::Ready); + assert_eq!(lm.acquire_shared(r_young, key("k")), AcquireOutcome::Ready); + assert_eq!( + lm.acquire(r_young, exclusive(&["k"])), + AcquireOutcome::Blocked ); - assert!(!lm.table.get(&key("k2")).unwrap().holders.contains(&r)); - // The older shared holder on k2 is untouched. - assert!(lm.table.get(&key("k2")).unwrap().holders.contains(&s2)); + let entry = lm.table.get(&key("k")).unwrap(); + assert_eq!(entry.mode, LockMode::Shared); + assert_eq!(entry.holders.as_slice(), &[r_old]); + assert!(entry.has_waiter(r_young)); + + assert_eq!(lm.release(r_old), vec![r_young]); + let entry = lm.table.get(&key("k")).unwrap(); + assert_eq!(entry.mode, LockMode::Exclusive); + assert_eq!(entry.holders.as_slice(), &[r_young]); } #[test] fn crossed_reservations_are_acyclic() { - // T1 holds shared K1 and wants exclusive K2; T2 holds shared K2 and - // wants exclusive K1. The older writer's exclusive acquire wounds the - // younger's shared holding, breaking the cycle — no deadlock. + // R1 reserves K1, R2 reserves K2. The older writer's exclusive acquire + // of K2 wounds the younger reservation, breaking any cycle. let mut lm = LockManager::new(); - let t1 = txn(1, 1); // older - let t2 = txn(1, 2); // younger - - assert_eq!(lm.acquire_shared(t1, key("k1")), AcquireOutcome::Ready); - assert_eq!(lm.acquire_shared(t2, key("k2")), AcquireOutcome::Ready); + let r1 = reservation(1, 1); + let r2 = reservation(1, 2); + let writer = txn(1, 0); - // T1 (older) acquires exclusive K2: wounds T2's shared holding and - // proceeds. - assert_eq!(lm.acquire(t1, keyset(&["k2"])), AcquireOutcome::Ready); + assert_eq!(lm.acquire_shared(r1, key("k1")), AcquireOutcome::Ready); + assert_eq!(lm.acquire_shared(r2, key("k2")), AcquireOutcome::Ready); + assert_eq!( + lm.acquire(writer, exclusive(&["k2"])), + AcquireOutcome::Ready + ); let k2 = lm.table.get(&key("k2")).unwrap(); assert_eq!(k2.mode, LockMode::Exclusive); - assert!(k2.holders.contains(&t1), "the older writer proceeds"); - assert!( - !k2.holders.contains(&t2), - "the younger's reservation is wounded away" - ); + assert_eq!(k2.holders.as_slice(), &[writer]); } } diff --git a/nodedb/src/control/cluster/calvin/scheduler/lock/mod.rs b/nodedb/src/control/cluster/calvin/scheduler/lock/mod.rs index 8f00d1d06..964bdeb07 100644 --- a/nodedb/src/control/cluster/calvin/scheduler/lock/mod.rs +++ b/nodedb/src/control/cluster/calvin/scheduler/lock/mod.rs @@ -9,4 +9,4 @@ pub mod reap; pub use hot_key_table::HotKeyTable; pub use lock_entry::{AcquireOutcome, LockMode}; pub use lock_key::{LockKey, TxnId}; -pub use manager::LockManager; +pub use manager::{KeyContention, LockManager}; diff --git a/nodedb/src/control/cluster/calvin/scheduler/lock/reap.rs b/nodedb/src/control/cluster/calvin/scheduler/lock/reap.rs index b741bc756..a47ff6470 100644 --- a/nodedb/src/control/cluster/calvin/scheduler/lock/reap.rs +++ b/nodedb/src/control/cluster/calvin/scheduler/lock/reap.rs @@ -10,6 +10,8 @@ //! replicated logical threshold — no wall clock, so every replica reaps //! identically given the same input order. +use std::collections::BTreeSet; + use super::lock_entry::LockMode; use super::lock_key::TxnId; use super::manager::LockManager; @@ -22,23 +24,21 @@ impl LockManager { /// /// Restricted to owners in the reservation band (`is_reservation()`) — a real /// transaction's lock owner is never in this band, so a real txn's locks can - /// never be reaped. Further restricted to owners ALL of whose held keys are - /// still `Shared`: an owner that self-upgraded a key to `Exclusive` (a commit - /// in flight under that reservation) is left alone. + /// never be reaped. A reservation that still waits for a key is reaped too, + /// so it never blocks the waiters queued behind it. An owner that holds or + /// requests any key in a mode other than `Shared` (a commit in flight under + /// that reservation) is left alone. pub fn reap_expired_shared(&mut self, epoch_threshold: u64) -> Vec { - let expired: Vec = self + let owners: BTreeSet = self .held_locks - .iter() - .filter(|(owner, keys)| { - owner.is_reservation() - && owner.epoch < epoch_threshold - && keys.iter().all(|k| { - self.table - .get(k) - .is_some_and(|e| e.mode == LockMode::Shared) - }) - }) - .map(|(owner, _)| *owner) + .keys() + .chain(self.pending_keys.keys()) + .copied() + .filter(|owner| owner.is_reservation() && owner.epoch < epoch_threshold) + .collect(); + let expired: Vec = owners + .into_iter() + .filter(|owner| self.holds_only_shared(*owner)) .collect(); let mut promoted = Vec::new(); @@ -50,15 +50,34 @@ impl LockManager { ); promoted.extend(self.release(owner)); } + promoted.sort(); + promoted.dedup(); promoted } + + /// Whether every key `owner` holds, and every key it waits for, is + /// `Shared`. A key whose entry a wound revoked from `owner` does not count. + fn holds_only_shared(&self, owner: TxnId) -> bool { + let held_shared = self.held_locks.get(&owner).is_none_or(|keys| { + keys.iter().all(|key| { + self.table + .get(key) + .is_none_or(|e| !e.holders.contains(&owner) || e.mode == LockMode::Shared) + }) + }); + let waits_shared = self + .pending_keys + .get(&owner) + .is_none_or(|keys| keys.values().all(|mode| *mode == LockMode::Shared)); + held_shared && waits_shared + } } // ── Tests ───────────────────────────────────────────────────────────────────── #[cfg(test)] mod tests { - use std::collections::BTreeSet; + use std::collections::BTreeMap; use std::sync::Arc; use super::*; @@ -72,8 +91,11 @@ mod tests { } } - fn keyset(names: &[&str]) -> BTreeSet { - names.iter().map(|n| key(n)).collect() + fn keyset(names: &[&str]) -> BTreeMap { + names + .iter() + .map(|n| (key(n), LockMode::Exclusive)) + .collect() } fn txn(epoch: u64, pos: u32) -> TxnId { @@ -176,4 +198,20 @@ mod tests { assert_eq!(entry.holders.len(), 1); assert_eq!(entry.holders[0], writer); } + + #[test] + fn reap_releases_a_waiting_reservation() { + let mut lm = LockManager::new(); + let writer = txn(1, 0); + let r = resv(1, 0); + let later = txn(2, 0); + + assert_eq!(lm.acquire(writer, keyset(&["k"])), AcquireOutcome::Ready); + assert_eq!(lm.acquire_shared(r, key("k")), AcquireOutcome::Blocked); + assert_eq!(lm.acquire(later, keyset(&["k"])), AcquireOutcome::Blocked); + + assert!(lm.reap_expired_shared(100).is_empty()); + // The reaped reservation no longer queues ahead of `later`. + assert_eq!(lm.release(writer), vec![later]); + } } diff --git a/nodedb/src/control/cluster/calvin/scheduler/metrics.rs b/nodedb/src/control/cluster/calvin/scheduler/metrics.rs index 3b5d4e62d..66df309e9 100644 --- a/nodedb/src/control/cluster/calvin/scheduler/metrics.rs +++ b/nodedb/src/control/cluster/calvin/scheduler/metrics.rs @@ -41,7 +41,7 @@ pub struct SchedulerMetrics { /// Times a staged txn parked in `AwaitingVerdict` passed its stall deadline /// with the durable global verdict still unknown. This is NOT an abort: the /// scheduler keeps waiting and holding locks (a unilateral abort while a - /// peer can already have flushed a commit will tear the transaction). A + /// peer can already have committed will tear the transaction). A /// non-zero, growing value flags a stuck sequencer / partitioned verdict /// path that needs operator attention, never a correctness action here. pub verdict_stall_count: AtomicU64, @@ -79,7 +79,9 @@ pub struct SchedulerMetrics { /// Owed sequencer entries re-proposed because their effect was not yet /// applied, by kind. Indexes are the constants in /// [`sequencer_propose_kind`]. - pub sequencer_propose_retry_counts: [AtomicU64; 4], + pub sequencer_propose_retry_counts: [AtomicU64; sequencer_propose_kind::LABELS.len()], + /// Committed txns this leader staged again after its own stage failed. + pub restage_count: AtomicU64, } /// Reason codes for `nodedb_calvin_infra_abort_total`. @@ -122,22 +124,28 @@ pub mod apply_halt_reason { pub const DISPATCH_REFUSED: usize = 1; pub const RESPONSE_DISCONNECTED: usize = 2; pub const RESOLVE_FAILED: usize = 3; - pub const FLUSH_FAILED: usize = 4; - pub const LOCAL_STAGE_FAILED: usize = 5; - pub const IDENTITY_BIND_FAILED: usize = 6; - pub const WAL_APPEND_FAILED: usize = 7; - pub const METADATA_GROUP_GONE: usize = 8; + pub const LOCAL_STAGE_FAILED: usize = 4; + pub const IDENTITY_BIND_FAILED: usize = 5; + pub const METADATA_GROUP_GONE: usize = 6; + pub const STAGE_VOTE_INVALID: usize = 7; + pub const REDO_PROPOSE_FAILED: usize = 8; + pub const REDO_INSTALL_REFUSED: usize = 9; + pub const REDO_APPLY_FAILED: usize = 10; + pub const CORE_FAIL_STOPPED: usize = 11; pub const LABELS: &[&str] = &[ "draining", "dispatch_refused", "response_disconnected", "resolve_failed", - "flush_failed", "local_stage_failed", "identity_bind_failed", - "wal_append_failed", "metadata_group_gone", + "stage_vote_invalid", + "redo_propose_failed", + "redo_install_refused", + "redo_apply_failed", + "core_fail_stopped", ]; } @@ -145,10 +153,8 @@ pub mod apply_halt_reason { pub mod sequencer_propose_kind { pub const VOTE: usize = 0; pub const COMPLETION_ACK: usize = 1; - pub const OLLP_MISMATCH: usize = 2; - pub const ROUTING_FAILED: usize = 3; - pub const LABELS: &[&str] = &["vote", "completion_ack", "ollp_mismatch", "routing_failed"]; + pub const LABELS: &[&str] = &["vote", "completion_ack"]; } impl SchedulerMetrics { @@ -207,6 +213,11 @@ impl SchedulerMetrics { self.catch_up_log_compacted.fetch_add(1, Ordering::Relaxed); } + /// Record that the leader staged a committed txn again. + pub fn record_restage(&self) { + self.restage_count.fetch_add(1, Ordering::Relaxed); + } + /// Record that the dispatcher refused a scheduler dispatch at capacity. pub fn record_dispatch_deferred(&self) { self.dispatch_deferred_count.fetch_add(1, Ordering::Relaxed); @@ -440,6 +451,7 @@ impl Default for SchedulerMetrics { apply_halted: AtomicU64::new(0), apply_halt_reason: AtomicU64::new(0), sequencer_propose_retry_counts: std::array::from_fn(|_| AtomicU64::new(0)), + restage_count: AtomicU64::new(0), } } } diff --git a/nodedb/src/control/cluster/calvin/scheduler/metrics_flow.rs b/nodedb/src/control/cluster/calvin/scheduler/metrics_flow.rs index 019437c27..5412b564e 100644 --- a/nodedb/src/control/cluster/calvin/scheduler/metrics_flow.rs +++ b/nodedb/src/control/cluster/calvin/scheduler/metrics_flow.rs @@ -1,7 +1,7 @@ // SPDX-License-Identifier: BUSL-1.1 //! Prometheus rendering of the scheduler's flow metrics: deferred dispatch, -//! the intake gate, the apply halt, and sequencer propose retries. +//! the intake gate, the apply halt, sequencer propose retries, and restages. use std::fmt::Write as _; use std::sync::atomic::Ordering; @@ -106,6 +106,18 @@ impl SchedulerMetrics { self.sequencer_propose_retry_counts[i].load(Ordering::Relaxed) ); } + + let _ = writeln!( + out, + "# HELP nodedb_calvin_restage_total \ + Committed txns the leader staged again after its own stage failed." + ); + let _ = writeln!(out, "# TYPE nodedb_calvin_restage_total counter"); + let _ = writeln!( + out, + "nodedb_calvin_restage_total{{{label}}} {}", + self.restage_count.load(Ordering::Relaxed) + ); } } @@ -128,8 +140,13 @@ mod tests { assert!(out.contains( "nodedb_calvin_sequencer_propose_retry_total{vshard=\"3\",kind=\"completion_ack\"} 1" )); - assert!(out.contains( - "nodedb_calvin_sequencer_propose_retry_total{vshard=\"3\",kind=\"routing_failed\"} 0" - )); + } + + #[test] + fn restage_counter_renders() { + let m = SchedulerMetrics::new(); + m.record_restage(); + let out = m.render_prometheus(4); + assert!(out.contains("nodedb_calvin_restage_total{vshard=\"4\"} 1")); } } diff --git a/nodedb/src/control/cluster/calvin/scheduler/mod.rs b/nodedb/src/control/cluster/calvin/scheduler/mod.rs index 26126e309..f9d862066 100644 --- a/nodedb/src/control/cluster/calvin/scheduler/mod.rs +++ b/nodedb/src/control/cluster/calvin/scheduler/mod.rs @@ -1,25 +1,33 @@ // SPDX-License-Identifier: BUSL-1.1 pub mod applied_gate; -pub mod applied_mirror; +pub mod applied_ledger; +pub mod barrier_log; +pub mod barrier_store; +pub mod caught_up; pub mod cut_floor; +pub mod cut_hold; pub mod driver; +pub mod inbox; pub mod lock; pub mod metrics; mod metrics_flow; +pub mod read_results; pub mod recovery; pub use applied_gate::AppliedGate; -pub use applied_mirror::{AppliedMirror, AppliedMirrors}; +pub use applied_ledger::{CalvinAppliedLedger, CalvinAppliedLedgers, ClaimRefusal}; +pub use barrier_log::{BarrierEvent, BarrierLog, BarrierOutcome, ReceivedReads}; +pub use caught_up::{CaughtUpHandle, CaughtUpRegistry}; pub use driver::{ - CalvinReadResultProposal, RaftSequencerProposer, ReadResultEvent, Scheduler, SchedulerConfig, - SchedulerParams, SequencerProposer, propose_calvin_read_result, + CalvinReadResultProposal, RaftSequencerProposer, Scheduler, SchedulerConfig, SchedulerParams, + SequencerProposer, propose_calvin_read_result, }; +pub use inbox::{CalvinApplyEvent, CalvinInboxes, CalvinVShardInbox, InboxHandle}; pub use lock::{AcquireOutcome, HotKeyTable, LockKey, LockManager, LockMode, TxnId}; // Existing call sites reference this module as `scheduler::lock_manager::…`; // keep that path stable via an alias while the module lives under `lock/`. pub use lock as lock_manager; pub use metrics::SchedulerMetrics; -pub use recovery::{ - AppliedRecovery, NOT_YET_APPLIED_EPOCH, read_applied_recovery, recover_applied, -}; +pub use read_results::{BufferedEvents, CalvinReadResults, VShardReadResults}; +pub use recovery::{AppliedRecovery, NOT_YET_APPLIED_EPOCH, recover_all_applied}; diff --git a/nodedb/src/control/cluster/calvin/scheduler/read_results.rs b/nodedb/src/control/cluster/calvin/scheduler/read_results.rs new file mode 100644 index 000000000..3815deef9 --- /dev/null +++ b/nodedb/src/control/cluster/calvin/scheduler/read_results.rs @@ -0,0 +1,511 @@ +// SPDX-License-Identifier: BUSL-1.1 + +//! Dependent-read barrier events that reached a vShard before its scheduler +//! took them. +//! +//! The data-group apply loop folds every `CalvinReadResult` and +//! `CalvinReadTimeout` entry into the txn's stored barrier row (see +//! [`super::barrier_store`]), then into the vShard's [`VShardReadResults`]. +//! The entry can apply before this node's scheduler granted the txn, or +//! before a scheduler for the vShard runs here at all. The events then wait +//! here. The scheduler takes a txn's events once it holds the txn: at the +//! grant, and on each push for a txn it holds. +//! +//! A txn's events wait in one of two places: +//! +//! - in memory, as a folded [`BarrierLog`]; +//! - in its stored row alone. Such a txn is "stored", and the scheduler reads +//! the row when it takes the txn. +//! +//! Memory holds at most `capacity` events per vShard. An event past it moves +//! its txn to stored: the row holds every saved event of the txn, so nothing +//! is dropped. Boot and a snapshot install mark the txns of the rows they +//! find as stored. +//! +//! An event whose row write failed stays in memory, past the bound. Its txn +//! becomes "unsaved": the row then holds a prefix of the txn's events, so +//! every later event of the txn stays in memory too and skips the row. The +//! row stays a prefix, so a restart that delivers the failed entry again +//! folds the events in log order. + +use std::collections::{BTreeMap, BTreeSet, HashMap, VecDeque}; +use std::sync::atomic::{AtomicU64, Ordering}; +use std::sync::{Arc, Mutex}; + +use tokio::sync::Notify; + +use super::barrier_log::{BarrierEvent, BarrierLog}; +use super::lock::TxnId; + +/// The waiting events of one txn, as [`VShardReadResults::take`] hands +/// them out. +#[derive(Debug, Default, PartialEq)] +pub struct BufferedEvents { + /// The events memory held, folded. + pub memory: BarrierLog, + /// Whether the txn's stored row holds events memory does not. Its + /// events are the row's, extended with `memory`. + pub stored: bool, +} + +#[derive(Debug, Default)] +struct Buffered { + logs: BTreeMap, + /// Sum of `event_count` over `logs`. + events: usize, + /// Txns whose stored row holds events memory does not. + stored: BTreeSet, + /// Txns with an event the stored row lacks. + unsaved: BTreeSet, +} + +impl Buffered { + fn remove_log(&mut self, txn: TxnId) -> Option { + let log = self.logs.remove(&txn)?; + self.events = self.events.saturating_sub(log.event_count()); + Some(log) + } + + /// Fold `log`, events after every event memory holds for `txn`. + fn extend_log(&mut self, txn: TxnId, later: BarrierLog) { + let mut log = self.remove_log(txn).unwrap_or_default(); + log.extend(later); + self.events += log.event_count(); + self.logs.insert(txn, log); + } + + /// Take `txn`'s waiting events, when it has any. + fn take(&mut self, txn: TxnId) -> Option { + let memory = self.remove_log(txn); + let stored = self.stored.remove(&txn); + if memory.is_none() && !stored { + return None; + } + Some(BufferedEvents { + memory: memory.unwrap_or_default(), + stored, + }) + } + + /// Whether `txn` has waiting events. + fn waits(&self, txn: TxnId) -> bool { + self.logs.contains_key(&txn) || self.stored.contains(&txn) + } +} + +/// One vShard's waiting barrier events. +#[derive(Debug)] +pub struct VShardReadResults { + vshard_id: u32, + state: Mutex, + /// Wakes the scheduler: an event arrived. + pushed: Notify, + /// Read-result entries the apply loop applied for the vShard, a finished + /// txn's included. + reads_applied: AtomicU64, + /// The `(group, log index)` of the latest read-result entries applied, + /// oldest first, at most [`READ_ENTRY_HISTORY`]. + read_entries: Mutex>, +} + +/// How many applied read-result entries a vShard's buffer names. +pub const READ_ENTRY_HISTORY: usize = 64; + +impl VShardReadResults { + fn new(vshard_id: u32) -> Self { + Self { + vshard_id, + state: Mutex::new(Buffered::default()), + pushed: Notify::new(), + reads_applied: AtomicU64::new(0), + read_entries: Mutex::new(VecDeque::new()), + } + } + + fn lock(&self) -> std::sync::MutexGuard<'_, Buffered> { + self.state.lock().unwrap_or_else(|p| p.into_inner()) + } + + /// The vShard this buffer serves. + pub fn vshard_id(&self) -> u32 { + self.vshard_id + } + + /// Count one read-result entry the apply loop applied, at `log_index` + /// of `group_id`. + pub fn note_read_applied(&self, group_id: u64, log_index: u64) { + self.reads_applied.fetch_add(1, Ordering::Relaxed); + let mut entries = self.read_entries.lock().unwrap_or_else(|p| p.into_inner()); + if entries.len() == READ_ENTRY_HISTORY { + entries.pop_front(); + } + entries.push_back((group_id, log_index)); + } + + /// The `(group, log index)` of the latest read-result entries applied + /// for the vShard since this buffer was created, oldest first. + pub fn read_entries(&self) -> Vec<(u64, u64)> { + self.read_entries + .lock() + .unwrap_or_else(|p| p.into_inner()) + .iter() + .copied() + .collect() + } + + /// How many read-result entries the apply loop applied for the vShard + /// since this buffer was created. + pub fn reads_applied(&self) -> u64 { + self.reads_applied.load(Ordering::Relaxed) + } + + /// Whether an event of `txn` is missing from its stored row. Every later + /// event of `txn` then skips the row. + pub fn is_unsaved(&self, txn: TxnId) -> bool { + self.lock().unsaved.contains(&txn) + } + + /// Whether some txn of the vShard has an event its stored row lacks. + pub fn has_unsaved(&self) -> bool { + !self.lock().unsaved.is_empty() + } + + /// Fold `event` of `txn`. `saved` says the event's row write + /// succeeded. + /// + /// A saved event of a stored txn stays in its row alone. A saved event + /// past `capacity` moves its txn to stored. An unsaved event stays in + /// memory whatever the bound, and marks its txn unsaved. + pub fn push(&self, txn: TxnId, event: BarrierEvent, capacity: usize, saved: bool) { + { + let mut state = self.lock(); + let in_row_alone = saved && !state.unsaved.contains(&txn); + if !in_row_alone { + state.unsaved.insert(txn); + } + let skip = if !in_row_alone { + false + } else if state.stored.contains(&txn) { + true + } else { + let grows = state.logs.get(&txn).is_none_or(|log| log.accepts(&event)); + if grows && state.events >= capacity { + state.remove_log(txn); + state.stored.insert(txn); + true + } else { + false + } + }; + if !skip { + let mut later = BarrierLog::default(); + later.note(event); + state.extend_log(txn, later); + } + } + self.pushed.notify_one(); + } + + /// Take `txn`'s waiting events. + pub fn take(&self, txn: TxnId) -> Option { + self.lock().take(txn) + } + + /// Take the waiting events of every txn `held` names, in txn order. + pub fn take_held(&self, held: impl Fn(TxnId) -> bool) -> Vec<(TxnId, BufferedEvents)> { + let mut state = self.lock(); + let txns: BTreeSet = state + .logs + .keys() + .chain(state.stored.iter()) + .copied() + .filter(|txn| held(*txn)) + .collect(); + txns.into_iter() + .filter_map(|txn| state.take(txn).map(|events| (txn, events))) + .collect() + } + + /// Hand back `earlier`, the events of `txn` a scheduler took and did + /// not finish. They come before any event buffered since. `stored` + /// says the txn's row holds events `earlier` lacks. A scheduler holds + /// at most its in-flight backlog of txns, so the hand back takes no + /// capacity check, and the next take returns them. It wakes no + /// scheduler: the events are ones a scheduler took already. + pub fn restore(&self, txn: TxnId, earlier: BufferedEvents) { + let mut state = self.lock(); + let later = state.remove_log(txn); + let mut log = earlier.memory; + if let Some(later) = later { + log.extend(later); + } + state.extend_log(txn, log); + if earlier.stored { + state.stored.insert(txn); + } + } + + /// Mark `txns` stored: their rows hold their events. + pub fn mark_stored(&self, txns: impl IntoIterator) { + self.lock().stored.extend(txns); + self.pushed.notify_one(); + } + + /// Replace every waiting event with the rows of `stored`. A snapshot + /// install replaced the vShard's rows with the builder's. + pub fn reset_to_stored(&self, stored: BTreeSet) { + { + let mut state = self.lock(); + *state = Buffered { + stored, + ..Buffered::default() + }; + } + self.pushed.notify_one(); + } + + /// Drop every waiting event of `txn`, which finished. Returns whether + /// it had any. + pub fn forget_txn(&self, txn: TxnId) -> bool { + let mut state = self.lock(); + let unsaved = state.unsaved.remove(&txn); + state.take(txn).is_some() || unsaved + } + + /// Drop the events of every txn `finished` names: its position is + /// applied, so no barrier of it opens again. + pub fn discard_finished(&self, finished: impl Fn(TxnId) -> bool) { + let mut state = self.lock(); + let done: Vec = state + .logs + .keys() + .chain(state.stored.iter()) + .chain(state.unsaved.iter()) + .copied() + .filter(|txn| finished(*txn)) + .collect(); + for txn in done { + state.take(txn); + state.unsaved.remove(&txn); + } + } + + /// Resolves once an event was pushed since the last wake. + pub async fn pushed(&self) { + self.pushed.notified().await; + } + + /// How many txns have waiting events, in memory or in their row. + pub fn waiting_txns(&self) -> usize { + let state = self.lock(); + state + .logs + .keys() + .filter(|txn| !state.stored.contains(txn)) + .count() + + state.stored.len() + } + + /// Whether `txn` has waiting events. + pub fn waits(&self, txn: TxnId) -> bool { + self.lock().waits(txn) + } +} + +/// The waiting barrier events of every vShard of this node. +#[derive(Debug, Default)] +pub struct CalvinReadResults { + by_vshard: Mutex>>, +} + +impl CalvinReadResults { + /// The buffer of `vshard_id`, created empty on first use. + pub fn vshard(&self, vshard_id: u32) -> Arc { + let mut by_vshard = self.by_vshard.lock().unwrap_or_else(|p| p.into_inner()); + Arc::clone( + by_vshard + .entry(vshard_id) + .or_insert_with(|| Arc::new(VShardReadResults::new(vshard_id))), + ) + } + + /// How many txns of `vshard_id` have waiting events. + pub fn waiting_txns(&self, vshard_id: u32) -> usize { + self.by_vshard + .lock() + .unwrap_or_else(|p| p.into_inner()) + .get(&vshard_id) + .map_or(0, |buffer| buffer.waiting_txns()) + } + + /// Drop the buffer of `vshard_id`, which left this node. + pub fn forget(&self, vshard_id: u32) { + self.by_vshard + .lock() + .unwrap_or_else(|p| p.into_inner()) + .remove(&vshard_id); + } +} + +#[cfg(test)] +mod tests { + use nodedb_physical::physical_plan::meta::PassiveReadKeyId; + use nodedb_types::{QualifiedCollection, Value}; + + use super::*; + + fn read(passive_vshard: u32) -> BarrierEvent { + BarrierEvent::Read { + passive_vshard, + values: vec![( + PassiveReadKeyId::surrogate(QualifiedCollection::from_stored("c".to_owned()), 1), + Value::Null, + )], + } + } + + /// Events of one txn fold into one log, in push order. + #[test] + fn events_of_a_txn_fold_in_push_order() { + let buffer = VShardReadResults::new(4); + let txn = TxnId::new(3, 1); + buffer.push(txn, read(1), 8, true); + buffer.push(txn, BarrierEvent::Timeout, 8, true); + buffer.push(txn, read(2), 8, true); + assert_eq!(buffer.waiting_txns(), 1); + let events = buffer.take(txn).expect("buffered log"); + assert!(!events.stored); + assert_eq!( + events.memory.missing(&[1, 2].into_iter().collect()), + vec![2] + ); + assert_eq!(buffer.waiting_txns(), 0); + } + + /// A saved event past the bound moves its txn to its row, and every + /// later saved event of it stays in the row alone. + #[test] + fn a_saved_event_past_the_bound_moves_its_txn_to_its_row() { + let buffer = VShardReadResults::new(4); + let first = TxnId::new(1, 0); + let second = TxnId::new(1, 1); + buffer.push(first, read(1), 1, true); + buffer.push(second, read(1), 1, true); + buffer.push(second, BarrierEvent::Timeout, 1, true); + assert_eq!(buffer.waiting_txns(), 2); + // A duplicate of a held event takes no room and moves nothing. + buffer.push(first, read(1), 1, true); + let first_events = buffer.take(first).expect("first txn"); + assert!(!first_events.stored); + assert_eq!(first_events.memory.event_count(), 1); + let second_events = buffer.take(second).expect("second txn"); + assert_eq!( + second_events, + BufferedEvents { + memory: BarrierLog::default(), + stored: true, + } + ); + } + + /// An unsaved event stays in memory past the bound, and every later + /// event of its txn stays in memory too. + #[test] + fn an_unsaved_event_stays_in_memory_with_every_later_one() { + let buffer = VShardReadResults::new(4); + let full = TxnId::new(1, 0); + let txn = TxnId::new(2, 0); + buffer.push(full, read(1), 1, true); + buffer.push(txn, read(1), 1, false); + assert!(buffer.is_unsaved(txn)); + assert!(buffer.has_unsaved()); + buffer.push(txn, BarrierEvent::Timeout, 1, true); + let events = buffer.take(txn).expect("unsaved txn"); + assert!(!events.stored); + assert_eq!(events.memory.event_count(), 2); + assert!(buffer.is_unsaved(txn), "the row still lacks the event"); + assert!(buffer.forget_txn(txn)); + assert!(!buffer.has_unsaved()); + } + + /// Only the txns the scheduler holds leave the buffer, stored ones too. + #[test] + fn take_held_takes_only_held_txns() { + let buffer = VShardReadResults::new(4); + buffer.push(TxnId::new(1, 0), read(1), 8, true); + buffer.push(TxnId::new(2, 0), read(1), 8, true); + buffer.mark_stored([TxnId::new(3, 0)]); + let taken = buffer.take_held(|txn| txn.epoch >= 2); + let txns: Vec = taken.iter().map(|(txn, _)| *txn).collect(); + assert_eq!(txns, vec![TxnId::new(2, 0), TxnId::new(3, 0)]); + assert!(taken[1].1.stored); + assert_eq!(buffer.waiting_txns(), 1); + } + + /// Events a stopping scheduler hands back come before the ones buffered + /// since, and a finished txn's events leave the buffer. + #[test] + fn restore_keeps_log_order_and_discard_drops_finished() { + let buffer = VShardReadResults::new(4); + let txn = TxnId::new(5, 0); + buffer.push(txn, BarrierEvent::Timeout, 8, true); + let mut earlier = BarrierLog::default(); + earlier.note(read(1)); + buffer.restore( + txn, + BufferedEvents { + memory: earlier, + stored: true, + }, + ); + let events = buffer.take(txn).expect("restored log"); + assert!(events.stored); + assert!(events.memory.missing(&[1].into_iter().collect()).is_empty()); + + buffer.push(TxnId::new(6, 0), read(1), 8, false); + buffer.mark_stored([TxnId::new(7, 0)]); + buffer.discard_finished(|txn| txn.epoch >= 6); + assert_eq!(buffer.waiting_txns(), 0); + assert!(!buffer.has_unsaved()); + } + + /// A snapshot install replaces every waiting event with its rows. + #[test] + fn a_reset_keeps_only_the_installed_rows() { + let buffer = VShardReadResults::new(4); + buffer.push(TxnId::new(1, 0), read(1), 8, false); + buffer.reset_to_stored(BTreeSet::from([TxnId::new(2, 0)])); + assert!(!buffer.has_unsaved()); + assert!(!buffer.waits(TxnId::new(1, 0))); + assert!(buffer.waits(TxnId::new(2, 0))); + } + + /// The buffer names the latest applied read-result entries, oldest + /// first, and drops the oldest past the history bound. + #[test] + fn applied_read_entries_keep_the_latest() { + let buffer = VShardReadResults::new(4); + for index in 0..(READ_ENTRY_HISTORY as u64 + 2) { + buffer.note_read_applied(3, index); + } + let entries = buffer.read_entries(); + assert_eq!(entries.len(), READ_ENTRY_HISTORY); + assert_eq!(entries.first(), Some(&(3, 2))); + assert_eq!(buffer.reads_applied(), READ_ENTRY_HISTORY as u64 + 2); + } + + /// A push wakes a waiting scheduler. + #[tokio::test] + async fn a_push_wakes_the_scheduler() { + let buffer = Arc::new(VShardReadResults::new(4)); + let waiter = { + let buffer = Arc::clone(&buffer); + tokio::spawn(async move { buffer.pushed().await }) + }; + buffer.push(TxnId::new(1, 0), read(1), 8, true); + tokio::time::timeout(std::time::Duration::from_secs(5), waiter) + .await + .expect("the push wakes the waiter") + .expect("waiter task"); + } +} diff --git a/nodedb/src/control/cluster/calvin/scheduler/recovery.rs b/nodedb/src/control/cluster/calvin/scheduler/recovery.rs index cac03b1e4..8b1779c27 100644 --- a/nodedb/src/control/cluster/calvin/scheduler/recovery.rs +++ b/nodedb/src/control/cluster/calvin/scheduler/recovery.rs @@ -2,16 +2,21 @@ //! Calvin scheduler WAL recovery. //! -//! Provides [`read_applied_recovery`] which scans the WAL for -//! `RecordType::CalvinApplied` records AND `RecordType::TransactionRedo` -//! records carrying a `calvin_stamp` (write-bearing Calvin transactions -//! journal the redo record as their applied-marker instead of a standalone -//! `CalvinApplied` marker), returning for a given vShard the union of applied -//! `(epoch, position)` pairs together with a fully-applied watermark. The -//! scheduler uses this on startup to seed its exactly-once applied gate -//! (see [`super::applied_gate::AppliedGate`]). +//! Provides [`recover_all_applied`], which scans the WAL for +//! `RecordType::TransactionRedo` records carrying a `calvin_stamp`. A +//! committed Calvin slice installs from a stamped redo entry, and its WAL +//! record is the slice's applied marker. The scan returns, per vShard, the +//! applied `(epoch, position)` pairs together with a fully-applied +//! watermark. //! -//! Each `CalvinApplied` marker is per `(epoch, position, vShard)` — one per +//! Boot runs [`recover_all_applied`] once, before the data-group apply loop +//! starts: one WAL pass yields every vShard's applied state, merged with the +//! state the last checkpoint saved. It fills the node's applied ledgers (see +//! [`super::applied_ledger::CalvinAppliedLedgers`]), and each scheduler seeds +//! its exactly-once applied gate (see [`super::applied_gate::AppliedGate`]) +//! from its vShard's ledger. +//! +//! Each stamp is per `(epoch, position, vShard)` — one per //! independent transaction position — so the scan preserves `position` rather //! than collapsing an epoch to a single "applied" bit. Collapsing to the max //! epoch will mark a whole epoch applied on the strength of its first committed @@ -26,10 +31,10 @@ //! re-fan-out supplies the counts. `max_applied_epoch` is the highest epoch with //! any marker — the rebuild-target cursor, independent of the exact gate. -use std::collections::BTreeSet; +use std::collections::{BTreeMap, BTreeSet}; +use nodedb_wal::WalRecord; use nodedb_wal::record::RecordType; -use nodedb_wal::{CalvinAppliedPayload, WalRecord}; use tracing::warn; use crate::wal::RedoRecord; @@ -39,7 +44,7 @@ use crate::wal::manager::WalManager; /// exists for this vShard. pub const NOT_YET_APPLIED_EPOCH: u64 = u64::MAX; -/// Result of scanning the WAL for a vShard's `CalvinApplied` markers. +/// Result of scanning the WAL for a vShard's stamped redo records. #[derive(Debug, Clone, PartialEq, Eq)] pub struct AppliedRecovery { /// Fully-applied watermark `W` to seed the applied gate. Conservative at @@ -53,74 +58,70 @@ pub struct AppliedRecovery { pub max_applied_epoch: u64, } -/// Scan the WAL and collect this vShard's applied `(epoch, position)` markers. +impl AppliedRecovery { + /// The state of a vShard with no applied marker. The watermark stays + /// conservative: the applied gate advances it once the re-fan-out + /// supplies the per-epoch expected counts. + fn empty() -> Self { + Self { + fully_applied_epoch: NOT_YET_APPLIED_EPOCH, + applied_tail: BTreeSet::new(), + max_applied_epoch: NOT_YET_APPLIED_EPOCH, + } + } + + /// Record one applied `(epoch, position)`. + fn note(&mut self, epoch: u64, position: u32) { + self.applied_tail.insert((epoch, position)); + if self.max_applied_epoch == NOT_YET_APPLIED_EPOCH || epoch > self.max_applied_epoch { + self.max_applied_epoch = epoch; + } + } +} + +/// Every vShard's applied `(epoch, position)` markers in one pass over the +/// WAL. `group_of` names a vShard's data group: a snapshot install of the +/// group drops the vShard's markers before it. The install replaced the +/// vShard's storage, and the applied state it brought replaced this node's +/// in the catalog. /// -/// Returns an empty tail and the [`NOT_YET_APPLIED_EPOCH`] sentinel for a -/// greenfield node (no `CalvinApplied` records exist). +/// A vShard with no stamped redo record is absent from the result. /// /// Records that fail to decode are logged and skipped — a corrupt record does /// not abort the scan. -pub fn read_applied_recovery(wal: &WalManager, vshard_id: u32) -> crate::Result { - scan_applied(wal, vshard_id, None) -} - -/// [`read_applied_recovery`] for a vShard of data group `group_id`: a -/// snapshot install of the group drops every marker before it. The install -/// replaced the vShard's storage, and the applied state it brought replaced -/// this node's in the catalog. -fn scan_applied( +fn scan_all( wal: &WalManager, - vshard_id: u32, - group_id: Option, -) -> crate::Result { + group_of: &dyn Fn(u32) -> Option, +) -> crate::Result> { let records = wal.replay()?; - let mut applied_tail = BTreeSet::new(); - let mut max_applied_epoch = NOT_YET_APPLIED_EPOCH; + let mut scans: BTreeMap = BTreeMap::new(); + let note = + |scans: &mut BTreeMap, vshard_id: u32, epoch: u64, position: u32| { + scans + .entry(vshard_id) + .or_insert_with(AppliedRecovery::empty) + .note(epoch, position); + }; for record in &records { match record_type_of(record) { Some(RecordType::SnapshotInstalled) => { - let installed = <[u8; 8]>::try_from(record.payload.as_slice()) + let Some(installed) = <[u8; 8]>::try_from(record.payload.as_slice()) .ok() - .map(u64::from_le_bytes); - if group_id.is_some() && installed == group_id { - applied_tail.clear(); - max_applied_epoch = NOT_YET_APPLIED_EPOCH; - } - } - Some(RecordType::CalvinApplied) => { - match CalvinAppliedPayload::from_bytes(&record.payload) { - Ok(p) if p.vshard_id == vshard_id => { - accumulate( - &mut applied_tail, - &mut max_applied_epoch, - p.epoch, - p.position, - ); - } - Ok(_) => { - // Different vshard — skip. - } - Err(e) => { - warn!( - lsn = record.header.lsn, - error = %e, - "calvin recovery: failed to decode CalvinApplied payload; skipping" - ); + .map(u64::from_le_bytes) + else { + continue; + }; + for (vshard_id, scan) in &mut scans { + if group_of(*vshard_id) == Some(installed) { + *scan = AppliedRecovery::empty(); } } } Some(RecordType::TransactionRedo) => match RedoRecord::from_bytes(&record.payload) { Ok(redo) => { - if let Some(stamp) = redo.calvin_stamp - && stamp.vshard_id == vshard_id - { - accumulate( - &mut applied_tail, - &mut max_applied_epoch, - stamp.epoch, - stamp.position, - ); + if let Some(stamp) = redo.calvin_stamp { + note(&mut scans, stamp.vshard_id, stamp.epoch, stamp.position); } } Err(e) => { @@ -134,19 +135,13 @@ fn scan_applied( _ => continue, } } - - Ok(AppliedRecovery { - // Conservative: the applied gate advances the watermark once the - // re-fan-out supplies the per-epoch expected counts. - fully_applied_epoch: NOT_YET_APPLIED_EPOCH, - applied_tail, - max_applied_epoch, - }) + Ok(scans) } -/// This vShard's applied state after a restart: the state the last -/// checkpoint saved in `catalog`, together with the markers the WAL still -/// holds. +/// Every vShard's applied state after a restart, in one pass over the WAL: +/// the state the last checkpoint saved in `catalog`, together with the +/// markers the WAL still holds. A vShard absent from the result has no +/// Calvin history on this node. /// /// A checkpoint deletes WAL segments that hold applied markers, while the /// sequencer log keeps delivering their entries after a restart. Without the @@ -154,19 +149,24 @@ fn scan_applied( /// one: its local stage refuses the rows it already wrote, the scheduler /// halts, and the transaction's completion ack never settles. /// -/// `group_id` is the data group of the vShard, when known: the markers +/// `group_of` names the data group of a vShard, when known: the markers /// before the group's last snapshot install no longer hold. -pub fn recover_applied( +pub fn recover_all_applied( wal: &WalManager, catalog: &crate::control::security::catalog::SystemCatalog, - vshard_id: u32, - group_id: Option, -) -> crate::Result { - let from_wal = scan_applied(wal, vshard_id, group_id)?; - let Some(saved) = catalog.load_calvin_applied(vshard_id)? else { - return Ok(from_wal); - }; - Ok(merge_saved(from_wal, saved.fully_applied_epoch, saved.tail)) + group_of: &dyn Fn(u32) -> Option, +) -> crate::Result> { + let mut recovered = scan_all(wal, group_of)?; + for saved in catalog.load_all_calvin_applied()? { + let from_wal = recovered + .remove(&saved.vshard_id) + .unwrap_or_else(AppliedRecovery::empty); + recovered.insert( + saved.vshard_id, + merge_saved(from_wal, saved.fully_applied_epoch, saved.tail), + ); + } + Ok(recovered) } /// Fold a saved `(fully_applied_epoch, tail)` into a WAL scan's result. @@ -207,37 +207,75 @@ fn record_type_of(record: &WalRecord) -> Option { RecordType::from_raw(raw_type) } -/// Feed one applied `(epoch, position)` pair into the shared tail / watermark -/// accumulation, used by both the `CalvinApplied` and `TransactionRedo` -/// (`calvin_stamp`) marker sources so the two unify into one applied set. -fn accumulate( - applied_tail: &mut BTreeSet<(u64, u32)>, - max_applied_epoch: &mut u64, - epoch: u64, - position: u32, -) { - applied_tail.insert((epoch, position)); - if *max_applied_epoch == NOT_YET_APPLIED_EPOCH || epoch > *max_applied_epoch { - *max_applied_epoch = epoch; - } -} - #[cfg(test)] mod tests { use super::*; use tempfile::TempDir; + use crate::types::{DatabaseId, TenantId, VShardId}; use crate::wal::manager::WalManager; + use crate::wal::{CalvinStamp, RedoRecord, RedoSubRecord}; fn open_wal(dir: &TempDir) -> WalManager { WalManager::open(dir.path(), false).expect("open wal") } + /// A redo record with one sub-record and the Calvin stamp `stamp`. + fn redo_record(stamp: Option) -> RedoRecord { + RedoRecord { + version: 1, + ops: vec![RedoSubRecord { + record_type: nodedb_wal::record::RecordType::Put as u32, + payload: vec![1, 2, 3], + }], + calvin_stamp: stamp, + cross_shard_applied: None, + row_sources: Vec::new(), + publishes: Vec::new(), + row_changes: Vec::new(), + } + } + + fn append_redo(wal: &WalManager, vshard: u32, record: &RedoRecord) { + wal.appender(crate::wal::manager::NO_APPLY_KEY) + .with_event_source(crate::event::EventSource::User) + .append_transaction_redo( + TenantId::new(0), + VShardId::new(vshard), + DatabaseId::DEFAULT, + record, + ) + .expect("append redo"); + } + + /// Append the stamped redo record of the Calvin slice at + /// `(epoch, position)` on `vshard`. + fn append_stamped(wal: &WalManager, vshard: u32, epoch: u64, position: u32) { + let stamp = CalvinStamp { + epoch, + position, + vshard_id: vshard, + }; + append_redo(wal, vshard, &redo_record(Some(stamp))); + } + + /// The recovered state of `vshard` alone, from a catalog with nothing + /// saved. `group_id` is the data group of `vshard`, when known. A vShard + /// with no marker recovers as `AppliedRecovery::empty()`. + fn recover_vshard(wal: &WalManager, vshard: u32, group_id: Option) -> AppliedRecovery { + let catalog = crate::control::security::catalog::SystemCatalog::open_in_memory().unwrap(); + let group_of = |v: u32| if v == vshard { group_id } else { None }; + recover_all_applied(wal, &catalog, &group_of) + .unwrap() + .remove(&vshard) + .unwrap_or_else(AppliedRecovery::empty) + } + #[test] fn greenfield_returns_sentinel_and_empty_tail() { let dir = TempDir::new().unwrap(); let wal = open_wal(&dir); - let rec = read_applied_recovery(&wal, 1).unwrap(); + let rec = recover_vshard(&wal, 1, None); assert_eq!(rec.fully_applied_epoch, NOT_YET_APPLIED_EPOCH); assert_eq!(rec.max_applied_epoch, NOT_YET_APPLIED_EPOCH); assert!(rec.applied_tail.is_empty()); @@ -248,21 +286,14 @@ mod tests { let dir = TempDir::new().unwrap(); let wal = open_wal(&dir); - use crate::types::VShardId; // Epoch 5: position 0 applied, position 1 NOT applied. Epoch 2: pos 0. - wal.appender(crate::wal::manager::NO_APPLY_KEY) - .append_calvin_applied(VShardId::new(1), 2, 0) - .unwrap(); - wal.appender(crate::wal::manager::NO_APPLY_KEY) - .append_calvin_applied(VShardId::new(1), 5, 0) - .unwrap(); + append_stamped(&wal, 1, 2, 0); + append_stamped(&wal, 1, 5, 0); // A different vshard (must be ignored). - wal.appender(crate::wal::manager::NO_APPLY_KEY) - .append_calvin_applied(VShardId::new(2), 99, 0) - .unwrap(); + append_stamped(&wal, 2, 99, 0); wal.sync().unwrap(); - let rec = read_applied_recovery(&wal, 1).unwrap(); + let rec = recover_vshard(&wal, 1, None); // The recovery API reports (5,0) applied and (5,1) NOT applied — the // exact per-position distinction the old max-epoch collapse destroyed. assert!(rec.applied_tail.contains(&(5, 0)), "(5,0) is applied"); @@ -274,7 +305,7 @@ mod tests { assert_eq!(rec.fully_applied_epoch, NOT_YET_APPLIED_EPOCH); assert!(rec.fully_applied_epoch == NOT_YET_APPLIED_EPOCH || rec.fully_applied_epoch < 5); - let rec2 = read_applied_recovery(&wal, 2).unwrap(); + let rec2 = recover_vshard(&wal, 2, None); assert!(rec2.applied_tail.contains(&(99, 0))); assert_eq!(rec2.max_applied_epoch, 99); } @@ -283,17 +314,14 @@ mod tests { fn multi_position_epoch_is_not_collapsed() { let dir = TempDir::new().unwrap(); let wal = open_wal(&dir); - use crate::types::VShardId; let vshard = 3u32; // Epoch 7 carries two independent positions on this vShard; only // position 0 committed before the crash. - wal.appender(crate::wal::manager::NO_APPLY_KEY) - .append_calvin_applied(VShardId::new(vshard), 7, 0) - .unwrap(); + append_stamped(&wal, vshard, 7, 0); wal.sync().unwrap(); - let rec = read_applied_recovery(&wal, vshard).unwrap(); + let rec = recover_vshard(&wal, vshard, None); assert!(rec.applied_tail.contains(&(7, 0))); assert!( !rec.applied_tail.contains(&(7, 1)), @@ -303,86 +331,20 @@ mod tests { } #[test] - fn transaction_redo_calvin_stamp_unions_with_calvin_applied() { - use crate::types::{DatabaseId, TenantId, VShardId}; - use crate::wal::{CalvinStamp, RedoRecord, RedoSubRecord}; - + fn only_stamped_transaction_redo_records_mark_positions_applied() { let dir = TempDir::new().unwrap(); let wal = open_wal(&dir); let vshard = 4u32; - // A pure-read/empty-ops txn still writes a standalone CalvinApplied - // marker at (epoch 1, position 0). - wal.appender(crate::wal::manager::NO_APPLY_KEY) - .append_calvin_applied(VShardId::new(vshard), 1, 0) - .unwrap(); - - // A write-bearing Calvin txn journals its applied-marker as a - // TransactionRedo record carrying a calvin_stamp at (epoch 1, position 1). - let write_bearing = RedoRecord { - version: 1, - ops: vec![RedoSubRecord { - record_type: nodedb_wal::record::RecordType::Put as u32, - payload: vec![1, 2, 3], - }], - calvin_stamp: Some(CalvinStamp { - epoch: 1, - position: 1, - vshard_id: vshard, - collections: Vec::new(), - sum_targets: Vec::new(), - }), - cross_shard_applied: None, - row_sources: Vec::new(), - publishes: Vec::new(), - row_changes: Vec::new(), - }; - wal.appender(crate::wal::manager::NO_APPLY_KEY) - .with_event_source(crate::event::EventSource::User) - .append_transaction_redo( - TenantId::new(0), - VShardId::new(vshard), - DatabaseId::DEFAULT, - &write_bearing, - ) - .unwrap(); - - // A single-shard TransactionRedo (calvin_stamp: None) must be ignored - // by Calvin recovery. - let single_shard = RedoRecord { - version: 1, - ops: vec![RedoSubRecord { - record_type: nodedb_wal::record::RecordType::Put as u32, - payload: vec![9, 9, 9], - }], - calvin_stamp: None, - cross_shard_applied: None, - row_sources: Vec::new(), - publishes: Vec::new(), - row_changes: Vec::new(), - }; - wal.appender(crate::wal::manager::NO_APPLY_KEY) - .with_event_source(crate::event::EventSource::User) - .append_transaction_redo( - TenantId::new(0), - VShardId::new(vshard), - DatabaseId::DEFAULT, - &single_shard, - ) - .unwrap(); - + append_stamped(&wal, vshard, 1, 0); + append_stamped(&wal, vshard, 1, 1); + // A session commit's record carries no stamp, and Calvin recovery + // ignores it. + append_redo(&wal, vshard, &redo_record(None)); wal.sync().unwrap(); - let rec = read_applied_recovery(&wal, vshard).unwrap(); - assert!( - rec.applied_tail.contains(&(1, 0)), - "CalvinApplied marker still contributes" - ); - assert!( - rec.applied_tail.contains(&(1, 1)), - "TransactionRedo calvin_stamp contributes its (epoch, position) too" - ); - assert_eq!(rec.applied_tail.len(), 2); + let rec = recover_vshard(&wal, vshard, None); + assert_eq!(rec.applied_tail, [(1, 0), (1, 1)].into_iter().collect()); assert_eq!(rec.max_applied_epoch, 1); } @@ -390,11 +352,8 @@ mod tests { fn saved_state_restores_what_a_truncated_wal_lost() { let dir = TempDir::new().unwrap(); let wal = open_wal(&dir); - use crate::types::VShardId; - // The WAL still holds only the marker written after the checkpoint. - wal.appender(crate::wal::manager::NO_APPLY_KEY) - .append_calvin_applied(VShardId::new(1), 6, 0) - .unwrap(); + // The WAL still holds only the record written after the checkpoint. + append_stamped(&wal, 1, 6, 0); wal.sync().unwrap(); let catalog_dir = TempDir::new().unwrap(); let catalog = crate::control::security::catalog::SystemCatalog::open( @@ -411,41 +370,100 @@ mod tests { ]) .unwrap(); - let rec = recover_applied(&wal, &catalog, 1, None).unwrap(); + let recovered = recover_all_applied(&wal, &catalog, &|_| None).unwrap(); + let rec = &recovered[&1]; assert_eq!(rec.fully_applied_epoch, 2); assert!(rec.applied_tail.contains(&(4, 1))); assert!(rec.applied_tail.contains(&(6, 0))); assert_eq!(rec.max_applied_epoch, 6); - // A vShard with nothing saved keeps the plain WAL scan. - let other = recover_applied(&wal, &catalog, 9, None).unwrap(); - assert_eq!(other, read_applied_recovery(&wal, 9).unwrap()); + // A vShard with nothing saved and no marker has no history. + assert!(!recovered.contains_key(&9)); + } + + fn empty_catalog(dir: &TempDir) -> crate::control::security::catalog::SystemCatalog { + crate::control::security::catalog::SystemCatalog::open(&dir.path().join("system.redb")) + .unwrap() + } + + /// The one boot pass yields, for every vShard, what a scan of that vShard + /// alone yields. + #[test] + fn one_pass_recovery_matches_per_vshard_scans() { + let dir = TempDir::new().unwrap(); + let wal = open_wal(&dir); + let appender = || wal.appender(crate::wal::manager::NO_APPLY_KEY); + append_stamped(&wal, 1, 1, 0); + append_stamped(&wal, 2, 1, 1); + appender().append_snapshot_installed(10).unwrap(); + append_stamped(&wal, 3, 4, 0); + append_stamped(&wal, 1, 2, 3); + appender().append_snapshot_installed(20).unwrap(); + append_stamped(&wal, 2, 6, 0); + wal.sync().unwrap(); + let group_of = |vshard: u32| match vshard { + 1 => Some(10), + 2 => Some(20), + _ => None, + }; + let catalog_dir = TempDir::new().unwrap(); + let catalog = empty_catalog(&catalog_dir); + + let all = recover_all_applied(&wal, &catalog, &group_of).unwrap(); + + assert_eq!(all.keys().copied().collect::>(), vec![1, 2, 3]); + for (vshard, recovered) in &all { + let alone = recover_vshard(&wal, *vshard, group_of(*vshard)); + assert_eq!(recovered, &alone, "vShard {vshard}"); + } + assert_eq!(all[&1].applied_tail, [(2, 3)].into_iter().collect()); + assert_eq!(all[&2].applied_tail, [(6, 0)].into_iter().collect()); + assert_eq!(all[&3].applied_tail, [(4, 0)].into_iter().collect()); + } + + /// A snapshot install record drops the markers of its own group's + /// vShards only. + #[test] + fn a_snapshot_install_record_clears_only_its_group() { + let dir = TempDir::new().unwrap(); + let wal = open_wal(&dir); + append_stamped(&wal, 1, 3, 0); + append_stamped(&wal, 2, 3, 1); + wal.appender(crate::wal::manager::NO_APPLY_KEY) + .append_snapshot_installed(8) + .unwrap(); + append_stamped(&wal, 1, 5, 2); + wal.sync().unwrap(); + let group_of = |vshard: u32| if vshard == 1 { Some(8) } else { Some(9) }; + let catalog_dir = TempDir::new().unwrap(); + let catalog = empty_catalog(&catalog_dir); + + let all = recover_all_applied(&wal, &catalog, &group_of).unwrap(); + + assert_eq!(all[&1].applied_tail, [(5, 2)].into_iter().collect()); + assert_eq!(all[&1].max_applied_epoch, 5); + assert_eq!(all[&2].applied_tail, [(3, 1)].into_iter().collect()); } /// A snapshot install of the vShard's group drops every marker before /// it: the install replaced the storage those positions wrote. #[test] fn a_group_snapshot_install_drops_the_markers_before_it() { - use crate::types::VShardId; let dir = TempDir::new().unwrap(); let wal = open_wal(&dir); let appender = || wal.appender(crate::wal::manager::NO_APPLY_KEY); - appender() - .append_calvin_applied(VShardId::new(1), 3, 0) - .unwrap(); + append_stamped(&wal, 1, 3, 0); appender().append_snapshot_installed(8).unwrap(); - appender() - .append_calvin_applied(VShardId::new(1), 5, 2) - .unwrap(); + append_stamped(&wal, 1, 5, 2); appender().append_snapshot_installed(9).unwrap(); wal.sync().unwrap(); - let in_group = scan_applied(&wal, 1, Some(8)).unwrap(); + let in_group = recover_vshard(&wal, 1, Some(8)); assert_eq!(in_group.applied_tail, [(5, 2)].into_iter().collect()); assert_eq!(in_group.max_applied_epoch, 5); - let other_group = scan_applied(&wal, 1, Some(4)).unwrap(); - assert_eq!(other_group, read_applied_recovery(&wal, 1).unwrap()); + let other_group = recover_vshard(&wal, 1, Some(4)); + assert_eq!(other_group, recover_vshard(&wal, 1, None)); assert!(other_group.applied_tail.contains(&(3, 0))); } } diff --git a/nodedb/src/control/cluster/calvin_snapshot/capture.rs b/nodedb/src/control/cluster/calvin_snapshot/capture.rs index ff95938c9..cc9fb8fea 100644 --- a/nodedb/src/control/cluster/calvin_snapshot/capture.rs +++ b/nodedb/src/control/cluster/calvin_snapshot/capture.rs @@ -17,6 +17,7 @@ use std::time::Duration; use nodedb_cluster::calvin::SequencerEntry; use crate::Error; +use crate::control::cluster::calvin::scheduler::barrier_store; use crate::control::security::catalog::calvin_base::CalvinBase; use crate::control::state::SharedState; use crate::types::{GroupCalvinCut, VShardCalvinState}; @@ -68,6 +69,7 @@ pub async fn await_calvin_cut( let marker = zerompk::to_msgpack_vec(&SequencerEntry::CutMarker { hlc, restore_point: 0, + barrier: None, }) .map_err(|error| Error::Internal { detail: format!("snapshot build: group {group_id}: encode the Calvin cut marker: {error}"), @@ -102,20 +104,24 @@ pub async fn await_calvin_cut( } /// The group's Calvin cut through `through`: the applied positions of every -/// group vShard here. The caller holds the group's apply gate exclusive, so -/// no Calvin install of these vShards runs. +/// group vShard here, and the stored barrier logs of their unfinished txns. +/// The caller holds the group's apply gate exclusive, so no Calvin install +/// of these vShards runs and no barrier entry of the group applies. +/// +/// Fails when the barrier logs cannot be read whole (see +/// [`barrier_store::capture_group`]). The build retries on the next +/// heartbeat. pub fn capture_calvin_cut( shared: &SharedState, group_vshards: &HashSet, through: u64, -) -> GroupCalvinCut { - let mirrors = shared.authorization_fence.calvin_mirrors(); - let mut ids: Vec = group_vshards.iter().copied().collect(); - ids.sort_unstable(); +) -> Result { + let ledgers = &shared.calvin.applied; + let ids: BTreeSet = group_vshards.iter().copied().collect(); let vshards = ids - .into_iter() - .filter_map(|vshard_id| { - let (fully_applied_epoch, tail) = mirrors.get(vshard_id)?.snapshot(); + .iter() + .filter_map(|&vshard_id| { + let (fully_applied_epoch, tail) = ledgers.get(vshard_id)?.snapshot(); Some(VShardCalvinState { vshard_id, fully_applied_epoch, @@ -123,5 +129,10 @@ pub fn capture_calvin_cut( }) }) .collect(); - GroupCalvinCut { through, vshards } + let barrier_logs = barrier_store::capture_group(shared, &ids)?; + Ok(GroupCalvinCut { + through, + vshards, + barrier_logs, + }) } diff --git a/nodedb/src/control/cluster/calvin_snapshot/install.rs b/nodedb/src/control/cluster/calvin_snapshot/install.rs index b528d7cbd..582ec801d 100644 --- a/nodedb/src/control/cluster/calvin_snapshot/install.rs +++ b/nodedb/src/control/cluster/calvin_snapshot/install.rs @@ -5,17 +5,21 @@ //! The snapshot replaced the storage of the group's vShards with the //! leader's, which holds exactly the Calvin positions the cut names. So //! each vShard's applied state here becomes the cut's: in the applied -//! mirror, which a checkpoint saves from, and in the catalog, which a -//! scheduler starts from. Its base becomes the cut's sequencer index, which -//! moves its generation: a scheduler started before the install installs -//! nothing more, and the next scheduler reconcile starts it again from the -//! installed state. +//! ledger, which a scheduler starts from and a checkpoint saves from, and +//! in the catalog, which boot recovery reads. Its base becomes the cut's +//! sequencer index, which moves its generation: a scheduler started before +//! the install installs nothing more, and the next scheduler reconcile +//! starts it again from the installed state. +//! +//! The stored dependent-read barrier logs of the group's vShards become the +//! cut's too: the snapshot covers barrier entries this node never applies. //! //! A WAL `SnapshotInstalled` record of the group, written before this runs, //! makes boot recovery drop the vShards' applied markers from before it. use std::collections::{BTreeSet, HashSet}; +use crate::control::cluster::calvin::scheduler::barrier_store; use crate::control::cluster::snapshot_install::SnapshotInstallError; use crate::control::security::catalog::calvin_applied::StoredCalvinApplied; use crate::control::state::SharedState; @@ -52,9 +56,12 @@ pub fn install_calvin_cut( }) .collect(); - let mirrors = shared.authorization_fence.calvin_mirrors(); for state in &states { - mirrors.register(state.vshard_id, state.fully_applied_epoch, &state.tail); + shared.calvin.applied.install( + state.vshard_id, + state.fully_applied_epoch, + state.tail.clone(), + ); } let catalog = shared.credentials.catalog(); let settle_error = |source| SnapshotInstallError::Settle { @@ -65,6 +72,10 @@ pub fn install_calvin_cut( catalog .replace_calvin_applied(&states) .map_err(settle_error)?; + // The receiver never applies the barrier entries the snapshot covers: + // the builder's stored logs of the unfinished txns replace this node's. + let vshards: BTreeSet = ids.iter().copied().collect(); + barrier_store::install_group(shared, &vshards, cut.barrier_logs).map_err(settle_error)?; shared .calvin .bases diff --git a/nodedb/src/control/cluster/calvin_snapshot/requirement.rs b/nodedb/src/control/cluster/calvin_snapshot/requirement.rs index 8a47eb1b5..a337335f8 100644 --- a/nodedb/src/control/cluster/calvin_snapshot/requirement.rs +++ b/nodedb/src/control/cluster/calvin_snapshot/requirement.rs @@ -18,6 +18,10 @@ //! sequencer log, and then keeps the base (see [`may_start`]). //! - The sequencer log keeps the range a waiting vShard will replay (see //! [`crate::control::state::CalvinBases::replay_floor`]). +//! +//! A data group whose chunked redo streams this node dropped when it left +//! the group requires a snapshot too, until one installs (see +//! `wal_replication::transaction_redo::chunks::owed`). use std::collections::BTreeSet; use std::sync::{Arc, Mutex, RwLock}; @@ -27,8 +31,9 @@ use nodedb_cluster::multi_raft::MultiRaft; use crate::control::security::catalog::calvin_base::CalvinBase; use crate::control::state::SharedState; -/// Load this node's Calvin bases and install the data-group snapshot -/// requirement on `multi_raft`, before its data groups take entries. +/// Load this node's Calvin bases and owed redo snapshots, and install the +/// data-group snapshot requirement on `multi_raft`, before its data groups +/// take entries. pub fn install_snapshot_requirement( shared: &Arc, routing: Arc>, @@ -39,6 +44,9 @@ pub fn install_snapshot_requirement( catalog.load_calvin_bases()?, catalog.load_calvin_sequencer_install()?, ); + shared + .redo_chunks + .load_owed(catalog.load_redo_snapshot_owed()?); let weak = Arc::downgrade(shared); multi_raft.set_snapshot_requirement(Arc::new(move |group_id, sequencer_first| { let Some(shared) = weak.upgrade() else { @@ -48,7 +56,9 @@ pub fn install_snapshot_requirement( .read() .unwrap_or_else(|p| p.into_inner()) .vshards_for_group(group_id); - !shared.calvin.bases.group_reaches(&vshards, sequencer_first) + // Evaluated first and always: it registers the vShards as waiting. + let reaches = shared.calvin.bases.group_reaches(&vshards, sequencer_first); + !reaches || shared.redo_chunks.owes_snapshot(group_id) })); Ok(()) } @@ -161,7 +171,6 @@ pub fn forget_left(shared: &SharedState, vshards: &[u32]) { return; } let catalog = shared.credentials.catalog(); - let mirrors = shared.authorization_fence.calvin_mirrors(); let resets: Vec = vshards .iter() @@ -175,7 +184,7 @@ pub fn forget_left(shared: &SharedState, vshards: &[u32]) { }) .collect(); for &vshard_id in vshards { - mirrors.remove(vshard_id); + shared.calvin.applied.remove(vshard_id); } let forgotten = catalog .replace_calvin_applied(&resets) diff --git a/nodedb/src/control/cluster/data_plane_error_wire.rs b/nodedb/src/control/cluster/data_plane_error_wire.rs index e4d6de18c..52b3c9395 100644 --- a/nodedb/src/control/cluster/data_plane_error_wire.rs +++ b/nodedb/src/control/cluster/data_plane_error_wire.rs @@ -38,6 +38,10 @@ impl From for DataPlaneErrorCode { } ErrorCode::RejectedPrevalidation { reason } => Self::RejectedPrevalidation { reason }, ErrorCode::RetryableRefusal { reason } => Self::RetryableRefusal { reason }, + ErrorCode::CoreFailStopped { core_id, detail } => Self::CoreFailStopped { + core_id: to_wire_count(core_id), + detail, + }, ErrorCode::SyncRejected { violation, applied_seq, @@ -186,6 +190,10 @@ impl From for ErrorCode { Self::RejectedPrevalidation { reason } } DataPlaneErrorCode::RetryableRefusal { reason } => Self::RetryableRefusal { reason }, + DataPlaneErrorCode::CoreFailStopped { core_id, detail } => Self::CoreFailStopped { + core_id: from_wire_count(core_id), + detail, + }, DataPlaneErrorCode::SyncRejected { violation, applied_seq, @@ -373,6 +381,16 @@ mod tests { } } + #[test] + fn a_core_fail_stop_roundtrips_verbatim() { + let original = ErrorCode::CoreFailStopped { + core_id: 3, + detail: "rollback failed".into(), + }; + let wire = DataPlaneErrorCode::from(original.clone()); + assert_eq!(ErrorCode::from(wire), original); + } + #[test] fn division_by_zero_survives_the_wire_hop() { let wire = DataPlaneErrorCode::from(ErrorCode::DivisionByZero); diff --git a/nodedb/src/control/cluster/metadata_applier/boot_seed.rs b/nodedb/src/control/cluster/metadata_applier/boot_seed.rs index 9df6fa735..98ca9f3dc 100644 --- a/nodedb/src/control/cluster/metadata_applier/boot_seed.rs +++ b/nodedb/src/control/cluster/metadata_applier/boot_seed.rs @@ -7,7 +7,7 @@ //! loads those rows into the in-memory state before any entry applies. //! State that no row holds starts fresh: //! - `last_applied_hlc` is the highest expiry over the seeded leases and drains. -//! - `metadata_ddl_applied_token` is 0. +//! - `metadata_ddl.applied_token` is 0. //! - `topology_log`, `routing_log`, and `catalog_entries_applied` are empty. //! //! A read error fails boot: running with a partial view of leases or drains @@ -45,7 +45,8 @@ pub fn seed_host_tables(shared: &SharedState) -> crate::Result<()> { .insert(record.token, record.objects, record.proposed_at); } *shared - .metadata_ddl_owner + .metadata_ddl + .owner .lock() .unwrap_or_else(|p| p.into_inner()) = owner.map( @@ -240,7 +241,8 @@ mod tests { assert_eq!(record.proposed_at, Hlc::new(60, 0)); assert_eq!( state - .metadata_ddl_owner + .metadata_ddl + .owner .lock() .unwrap() .map(|owner| (owner.token, owner.node_id)), @@ -248,7 +250,8 @@ mod tests { ); assert_eq!( state - .metadata_ddl_applied_token + .metadata_ddl + .applied_token .load(std::sync::atomic::Ordering::Acquire), 0 ); diff --git a/nodedb/src/control/cluster/metadata_applier/catalog_ddl.rs b/nodedb/src/control/cluster/metadata_applier/catalog_ddl.rs index be9bfdc24..c71d79712 100644 --- a/nodedb/src/control/cluster/metadata_applier/catalog_ddl.rs +++ b/nodedb/src/control/cluster/metadata_applier/catalog_ddl.rs @@ -89,12 +89,16 @@ impl MetadataCommitApplier { stamped, catalog_entry::CatalogEntry::PutBackupScheduleMark(_) ) { - nodedb_types::fail_point_err!(&backup_mark_fail_point(shared.node_id), |detail| { - crate::Error::Storage { - engine: "catalog".into(), - detail, + nodedb_types::fail_point_err!( + nodedb_types::fail_point::FailScope::Node(shared.node_id), + BACKUP_MARK_FAIL_POINT, + |detail| { + crate::Error::Storage { + engine: "catalog".into(), + detail, + } } - }); + ); } // Descriptor versions (and the constraint_version / @@ -175,8 +179,6 @@ impl MetadataCommitApplier { } } -/// The fail point that holds back node `node_id`'s apply of backup schedule -/// marks. -pub fn backup_mark_fail_point(node_id: u64) -> String { - format!("backup_schedule::mark_apply::node{node_id}") -} +/// The fail point that holds back a node's apply of backup schedule marks. +/// A test arms it for one node. +pub const BACKUP_MARK_FAIL_POINT: &str = "backup_schedule::mark_apply"; diff --git a/nodedb/src/control/cluster/metadata_applier/dispatch.rs b/nodedb/src/control/cluster/metadata_applier/dispatch.rs index 34dabf8c1..fdf8343f5 100644 --- a/nodedb/src/control/cluster/metadata_applier/dispatch.rs +++ b/nodedb/src/control/cluster/metadata_applier/dispatch.rs @@ -261,7 +261,8 @@ impl MetadataCommitApplier { } self.apply_host_side_effects(entry, raft_index).await?; shared - .metadata_ddl_applied_token + .metadata_ddl + .applied_token .store(token, std::sync::atomic::Ordering::Release); Ok(()) } @@ -314,10 +315,9 @@ impl MetadataCommitApplier { } } -/// The fail point that holds back node `node_id`'s whole metadata apply. -pub fn metadata_apply_hold_point(node_id: u64) -> String { - format!("metadata_apply::hold::node{node_id}") -} +/// The fail point that holds back a node's whole metadata apply. A test arms +/// it for one node. +pub const METADATA_APPLY_HOLD_POINT: &str = "metadata_apply::hold"; #[async_trait::async_trait] impl MetadataApplier for MetadataCommitApplier { @@ -509,7 +509,7 @@ mod tests { /// `DdlPrepareAcquire` does. A pending propose and finalize apply only /// under the owner's token. fn own_ddl_lease(state: &SharedState, token: u64) { - *state.metadata_ddl_owner.lock().unwrap() = + *state.metadata_ddl.owner.lock().unwrap() = Some(crate::control::metadata_proposer::DdlPrepareOwner { token, node_id: state.node_id, @@ -564,7 +564,8 @@ mod tests { } assert_ne!( state - .metadata_ddl_applied_token + .metadata_ddl + .applied_token .load(std::sync::atomic::Ordering::Acquire), dead, "the dead owner's proposer must see its entries superseded" diff --git a/nodedb/src/control/cluster/metadata_applier/host_state.rs b/nodedb/src/control/cluster/metadata_applier/host_state.rs index c47f7d6f6..8177d65f8 100644 --- a/nodedb/src/control/cluster/metadata_applier/host_state.rs +++ b/nodedb/src/control/cluster/metadata_applier/host_state.rs @@ -32,7 +32,8 @@ impl MetadataCommitApplier { ) -> Result<(), crate::Error> { let shared = self.shared_state()?; let mut owner = shared - .metadata_ddl_owner + .metadata_ddl + .owner .lock() .unwrap_or_else(|poisoned| poisoned.into_inner()); if owner.is_none() { @@ -53,7 +54,8 @@ impl MetadataCommitApplier { pub(super) fn apply_ddl_prepare_release(&self, token: u64) -> Result<(), crate::Error> { let shared = self.shared_state()?; let mut owner = shared - .metadata_ddl_owner + .metadata_ddl + .owner .lock() .unwrap_or_else(|poisoned| poisoned.into_inner()); if owner.is_some_and(|current| current.token == token) { diff --git a/nodedb/src/control/cluster/metadata_applier/mod.rs b/nodedb/src/control/cluster/metadata_applier/mod.rs index e5c28f67e..68f75ea0f 100644 --- a/nodedb/src/control/cluster/metadata_applier/mod.rs +++ b/nodedb/src/control/cluster/metadata_applier/mod.rs @@ -62,7 +62,7 @@ mod types; mod wedge; pub use boot_seed::{seed_host_tables, seed_metadata_cache}; -pub use catalog_ddl::backup_mark_fail_point; -pub use dispatch::metadata_apply_hold_point; +pub use catalog_ddl::BACKUP_MARK_FAIL_POINT; +pub use dispatch::METADATA_APPLY_HOLD_POINT; pub use types::{CATALOG_CHANNEL_CAPACITY, CatalogChangeEvent, MetadataCommitApplier}; pub use wedge::{ApplyFailureClass, MetadataApplyWedge, WedgeReport, classify}; diff --git a/nodedb/src/control/cluster/metadata_applier/pending_ddl.rs b/nodedb/src/control/cluster/metadata_applier/pending_ddl.rs index c9a415c0d..acc8563f9 100644 --- a/nodedb/src/control/cluster/metadata_applier/pending_ddl.rs +++ b/nodedb/src/control/cluster/metadata_applier/pending_ddl.rs @@ -78,7 +78,7 @@ impl MetadataCommitApplier { /// for the next re-delivery instead of silently skipping the rest. /// /// Applies only while `token` owns the preparation lease. A finalize - /// that applies records `token` in `metadata_ddl_applied_token`, which + /// that applies records `token` in `metadata_ddl.applied_token`, which /// is how its proposer learns the objects landed. pub(super) async fn apply_ddl_pending_finalize( &self, @@ -104,7 +104,8 @@ impl MetadataCommitApplier { self.credentials.catalog().remove_pending_ddl(token)?; shared.pending_ddl.take(token); shared - .metadata_ddl_applied_token + .metadata_ddl + .applied_token .store(token, std::sync::atomic::Ordering::Release); Ok(()) } diff --git a/nodedb/src/control/cluster/mod.rs b/nodedb/src/control/cluster/mod.rs index 8f8e0abdf..de6a5717e 100644 --- a/nodedb/src/control/cluster/mod.rs +++ b/nodedb/src/control/cluster/mod.rs @@ -27,6 +27,7 @@ pub mod metadata_stamp; pub mod pem_io; pub mod read_index; pub mod recovery_check; +pub mod redo_stream_membership; pub mod sequencer_compaction; pub mod sequencer_halt; pub mod sequencer_snapshot; diff --git a/nodedb/src/control/cluster/read_index.rs b/nodedb/src/control/cluster/read_index.rs index d2dcd06ec..86493a307 100644 --- a/nodedb/src/control/cluster/read_index.rs +++ b/nodedb/src/control/cluster/read_index.rs @@ -118,6 +118,7 @@ fn refusal_of(error: ClusterError) -> ReadIndexRefusal { | ClusterError::GhostNotFound { .. } | ClusterError::Transport { .. } | ClusterError::ShardTimeout { .. } + | ClusterError::Unanswered { .. } | ClusterError::StreamTerminal { .. } | ClusterError::Storage { .. } | ClusterError::DataPlane { .. } diff --git a/nodedb/src/control/cluster/redo_stream_membership.rs b/nodedb/src/control/cluster/redo_stream_membership.rs new file mode 100644 index 000000000..fdcc58ee0 --- /dev/null +++ b/nodedb/src/control/cluster/redo_stream_membership.rs @@ -0,0 +1,92 @@ +// SPDX-License-Identifier: BUSL-1.1 + +//! The membership pass over this node's chunked redo streams. +//! +//! - A data group this node no longer mounts gets no later entry here, so +//! no final entry, abandon, or later term closes its streams. They drop +//! here, with their WAL floor holds. The group then owes a snapshot +//! install before it applies here again. A remount resumes its old log, +//! which can still name a dropped stream. +//! - A mounted group that owes a snapshot refuses log entries until one +//! installs, and its leadership here moves to a replica that can lead. +//! This covers a group that mounted again, and a group whose apply here +//! held an entry of a dropped stream. + +use std::sync::Mutex; + +use nodedb_cluster::multi_raft::MultiRaft; + +use crate::control::state::SharedState; + +/// Run the pass once. The membership reconcile calls it on every tick. +pub fn reconcile_redo_streams(shared: &SharedState, multi_raft: &Mutex) { + drop_unmounted(shared, multi_raft); + require_owed_snapshots(shared, multi_raft); +} + +/// Drop the streams of every data group this node no longer mounts. +/// +/// The `MultiRaft` lock is held throughout. A group that mounts meanwhile +/// keeps its streams, and a remount sees the group's debt. The debt write +/// is a catalog commit under that lock. It runs only when a left group +/// still holds a stream. +fn drop_unmounted(shared: &SharedState, multi_raft: &Mutex) { + let dropped = { + let multi_raft = multi_raft.lock().unwrap_or_else(|p| p.into_inner()); + shared.redo_chunks.drop_unhosted_groups( + |group_id| multi_raft.contains_group(group_id), + shared.credentials.catalog(), + ) + }; + match dropped { + Ok(0) => {} + Ok(dropped) => tracing::info!( + dropped, + "redo chunks: dropped the open streams of data groups this node left; each such \ + group installs a snapshot before it applies here again" + ), + Err(error) => { + tracing::error!( + %error, + "redo chunks: the snapshot debt of data groups this node left did not \ + persist; their streams stay until a later pass records it" + ); + crate::diag::redo_snapshot_debt_not_recorded(&error); + } + } +} + +/// Make every mounted group that owes a snapshot refuse log entries, and +/// move a leadership of it this node holds: a leader takes no snapshot. +/// +/// The debt is read under the `MultiRaft` lock. An install settles the debt +/// before it refreshes the group's requirement under the same lock, so this +/// pass never raises a requirement an install has paid. +fn require_owed_snapshots(shared: &SharedState, multi_raft: &Mutex) { + let owed = shared.redo_chunks.owed_groups(); + if owed.is_empty() { + return; + } + let mut multi_raft = multi_raft.lock().unwrap_or_else(|p| p.into_inner()); + for group_id in owed { + if !multi_raft.contains_group(group_id) || !shared.redo_chunks.owes_snapshot(group_id) { + continue; + } + if !multi_raft.snapshot_required(group_id) { + multi_raft.set_snapshot_required(group_id, true); + tracing::warn!( + group_id, + "redo chunks: the group's replica here owes a snapshot install; it refuses \ + log entries until one installs" + ); + } + if let Some(target) = multi_raft.hand_off_leadership(group_id) { + tracing::info!( + group_id, + target, + "redo chunks: the group's replica here owes a snapshot install; its \ + leadership moves to a replica that can lead" + ); + } + } +} diff --git a/nodedb/src/control/cluster/sequencer_halt.rs b/nodedb/src/control/cluster/sequencer_halt.rs index ff4d9e466..f4c3ff903 100644 --- a/nodedb/src/control/cluster/sequencer_halt.rs +++ b/nodedb/src/control/cluster/sequencer_halt.rs @@ -122,9 +122,9 @@ mod tests { vshard_id, epoch: 9, position: 1, - reason: "flush_failed", - step: "flush", - error: "flush returned Error".to_string(), + reason: "redo_apply_failed", + step: "redo_apply", + error: "redo install did not become durable".to_string(), } } diff --git a/nodedb/src/control/cluster/sequencer_snapshot.rs b/nodedb/src/control/cluster/sequencer_snapshot.rs index 96c7bdec6..f25219720 100644 --- a/nodedb/src/control/cluster/sequencer_snapshot.rs +++ b/nodedb/src/control/cluster/sequencer_snapshot.rs @@ -104,28 +104,44 @@ impl SequencerSnapshotStore { /// The log then delivers only the entries after the file's index again: /// the state machine skips every entry at or below the index it /// restored. + /// + /// A boundary above 0 with no usable file leaves the entries through the + /// boundary unrecoverable. The state machine then records its history as + /// unknown, so its leader never seeds an epoch from partial state. pub fn restore_at_boot( &self, multi_raft: &nodedb_cluster::multi_raft::MultiRaft, ) -> crate::Result { + let Ok((_, boundary, _)) = multi_raft.snapshot_metadata(SEQUENCER_GROUP_ID) else { + return Ok(false); + }; + let snapshot = self.read_kept()?; + let mut state_machine = self.state_machine.lock().unwrap_or_else(|p| p.into_inner()); + match snapshot { + Some(snapshot) if snapshot.applied_index() >= boundary => { + state_machine.restore_snapshot(snapshot); + Ok(true) + } + _ => { + if boundary > 0 { + state_machine.mark_history_unknown(); + } + Ok(false) + } + } + } + + /// The kept snapshot, or `None` when no file exists. + fn read_kept(&self) -> crate::Result> { let path = self.dir.join(SNAPSHOT_FILE); let bytes = match std::fs::read(&path) { Ok(bytes) => bytes, - Err(e) if e.kind() == std::io::ErrorKind::NotFound => return Ok(false), + Err(e) if e.kind() == std::io::ErrorKind::NotFound => return Ok(None), Err(e) => return Err(storage_error(&path, "read", e)), }; - let snapshot = SequencerSnapshot::decode(&bytes).map_err(codec_error)?; - let Ok((_, boundary, _)) = multi_raft.snapshot_metadata(SEQUENCER_GROUP_ID) else { - return Ok(false); - }; - if snapshot.applied_index() < boundary { - return Ok(false); - } - self.state_machine - .lock() - .unwrap_or_else(|p| p.into_inner()) - .restore_snapshot(snapshot); - Ok(true) + SequencerSnapshot::decode(&bytes) + .map(Some) + .map_err(codec_error) } } diff --git a/nodedb/src/control/cluster/snapshot_applier.rs b/nodedb/src/control/cluster/snapshot_applier.rs index e75512b64..76d1def3e 100644 --- a/nodedb/src/control/cluster/snapshot_applier.rs +++ b/nodedb/src/control/cluster/snapshot_applier.rs @@ -50,7 +50,7 @@ use crate::bridge::envelope::PhysicalPlan; use crate::control::cluster::snapshot_install::{ CoreMap, CoreShares, SettleStep, SnapshotInstallError, append_install_barrier, append_install_marker, clear_targets_per_core, group_collections, install_on_every_core, - split_by_core, + split_by_core, version_floor, }; use crate::control::state::SharedState; use crate::types::{SurrogateBindEntry, TenantDataSnapshot, TenantId, VShardId}; @@ -167,6 +167,7 @@ impl DataPlaneSnapshotApplier { cut_index, event_lane, calvin, + redo_streams, } = split_by_core(group_id, snap, &cores)?; // Refused before any core changes: storage installed without its // Calvin cut will hold Calvin transactions no applied state names. @@ -179,6 +180,7 @@ impl DataPlaneSnapshotApplier { // vShards. A store with none of them stays as it is. let mut vshards: Vec = group_vshards.iter().copied().collect(); vshards.sort_unstable(); + let version_floor = version_floor(&self.shared, group_id, &vshards, cut_index); let plans = per_core .into_iter() @@ -197,6 +199,7 @@ impl DataPlaneSnapshotApplier { replace_mode: true, collections_to_clear, group_vshards: vshards.clone(), + version_floor: version_floor.clone(), })) }) .collect::, SnapshotInstallError>>()?; @@ -214,6 +217,17 @@ impl DataPlaneSnapshotApplier { &group_vshards, calvin, )?; + // After the install marker too: boot drops the group's earlier streams + // there and rebuilds these. + self.shared + .redo_chunks + .install_group(group_id, redo_streams) + .await + .map_err(|source| SnapshotInstallError::Settle { + group_id, + step: SettleStep::RedoStreams, + source: source.into(), + })?; crate::control::pitr::force_base_after_install(&self.shared); self.settle( group_id, @@ -243,6 +257,17 @@ impl DataPlaneSnapshotApplier { if let Some(tracker) = self.shared.propose_tracker.get() { tracker.cover_through(group_id, cut_index); } + // Last: the group's streams here are the snapshot's now, so a + // snapshot this node owed the group is paid. An install that fails + // earlier leaves the debt, and the group keeps requiring one. + self.shared + .redo_chunks + .settle_owed(self.shared.credentials.catalog(), group_id) + .map_err(|source| SnapshotInstallError::Settle { + group_id, + step: SettleStep::RedoStreams, + source, + })?; Ok(()) } diff --git a/nodedb/src/control/cluster/snapshot_builder.rs b/nodedb/src/control/cluster/snapshot_builder.rs index debd6ddcd..c67b429ca 100644 --- a/nodedb/src/control/cluster/snapshot_builder.rs +++ b/nodedb/src/control/cluster/snapshot_builder.rs @@ -358,7 +358,8 @@ impl nodedb_cluster::SnapshotBuilder for DataPlaneSnapshotBuilder { .map_err(|e| Box::new(e) as Box)?; // Read under the fence: the storage captured below holds exactly the // Calvin positions this names. - let calvin_cut = capture_calvin_cut(&self.shared, &group_vshards, calvin_through); + let calvin_cut = capture_calvin_cut(&self.shared, &group_vshards, calvin_through) + .map_err(|e| Box::new(e) as Box)?; // Enumerate tenants and their databases from the system catalog — the // same source the backup orchestrator's catalog sections use. Every @@ -424,6 +425,9 @@ impl nodedb_cluster::SnapshotBuilder for DataPlaneSnapshotBuilder { } merged.group_cut_index = cut_index; merged.group_calvin = Some(calvin_cut); + // Read under the fence, so the streams hold the chunks of the entries + // at or below the cut. + merged.group_redo_streams = self.shared.redo_chunks.capture_group(group_id); merged.group_event_lane = lane_snapshot::capture(&self.shared, &group_vshards) .await .map_err(|e| Box::new(e) as Box)?; diff --git a/nodedb/src/control/cluster/snapshot_install/dispatch.rs b/nodedb/src/control/cluster/snapshot_install/dispatch.rs index c894970ca..829f243ad 100644 --- a/nodedb/src/control/cluster/snapshot_install/dispatch.rs +++ b/nodedb/src/control/cluster/snapshot_install/dispatch.rs @@ -58,6 +58,7 @@ pub async fn install_on_every_core( wal_lsn: None, resolved_now_ms: None, commit_hlc: None, + entry_version: None, admission: Admission::Exempt(ExemptReason::AlreadyOrdered), }; let rx = state.tracker.register(request_id); @@ -136,13 +137,14 @@ fn installed(resp: Response) -> crate::Result<()> { } } -/// Fail point `snapshot_install::node::core`: core `C` of node `N` -/// reports its share as failed after it installed it. +/// Fail point `snapshot_install::core`, scoped to node `node_id`: core +/// `C` of the node reports its share as failed after it installed it. fn injected_fault(node_id: u64, core_id: usize) -> crate::Result<()> { #[cfg(feature = "failpoints")] - if let Some(detail) = - crate::fail_point::eval_fail(&format!("snapshot_install::node{node_id}::core{core_id}")) - { + if let Some(detail) = crate::fail_point::eval_fail( + crate::fail_point::FailScope::Node(node_id), + &format!("snapshot_install::core{core_id}"), + ) { return Err(crate::Error::DataPlane(ErrorCode::Internal { detail })); } #[cfg(not(feature = "failpoints"))] diff --git a/nodedb/src/control/cluster/snapshot_install/error.rs b/nodedb/src/control/cluster/snapshot_install/error.rs index f07b043c5..a433c4b04 100644 --- a/nodedb/src/control/cluster/snapshot_install/error.rs +++ b/nodedb/src/control/cluster/snapshot_install/error.rs @@ -16,6 +16,8 @@ pub enum SettleStep { EventLane, /// Replacing the group vShards' Calvin applied state and base. CalvinState, + /// Replacing the group's open chunked redo streams. + RedoStreams, } impl std::fmt::Display for SettleStep { @@ -26,6 +28,7 @@ impl std::fmt::Display for SettleStep { Self::ProposalKeys => "proposal-key persist", Self::EventLane => "event lane install", Self::CalvinState => "Calvin state install", + Self::RedoStreams => "redo stream install", }) } } diff --git a/nodedb/src/control/cluster/snapshot_install/mod.rs b/nodedb/src/control/cluster/snapshot_install/mod.rs index 10e49e144..073ef3175 100644 --- a/nodedb/src/control/cluster/snapshot_install/mod.rs +++ b/nodedb/src/control/cluster/snapshot_install/mod.rs @@ -7,9 +7,11 @@ pub mod clear; pub mod dispatch; pub mod error; pub mod split; +pub mod version_floor; pub use barrier::{append_install_barrier, append_install_marker}; pub use clear::{GroupCollection, clear_targets_per_core, group_collections}; pub use dispatch::install_on_every_core; pub use error::{SettleStep, SnapshotInstallError}; pub use split::{CoreMap, CoreShares, split_by_core}; +pub use version_floor::version_floor; diff --git a/nodedb/src/control/cluster/snapshot_install/split.rs b/nodedb/src/control/cluster/snapshot_install/split.rs index b059a7f97..8a0488705 100644 --- a/nodedb/src/control/cluster/snapshot_install/split.rs +++ b/nodedb/src/control/cluster/snapshot_install/split.rs @@ -90,6 +90,8 @@ pub struct CoreShares { pub event_lane: crate::types::snapshot::GroupEventLane, /// The Calvin cut of the group's vShards. pub calvin: Option, + /// The group's open chunked redo streams. + pub redo_streams: Vec, } /// `(database, collection)` of a `"{db}:{tid}:{collection}[:suffix]"` key. @@ -211,6 +213,7 @@ pub fn split_by_core( group_cut_index, group_event_lane, group_calvin, + group_redo_streams, } = snap; let mut s = Splitter { @@ -346,6 +349,7 @@ pub fn split_by_core( cut_index: group_cut_index, event_lane: group_event_lane, calvin: group_calvin, + redo_streams: group_redo_streams, }) } diff --git a/nodedb/src/control/cluster/snapshot_install/version_floor.rs b/nodedb/src/control/cluster/snapshot_install/version_floor.rs new file mode 100644 index 000000000..d1b97b231 --- /dev/null +++ b/nodedb/src/control/cluster/snapshot_install/version_floor.rs @@ -0,0 +1,32 @@ +// SPDX-License-Identifier: BUSL-1.1 + +//! The write version a data-group snapshot install lands on each vShard. + +use nodedb_types::{ShardVersion, WriteVersion}; + +use crate::control::state::SharedState; + +/// The version each of `vshards` holds once a snapshot of `group_id` cut at +/// `cut_index` lands: the cut's log position in the vShard's epoch. +/// +/// The installed rows carry no per-row versions. Every write the snapshot +/// holds sits at or below the cut, so a read the cut does not cover no longer +/// validates on this node. +pub fn version_floor( + state: &SharedState, + group_id: u64, + vshards: &[u32], + cut_index: u64, +) -> Vec { + vshards + .iter() + .map(|&vshard| { + let position = + crate::event::cdc::position::entry_position(state, vshard, group_id, cut_index); + ShardVersion { + vshard, + version: WriteVersion::logged(position.epoch, position.log_index), + } + }) + .collect() +} diff --git a/nodedb/src/control/cluster/start_raft/core.rs b/nodedb/src/control/cluster/start_raft/core.rs index 0e4d5d2e7..1628977d6 100644 --- a/nodedb/src/control/cluster/start_raft/core.rs +++ b/nodedb/src/control/cluster/start_raft/core.rs @@ -66,9 +66,11 @@ pub async fn start_raft( &loop_build.raft_loop, loop_build.tracker, loop_build.apply_rx, - loop_build.calvin_read_result_senders, loop_build.sequencer_state_machine, )?; + // The async proposer is installed: a failed redo stream abandon can be + // retried from here on. + crate::control::wal_replication::transaction_redo::spawn_redo_abandoner(&shared); let ready_rx = finish_observability( handle, diff --git a/nodedb/src/control/cluster/start_raft/group_setup.rs b/nodedb/src/control/cluster/start_raft/group_setup.rs index bdf8aebdf..389644a16 100644 --- a/nodedb/src/control/cluster/start_raft/group_setup.rs +++ b/nodedb/src/control/cluster/start_raft/group_setup.rs @@ -6,18 +6,16 @@ //! snapshot-transfer / replication-factor config that must be read before //! `pending_subsystems` is consumed. -use std::collections::BTreeMap; use std::sync::{Arc, Mutex}; use std::time::Duration; -use tokio::sync::mpsc::{self, Sender}; +use tokio::sync::mpsc; use nodedb_cluster::calvin::{ CalvinCompletionRegistry, SEQUENCER_GROUP_ID, SequencerStateMachine, TxnId, VerdictOutcome, }; use crate::control::cluster::array_executor::DataPlaneArrayExecutor; -use crate::control::cluster::calvin::ReadResultEvent; use crate::control::cluster::handle::ClusterHandle; use crate::control::cluster::metadata_applier::{MetadataCommitApplier, seed_metadata_cache}; use crate::control::cluster::spsc_applier::SpscCommitApplier; @@ -38,7 +36,6 @@ pub(super) struct GroupSetup { /// vote tally completes. pub(super) calvin_verdict_rx: mpsc::Receiver<(TxnId, VerdictOutcome)>, pub(super) sequencer_state_machine: Arc>, - pub(super) calvin_read_result_senders: Arc>>>, pub(super) metadata_applier: Arc, pub(super) token_state: nodedb_cluster::SharedTokenStateMirror, pub(super) plan_executor: Arc, @@ -110,7 +107,6 @@ pub(super) fn build_group_setup( calvin_completion_registry: calvin.completion_registry, calvin_verdict_rx: calvin.verdict_rx, sequencer_state_machine: calvin.sequencer_state_machine, - calvin_read_result_senders: calvin.read_result_senders, metadata_applier, token_state, plan_executor, @@ -165,7 +161,6 @@ struct CalvinState { completion_registry: Arc, verdict_rx: mpsc::Receiver<(TxnId, VerdictOutcome)>, sequencer_state_machine: Arc>, - read_result_senders: Arc>>>, } /// Build the Calvin completion registry, its verdict channel, and the @@ -238,12 +233,10 @@ fn build_calvin_state(handle: &ClusterHandle, shared: &Arc) -> Calv shared, ))), )); - let read_result_senders = Arc::new(Mutex::new(BTreeMap::>::new())); CalvinState { completion_registry: calvin_completion_registry, verdict_rx: calvin_verdict_rx, sequencer_state_machine, - read_result_senders, } } diff --git a/nodedb/src/control/cluster/start_raft/hooks.rs b/nodedb/src/control/cluster/start_raft/hooks.rs index e838d80d8..a68f13243 100644 --- a/nodedb/src/control/cluster/start_raft/hooks.rs +++ b/nodedb/src/control/cluster/start_raft/hooks.rs @@ -28,6 +28,8 @@ pub(super) struct Hooks { pub(super) calvin_submit_inbox: Arc, pub(super) reserve_read: Arc, pub(super) release_reservation: Arc, + /// The write gate a data-group leader runs on every proposal to its group. + pub(super) data_propose_gate: Arc, /// The sequencer group's kept snapshot, which its own compaction writes. pub(super) sequencer_snapshots: Arc, @@ -54,7 +56,8 @@ pub(super) async fn build_hooks( // The sequencer group's snapshot: its state machine, captured on the // send path and kept durably on the receive path. A node whose sequencer // log starts right after an installed snapshot restores the state - // machine from it before any entry applies. + // machine from it before any entry applies. A log that starts above its + // first entry with no usable snapshot marks the history unknown. let sequencer_snapshots = Arc::new( crate::control::cluster::sequencer_snapshot::SequencerSnapshotStore::new( Arc::clone(sequencer_state_machine), @@ -77,6 +80,47 @@ pub(super) async fn build_hooks( multi_raft, )?; + // Every vShard's Calvin applied ledger, from one WAL pass and the state + // the last checkpoint saved. The data-group apply loop and the + // schedulers read the ledgers, so they fill before either starts. A + // boot completion of a snapshot install below replaces its group's. + let vshard_to_group = handle + .routing + .read() + .unwrap_or_else(|p| p.into_inner()) + .vshard_to_group() + .to_vec(); + let group_of = |vshard_id: u32| vshard_to_group.get(vshard_id as usize).copied(); + let recovered = crate::control::cluster::calvin::scheduler::recover_all_applied( + &shared.wal, + shared.credentials.catalog(), + &group_of, + )?; + for (vshard_id, state) in recovered { + shared + .calvin + .applied + .install(vshard_id, state.fully_applied_epoch, state.applied_tail); + } + // A chunk stream of an applied position installs nothing more: a copy + // of the position's redo installed already. + shared + .redo_chunks + .drop_applied_calvin_streams(|vshard_id, epoch, position| { + shared + .calvin + .applied + .get(vshard_id) + .is_some_and(|ledger| ledger.is_applied(epoch, position)) + }); + + // Every stored dependent-read barrier log of an unfinished txn waits in + // its vShard's buffer, and the rows of finished txns go. A barrier entry + // at or below a group's durable applied floor is never delivered again, + // so its row is what a barrier reads from now on. A boot completion of a + // snapshot install below replaces its group's rows. + crate::control::cluster::calvin::scheduler::barrier_store::restore_at_boot(shared)?; + // Per-group snapshot builder for the SEND path: on the leader, build the // real serialized engine state for a lagging follower's group vshards // (replacing the prior empty stub bytes). @@ -181,6 +225,12 @@ pub(super) async fn build_hooks( crate::control::server::reservation::RegistryReleaseReservation::new(shared.clone()), ); + // Leader write gate: when this node leads a data group, every proposal + // to it, local or forwarded, takes its lock keys on the vShard's Calvin + // lock table before the entry enters the log. + let data_propose_gate: Arc = + Arc::new(crate::control::server::shared::write_admission::LeaderWriteGate::new(shared)); + Ok(Hooks { quarantine_hook, snapshot_builder, @@ -194,6 +244,7 @@ pub(super) async fn build_hooks( calvin_submit_inbox, reserve_read, release_reservation, + data_propose_gate, sequencer_snapshots, }) } diff --git a/nodedb/src/control/cluster/start_raft/loop_build.rs b/nodedb/src/control/cluster/start_raft/loop_build.rs index 2c53e7195..527aca8ab 100644 --- a/nodedb/src/control/cluster/start_raft/loop_build.rs +++ b/nodedb/src/control/cluster/start_raft/loop_build.rs @@ -5,19 +5,18 @@ //! service, spawn the vShard schedulers, and start the cluster subsystems //! (health/gossip/etc.) that share the loop's `MultiRaft`. -use std::collections::BTreeMap; use std::sync::{Arc, Mutex}; -use tokio::sync::mpsc::{self, Sender}; +use tokio::sync::mpsc; use nodedb_cluster::calvin::{ CalvinCompletionRegistry, SequencerConfig, SequencerReceivers, SequencerService, SequencerStateMachine, new_inbox, new_reservation_inbox, }; +use crate::control::cluster::calvin::SchedulerConfig; use crate::control::cluster::calvin::executor::ollp::OllpConfig; use crate::control::cluster::calvin::executor::ollp::orchestrator::OllpOrchestrator; -use crate::control::cluster::calvin::{ReadResultEvent, SchedulerConfig}; use crate::control::cluster::handle::ClusterHandle; use crate::control::cluster::start_raft_helpers::{ SpawnVshardSchedulersParams, spawn_vshard_schedulers, @@ -47,7 +46,6 @@ pub(super) struct LoopBuild { pub(super) ollp_orchestrator: Arc, pub(super) tracker: Arc, pub(super) apply_rx: mpsc::Receiver, - pub(super) calvin_read_result_senders: Arc>>>, pub(super) calvin_completion_registry: Arc, pub(super) token_state: nodedb_cluster::SharedTokenStateMirror, /// Shared with the compactor wiring so sequencer-group log compaction can @@ -78,7 +76,6 @@ pub(super) async fn build_raft_loop( calvin_completion_registry, calvin_verdict_rx, sequencer_state_machine, - calvin_read_result_senders, metadata_applier, token_state, plan_executor, @@ -156,6 +153,7 @@ pub(super) async fn build_raft_loop( .with_calvin_submit_inbox(hooks.calvin_submit_inbox) .with_reserve_read(hooks.reserve_read) .with_release_reservation(hooks.release_reservation) + .with_data_propose_gate(hooks.data_propose_gate) .with_auth_lease(lease_service) .with_data_dir(data_dir.to_path_buf()) .with_snapshot_chunk_bytes(snapshot_chunk_bytes) @@ -185,6 +183,8 @@ pub(super) async fn build_raft_loop( .map_err(|e| crate::Error::Config { detail: e.to_string(), })?; + // Each scheduler runs on the epoch duration of the sequencer feeding it. + let scheduler_config = SchedulerConfig::from_tuning(&shared.tuning.calvin, &sequencer_config); let (sequencer_inbox, sequencer_inbox_rx) = new_inbox(10_000, &sequencer_config); let (reservation_inbox, reservation_inbox_rx) = new_reservation_inbox(10_000); let ollp_orchestrator = Arc::new(OllpOrchestrator::new(OllpConfig::default())); @@ -208,13 +208,11 @@ pub(super) async fn build_raft_loop( ); let sequencer_metrics = Arc::clone(&sequencer_service.metrics); - let scheduler_config = SchedulerConfig::default(); spawn_vshard_schedulers(SpawnVshardSchedulersParams { handle, shared, raft_loop_handle: raft_loop_handle.clone(), sequencer_state_machine: &sequencer_state_machine, - calvin_read_result_senders: &calvin_read_result_senders, calvin_completion_registry: &calvin_completion_registry, scheduler_config: &scheduler_config, })?; @@ -264,7 +262,6 @@ pub(super) async fn build_raft_loop( ollp_orchestrator, tracker, apply_rx, - calvin_read_result_senders, calvin_completion_registry, token_state, sequencer_state_machine, diff --git a/nodedb/src/control/cluster/start_raft/mod.rs b/nodedb/src/control/cluster/start_raft/mod.rs index 76838459d..04e22ebf9 100644 --- a/nodedb/src/control/cluster/start_raft/mod.rs +++ b/nodedb/src/control/cluster/start_raft/mod.rs @@ -14,6 +14,8 @@ //! cluster subsystems. //! - [`proposer_wiring`]: install the sync/async Raft proposer + compactor //! closures and spawn the apply loop. +//! - [`sequencer_route`]: run a proposal the leader's write gate refused +//! through the Calvin sequencer. //! - [`observability`]: publish observability handles and spawn the tick //! loop, sequencer service, RPC server, and health monitor. @@ -24,5 +26,6 @@ mod loop_build; mod observability; mod propose_error; mod proposer_wiring; +mod sequencer_route; pub use core::start_raft; diff --git a/nodedb/src/control/cluster/start_raft/observability.rs b/nodedb/src/control/cluster/start_raft/observability.rs index f17d0f41d..53357fe5e 100644 --- a/nodedb/src/control/cluster/start_raft/observability.rs +++ b/nodedb/src/control/cluster/start_raft/observability.rs @@ -175,6 +175,13 @@ fn publish_raft_status( if shared.raft_read_gate.set(gate).is_err() { tracing::warn!("raft_read_gate already set — start_raft appears to have run twice"); } + if shared + .multi_raft + .set(raft_loop.multi_raft_handle()) + .is_err() + { + tracing::warn!("multi_raft already set — start_raft appears to have run twice"); + } } /// Renew this node's authorization lease with the metadata leader. Its diff --git a/nodedb/src/control/cluster/start_raft/propose_error.rs b/nodedb/src/control/cluster/start_raft/propose_error.rs index 35240fa44..54001390e 100644 --- a/nodedb/src/control/cluster/start_raft/propose_error.rs +++ b/nodedb/src/control/cluster/start_raft/propose_error.rs @@ -2,20 +2,31 @@ //! The error an async Raft propose returns to its statement. -use nodedb_cluster::ClusterError; +use nodedb_cluster::{CalvinError, ClusterError}; use nodedb_raft::RaftError; +use crate::bridge::envelope::ErrorCode; use crate::types::VShardId; +/// The error of a write whose deadline passed before any leader proposed it. +/// +/// A definite refusal: nothing entered the log, so nothing applies. A client +/// sees the same deadline SQLSTATE as an unknown outcome. A caller that +/// matches the code can tell the two apart. +pub(super) fn expired_before_propose() -> crate::Error { + crate::Error::DataPlane(ErrorCode::ExpiredBeforeExecution) +} + /// The error an async propose returns for a cluster error. /// /// A group with no leader to take the proposal right now accepts the same /// proposal once it has one, so the proposal is retried: /// [`crate::Error::NoLeader`]. That covers an election, a leadership transfer /// in flight, a leader that stepped down after this node or a forwarding node -/// chose it, and a vShard whose owner is moving. A forwarded refusal arrives -/// here with its typed Raft error (`DataProposeResponse::refusal_error`). A -/// typed verdict keeps its class. Every other failure is final here. +/// chose it, a vShard whose owner is moving, and a leader this node cannot +/// reach. A forwarded refusal arrives here with its typed Raft error +/// (`DataProposeResponse::refusal_error`). A typed verdict keeps its class. +/// Every other failure is final here. pub(super) fn async_propose_error(vshard_id: u32, error: ClusterError) -> crate::Error { match error { ClusterError::Raft( @@ -23,15 +34,38 @@ pub(super) fn async_propose_error(vshard_id: u32, error: ClusterError) -> crate: ) | ClusterError::ReadIndexNotLeader { .. } | ClusterError::MigrationInProgress { .. } - | ClusterError::WrongOwner { .. } => crate::Error::NoLeader { + | ClusterError::WrongOwner { .. } + // The forward never reached the leader whole: the leader is not in + // this node's topology, or the connect, stream open or write failed. + // Nothing was proposed. A written forward ends as `Unanswered`. + | ClusterError::Transport { .. } => crate::Error::NoLeader { vshard_id: VShardId::new(vshard_id), }, - // The forward did not answer before its timeout. The statement's - // deadline class, the same class the array fan-out gives it. - ClusterError::ShardTimeout { .. } => crate::Error::DeadlineExceeded { - request_id: crate::types::RequestId::new(0), - }, + // The forward did not answer before its timeout. The leader can have + // proposed the write, so its outcome is unknown: the statement's + // deadline class, the same class the array fan-out gives it. A + // forward whose stream failed after the request was written can + // also have been proposed, so it takes the same class. + error @ (ClusterError::ShardTimeout { .. } | ClusterError::Unanswered { .. }) => { + tracing::warn!( + vshard_id, + %error, + "raft proposal forward reached its deadline with no answer from the leader" + ); + crate::Error::DeadlineExceeded { + request_id: crate::types::RequestId::new(0), + } + } ClusterError::DataPlane { code } => crate::Error::DataPlane(code.into()), + // The leader's write gate waited for the write's lock keys until the + // caller's deadline, and proposed nothing. + ClusterError::Calvin(CalvinError::AdmissionTimedOut) => { + tracing::warn!( + vshard_id, + "raft proposal reached its deadline at the leader's write gate" + ); + expired_before_propose() + } ClusterError::ShardExecution { error, .. } | ClusterError::StreamTerminal { error, .. } => { crate::Error::from(*error) } @@ -53,7 +87,6 @@ pub(super) fn async_propose_error(vshard_id: u32, error: ClusterError) -> crate: | ClusterError::MigrationPauseBudgetExceeded { .. } | ClusterError::NodeUnreachable { .. } | ClusterError::GhostNotFound { .. } - | ClusterError::Transport { .. } | ClusterError::Storage { .. } | ClusterError::Codec { .. } | ClusterError::UnsupportedWireVersion { .. } @@ -113,15 +146,66 @@ mod tests { )); } + /// An admission timeout proposed nothing: a definite refusal, never an + /// unknown outcome. #[test] - fn a_forward_timeout_is_a_deadline() { - let error = ClusterError::ShardTimeout { - vshard_id: 3, - elapsed_ms: 50, + fn an_admission_timeout_is_a_definite_refusal() { + let error = async_propose_error(3, ClusterError::Calvin(CalvinError::AdmissionTimedOut)); + let crate::Error::DataPlane(code) = &error else { + panic!("expected a Data-Plane verdict, got {error:?}"); }; - assert!(matches!( - async_propose_error(3, error), - crate::Error::DeadlineExceeded { .. } - )); + assert_eq!(code, &ErrorCode::ExpiredBeforeExecution); + assert!(crate::control::server::dispatch_utils::write_definitely_not_applied(code)); + } + + /// A forward with no reply can have been proposed. Its timeout stays an + /// unknown outcome, never a definite refusal. + #[test] + fn a_forward_timeout_is_an_unknown_outcome() { + let error = async_propose_error( + 3, + ClusterError::ShardTimeout { + vshard_id: 3, + elapsed_ms: 50, + }, + ); + assert!(matches!(error, crate::Error::DeadlineExceeded { .. })); + assert!( + !crate::control::server::dispatch_utils::write_definitely_not_applied( + &ErrorCode::from(error) + ) + ); + } + + /// A forward whose stream failed after the request was written can have + /// been proposed. It is an unknown outcome, never an internal error. + #[test] + fn a_sent_forward_with_no_answer_is_an_unknown_outcome() { + let error = async_propose_error( + 3, + ClusterError::Unanswered { + node_id: 2, + detail: "connection lost".into(), + }, + ); + assert!(matches!(error, crate::Error::DeadlineExceeded { .. })); + assert!( + !crate::control::server::dispatch_utils::write_definitely_not_applied( + &ErrorCode::from(error) + ) + ); + } + + /// A forward that never went out proposed nothing. The proposal is + /// retried, never reported as an unknown outcome. + #[test] + fn an_unsent_forward_is_retried() { + let error = async_propose_error( + 3, + ClusterError::Transport { + detail: "connect to node 2 refused".into(), + }, + ); + assert!(matches!(error, crate::Error::NoLeader { .. })); } } diff --git a/nodedb/src/control/cluster/start_raft/proposer_wiring.rs b/nodedb/src/control/cluster/start_raft/proposer_wiring.rs index 9f66a278c..264ae0d08 100644 --- a/nodedb/src/control/cluster/start_raft/proposer_wiring.rs +++ b/nodedb/src/control/cluster/start_raft/proposer_wiring.rs @@ -5,12 +5,10 @@ //! background apply loop that drains `DistributedApplier::apply_committed` //! into the Data Plane and notifies propose waiters. -use std::collections::BTreeMap; use std::sync::{Arc, Mutex}; -use tokio::sync::mpsc::{self, Sender}; +use tokio::sync::mpsc; -use crate::control::cluster::calvin::ReadResultEvent; use crate::control::distributed_applier::{ApplyBatch, ProposeTracker, run_apply_loop}; use crate::control::state::SharedState; @@ -25,7 +23,6 @@ pub(super) fn wire_proposers( raft_loop: &Arc, tracker: Arc, apply_rx: mpsc::Receiver, - calvin_read_result_senders: Arc>>>, sequencer_state_machine: Arc>, ) -> crate::Result<()> { install_sync_proposer(shared, raft_loop); @@ -33,7 +30,7 @@ pub(super) fn wire_proposers( install_applied_index_sink(shared, raft_loop); install_apply_gates(shared, raft_loop); install_async_proposer(shared, raft_loop, &tracker)?; - spawn_apply_loop(shared, tracker, apply_rx, calvin_read_result_senders); + spawn_apply_loop(shared, tracker, apply_rx); Ok(()) } @@ -116,13 +113,13 @@ fn install_compactor( .ok_or_else(|| crate::Error::Internal { detail: "raft log compaction: shared state dropped".into(), })?; - let mirrors = shared.authorization_fence.calvin_mirrors(); + let ledgers = &shared.calvin.applied; let mut sm = sm_for_compact.lock().unwrap_or_else(|p| p.into_inner()); let undurable = sm.undurable_floor(|vshard, epoch, position| { - // A vShard with no scheduler here holds no state to lose. - mirrors + // A vShard with no ledger here holds no state to lose. + ledgers .get(vshard) - .is_none_or(|mirror| mirror.is_applied(epoch, position)) + .is_none_or(|ledger| ledger.is_applied(epoch, position)) }); let floor = [ sm.min_catch_up_from(), @@ -238,14 +235,23 @@ fn install_async_proposer( detail: "raft propose (async): cluster not running".into(), })?; // The attempt gets only what remains of the caller's deadline. + // A deadline that passed before the propose refuses the write. if tokio::time::Instant::now() >= deadline { - return Err(propose_deadline_exceeded()); + return Err(super::propose_error::expired_before_propose()); } - let (group_id, log_index) = - tokio::time::timeout_at(deadline, rl.propose_via_data_leader(vshard_id, data)) - .await - .map_err(|_| propose_deadline_exceeded())? - .map_err(|e| async_propose_error(vshard_id, e))?; + // The leader's write gate stops at `deadline`. The call waits + // a bounded margin past it for the leader's verdict. An + // unproposed write then reports a refusal, not an unknown outcome. + let proposed = rl.propose_via_data_leader(vshard_id, &data, deadline).await; + let (group_id, log_index) = match proposed { + Ok(at) => at, + // The leader's write gate found a key of the write held. + // The Calvin sequencer orders it behind the holder. + Err(nodedb_cluster::ClusterError::Calvin( + nodedb_cluster::CalvinError::RouteToSequencer, + )) => return Ok(super::sequencer_route::routed_write(&state_weak, data)), + Err(e) => return Err(async_propose_error(vshard_id, e)), + }; // Register the waiter with the proposer's idempotency // key. The apply path compares against the committed @@ -282,16 +288,9 @@ fn install_async_proposer( e } }) - // Carry out the write-version the APPLY side stamped, not - // `log_index`. The tracker resolves on the node that applied - // the entry locally, so `write_version` is this replica's own - // post-write `coll_write_lsn` — a WAL LSN, the same domain - // every other feed of that map records in, and the only - // domain the shard-local OCC read validator compares in. The - // raft log index is a per-group counter on a different scale - // entirely; publishing it here made reads validate a WAL LSN - // against a log index. - .map(|applied| (applied.payload, applied.write_version)); + // Carry out the versions the apply side stamped: the + // entry's log position on every vShard the write touched. + .map(|applied| (applied.payload, applied.write_versions)); let applied = applied?; // A write to a vShard homing a permission-tree source is // acknowledged only once every lease holder covers it, or its @@ -330,7 +329,6 @@ fn spawn_apply_loop( shared: &Arc, tracker: Arc, apply_rx: mpsc::Receiver, - calvin_read_result_senders: Arc>>>, ) { // Spawn the background apply loop. It reads from the mpsc channel // pushed by `DistributedApplier::apply_committed`, dispatches to the @@ -342,7 +340,6 @@ fn spawn_apply_loop( // because its applies dispatch to the Data Plane. let apply_state = shared.clone(); let apply_tracker = tracker; - let apply_calvin_read_result_senders = calvin_read_result_senders; crate::control::shutdown::spawn_loop_no_abort( &shared.loop_registry, &shared.shutdown, @@ -355,12 +352,7 @@ fn spawn_apply_loop( // cuts the loop off mid-apply. tokio::select! { biased; - _ = run_apply_loop( - apply_rx, - apply_state, - apply_tracker, - apply_calvin_read_result_senders, - ) => {} + _ = run_apply_loop(apply_rx, apply_state, apply_tracker) => {} _ = shutdown.wait_cancelled() => {} } }, @@ -467,6 +459,7 @@ async fn await_local_apply( tracing::warn!( group_id, log_index, + vshard_id, progress = %apply_progress(state.as_deref(), group_id), oldest_unfinished = %tracker .applying(group_id) diff --git a/nodedb/src/control/cluster/start_raft/sequencer_route.rs b/nodedb/src/control/cluster/start_raft/sequencer_route.rs new file mode 100644 index 000000000..e182e2d12 --- /dev/null +++ b/nodedb/src/control/cluster/start_raft/sequencer_route.rs @@ -0,0 +1,104 @@ +// SPDX-License-Identifier: BUSL-1.1 + +//! A proposal the data-group leader's write gate routed to the Calvin +//! sequencer. +//! +//! The leader refuses the propose with `RouteToSequencer` when a key of the +//! write is held and a Calvin transaction can sequence it. The proposer then +//! submits the same write, decoded from the entry it encoded, through the +//! scheduler. The scheduler queues it FIFO behind the holder on every +//! replica. Its answer stands for the apply the propose would have returned. + +use std::sync::Weak; + +use crate::bridge::envelope::{Response, Status}; +use crate::control::server::shared::write_admission::{bare_ok_response, route_write_to_calvin}; +use crate::control::state::SharedState; +use crate::control::wal_replication::ReplicatedEntry; +use crate::control::wal_replication::decode::decode_parsed_entry; +use crate::control::wal_replication::{AppliedWait, ProposedWrite}; +use crate::types::RequestId; + +/// The proposal of the encoded entry `data`, run through the Calvin +/// sequencer. No entry landed in the data-group log. +pub(super) fn routed_write(state: &Weak, data: Vec) -> ProposedWrite { + let state = state.clone(); + let applied: AppliedWait = Box::pin(async move { + let state = state.upgrade().ok_or_else(|| crate::Error::Internal { + detail: "calvin route of a gated proposal: node is shutting down".into(), + })?; + let entry = ReplicatedEntry::from_bytes(&data).ok_or_else(|| crate::Error::Internal { + detail: "calvin route of a gated proposal: the proposer's own entry does not decode" + .into(), + })?; + let Some((database_id, (tenant_id, vshard_id, plan, _))) = decode_parsed_entry(&entry)? + else { + return Err(crate::Error::Internal { + detail: "calvin route of a gated proposal: the entry carries no plan, yet the \ + leader's write gate named it sequenceable" + .into(), + }); + }; + let event_source = crate::event::EventSource::from(entry.event_source); + let response = route_write_to_calvin( + &state, + tenant_id, + database_id, + vshard_id, + plan, + event_source, + ) + .await? + .unwrap_or_else(|| bare_ok_response(RequestId::new(0))); + applied_result(&response) + }); + ProposedWrite { at: None, applied } +} + +/// The apply result a routed write's answer stands for: its payload and +/// written version, or its typed Data-Plane verdict. +fn applied_result( + response: &Response, +) -> crate::Result { + if response.status == Status::Ok { + return Ok((response.payload.to_vec(), response.read_versions.clone())); + } + match response.error_code.as_deref() { + Some(code) => Err(crate::Error::DataPlane(code.clone())), + None => Err(crate::Error::Internal { + detail: "calvin route of a gated proposal: the scheduler answered an error with no \ + error code" + .into(), + }), + } +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::bridge::envelope::ErrorCode; + + #[test] + fn an_ok_answer_carries_its_payload_and_version() { + let mut response = bare_ok_response(RequestId::new(1)); + response.payload = crate::bridge::envelope::Payload::from_vec(vec![7]); + let version = nodedb_types::WriteVersion::logged(2, 9); + response + .read_versions + .note(crate::types::VShardId::new(4), version); + let (payload, versions) = applied_result(&response).expect("ok"); + assert_eq!(payload, vec![7]); + assert_eq!(versions.of(crate::types::VShardId::new(4)), Some(version)); + } + + #[test] + fn an_error_answer_keeps_its_verdict() { + let mut response = bare_ok_response(RequestId::new(1)); + response.status = Status::Error; + response.error_code = Some(Box::new(ErrorCode::OllpRetryRequired)); + assert!(matches!( + applied_result(&response), + Err(crate::Error::DataPlane(ErrorCode::OllpRetryRequired)) + )); + } +} diff --git a/nodedb/src/control/cluster/start_raft_helpers.rs b/nodedb/src/control/cluster/start_raft_helpers.rs index bfcece5ec..04be91339 100644 --- a/nodedb/src/control/cluster/start_raft_helpers.rs +++ b/nodedb/src/control/cluster/start_raft_helpers.rs @@ -1,22 +1,20 @@ // SPDX-License-Identifier: BUSL-1.1 +use std::collections::BTreeSet; use std::sync::{Arc, Mutex, RwLock}; use nodedb_cluster::calvin::{CalvinCompletionRegistry, SequencerStateMachine}; use crate::control::cluster::calvin::scheduler::metrics::SchedulerMetrics; -use crate::control::cluster::calvin::scheduler::recover_applied; use crate::control::cluster::calvin::{ - RaftSequencerProposer, ReadResultEvent, Scheduler, SchedulerConfig, SchedulerParams, - SequencerProposer, + RaftSequencerProposer, Scheduler, SchedulerConfig, SchedulerParams, SequencerProposer, }; use crate::control::cluster::calvin_snapshot; use crate::control::cluster::handle::ClusterHandle; use crate::control::state::SharedState; -/// Type alias for the shared per-vShard read-result sender registry. -type ReadResultSenders = - Arc>>>; +/// The vShards whose scheduler this node started and has not stopped. +type ServedVShards = Arc>>; /// The vShards this node currently hosts: the union of `vshards_for_group` over /// every Raft group whose member set includes this node, read from the live @@ -42,20 +40,19 @@ fn hosted_vshards(routing: &RwLock, node_id: u64) /// them. A snapshot install replaced the state a running scheduler started /// from. Dropping the vShard's sequencer sender closes the scheduler's /// intake, and the scheduler exits. Its other channels and its lock table go -/// too. The next reconcile that finds the vShard hosted spawns a new +/// too. It hands the barrier events of the txns it did not finish back to +/// the vShard's read-result buffer. The next reconcile that finds the vShard hosted spawns a new /// scheduler from the vShard's base. fn retire_vshard_schedulers( hosted: &[u32], shared: &Arc, sequencer_state_machine: &Arc>, - calvin_read_result_senders: &ReadResultSenders, + served_vshards: &ServedVShards, calvin_completion_registry: &Arc, ) { let (left, rebased): (Vec, Vec) = { - let mut senders = calvin_read_result_senders - .lock() - .unwrap_or_else(|p| p.into_inner()); - let served: Vec = senders.keys().copied().collect(); + let mut served_set = served_vshards.lock().unwrap_or_else(|p| p.into_inner()); + let served: Vec = served_set.iter().copied().collect(); let left: Vec = served .iter() .copied() @@ -66,11 +63,28 @@ fn retire_vshard_schedulers( .filter(|vshard| !left.contains(vshard)) .collect(); for vshard in left.iter().chain(&rebased) { - senders.remove(vshard); + served_set.remove(vshard); } (left, rebased) }; calvin_snapshot::forget_left(shared, &left); + // A vShard that left this node keeps no barrier events here, in memory + // or stored. A rebased vShard keeps them for the scheduler that starts + // again. + for &vshard in &left { + if let Err(e) = + crate::control::cluster::calvin::scheduler::barrier_store::forget_vshard(shared, vshard) + { + // The rows stay. Boot marks them stored for a vShard with no + // scheduler, and a snapshot install of the vShard replaces them. + tracing::warn!( + vshard_id = vshard, + error = %e, + "calvin: the barrier rows of a vShard that left were not removed" + ); + crate::diag::calvin_barrier_log_store_failed(vshard, None, "remove", &e); + } + } for &vshard in &rebased { tracing::info!( vshard_id = vshard, @@ -102,6 +116,11 @@ fn retire_vshard_schedulers( .lock() .unwrap_or_else(|p| p.into_inner()) .remove(&vshard); + // The write gate's holds on the stopped lock table order nothing. + shared + .calvin + .admission_holds + .release_vshard(crate::types::VShardId::new(vshard)); calvin_completion_registry.unregister_verdict_signal_sender(vshard); // A cut on this node waits only on the schedulers it runs. shared.calvin.cuts.unregister(vshard); @@ -118,7 +137,7 @@ struct ReconcileSchedulersParams<'a> { /// One proposer per node, so its forward limit bounds the whole node. sequencer_proposer: &'a Arc, sequencer_state_machine: &'a Arc>, - calvin_read_result_senders: &'a ReadResultSenders, + served_vshards: &'a ServedVShards, calvin_completion_registry: &'a Arc, scheduler_config: &'a SchedulerConfig, } @@ -126,10 +145,9 @@ struct ReconcileSchedulersParams<'a> { /// Idempotently ensure a Calvin `Scheduler` runs for exactly the vShards this /// node currently hosts. /// -/// A vShard is considered already-served iff it has a registered read-result -/// sender (the schedulers' presence registry). A served vShard this node no -/// longer hosts has its scheduler stopped first (see -/// [`retire_left_vshard_schedulers`]). Only newly-hosted vShards get a fresh +/// A vShard is already served when `served_vshards` holds it. A served +/// vShard this node no longer hosts has its scheduler stopped first (see +/// [`retire_vshard_schedulers`]). Only newly-hosted vShards get a fresh /// scheduler — this pass never double-spawns. Returns the number of NEW /// schedulers started. fn reconcile_vshard_schedulers(params: ReconcileSchedulersParams<'_>) -> crate::Result { @@ -140,7 +158,7 @@ fn reconcile_vshard_schedulers(params: ReconcileSchedulersParams<'_>) -> crate:: raft_loop_handle, sequencer_proposer, sequencer_state_machine, - calvin_read_result_senders, + served_vshards, calvin_completion_registry, scheduler_config, } = params; @@ -150,18 +168,22 @@ fn reconcile_vshard_schedulers(params: ReconcileSchedulersParams<'_>) -> crate:: &hosted, shared, sequencer_state_machine, - calvin_read_result_senders, + served_vshards, calvin_completion_registry, ); calvin_snapshot::retain_mounted(shared, raft_loop_handle, routing); + crate::control::cluster::redo_stream_membership::reconcile_redo_streams( + shared, + raft_loop_handle, + ); let mut spawned = 0usize; let mut kept = Vec::new(); for vshard_id in hosted { // Already-served vShards keep their running scheduler untouched. - if calvin_read_result_senders + if served_vshards .lock() .unwrap_or_else(|p| p.into_inner()) - .contains_key(&vshard_id) + .contains(&vshard_id) { continue; } @@ -193,28 +215,11 @@ fn reconcile_vshard_schedulers(params: ReconcileSchedulersParams<'_>) -> crate:: continue; }; - // The applied state the last checkpoint saved, with the markers the - // WAL still holds: a checkpoint deletes the segments that held older - // markers, and the sequencer log delivers their entries again. - // A failed read leaves the vShard for the next pass. The pass goes - // on, so the schedulers it started keep their bases. - let recovery = match recover_applied( - &shared.wal, - shared.credentials.catalog(), - vshard_id, - group_id, - ) { - Ok(recovery) => recovery, - Err(error) => { - tracing::warn!( - vshard_id, - %error, - "calvin: the vShard's applied state did not load; its scheduler starts \ - on a later pass" - ); - continue; - } - }; + // The vShard's applied ledger: boot recovery or a snapshot install + // filled it, and a vShard with no Calvin history here gets an empty + // one. The scheduler rebuilds up to its highest applied epoch. + let ledger = shared.calvin.applied.get_or_create(vshard_id); + let rebuild_target_epoch = ledger.max_applied_epoch(); let (sequenced_tx, sequenced_rx) = tokio::sync::mpsc::channel(scheduler_config.channel_capacity); @@ -241,12 +246,10 @@ fn reconcile_vshard_schedulers(params: ReconcileSchedulersParams<'_>) -> crate:: // The armed catch-up holds the log from here, so the base is kept. kept.push((vshard_id, first_available)); - let (read_result_tx, read_result_rx) = - tokio::sync::mpsc::channel(scheduler_config.channel_capacity); - calvin_read_result_senders + served_vshards .lock() .unwrap_or_else(|p| p.into_inner()) - .insert(vshard_id, read_result_tx); + .insert(vshard_id); // The deterministic lock table is shared between this scheduler and the // Control-Plane write-admission gate: build it once and register the @@ -294,12 +297,10 @@ fn reconcile_vshard_schedulers(params: ReconcileSchedulersParams<'_>) -> crate:: multi_raft: raft_loop_handle.clone(), sequencer_proposer: Arc::clone(sequencer_proposer), sequencer_state_machine: Arc::clone(sequencer_state_machine), - fully_applied_epoch: recovery.fully_applied_epoch, - applied_tail: recovery.applied_tail, - rebuild_target_epoch: recovery.max_applied_epoch, + ledger, + rebuild_target_epoch, config: scheduler_config.clone(), metrics: SchedulerMetrics::new(), - read_result_rx, lock_manager, promotion_rx, registry: Arc::clone(calvin_completion_registry), @@ -343,7 +344,6 @@ pub(super) struct SpawnVshardSchedulersParams<'a> { pub(super) shared: &'a Arc, pub(super) raft_loop_handle: Arc>, pub(super) sequencer_state_machine: &'a Arc>, - pub(super) calvin_read_result_senders: &'a ReadResultSenders, pub(super) calvin_completion_registry: &'a Arc, pub(super) scheduler_config: &'a SchedulerConfig, } @@ -363,7 +363,9 @@ pub(super) struct SpawnVshardSchedulersParams<'a> { /// (covers the bootstrap node, which already sees its membership) and then /// spawns a background task that re-reconciles on a short interval until /// shutdown. Reconcile is idempotent: it starts schedulers for vShards this -/// node gained and stops those of vShards it left. +/// node gained and stops those of vShards it left. Each pass also runs the +/// membership pass over this node's chunked redo streams (see +/// [`crate::control::cluster::redo_stream_membership`]). pub(super) fn spawn_vshard_schedulers( params: SpawnVshardSchedulersParams<'_>, ) -> crate::Result<()> { @@ -372,12 +374,12 @@ pub(super) fn spawn_vshard_schedulers( shared, raft_loop_handle, sequencer_state_machine, - calvin_read_result_senders, calvin_completion_registry, scheduler_config, } = params; let node_id = handle.node_id; + let served_vshards: ServedVShards = Arc::new(Mutex::new(BTreeSet::new())); let routing = Arc::clone(&handle.routing); let sequencer_proposer: Arc = Arc::new(RaftSequencerProposer::new( node_id, @@ -402,7 +404,7 @@ pub(super) fn spawn_vshard_schedulers( raft_loop_handle: &raft_loop_handle, sequencer_proposer: &sequencer_proposer, sequencer_state_machine, - calvin_read_result_senders, + served_vshards: &served_vshards, calvin_completion_registry, scheduler_config, })?; @@ -414,7 +416,7 @@ pub(super) fn spawn_vshard_schedulers( // a no-op once the set has converged. let shared_task = Arc::clone(shared); let sm_task = Arc::clone(sequencer_state_machine); - let rr_task = Arc::clone(calvin_read_result_senders); + let served_task = Arc::clone(&served_vshards); let registry_task = Arc::clone(calvin_completion_registry); let cfg_task = scheduler_config.clone(); crate::control::shutdown::spawn_loop( @@ -437,7 +439,7 @@ pub(super) fn spawn_vshard_schedulers( raft_loop_handle: &raft_loop_handle, sequencer_proposer: &sequencer_proposer, sequencer_state_machine: &sm_task, - calvin_read_result_senders: &rr_task, + served_vshards: &served_task, calvin_completion_registry: ®istry_task, scheduler_config: &cfg_task, }) { diff --git a/nodedb/src/control/crdt_admission.rs b/nodedb/src/control/crdt_admission.rs index c366b5e0b..560ff47ed 100644 --- a/nodedb/src/control/crdt_admission.rs +++ b/nodedb/src/control/crdt_admission.rs @@ -54,7 +54,9 @@ pub struct CrdtApplyAdmissionRequest<'a> { pub struct CrdtAdmissionOutcome { pub payload: Vec, - pub write_version: crate::types::Lsn, + /// The versions the apply stamped: the entry's log position on the + /// written vShard. + pub write_versions: crate::types::ReadVersions, /// Operations the admitted delta encoded that the target document already /// knew, measured by the preview that fenced this apply. Distinguishes a /// client whose writes land from one whose writes are being absorbed — @@ -582,7 +584,12 @@ async fn apply_fenced( .with_event_source(workflow.event_source); let outcome = tokio::time::timeout( workflow.timeout, - crate::control::wal_replication::propose_replicated_entry(workflow.state, raw, entry), + crate::control::wal_replication::propose_replicated_entry( + workflow.state, + raw, + entry, + tokio::time::Instant::now() + workflow.timeout, + ), ) .await .map_err(|_| crate::Error::CrdtAdmissionTimeout { @@ -592,7 +599,7 @@ async fn apply_fenced( // This node's apply of the entry recorded its commit HLC. Ok(CrdtAdmissionOutcome { payload: outcome.0, - write_version: outcome.1, + write_versions: outcome.1, trimmed_ops: 0, }) } @@ -678,8 +685,8 @@ mod tests { payload: payload.into(), watermark_lsn: Lsn::ZERO, error_code: None, - read_set_valid: None, - read_version_lsn: Lsn::ZERO, + stage_vote: None, + read_versions: crate::types::ReadVersions::new(), write_set: Vec::new(), } } @@ -781,7 +788,7 @@ mod tests { .expect("fences lock") .push(expected_frontier_digest); } - Ok((Vec::new(), Lsn::ZERO)) + Ok((Vec::new(), crate::types::ReadVersions::new())) }) }); crate::control::vshard_admission::install_async_raft_proposer( @@ -880,7 +887,7 @@ mod tests { } other => panic!("unexpected replicated write: {other:?}"), } - Ok((Vec::new(), Lsn::ZERO)) + Ok((Vec::new(), crate::types::ReadVersions::new())) }) }); crate::control::vshard_admission::install_async_raft_proposer( @@ -930,7 +937,7 @@ mod tests { actual: [0; 32], })) } else { - Ok((Vec::new(), Lsn::ZERO)) + Ok((Vec::new(), crate::types::ReadVersions::new())) } }) }); @@ -997,7 +1004,7 @@ mod tests { actual: [0; 32], })) } else { - Ok((Vec::new(), Lsn::ZERO)) + Ok((Vec::new(), crate::types::ReadVersions::new())) } }) }) diff --git a/nodedb/src/control/distributed_applier/applied_index.rs b/nodedb/src/control/distributed_applier/applied_index.rs index 6ab3c790c..7a9314dc2 100644 --- a/nodedb/src/control/distributed_applier/applied_index.rs +++ b/nodedb/src/control/distributed_applier/applied_index.rs @@ -83,7 +83,13 @@ pub fn save_applied_index(state: &Arc, group_id: u64, applied_index #[derive(Debug, Default)] pub struct AppliedPrefix { floor: Option, - broken: bool, + /// `(first, last)` non-durable entries above `covered_through`, when any. + /// `first` can sit below the true first break, never above it. + breaks: Option<(u64, u64)>, + /// The highest index recorded above `covered_through`. + last_recorded: Option, + /// An installed snapshot holds every entry at or below this index. + covered_through: u64, } impl AppliedPrefix { @@ -96,19 +102,55 @@ impl AppliedPrefix { /// /// A failure breaks the prefix permanently for this batch: every later /// entry, successful or not, is past the gap and can no longer be covered - /// by the floor. Call this ONLY for entries whose success means "this - /// entry's redo record is WAL-fsync-durable" — branches that apply no + /// by the floor. Only [`Self::cover_through`] heals a break. Call this + /// ONLY for entries whose success means "this entry's redo record is + /// WAL-fsync-durable" — branches that apply no /// durable state must not call it at all (see [`Self::skip`]). pub fn record(&mut self, index: u64, applied_ok: bool) { + // The installed snapshot holds the entry's state, whatever its apply did. + if index <= self.covered_through { + return; + } + self.last_recorded = Some(index); if !applied_ok { - self.broken = true; + self.breaks = Some(match self.breaks { + None => (index, index), + Some((first, _)) => (first, index), + }); return; } - if !self.broken { + if self.breaks.is_none() { self.floor = Some(index); } } + /// Count every entry through `index` durable: an installed snapshot holds + /// their state. + /// + /// A break at or below `index` heals. A break above it stays. A later + /// record at or below `index` changes nothing. + pub fn cover_through(&mut self, index: u64) { + if index <= self.covered_through { + return; + } + self.covered_through = index; + self.breaks = match self.breaks { + Some((_, last)) if last <= index => None, + // The breaks above `index` start no lower than `index + 1`. + Some((first, last)) if first <= index => Some((index.saturating_add(1), last)), + unchanged => unchanged, + }; + // Every record above `index` came after the last healed break, so + // each one was durable when no break remains. + let reach = match self.breaks { + None => self.last_recorded.map_or(index, |last| last.max(index)), + Some(_) => index, + }; + if self.floor.is_none_or(|floor| floor < reach) { + self.floor = Some(reach); + } + } + /// Note an entry that carries no durable state — it neither advances the /// prefix nor breaks it. /// @@ -188,4 +230,51 @@ mod tests { broken.record(4, true); assert_eq!(broken.floor(), Some(1)); } + + #[test] + fn a_covering_snapshot_heals_a_break_at_or_below_it() { + let mut prefix = AppliedPrefix::new(); + prefix.record(1, true); + prefix.record(2, false); + prefix.record(3, true); + assert_eq!(prefix.floor(), Some(1)); + + prefix.cover_through(5); + assert_eq!(prefix.floor(), Some(5)); + prefix.record(4, false); + assert_eq!(prefix.floor(), Some(5), "a covered entry cannot break"); + prefix.record(6, true); + assert_eq!(prefix.floor(), Some(6)); + } + + #[test] + fn a_break_above_the_snapshot_still_holds_the_floor() { + let mut prefix = AppliedPrefix::new(); + prefix.record(1, true); + prefix.record(7, false); + prefix.cover_through(5); + assert_eq!(prefix.floor(), Some(5)); + prefix.record(8, true); + assert_eq!(prefix.floor(), Some(5), "entry 7 is not durable"); + + // Breaks on both sides: the one above the snapshot still holds. + let mut split = AppliedPrefix::new(); + split.record(2, false); + split.record(9, false); + split.cover_through(5); + split.record(10, true); + assert_eq!(split.floor(), Some(5)); + split.cover_through(9); + assert_eq!(split.floor(), Some(10), "entry 10 settled durable"); + } + + #[test] + fn a_lower_cover_changes_nothing() { + let mut prefix = AppliedPrefix::new(); + prefix.cover_through(5); + prefix.record(6, false); + prefix.cover_through(4); + prefix.record(7, true); + assert_eq!(prefix.floor(), Some(5)); + } } diff --git a/nodedb/src/control/distributed_applier/apply_loop/barrier_prefold.rs b/nodedb/src/control/distributed_applier/apply_loop/barrier_prefold.rs new file mode 100644 index 000000000..06d8ebe57 --- /dev/null +++ b/nodedb/src/control/distributed_applier/apply_loop/barrier_prefold.rs @@ -0,0 +1,40 @@ +// SPDX-License-Identifier: BUSL-1.1 + +//! Fold the dependent-read barrier entries of a waiting lane ahead of their +//! turn. +//! +//! A group's entries start in log order. A write held for this node's +//! metadata apply (see [`super::metadata_floor`]) keeps every later entry of +//! its group in the backlog until the catalog catches up. A barrier entry +//! names no collection and needs no catalog. Behind such a hold it would +//! still wait, and the txn's barrier on this replica would lack an event its +//! peers folded. +//! +//! So each pump folds every barrier entry the backlog holds past the last +//! one it folded, in log order. Barrier entries fold in log order among +//! themselves, which is the only order a barrier log depends on. Each entry +//! still concludes at its turn (see +//! [`super::calvin_read_result::prefold_barrier_entry`]). + +use super::calvin_read_result::prefold_barrier_entry; +use super::context::ApplyContext; +use super::lane::Lane; + +/// Fold each barrier entry of `lane`'s backlog above +/// `lane.prefolded_through`, in log order, and move the mark past every +/// backlog entry. +pub(super) fn prefold_backlog(ctx: ApplyContext<'_>, lane: &mut Lane, group_id: u64) { + let through = lane.prefolded_through; + let mut last = through; + for queued in lane.backlog.iter() { + let index = queued.entry.index; + if index <= through { + continue; + } + last = last.max(index); + if let Some(decoded) = queued.decoded.as_ref() { + prefold_barrier_entry(ctx, group_id, index, decoded); + } + } + lane.prefolded_through = last; +} diff --git a/nodedb/src/control/distributed_applier/apply_loop/calvin_read_result.rs b/nodedb/src/control/distributed_applier/apply_loop/calvin_read_result.rs index 998e8aa74..2ad1f2fca 100644 --- a/nodedb/src/control/distributed_applier/apply_loop/calvin_read_result.rs +++ b/nodedb/src/control/distributed_applier/apply_loop/calvin_read_result.rs @@ -1,25 +1,30 @@ // SPDX-License-Identifier: BUSL-1.1 -//! Forward a committed `ReplicatedWrite::CalvinReadResult` entry to the local -//! Calvin scheduler's read-result channel for its target vShard. +//! Fold a committed `CalvinReadResult` or `CalvinReadTimeout` entry into the +//! barrier log of its txn on its target vShard. //! -//! A read result is forwarded to an in-memory Calvin scheduler and writes -//! nothing durable, so the caller neither advances the applied prefix nor -//! breaks it for this entry. Advancing on it would assert a redo record that -//! does not exist; breaking on it would stall the floor behind an entry that a -//! re-delivery could not usefully replay anyway — the epoch it belongs to does -//! not survive a restart — and force every later write in the batch to be -//! applied twice on the next boot. - -use std::collections::BTreeMap; -use std::sync::{Arc, Mutex}; - -use tokio::sync::mpsc; +//! The event lands in the txn's stored row first (see +//! [`crate::control::cluster::calvin::scheduler::barrier_store`]), then in +//! the vShard's read-result buffer, which wakes the scheduler. An event that +//! applies before this node granted the txn, or before a scheduler for the +//! vShard runs here, waits in the buffer or its row: none is dropped. An +//! event for a position the vShard's applied ledger holds is dropped: the +//! txn finished, and no barrier of it opens again. +//! +//! The entry counts as durably applied once its row holds the event. Boot +//! resumes Raft delivery above the group's durable applied floor, so the +//! row is what a barrier reads after a restart. A row write that fails +//! leaves the event in memory alone, and the entry breaks the applied +//! prefix: a restart delivers it again. use crate::control::array_sync::raft_apply::AppliedPosition; -use crate::control::cluster::calvin::ReadResultEvent; -use crate::control::distributed_applier::propose_tracker::{AppliedWrite, ProposeTracker}; -use crate::types::TenantId; +use crate::control::cluster::calvin::BarrierEvent; +use crate::control::cluster::calvin::scheduler::barrier_store; +use crate::control::cluster::calvin::scheduler::lock_manager::TxnId; +use crate::control::distributed_applier::propose_tracker::AppliedWrite; + +use super::context::ApplyContext; +use super::proposal_gate::EntryOutcome; /// Fields extracted from a `ReplicatedWrite::CalvinReadResult` entry. pub(super) struct CalvinReadResultFields<'a> { @@ -27,80 +32,452 @@ pub(super) struct CalvinReadResultFields<'a> { pub epoch: u64, pub position: u32, pub passive_vshard: u32, - pub tenant_id: u64, pub values: &'a [u8], } -/// Decode `fields.values`, forward the resulting [`ReadResultEvent`] to the -/// scheduler registered for `fields.target_vshard`, and complete the propose -/// waiter. +/// Decode `fields.values` and fold the read result into the barrier log of +/// its txn on `fields.target_vshard`, then complete the propose waiter. pub(super) fn forward_calvin_read_result( - tracker: &Arc, - calvin_read_result_senders: &Arc>>>, + ctx: ApplyContext<'_>, pos: AppliedPosition, fields: CalvinReadResultFields<'_>, -) { - let AppliedPosition { - group_id, - log_index, - applied_key, - .. - } = pos; - - let decoded_values: Vec<( +) -> EntryOutcome { + let values: Vec<( nodedb_physical::physical_plan::meta::PassiveReadKeyId, nodedb_types::Value, )> = match zerompk::from_msgpack(fields.values) { Ok(decoded) => decoded, Err(e) => { + // Every replica decodes the same bytes and refuses them alike, + // so each barrier of the txn lacks this result and waits for + // the log's timeout entry. A restart refuses them again, so the + // refusal holds nothing to keep. tracing::warn!( - group_id, - index = log_index, + group_id = pos.group_id, + index = pos.log_index, error = %e, "failed to decode CalvinReadResult payload" ); - tracker.complete( - group_id, - log_index, - applied_key, - Err(crate::Error::Internal { - detail: format!("decode CalvinReadResult payload: {e}"), + ctx.tracker.complete( + pos.group_id, + pos.log_index, + pos.applied_key, + Err(crate::Error::Serialization { + format: "msgpack".into(), + detail: format!("CalvinReadResult payload: {e}"), }), ); - // Prefix-neutral, like the forward below: a read result mints no - // durable state either way, so there is nothing a re-delivery - // could restore and nothing later entries must wait behind. - return; + return EntryOutcome::Applied { + durable: true, + result: None, + }; } }; + fold_barrier_event( + ctx, + pos, + fields.target_vshard, + TxnId::new(fields.epoch, fields.position), + BarrierEvent::Read { + passive_vshard: fields.passive_vshard, + values, + }, + ) +} + +/// Fold a `CalvinReadTimeout` entry for txn `(epoch, position)` into its +/// barrier log on `target_vshard`, then complete the propose waiter. +pub(super) fn forward_calvin_read_timeout( + ctx: ApplyContext<'_>, + pos: AppliedPosition, + target_vshard: u32, + epoch: u64, + position: u32, +) -> EntryOutcome { + fold_barrier_event( + ctx, + pos, + target_vshard, + TxnId::new(epoch, position), + BarrierEvent::Timeout, + ) +} + +/// Fold `event` for `txn` into its stored row and `target_vshard`'s buffer, +/// unless the txn finished on this replica, and complete the entry's +/// propose waiter. +fn fold_barrier_event( + ctx: ApplyContext<'_>, + pos: AppliedPosition, + target_vshard: u32, + txn: TxnId, + event: BarrierEvent, +) -> EntryOutcome { + if matches!(event, BarrierEvent::Read { .. }) { + ctx.state + .calvin + .read_results + .vshard(target_vshard) + .note_read_applied(pos.group_id, pos.log_index); + } + let saved = fold_unless_finished(ctx, target_vshard, txn, event); + tracing::debug!( + node_id = ctx.state.node_id, + group_id = pos.group_id, + index = pos.log_index, + vshard_id = target_vshard, + epoch = txn.epoch, + position = txn.position, + saved, + "calvin: barrier entry applied" + ); + // The event reached this replica's barrier either way: the waiter + // learns the entry committed. + ctx.tracker.complete( + pos.group_id, + pos.log_index, + pos.applied_key, + Ok(AppliedWrite::unversioned(Vec::new())), + ); + EntryOutcome::Applied { + durable: saved, + result: None, + } +} + +/// Fold `event` of `txn` into its stored row and `vshard_id`'s buffer, +/// unless the txn finished on this replica. Returns whether the row holds +/// the event, or needs none. +fn fold_unless_finished( + ctx: ApplyContext<'_>, + vshard_id: u32, + txn: TxnId, + event: BarrierEvent, +) -> bool { + if is_finished(ctx, vshard_id, txn) { + return true; + } + let saved = save_and_buffer(ctx, vshard_id, txn, event); + // The scheduler marks the ledger before it drops a finished txn's + // entries. A finish between the check above and the save saw no entry + // to drop, so this drops the one just saved. + if is_finished(ctx, vshard_id, txn) { + drop_finished(ctx, vshard_id, txn); + } + saved +} - let event = ReadResultEvent { - epoch: fields.epoch, - position: fields.position, - passive_vshard: fields.passive_vshard, - tenant_id: TenantId::new(fields.tenant_id), - values: decoded_values, +/// Fold the barrier event of `entry`, at `log_index` of `group_id`, ahead +/// of its turn in the group's lane. Entries that are not barrier entries +/// and payloads that do not decode change nothing. +/// +/// An earlier entry of the group can wait for this node's metadata apply, +/// and every later entry waits behind it. A barrier event names no +/// collection and needs no catalog, so it folds while the lane waits. Its +/// entry still concludes in log order, and that conclusion folds it again: +/// a second fold of an event the log already holds changes nothing. The +/// waiter, the applied counter and the durable prefix move only at the +/// conclusion. +/// +/// A barrier decision never runs ahead of the lane: a leader stages only +/// once its group applied its term's first entry. +pub(super) fn prefold_barrier_entry( + ctx: ApplyContext<'_>, + group_id: u64, + log_index: u64, + entry: &crate::control::wal_replication::ReplicatedEntry, +) { + use crate::control::wal_replication::ReplicatedWrite; + let (txn, event) = match &entry.write { + ReplicatedWrite::CalvinReadResult { + epoch, + position, + passive_vshard, + values, + .. + } => { + let Ok(values) = zerompk::from_msgpack(values) else { + // The conclusion reports the payload. + return; + }; + ( + TxnId::new(*epoch, *position), + BarrierEvent::Read { + passive_vshard: *passive_vshard, + values, + }, + ) + } + ReplicatedWrite::CalvinReadTimeout { + epoch, position, .. + } => (TxnId::new(*epoch, *position), BarrierEvent::Timeout), + _ => return, }; + let saved = fold_unless_finished(ctx, entry.vshard_id, txn, event); + tracing::debug!( + node_id = ctx.state.node_id, + group_id, + index = log_index, + vshard_id = entry.vshard_id, + epoch = txn.epoch, + position = txn.position, + saved, + "calvin: barrier entry folded ahead of its lane" + ); +} - let send_result = calvin_read_result_senders - .lock() - .unwrap_or_else(|p| p.into_inner()) - .get(&fields.target_vshard) - .cloned() - .map(|sender| sender.try_send(event)); +/// Whether `txn` finished on `vshard_id` here: its position is applied. +fn is_finished(ctx: ApplyContext<'_>, vshard_id: u32, txn: TxnId) -> bool { + ctx.state + .calvin + .applied + .get(vshard_id) + .is_some_and(|ledger| ledger.is_applied(txn.epoch, txn.position)) +} - if let Some(Err(e)) = send_result { +/// Drop the buffered entries and the row of `txn` on `vshard_id`, which +/// finished. A failed remove leaves a row no barrier reads, and the +/// scheduler's stall-tick sweep removes it. +fn drop_finished(ctx: ApplyContext<'_>, vshard_id: u32, txn: TxnId) { + ctx.state + .calvin + .read_results + .vshard(vshard_id) + .forget_txn(txn); + if let Err(e) = barrier_store::remove_log(ctx.state.credentials.catalog(), vshard_id, txn) { tracing::warn!( - group_id, - index = log_index, + vshard_id, + epoch = txn.epoch, + position = txn.position, error = %e, - "failed to forward CalvinReadResult to scheduler" + "calvin: the barrier row of a finished txn was not removed" + ); + crate::diag::calvin_barrier_log_store_failed( + vshard_id, + Some((txn.epoch, txn.position)), + "remove", + &e, ); } - tracker.complete( - group_id, - log_index, - applied_key, - Ok(AppliedWrite::unversioned(Vec::new())), +} + +/// Save `event` in the row of `txn` on `vshard_id`, then hand it to the +/// vShard's buffer. Returns whether the row holds it. +/// +/// A txn with an event its row lacks skips the row: the row stays a prefix +/// of the txn's events, so a restart that delivers the missing entry again +/// folds them in log order. +fn save_and_buffer(ctx: ApplyContext<'_>, vshard_id: u32, txn: TxnId, event: BarrierEvent) -> bool { + let buffer = ctx.state.calvin.read_results.vshard(vshard_id); + let saved = !buffer.is_unsaved(txn) + && match barrier_store::save_event(ctx.state.credentials.catalog(), vshard_id, txn, &event) + { + Ok(()) => true, + Err(e) => { + tracing::error!( + vshard_id, + epoch = txn.epoch, + position = txn.position, + error = %e, + "calvin: the barrier event's row write failed; the event stays in memory" + ); + crate::diag::calvin_barrier_log_store_failed( + vshard_id, + Some((txn.epoch, txn.position)), + "save", + &e, + ); + false + } + }; + buffer.push( + txn, + event, + ctx.state.tuning.calvin.max_inflight_backlog, + saved, ); + saved +} + +#[cfg(test)] +mod tests { + use std::sync::Arc; + + use nodedb_physical::physical_plan::meta::PassiveReadKeyId; + use nodedb_types::{QualifiedCollection, Value}; + + use super::*; + use crate::bridge::dispatch::Dispatcher; + use crate::control::distributed_applier::propose_tracker::ProposeTracker; + use crate::control::state::SharedState; + use crate::wal::WalManager; + + const VSHARD: u32 = 6; + + fn pos(log_index: u64) -> AppliedPosition { + AppliedPosition { + group_id: 2, + log_index, + applied_key: 0, + commit_hlc: 0, + } + } + + fn values() -> Vec { + let values = vec![( + PassiveReadKeyId::kv( + QualifiedCollection::from_stored("items".to_owned()), + b"k".to_vec(), + ), + Value::Bytes(b"v".to_vec()), + )]; + zerompk::to_msgpack_vec(&values).expect("encode values") + } + + fn state() -> (Arc, tempfile::TempDir) { + let dir = tempfile::tempdir().expect("tempdir"); + let wal = Arc::new(WalManager::open_for_testing(&dir.path().join("wal")).expect("wal")); + let (dispatcher, _sides) = Dispatcher::new(1, 64); + ( + SharedState::new(dispatcher, wal).expect("shared state"), + dir, + ) + } + + /// A read result for a txn no scheduler here holds waits in the buffer. + #[test] + fn a_result_before_the_grant_waits_in_the_buffer() { + let (state, _dir) = state(); + let tracker = Arc::new(ProposeTracker::new()); + let ctx = ApplyContext { + state: &state, + tracker: &tracker, + }; + let bytes = values(); + let outcome = forward_calvin_read_result( + ctx, + pos(10), + CalvinReadResultFields { + target_vshard: VSHARD, + epoch: 4, + position: 0, + passive_vshard: 9, + values: &bytes, + }, + ); + assert!(matches!( + outcome, + EntryOutcome::Applied { + durable: true, + result: None + } + )); + forward_calvin_read_timeout(ctx, pos(11), VSHARD, 4, 0); + assert_eq!( + state.calvin.read_results.vshard(VSHARD).read_entries(), + vec![(2, 10)], + "the buffer names the read result's entry, not the timeout's" + ); + assert_eq!(state.calvin.read_results.waiting_txns(VSHARD), 1); + let events = state + .calvin + .read_results + .vshard(VSHARD) + .take(TxnId::new(4, 0)) + .expect("the buffered log"); + assert!(!events.stored); + assert!(events.memory.missing(&[9].into_iter().collect()).is_empty()); + // The row holds both events: a restart reads them from it. + let stored = barrier_store::load_log(state.credentials.catalog(), VSHARD, TxnId::new(4, 0)) + .expect("load the row"); + assert_eq!(stored, events.memory); + } + + /// A barrier entry folds ahead of its turn while its lane waits, and + /// its conclusion folds it again with no change. Only the conclusion + /// counts the entry and completes its waiter. + #[test] + fn a_prefolded_entry_concludes_without_a_second_change() { + let (state, _dir) = state(); + let tracker = Arc::new(ProposeTracker::new()); + let ctx = ApplyContext { + state: &state, + tracker: &tracker, + }; + let bytes = values(); + let entry = crate::control::wal_replication::ReplicatedEntry::new( + 1, + 0, + VSHARD, + crate::control::wal_replication::ReplicatedWrite::CalvinReadResult { + epoch: 5, + position: 0, + passive_vshard: 9, + tenant_id: 1, + values: bytes.clone(), + }, + ); + prefold_barrier_entry(ctx, 2, 12, &entry); + let buffer = state.calvin.read_results.vshard(VSHARD); + assert_eq!(buffer.waiting_txns(), 1, "the event waits before its turn"); + assert!(buffer.read_entries().is_empty(), "a prefold counts nothing"); + let stored = barrier_store::load_log(state.credentials.catalog(), VSHARD, TxnId::new(5, 0)) + .expect("load the row"); + assert!(stored.missing(&[9].into_iter().collect()).is_empty()); + + let outcome = forward_calvin_read_result( + ctx, + pos(12), + CalvinReadResultFields { + target_vshard: VSHARD, + epoch: 5, + position: 0, + passive_vshard: 9, + values: &bytes, + }, + ); + assert!(matches!( + outcome, + EntryOutcome::Applied { + durable: true, + result: None + } + )); + assert_eq!(buffer.read_entries(), vec![(2, 12)]); + assert_eq!( + barrier_store::load_log(state.credentials.catalog(), VSHARD, TxnId::new(5, 0)) + .expect("load the row"), + stored + ); + let events = buffer.take(TxnId::new(5, 0)).expect("buffered events"); + assert_eq!(events.memory, stored); + } + + /// An event for a position the vShard's ledger holds is dropped: the + /// txn finished. + #[test] + fn an_event_of_a_finished_txn_is_dropped() { + let (state, _dir) = state(); + state + .calvin + .applied + .get_or_create(VSHARD) + .mark_terminal(4, 0); + let tracker = Arc::new(ProposeTracker::new()); + let ctx = ApplyContext { + state: &state, + tracker: &tracker, + }; + forward_calvin_read_timeout(ctx, pos(11), VSHARD, 4, 0); + assert_eq!(state.calvin.read_results.waiting_txns(VSHARD), 0); + assert!( + state + .credentials + .catalog() + .load_calvin_barrier_log(VSHARD, 4, 0) + .expect("load") + .is_none(), + "a finished txn's event writes no row" + ); + } } diff --git a/nodedb/src/control/distributed_applier/apply_loop/calvin_redo.rs b/nodedb/src/control/distributed_applier/apply_loop/calvin_redo.rs new file mode 100644 index 000000000..bfdec2140 --- /dev/null +++ b/nodedb/src/control/distributed_applier/apply_loop/calvin_redo.rs @@ -0,0 +1,439 @@ +// SPDX-License-Identifier: BUSL-1.1 + +//! The Calvin part of a stamped `TransactionRedo` entry's apply. +//! +//! A committed Calvin slice installs only from its vShard's data-group log. +//! Every replica applies its entries in log order, so every replica decides +//! each copy of a slice's redo alike: +//! +//! 1. At entry start, in log order, the apply claims `(epoch, position)` in +//! the vShard's applied ledger. A claim refused because the position is +//! applied, or another copy holds it, concludes the entry applied and +//! installs nothing: a re-proposed copy never installs twice. +//! 2. The install runs like every committed redo's. +//! 3. When the install is durable, the ledger marks the position, the +//! position's open chunk streams drop, and the vShard's scheduler hears +//! `RedoApplied`. All of it happens before the entry concludes, so the +//! ledger holds the position before the group's applied index passes +//! the entry. +//! 4. A final refusal keeps the claim: every replica refuses the same entry, +//! and no later copy installs. An install that is not durable releases +//! the claim: a restart applies the entry again. +//! +//! A full inbox makes the apply wait for room. The entry holds its own +//! place in the group's lane meanwhile, as a parked write does. A vShard +//! with no running scheduler gets no event: the next scheduler seeds from +//! the ledger. + +use std::sync::Arc; +use std::sync::atomic::Ordering; + +use crate::bridge::envelope::{ErrorCode, Payload, Response, Status}; +use crate::control::cluster::calvin::scheduler::{CalvinAppliedLedger, CalvinApplyEvent}; +use crate::control::server::dispatch_utils::{SubmitOutcome, refusal_is_final}; +use crate::control::state::SharedState; +use crate::control::wal_replication::types::CalvinRedoMeta; +use crate::types::{Lsn, RequestId}; +use crate::wal::RedoStreamId; + +/// A Calvin position this entry holds the claim on. +pub(super) struct CalvinClaim { + vshard_id: u32, + epoch: u64, + position: u32, + primary_write: bool, + returning: bool, + ledger: Arc, +} + +/// What a stamped redo entry's claim decided. +pub(super) enum ClaimOutcome { + /// The entry carries no Calvin slice. + NotCalvin, + /// The entry installs the position. + Claimed(CalvinClaim), + /// The position is applied, or another copy holds it: the entry + /// installs nothing. + Refused, + /// The entry names a Calvin slice but no position, so no replica can + /// install it. + Malformed(crate::Error), +} + +/// Claim the position of an inline Calvin redo: its record's stamp. +pub(super) fn claim_inline( + state: &SharedState, + meta: Option<&CalvinRedoMeta>, + stamp: Option<&crate::wal::CalvinStamp>, +) -> ClaimOutcome { + let Some(meta) = meta else { + return ClaimOutcome::NotCalvin; + }; + match stamp { + Some(stamp) => claim(state, meta, stamp.vshard_id, stamp.epoch, stamp.position), + None => ClaimOutcome::Malformed(crate::Error::Internal { + detail: "a Calvin redo entry carries no calvin_stamp, so it names no position".into(), + }), + } +} + +/// Claim the position of a chunked Calvin redo: its stream's. +pub(super) fn claim_chunked( + state: &SharedState, + meta: Option<&CalvinRedoMeta>, + stream: &RedoStreamId, +) -> ClaimOutcome { + let Some(meta) = meta else { + return ClaimOutcome::NotCalvin; + }; + match *stream { + RedoStreamId::Calvin { + vshard, + epoch, + position, + .. + } => claim(state, meta, vshard, epoch, position), + RedoStreamId::Session { .. } => ClaimOutcome::Malformed(crate::Error::Internal { + detail: format!("a Calvin redo entry names the session stream {stream:?}"), + }), + } +} + +fn claim( + state: &SharedState, + meta: &CalvinRedoMeta, + vshard_id: u32, + epoch: u64, + position: u32, +) -> ClaimOutcome { + let ledger = state.calvin.applied.get_or_create(vshard_id); + match ledger.claim(epoch, position) { + Ok(()) => ClaimOutcome::Claimed(CalvinClaim { + vshard_id, + epoch, + position, + primary_write: meta.primary_write, + returning: meta.returning, + ledger, + }), + Err(refusal) => { + state + .calvin + .counters + .redo_copies_skipped + .fetch_add(1, Ordering::Relaxed); + tracing::debug!( + vshard_id, + epoch, + position, + ?refusal, + "calvin redo: a copy of an applied or claimed position installs nothing" + ); + ClaimOutcome::Refused + } + } +} + +impl CalvinClaim { + /// Conclude the claim from the install's outcome, and return what the + /// vShard's scheduler hears. + pub(super) fn settle( + &self, + state: &SharedState, + submitted: &crate::Result, + ) -> CalvinApplyEvent { + match submitted { + Ok(outcome) if outcome.response.status == Status::Ok => { + self.applied(state, outcome.response.clone()) + } + Ok(outcome) + if outcome + .response + .error_code + .as_deref() + .is_some_and(refusal_is_final) => + { + CalvinApplyEvent::RedoRefused { + error: format!("{:?}", outcome.response.error_code.as_deref()), + } + } + Ok(outcome) => self.not_applied(format!( + "the install answered {:?} with {:?}", + outcome.response.status, + outcome.response.error_code.as_deref() + )), + Err(error) => self.not_applied(error.to_string()), + } + } + + /// The entry's collection is gone on this replica: nothing installs, + /// and the position is done. + pub(super) fn superseded(&self, state: &SharedState) -> CalvinApplyEvent { + let reply = Response { + request_id: RequestId::new(0), + status: Status::Error, + attempt: 1, + partial: false, + payload: Payload::empty(), + watermark_lsn: Lsn::ZERO, + error_code: Some(Box::new(ErrorCode::NotFound)), + stage_vote: None, + read_versions: crate::types::ReadVersions::new(), + write_set: Vec::new(), + }; + self.applied(state, reply) + } + + /// The install did not run, or did not become durable. + pub(super) fn not_applied(&self, error: String) -> CalvinApplyEvent { + self.ledger.release_claim(self.epoch, self.position); + CalvinApplyEvent::RedoNotApplied { error } + } + + fn applied(&self, state: &SharedState, reply: Response) -> CalvinApplyEvent { + self.ledger.mark_applied(self.epoch, self.position); + state + .redo_chunks + .drop_calvin_position(self.vshard_id, self.epoch, self.position); + // The install recorded the slice's write versions. + state + .calvin + .counters + .write_versions_recorded + .fetch_add(1, Ordering::Relaxed); + CalvinApplyEvent::RedoApplied { + reply, + primary_write: self.primary_write, + returning: self.returning, + } + } + + /// Hand `event` to the vShard's scheduler, waiting for room in its + /// inbox. A vShard with no running scheduler takes nothing. + pub(super) async fn report(&self, state: &SharedState, event: CalvinApplyEvent) { + let Some(inbox) = state.calvin.inboxes.get(self.vshard_id) else { + return; + }; + inbox.push(self.epoch, self.position, event).await; + } + + /// Hand an install's `event` to the vShard's scheduler. `collections` + /// are the slice's collections. + /// + /// With fail points, the gate `calvin::before_redo_applied_push::` on + /// one of them withholds a `RedoApplied`. The entry concludes, and its + /// own task pushes the event once the gate releases. Until then the + /// leader sees its group apply past the redo with no install heard, and + /// proposes the redo again. + pub(super) async fn report_install( + &self, + state: &SharedState, + event: CalvinApplyEvent, + collections: &[String], + ) { + #[cfg(feature = "failpoints")] + let Some(event) = self.withhold_applied(state, event, collections) else { + return; + }; + #[cfg(not(feature = "failpoints"))] + let _ = collections; + self.report(state, event).await; + } + + /// Hand a held `RedoApplied` to a task that pushes it once its gate + /// releases. Returns any other event, or one no gate holds. + #[cfg(feature = "failpoints")] + fn withhold_applied( + &self, + state: &SharedState, + event: CalvinApplyEvent, + collections: &[String], + ) -> Option { + if !matches!(event, CalvinApplyEvent::RedoApplied { .. }) { + return Some(event); + } + let Some(gate) = + crate::control::fail_gate::holds_redo_applied_push(state.node_id, collections) + else { + return Some(event); + }; + let Some(inbox) = state.calvin.inboxes.get(self.vshard_id) else { + return Some(event); + }; + let (epoch, position) = (self.epoch, self.position); + let fail_scope = crate::fail_point::FailScope::Node(state.node_id); + tracing::info!( + vshard_id = self.vshard_id, + epoch, + position, + gate = %gate, + "calvin: RedoApplied withheld at a fail point" + ); + tokio::spawn(async move { + crate::control::fail_gate::released(fail_scope, &gate).await; + inbox.push(epoch, position, event).await; + }); + None + } +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::bridge::dispatch::Dispatcher; + use crate::wal::WalManager; + use nodedb_physical::physical_plan::CalvinReplySpec; + + fn state() -> (Arc, tempfile::TempDir) { + let dir = tempfile::tempdir().expect("tempdir"); + let wal = Arc::new(WalManager::open_for_testing(&dir.path().join("wal")).expect("wal")); + let (dispatcher, _sides) = Dispatcher::new(1, 64); + (SharedState::new(dispatcher, wal).expect("state"), dir) + } + + fn meta() -> CalvinRedoMeta { + CalvinRedoMeta { + epoch_system_ms: 0, + reply: CalvinReplySpec::Count(Vec::new()), + primary_write: true, + user_write: true, + returning: false, + } + } + + fn stamp(position: u32) -> crate::wal::CalvinStamp { + crate::wal::CalvinStamp { + epoch: 4, + position, + vshard_id: 3, + } + } + + fn ok() -> crate::Result { + Ok(SubmitOutcome { + response: Response { + request_id: RequestId::new(1), + status: Status::Ok, + attempt: 1, + partial: false, + payload: Payload::empty(), + watermark_lsn: Lsn::ZERO, + error_code: None, + stage_vote: None, + read_versions: crate::types::ReadVersions::new(), + write_set: Vec::new(), + }, + }) + } + + /// A second stamped copy of a position finds the claim of the first, or + /// the position applied, and installs nothing. + #[test] + fn a_second_stamped_copy_installs_nothing() { + let (state, _dir) = state(); + let ClaimOutcome::Claimed(first) = claim_inline(&state, Some(&meta()), Some(&stamp(1))) + else { + panic!("the first copy claims the position"); + }; + assert!(matches!( + claim_inline(&state, Some(&meta()), Some(&stamp(1))), + ClaimOutcome::Refused + )); + let event = first.settle(&state, &ok()); + assert!(matches!(event, CalvinApplyEvent::RedoApplied { .. })); + assert!(matches!( + claim_inline(&state, Some(&meta()), Some(&stamp(1))), + ClaimOutcome::Refused + )); + assert!(matches!( + claim_chunked( + &state, + Some(&meta()), + &RedoStreamId::Calvin { + vshard: 3, + epoch: 4, + position: 1, + attempt: 2, + } + ), + ClaimOutcome::Refused + )); + } + + /// The ledger holds a durable install's position by the time its + /// settle returns, before the entry concludes and the group's applied + /// index passes it. An install that is not durable releases the claim. + #[test] + fn the_ledger_marks_before_the_applied_index_passes() { + let (state, _dir) = state(); + let ledger = state.calvin.applied.get_or_create(3); + let ClaimOutcome::Claimed(claim) = claim_inline(&state, Some(&meta()), Some(&stamp(2))) + else { + panic!("claims"); + }; + assert!( + !ledger.is_applied(4, 2), + "a claim is not an applied position" + ); + claim.settle(&state, &ok()); + assert!(ledger.is_applied(4, 2)); + + let ClaimOutcome::Claimed(failing) = claim_inline(&state, Some(&meta()), Some(&stamp(5))) + else { + panic!("claims"); + }; + let event = failing.settle( + &state, + &Err(crate::Error::Internal { + detail: "fsync failed".into(), + }), + ); + assert!(matches!(event, CalvinApplyEvent::RedoNotApplied { .. })); + assert!(!ledger.is_applied(4, 5)); + assert!( + matches!( + claim_inline(&state, Some(&meta()), Some(&stamp(5))), + ClaimOutcome::Claimed(_) + ), + "a released claim lets the next copy install" + ); + } + + /// An entry with no Calvin meta claims nothing, and a Calvin meta with + /// no position is malformed. + #[test] + fn only_a_stamped_calvin_entry_claims() { + let (state, _dir) = state(); + assert!(matches!( + claim_inline(&state, None, Some(&stamp(1))), + ClaimOutcome::NotCalvin + )); + assert!(matches!( + claim_inline(&state, Some(&meta()), None), + ClaimOutcome::Malformed(_) + )); + } + + /// A vShard with no running scheduler hears nothing, and its ledger + /// still holds the position. + #[tokio::test] + async fn no_scheduler_takes_no_event() { + let (state, _dir) = state(); + let ClaimOutcome::Claimed(claim) = claim_inline(&state, Some(&meta()), Some(&stamp(3))) + else { + panic!("claims"); + }; + let event = claim.settle(&state, &ok()); + claim.report(&state, event).await; + assert!(state.calvin.applied.get_or_create(3).is_applied(4, 3)); + + let handle = state.calvin.inboxes.register(3, 4); + let ClaimOutcome::Claimed(next) = claim_inline(&state, Some(&meta()), Some(&stamp(4))) + else { + panic!("claims"); + }; + let event = next.settle(&state, &ok()); + next.report(&state, event).await; + assert_eq!(handle.inbox().take_all().len(), 1); + } +} diff --git a/nodedb/src/control/distributed_applier/apply_loop/context.rs b/nodedb/src/control/distributed_applier/apply_loop/context.rs index 21b27a800..3c7996901 100644 --- a/nodedb/src/control/distributed_applier/apply_loop/context.rs +++ b/nodedb/src/control/distributed_applier/apply_loop/context.rs @@ -3,28 +3,20 @@ //! What every entry's apply borrows from the loop, and the futures the loop //! collects while it prepares later entries. -use std::collections::BTreeMap; use std::future::Future; use std::pin::Pin; -use std::sync::{Arc, Mutex}; +use std::sync::Arc; -use tokio::sync::mpsc; - -use crate::control::cluster::calvin::ReadResultEvent; use crate::control::distributed_applier::propose_tracker::ProposeTracker; use crate::control::state::SharedState; use super::proposal_gate::EntryOutcome; -/// Senders to each local Calvin scheduler's read-result channel, by vShard. -pub(super) type CalvinReadResultSenders = Arc>>>; - /// The loop-owned handles an entry's apply borrows. #[derive(Clone, Copy)] pub(super) struct ApplyContext<'a> { pub state: &'a Arc, pub tracker: &'a Arc, - pub calvin_read_result_senders: &'a CalvinReadResultSenders, } /// An entry whose apply finished: its waiter is resolved, and its outcome diff --git a/nodedb/src/control/distributed_applier/apply_loop/cut_barrier.rs b/nodedb/src/control/distributed_applier/apply_loop/cut_barrier.rs new file mode 100644 index 000000000..486595688 --- /dev/null +++ b/nodedb/src/control/distributed_applier/apply_loop/cut_barrier.rs @@ -0,0 +1,202 @@ +// SPDX-License-Identifier: BUSL-1.1 + +//! A backup cut's barrier entry: the group's floor above the cut, the +//! restore point's place, and a database backup's capture. +//! +//! The apply notes each barrier in the node's cut registry in log order, at +//! the entry's start. A cut waiting on this node reads where the group's +//! barrier sits, and a leader's schedulers release the Calvin redo they held +//! for the cut. + +use crate::control::array_sync::raft_apply::AppliedPosition; +use crate::control::backup::cut_order::OrderedCut; +use crate::control::state::SharedState; +use crate::control::wal_replication::{ReplicatedEntry, ReplicatedWrite}; + +use super::context::{ApplyContext, FinishedApply}; +use super::lane::QueuedEntry; +use super::proposal_gate::EntryOutcome; +use super::start::Prepared; + +/// Where a cut barrier cuts its group. +#[derive(Clone, Copy)] +struct CutBarrierPoint { + hlc: u64, + restore_point: u64, + log_term: u64, +} + +/// Prepare the barrier of `cut` at `pos`, whose entry has term `log_term`. +pub(super) fn prepare_cut_barrier<'a>( + ctx: ApplyContext<'a>, + pos: AppliedPosition, + log_term: u64, + cut: OrderedCut, +) -> Prepared<'a> { + ctx.state + .calvin + .cut_barriers + .note_applied(&cut, pos.group_id, pos.log_index); + let barrier = CutBarrierPoint { + hlc: cut.hlc, + restore_point: cut.restore_point, + log_term, + }; + match cut.capture { + Some(request) => prepare_capture_barrier(ctx, pos, barrier, request), + None => prepare_plain_barrier(ctx, pos, barrier), + } +} + +/// Note a cut barrier an installed snapshot covers. The entry is committed, +/// and the snapshot holds every entry before it, so a cut waiting here and a +/// leader's held redo read the barrier as applied. +pub(super) fn note_covered_barrier(state: &SharedState, group_id: u64, queued: &QueuedEntry) { + if let Some(ReplicatedEntry { + write: + ReplicatedWrite::CutBarrier { + hlc, + restore_point, + capture, + }, + .. + }) = &queued.decoded + { + let cut = OrderedCut { + hlc: *hlc, + restore_point: *restore_point, + capture: capture.clone(), + }; + state + .calvin + .cut_barriers + .note_applied(&cut, group_id, queued.entry.index); + } +} + +/// Prepare a backup capture's cut barrier. The lane starts a barrier with +/// nothing else of its group in flight, and starts no later entry until it +/// finishes: the floor is durable, and the capture holds every entry at or +/// below it and none above. +fn prepare_capture_barrier<'a>( + ctx: ApplyContext<'a>, + pos: AppliedPosition, + barrier: CutBarrierPoint, + request: nodedb_physical::physical_plan::CutCaptureRequest, +) -> Prepared<'a> { + let AppliedPosition { + group_id, + log_index, + applied_key, + .. + } = pos; + let CutBarrierPoint { + hlc, + restore_point, + log_term, + } = barrier; + Prepared::Exclusive(Box::pin(async move { + persist_floor_durably(ctx, group_id, log_index, hlc).await; + record_restore_point(ctx, group_id, restore_point, hlc, log_index, log_term); + crate::control::backup::cut_capture::apply::capture_at_barrier( + ctx.state, group_id, &request, + ) + .await; + finish_barrier(ctx, group_id, log_index, applied_key) + })) +} + +/// Prepare a cut barrier with no capture. A failed floor write holds the +/// group until the floor is durable. +fn prepare_plain_barrier<'a>( + ctx: ApplyContext<'a>, + pos: AppliedPosition, + barrier: CutBarrierPoint, +) -> Prepared<'a> { + let AppliedPosition { + group_id, + log_index, + applied_key, + .. + } = pos; + let CutBarrierPoint { + hlc, + restore_point, + log_term, + } = barrier; + let persisted = + crate::control::pitr::restore_point::persist_cut_floor(ctx.state, group_id, log_index, hlc); + if persisted.is_ok() { + record_restore_point(ctx, group_id, restore_point, hlc, log_index, log_term); + return Prepared::Barrier; + } + Prepared::Exclusive(Box::pin(async move { + persist_floor_durably(ctx, group_id, log_index, hlc).await; + record_restore_point(ctx, group_id, restore_point, hlc, log_index, log_term); + finish_barrier(ctx, group_id, log_index, applied_key) + })) +} + +/// Record the group's place at the cluster restore point `restore_point` a +/// cut barrier takes. `0` takes none. +fn record_restore_point( + ctx: ApplyContext<'_>, + group_id: u64, + restore_point: u64, + hlc: u64, + log_index: u64, + log_term: u64, +) { + if restore_point == 0 { + return; + } + crate::control::pitr::restore_point::record_group_point( + ctx.state, + nodedb_wal::record::RestorePointPayload { + id: restore_point, + hlc, + group_id, + applied_index: log_index, + term: log_term, + next_epoch: 0, + epoch_system_ms: 0, + vshards: Vec::new(), + }, + ); +} + +/// Persist the floor of the barrier at `log_index`, retrying until it holds. +/// A floor write that keeps failing wedges the node until it succeeds. +async fn persist_floor_durably(ctx: ApplyContext<'_>, group_id: u64, log_index: u64, hlc: u64) { + crate::control::pitr::restore_point::persist_until_durable( + &ctx.state.metadata_apply_wedge, + group_id, + log_index, + || { + crate::control::pitr::restore_point::persist_cut_floor( + ctx.state, group_id, log_index, hlc, + ) + }, + ) + .await; +} + +/// Resolve a barrier's waiter once every step of the barrier is durable. +fn finish_barrier( + ctx: ApplyContext<'_>, + group_id: u64, + log_index: u64, + applied_key: u64, +) -> FinishedApply { + ctx.tracker.complete( + group_id, + log_index, + applied_key, + Ok(crate::control::distributed_applier::AppliedWrite::unversioned(Vec::new())), + ); + FinishedApply { + group_id, + log_index, + outcome: EntryOutcome::Skipped, + } +} diff --git a/nodedb/src/control/distributed_applier/apply_loop/driver.rs b/nodedb/src/control/distributed_applier/apply_loop/driver.rs index 865cf9ffe..abdba3b87 100644 --- a/nodedb/src/control/distributed_applier/apply_loop/driver.rs +++ b/nodedb/src/control/distributed_applier/apply_loop/driver.rs @@ -8,7 +8,6 @@ use std::sync::Arc; use tokio::sync::{mpsc, watch}; -use crate::control::cluster::calvin::ReadResultEvent; use crate::control::distributed_applier::applier::ApplyBatch; use crate::control::distributed_applier::proposal_ledger::{ PROPOSAL_LEDGER_CAPACITY, ProposalLedger, @@ -27,9 +26,6 @@ pub async fn run_apply_loop( mut apply_rx: mpsc::Receiver, state: Arc, tracker: Arc, - calvin_read_result_senders: Arc< - std::sync::Mutex>>, - >, ) { // Proposals this node already applied, recovered from its WAL before any // entry is delivered: every record an entry's apply appended carries the @@ -57,7 +53,6 @@ pub async fn run_apply_loop( let ctx = ApplyContext { state: &state, tracker: &tracker, - calvin_read_result_senders: &calvin_read_result_senders, }; let mut pipeline = Pipeline::new(ctx, ledger); let mut install_released = state.raft_apply_gates.get().map(|g| g.subscribe_released()); diff --git a/nodedb/src/control/distributed_applier/apply_loop/lane.rs b/nodedb/src/control/distributed_applier/apply_loop/lane.rs index 71728e455..9cf667612 100644 --- a/nodedb/src/control/distributed_applier/apply_loop/lane.rs +++ b/nodedb/src/control/distributed_applier/apply_loop/lane.rs @@ -18,6 +18,7 @@ use crate::control::distributed_applier::applied_index::AppliedPrefix; use crate::control::distributed_applier::propose_tracker::{ApplyingEntry, ProposeTracker}; use crate::control::server::shared::write_admission::plan_writes_user_data; use crate::control::state::tenant_marks::{MarkSite, TenantMarks}; +use crate::control::wal_replication::types::CalvinRedoMeta; use crate::control::wal_replication::{ReplicatedEntry, ReplicatedWrite, decode_replicated_entry}; use super::proposal_gate::PrefixStep; @@ -52,9 +53,10 @@ impl QueuedEntry { } /// `(tenant_id, write_hlc, restore_id)` of an entry that writes a - /// tenant's data. A cut barrier, a Calvin read result and a surrogate - /// bind write no data, and an entry with no proposer stamp has no commit - /// HLC to record. + /// tenant's data. A cut barrier, a Calvin read result or read timeout + /// and a surrogate bind write no data, a Calvin slice that raises no write mark writes + /// only derived rows or schema, and an entry with no proposer stamp has + /// no commit HLC to record. pub fn write_stamp(&self) -> Option<(u64, u64, u64)> { let decoded = self.decoded.as_ref()?; if decoded.write_hlc == 0 @@ -62,7 +64,17 @@ impl QueuedEntry { decoded.write, ReplicatedWrite::CutBarrier { .. } | ReplicatedWrite::CalvinReadResult { .. } + | ReplicatedWrite::CalvinReadTimeout { .. } | ReplicatedWrite::SurrogateBind { .. } + | ReplicatedWrite::RedoChunk { .. } + | ReplicatedWrite::RedoAbandon { .. } + | ReplicatedWrite::TransactionRedo { + calvin: Some(CalvinRedoMeta { + user_write: false, + .. + }), + .. + } ) { return None; @@ -77,15 +89,20 @@ impl QueuedEntry { let Some(decoded) = self.decoded.as_ref() else { return false; }; - match decoded.write { + match &decoded.write { + ReplicatedWrite::TransactionRedo { calvin, .. } => { + calvin.as_ref().is_none_or(|meta| meta.user_write) + } ReplicatedWrite::ArrayOp { .. } | ReplicatedWrite::ArrayCellPut { .. } - | ReplicatedWrite::ArrayCellDelete { .. } - | ReplicatedWrite::TransactionRedo { .. } => true, + | ReplicatedWrite::ArrayCellDelete { .. } => true, ReplicatedWrite::ArraySchema { .. } | ReplicatedWrite::CutBarrier { .. } | ReplicatedWrite::CalvinReadResult { .. } - | ReplicatedWrite::SurrogateBind { .. } => false, + | ReplicatedWrite::CalvinReadTimeout { .. } + | ReplicatedWrite::SurrogateBind { .. } + | ReplicatedWrite::RedoChunk { .. } + | ReplicatedWrite::RedoAbandon { .. } => false, _ => matches!( decode_replicated_entry(&self.entry.data), Ok(Some((_, _, plan, _))) if plan_writes_user_data(&plan) @@ -99,7 +116,8 @@ impl QueuedEntry { /// and a schema import must follow every earlier entry's apply. A topic /// publication awaits its proposal marker's fsync. A cut barrier /// persists its floor, and a capturing one snapshots its tenants, after - /// every earlier entry and before every later one. + /// every earlier entry and before every later one. A redo chunk or + /// abandon changes the stream a later entry of its group reads. pub fn is_exclusive(&self) -> bool { self.decoded.as_ref().is_some_and(|e| { matches!( @@ -110,6 +128,8 @@ impl QueuedEntry { | ReplicatedWrite::ArrayCellDelete { .. } | ReplicatedWrite::TopicPublish { .. } | ReplicatedWrite::CutBarrier { .. } + | ReplicatedWrite::RedoChunk { .. } + | ReplicatedWrite::RedoAbandon { .. } ) }) } @@ -153,13 +173,16 @@ pub(super) struct Lane { /// runs, or an exclusive entry that runs. pub blocking: Option, /// The durable prefix over every entry this process settled for the - /// group. A break holds for the life of the process: an index saved past - /// a non-durable entry lets the next boot skip it. + /// group. A break holds until a snapshot install covers it: an index + /// saved past a non-durable entry lets the next boot skip it. prefix: AppliedPrefix, /// The floor last saved for the group. saved_floor: Option, /// The entry the last settle ended at. last_settled: Option, + /// The highest backlog index whose barrier event folded ahead of its + /// turn (see `barrier_prefold`). + pub prefolded_through: u64, } impl Lane { @@ -172,6 +195,7 @@ impl Lane { prefix: AppliedPrefix::new(), saved_floor: None, last_settled: None, + prefolded_through: 0, } } @@ -326,6 +350,17 @@ impl Lane { settled } + /// Count every entry of the group through `installed_through` durable. + /// An installed snapshot holds their state, so a break at or below it + /// heals. + /// + /// The caller passes the index it read under the group's apply gate, + /// before it admits an entry. Every entry above the index reaches the + /// lane after the install, so none settles before this call. + pub fn cover_through(&mut self, installed_through: u64) { + self.prefix.cover_through(installed_through); + } + /// Whether the durable floor moved past the floor last saved. pub fn floor_pending(&self) -> bool { self.prefix @@ -410,6 +445,48 @@ mod tests { ); } + #[test] + fn a_covering_snapshot_lets_the_floor_pass_a_held_entry() { + let tracker = ProposeTracker::new(); + let marks = TenantMarks::default(); + let mut lane = Lane::new(1); + lane.push(slot(1, SlotState::Concluded(PrefixStep::Record(true)))); + lane.push(slot(2, SlotState::Concluded(PrefixStep::Record(false)))); + lane.push(slot(3, SlotState::Running)); + lane.settle(&tracker, &marks); + assert_eq!(lane.take_floor_to_save(), Some(1)); + + // A snapshot at index 4 installs while entry 3 still runs. + lane.cover_through(4); + assert_eq!(lane.take_floor_to_save(), Some(4)); + + assert!(lane.conclude(3, false, |_| PrefixStep::Record(false))); + lane.push(slot(5, SlotState::Concluded(PrefixStep::Record(true)))); + lane.settle(&tracker, &marks); + assert_eq!( + lane.take_floor_to_save(), + Some(5), + "the snapshot holds entries 2 and 3" + ); + } + + #[test] + fn a_held_entry_above_the_snapshot_still_holds_the_floor() { + let tracker = ProposeTracker::new(); + let marks = TenantMarks::default(); + let mut lane = Lane::new(1); + lane.cover_through(4); + lane.push(slot(5, SlotState::Concluded(PrefixStep::Record(false)))); + lane.push(slot(6, SlotState::Concluded(PrefixStep::Record(true)))); + lane.settle(&tracker, &marks); + assert_eq!(lane.take_floor_to_save(), Some(4)); + assert_eq!( + lane.take_floor_to_save(), + None, + "a floor past entry 5 would let the next boot skip it" + ); + } + #[test] fn an_enqueue_that_returns_unblocks_the_next_start() { let tracker = ProposeTracker::new(); @@ -506,4 +583,55 @@ mod tests { lane.settle(&tracker, &TenantMarks::default()); assert!(matches!(barrier.try_recv(), Ok(Ok(_)))); } + + /// The stamped redo entry of a Calvin slice that raises the write mark + /// when `user_write` holds, proposed with commit HLC 77. + fn calvin_redo_entry(user_write: bool) -> QueuedEntry { + let payload = crate::control::wal_replication::transaction_redo::TransactionRedoPayload { + redo: crate::wal::RedoRecord { + version: 1, + ops: Vec::new(), + calvin_stamp: None, + cross_shard_applied: None, + row_sources: Vec::new(), + publishes: Vec::new(), + row_changes: Vec::new(), + }, + collections: vec!["orders".into()], + sum_targets: Vec::new(), + identities: Vec::new(), + event_source: crate::event::EventSource::User, + origin: nodedb_physical::physical_plan::RedoOrigin::Commit, + calvin: Some(CalvinRedoMeta { + epoch_system_ms: 0, + reply: nodedb_physical::physical_plan::CalvinReplySpec::Count(Vec::new()), + primary_write: true, + user_write, + returning: false, + }), + }; + let mut entry = crate::control::wal_replication::encode::transaction_redo_entry( + crate::types::TenantId::new(4), + crate::types::DatabaseId::DEFAULT, + crate::types::VShardId::new(2), + &payload, + ); + entry.write_hlc = 77; + QueuedEntry::new(LogEntry { + term: 1, + index: 9, + data: entry.to_bytes(), + }) + } + + /// A Calvin slice that writes only derived rows or schema raises no write + /// mark, even as the slice that deposits the reply. A slice that changes + /// a tenant row raises its commit HLC. + #[test] + fn only_a_calvin_slice_that_changes_a_row_carries_a_write_stamp() { + assert_eq!(calvin_redo_entry(true).write_stamp(), Some((4, 77, 0))); + assert_eq!(calvin_redo_entry(false).write_stamp(), None); + assert!(calvin_redo_entry(true).plan_writes_user_data()); + assert!(!calvin_redo_entry(false).plan_writes_user_data()); + } } diff --git a/nodedb/src/control/distributed_applier/apply_loop/metadata_floor.rs b/nodedb/src/control/distributed_applier/apply_loop/metadata_floor.rs index d0b870dc8..9194ee802 100644 --- a/nodedb/src/control/distributed_applier/apply_loop/metadata_floor.rs +++ b/nodedb/src/control/distributed_applier/apply_loop/metadata_floor.rs @@ -78,6 +78,13 @@ pub(super) fn hold_for_metadata<'a>( /// order it holds. It fails only when the metadata group left this node. async fn await_metadata_floor(state: &SharedState, held: HeldEntry) -> crate::Result<()> { let watcher = state.applied_index_watcher(METADATA_GROUP_ID); + tracing::debug!( + group_id = held.group_id, + log_index = held.log_index, + metadata_floor = held.metadata_floor, + metadata_applied = watcher.current(), + "a replicated write holds its group's lane for this node's metadata apply" + ); loop { let waiting = Arc::clone(&watcher); let floor = held.metadata_floor; diff --git a/nodedb/src/control/distributed_applier/apply_loop/mod.rs b/nodedb/src/control/distributed_applier/apply_loop/mod.rs index 00d25c0f8..031510b6d 100644 --- a/nodedb/src/control/distributed_applier/apply_loop/mod.rs +++ b/nodedb/src/control/distributed_applier/apply_loop/mod.rs @@ -19,13 +19,22 @@ //! - [`pipeline`]: every group's lane and the applies that run. //! - [`lane`]: one group's queued and started entries, settled in log order. //! - [`start`]: prepares one entry and routes it to its apply path. +//! - [`cut_barrier`]: a backup cut's barrier: the group's cut floor, the +//! restore point's place, and a database backup's capture. //! - [`context`]: the handles an apply borrows, and the futures the loop //! collects. -//! - [`calvin_read_result`]: forwards a committed `CalvinReadResult` entry to -//! the local Calvin scheduler. +//! - [`barrier_prefold`]: folds the barrier entries of a waiting lane ahead +//! of their turn. +//! - [`calvin_read_result`]: folds a committed `CalvinReadResult` or +//! `CalvinReadTimeout` entry into its txn's stored barrier row and its +//! vShard's read-result buffer. +//! - [`calvin_redo`]: claims a stamped Calvin redo's position in the applied +//! ledger and reports its install to the vShard's scheduler. //! - [`write_dispatch`]: the generic decode + write-funnel enqueue path. //! - [`transaction_redo`]: a committed transaction's redo, stamped with its //! Raft entry and applied through the WAL replay arms. +//! - [`redo_chunk`]: the chunks, the abandon and the final entry of a +//! chunked redo stream. //! - [`topic_publish`]: a committed durable-topic publication, appended to //! this replica's topic log at the entry's position. //! - [`proposal_gate`]: skips a second committed copy of an applied proposal @@ -43,10 +52,13 @@ //! collection it names holds the incarnation its proposer planned against. mod array_cell_route; +mod barrier_prefold; mod bookkeeping; mod calvin_read_result; +mod calvin_redo; mod collection_route; mod context; +mod cut_barrier; mod driver; mod group_watch; mod helpers; @@ -54,6 +66,7 @@ mod lane; mod metadata_floor; mod pipeline; mod proposal_gate; +mod redo_chunk; mod snapshot_gate; mod start; mod surrogate_bind; diff --git a/nodedb/src/control/distributed_applier/apply_loop/pipeline.rs b/nodedb/src/control/distributed_applier/apply_loop/pipeline.rs index 89dccd45d..f99e02c5e 100644 --- a/nodedb/src/control/distributed_applier/apply_loop/pipeline.rs +++ b/nodedb/src/control/distributed_applier/apply_loop/pipeline.rs @@ -124,6 +124,9 @@ impl<'a> Pipeline<'a> { (finished.group_id, finished.log_index, handled) } }; + // The entry's enqueue returned, or its exclusive apply finished: + // every entry of the group through it reached its core. + release_admission_holds(self.ctx.state, group_id, log_index); if !handled { // Every enqueue and apply the pipeline runs has a slot in its // group's lane in the matching state: the pump pushes both @@ -180,6 +183,13 @@ impl<'a> Pipeline<'a> { .collect(); for group_id in groups { self.pump_group(group_id); + // The entries left wait behind an earlier one. Their barrier + // events fold now: a barrier needs no catalog. + if !self.install_blocked + && let Some(lane) = self.lanes.get_mut(&group_id) + { + super::barrier_prefold::prefold_backlog(self.ctx, lane, group_id); + } } } @@ -190,7 +200,13 @@ impl<'a> Pipeline<'a> { let log_index = queued.entry.index; let proposal_key = queued.proposal_key(); let covered_through = self.ctx.tracker.covered_through(group_id); - let permit = match admit_entry(gates, group_id, log_index, covered_through) { + let admission = admit_entry(gates, group_id, log_index, covered_through); + // Before the entry joins the lane: no entry above the installed + // index settles before the lane covers it. + if let Some(lane) = self.lanes.get_mut(&group_id) { + lane.cover_through(admission.installed_through); + } + let permit = match admission.decision { EntryAdmission::Installing => { if let Some(lane) = self.lanes.get_mut(&group_id) { lane.backlog.push_front(queued); @@ -202,6 +218,7 @@ impl<'a> Pipeline<'a> { // At or below the snapshot's Raft index: the installed state // and the Raft boundary both hold the entry. EntryAdmission::Covered => { + super::cut_barrier::note_covered_barrier(shared, group_id, &queued); if !self.conclude_covered( group_id, log_index, @@ -217,6 +234,7 @@ impl<'a> Pipeline<'a> { // applying and extends the durable prefix, so a restart never // delivers it again. EntryAdmission::CoveredByCut => { + super::cut_barrier::note_covered_barrier(shared, group_id, &queued); if !self.conclude_covered( group_id, log_index, @@ -253,6 +271,11 @@ impl<'a> Pipeline<'a> { proposal_key, metadata_floor: queued.metadata_floor(), }; + // An entry of a later term ends every earlier term of the group: + // no earlier term adds a chunk or a final entry after it. + shared + .redo_chunks + .end_terms_before(group_id, queued.entry.term); let prepared = hold_for_metadata( self.ctx.state, self.ctx.tracker, @@ -261,6 +284,7 @@ impl<'a> Pipeline<'a> { ); let (state, blocks, user_write) = match prepared { Prepared::Concluded(outcome) => { + release_admission_holds(shared, group_id, log_index); let user_write = matches!(outcome, EntryOutcome::Repeat) && repeat_writes; ( SlotState::Concluded(self.gate.conclude(proposal_key, false, outcome)), @@ -268,7 +292,10 @@ impl<'a> Pipeline<'a> { user_write, ) } - Prepared::Barrier => (SlotState::Barrier, false, false), + Prepared::Barrier => { + release_admission_holds(shared, group_id, log_index); + (SlotState::Barrier, false, false) + } Prepared::Enqueue(enqueue) => { self.gate.open(proposal_key); self.ctx.tracker.note_dispatched(group_id, log_index); @@ -319,6 +346,7 @@ impl<'a> Pipeline<'a> { outcome: EntryOutcome, ) -> bool { self.ctx.tracker.note_started(group_id, log_index); + release_admission_holds(self.ctx.state, group_id, log_index); self.ctx .tracker .complete_covered(group_id, log_index, proposal_key); @@ -404,6 +432,15 @@ impl<'a> Pipeline<'a> { } } +/// Release the lock keys this node's write gate holds for entries of +/// `group_id` through `log_index`: their applies started, in log order. +fn release_admission_holds(state: &SharedState, group_id: u64, log_index: u64) { + state + .calvin + .admission_holds + .release_through(group_id, log_index); +} + fn finished_event(apply: ApplyFuture<'_>) -> LoopFuture<'_> { Box::pin(apply.map(LoopEvent::Finished)) } diff --git a/nodedb/src/control/distributed_applier/apply_loop/redo_chunk.rs b/nodedb/src/control/distributed_applier/apply_loop/redo_chunk.rs new file mode 100644 index 000000000..335ed0c61 --- /dev/null +++ b/nodedb/src/control/distributed_applier/apply_loop/redo_chunk.rs @@ -0,0 +1,440 @@ +// SPDX-License-Identifier: BUSL-1.1 + +//! Apply path for the entries of a chunked redo stream: a `RedoChunk`, a +//! `RedoAbandon`, and the final `TransactionRedo` entry that names the +//! stream. +//! +//! Every replica decides each of them from the log alone (see +//! `wal_replication::transaction_redo::chunks`). A chunk and an abandon +//! apply with nothing else of their group in flight, so the stream a later +//! entry reads is the one every earlier entry left. A refusal applied +//! nothing on any replica, so it advances the durable prefix like a success. +//! +//! A group whose replica here owes a snapshot install is the exception (see +//! `chunks::owed`). This node dropped streams of it that the other replicas +//! still hold. A refusal there can be this replica's alone, so it holds the +//! group's durable floor until the snapshot installs. + +use crate::control::array_sync::raft_apply::AppliedPosition; +use crate::control::distributed_applier::propose_tracker::{AppliedWrite, ProposeTracker}; +use crate::control::wal_replication::decode::ChunkedRedo; +use crate::control::wal_replication::transaction_redo::RedoTarget; +use crate::control::wal_replication::transaction_redo::chunks::{ChunkApply, RedoChunkError}; +use crate::control::wal_replication::types::RedoContent; +use crate::control::wal_replication::{CollectionIncarnation, ReplicatedEntry, ReplicatedWrite}; +use crate::types::{DatabaseId, TenantId, VShardId}; +use crate::wal::RedoStreamId; +use crate::wal::manager::RedoChunkPlacement; + +use super::calvin_redo::{ClaimOutcome, claim_chunked}; +use super::context::{ApplyContext, FinishedApply}; +use super::proposal_gate::{EntryOutcome, ledger_outcome}; +use super::start::Prepared; +use super::transaction_redo::{ + RedoApply, conclude_held, conclude_held_claimed, conclude_installed_copy, enqueue_payload, + report_then_conclude, +}; + +/// Prepare a committed `RedoChunk` or `RedoAbandon` entry of term `term`. +pub(super) fn prepare_stream_entry<'a>( + ctx: ApplyContext<'a>, + pos: AppliedPosition, + term: u64, + entry: ReplicatedEntry, +) -> Prepared<'a> { + let placement = RedoChunkPlacement { + tenant_id: TenantId::new(entry.tenant_id), + vshard_id: VShardId::new(entry.vshard_id), + database_id: DatabaseId::new(entry.database_id), + }; + match entry.write { + ReplicatedWrite::RedoChunk { + stream, + index, + len, + bytes, + } => { + let chunk = ChunkApply { + group_id: pos.group_id, + term, + apply_key: pos.applied_key, + placement, + stream, + index, + len, + bytes, + }; + Prepared::Exclusive(Box::pin(async move { + let applied = ctx.state.redo_chunks.apply_chunk(chunk).await; + // The chunk's records are durable here, its entry not yet + // concluded. + crate::fail_point!( + crate::fail_point::FailScope::Node(ctx.state.node_id), + "redo_chunk::after_chunk_durable" + ); + finished(ctx, pos, stream, applied) + })) + } + ReplicatedWrite::RedoAbandon { stream } => Prepared::Exclusive(Box::pin(async move { + let abandoned = ctx + .state + .redo_chunks + .abandon(stream, pos.applied_key, placement) + .await; + finished(ctx, pos, stream, abandoned) + })), + other => conclude_held( + ctx, + pos, + crate::Error::Internal { + detail: format!("redo stream apply called with another write: {other:?}"), + }, + None, + ), + } +} + +/// Resolve a chunk or abandon entry of `stream`'s waiter and conclude it. +fn finished( + ctx: ApplyContext<'_>, + pos: AppliedPosition, + stream: RedoStreamId, + applied: Result<(), RedoChunkError>, +) -> FinishedApply { + let outcome = match applied { + Err(refusal) + if refusal.is_refusal() && ctx.state.redo_chunks.owes_snapshot(pos.group_id) => + { + let error = stream_lost_here(pos, stream, LostEntry::Chunk, &refusal); + held(ctx.tracker, pos, error) + } + applied => concluded(ctx.tracker, pos, applied), + }; + FinishedApply { + group_id: pos.group_id, + log_index: pos.log_index, + outcome, + } +} + +/// Conclude a chunk or abandon entry by its stream's verdict: a success or +/// a refusal passes the entry, a storage error holds it. +fn concluded( + tracker: &ProposeTracker, + pos: AppliedPosition, + applied: Result<(), RedoChunkError>, +) -> EntryOutcome { + let durable = applied + .as_ref() + .err() + .is_none_or(RedoChunkError::is_refusal); + let result = applied + .map(|()| AppliedWrite::unversioned(Vec::new())) + .map_err(crate::Error::from); + let outcome = EntryOutcome::Applied { + durable, + result: Some(ledger_outcome(&result)), + }; + tracker.complete(pos.group_id, pos.log_index, pos.applied_key, result); + outcome +} + +/// Hold a chunk entry this replica cannot decide as the other replicas do. +/// No ledger outcome records it, so a re-delivery applies it again. +fn held(tracker: &ProposeTracker, pos: AppliedPosition, error: crate::Error) -> EntryOutcome { + tracker.complete(pos.group_id, pos.log_index, pos.applied_key, Err(error)); + EntryOutcome::Applied { + durable: false, + result: None, + } +} + +/// Which entry of a stream refused in a group that owes a snapshot. +#[derive(Debug, Clone, Copy)] +enum LostEntry { + Chunk, + Final, +} + +impl LostEntry { + fn as_str(self) -> &'static str { + match self { + Self::Chunk => "chunk", + Self::Final => "final", + } + } +} + +/// Report a refusal of `stream`'s `entry` at `pos` on a replica that owes +/// its group a snapshot install. Returns the error the entry's waiter gets. +fn stream_lost_here( + pos: AppliedPosition, + stream: RedoStreamId, + entry: LostEntry, + refusal: &RedoChunkError, +) -> crate::Error { + tracing::error!( + group_id = pos.group_id, + log_index = pos.log_index, + ?stream, + entry = entry.as_str(), + %refusal, + "redo stream: this replica dropped streams of the group when it left it; the entry \ + holds until a snapshot of the group installs" + ); + crate::diag::redo_stream_lost_here( + pos.group_id, + pos.log_index, + stream, + entry.as_str(), + refusal, + ); + crate::Error::Internal { + detail: format!( + "group {}: entry {} of redo stream {stream:?} is held on this replica until a \ + snapshot of the group installs: {refusal}", + pos.group_id, pos.log_index + ), + } +} + +/// Prepare the final entry of a chunked redo: take its stream, assemble the +/// content, and apply it as an inline redo. A stream that is gone or that +/// holds other chunks than the entry names refuses the entry. In a group +/// whose replica here owes a snapshot install, it holds the entry instead. +pub(super) fn prepare_chunked_final<'a>( + ctx: ApplyContext<'a>, + pos: AppliedPosition, + target: RedoTarget, + chunked: ChunkedRedo, + incarnations: Vec, +) -> Prepared<'a> { + let claim = claim_chunked(ctx.state, chunked.calvin(), &chunked.stream); + let taken = ctx.state.redo_chunks.take_for_final(&chunked.stream); + let calvin = match claim { + ClaimOutcome::NotCalvin => None, + ClaimOutcome::Claimed(claim) => Some(claim), + // Another copy of the position installed, or installs now: this + // copy's stream drops. + ClaimOutcome::Refused => return conclude_installed_copy(ctx, pos, taken), + ClaimOutcome::Malformed(error) => return conclude_held(ctx, pos, error, taken), + }; + // Every chunk is durable and the final entry committed; nothing of the + // final entry applied yet. + crate::fail_point!( + crate::fail_point::FailScope::Node(ctx.state.node_id), + "redo_chunk::before_final_install" + ); + let assembled = match &taken { + Some(open) => open.assemble(chunked.count, chunked.len), + None => Err(RedoChunkError::NotOpen { + stream: chunked.stream, + }), + }; + let bytes = match assembled { + Ok(bytes) => bytes, + Err(refusal) if ctx.state.redo_chunks.owes_snapshot(pos.group_id) => { + let error = stream_lost_here(pos, chunked.stream, LostEntry::Final, &refusal); + return match calvin { + Some(claim) => conclude_held_claimed(ctx, pos, error, taken, claim), + None => conclude_held(ctx, pos, error, taken), + }; + } + Err(refusal) => { + // Every replica refuses the final entry alike. The claim stays, + // and the scheduler hears the refusal once the entry starts. + let result = Err(crate::Error::from(refusal)); + let outcome = EntryOutcome::Applied { + durable: true, + result: Some(ledger_outcome(&result)), + }; + ctx.tracker + .complete(pos.group_id, pos.log_index, pos.applied_key, result); + return match calvin { + Some(claim) => { + let event = + crate::control::cluster::calvin::scheduler::CalvinApplyEvent::RedoRefused { + error: "the final entry's chunk stream does not hold the chunks it \ + names" + .into(), + }; + report_then_conclude(ctx, pos, claim, event, outcome) + } + None => Prepared::Concluded(outcome), + }; + } + }; + match RedoContent::from_bytes(&bytes) { + Ok(content) => Prepared::Enqueue(enqueue_payload( + ctx, + pos, + target, + RedoApply { + payload: chunked.payload(content), + incarnations, + stream: taken, + calvin, + }, + )), + Err(error) => match calvin { + Some(claim) => conclude_held_claimed(ctx, pos, error, taken, claim), + None => conclude_held(ctx, pos, error, taken), + }, + } +} + +#[cfg(test)] +mod tests { + use std::sync::Arc; + + use nodedb_physical::physical_plan::RedoOrigin; + + use super::*; + use crate::bridge::dispatch::Dispatcher; + use crate::control::state::SharedState; + use crate::control::wal_replication::decode::{ + DecodedTransactionRedo, decode_transaction_redo, + }; + use crate::control::wal_replication::types::{RedoBody, ReplicatedEventSource}; + use crate::wal::WalManager; + + const GROUP: u64 = 4; + + fn stream() -> RedoStreamId { + RedoStreamId::Session { + vshard: 2, + idempotency_key: 77, + } + } + + fn pos(group_id: u64, log_index: u64) -> AppliedPosition { + AppliedPosition { + group_id, + log_index, + applied_key: 1_000 + log_index, + commit_hlc: 0, + } + } + + fn chunk_entry(index: u32, bytes: &[u8]) -> ReplicatedEntry { + ReplicatedEntry::new( + 1, + 0, + 2, + ReplicatedWrite::RedoChunk { + stream: stream(), + index, + len: 4, + bytes: bytes.to_vec(), + }, + ) + } + + fn final_entry() -> ChunkedRedo { + let write = ReplicatedWrite::TransactionRedo { + body: RedoBody::Chunked { + stream: stream(), + count: 2, + len: 4, + }, + collections: Vec::new(), + event_source: ReplicatedEventSource::User, + origin: RedoOrigin::Commit, + calvin: None, + }; + match decode_transaction_redo(&write).expect("decode") { + DecodedTransactionRedo::Chunked(chunked) => chunked, + DecodedTransactionRedo::Inline(_) => panic!("a chunked body decodes chunked"), + } + } + + fn target() -> RedoTarget { + RedoTarget { + tenant_id: TenantId::new(1), + database_id: DatabaseId::DEFAULT, + vshard_id: VShardId::new(2), + } + } + + async fn apply_chunk(ctx: ApplyContext<'_>, pos: AppliedPosition, index: u32) -> EntryOutcome { + let Prepared::Exclusive(apply) = + prepare_stream_entry(ctx, pos, 3, chunk_entry(index, b"ab")) + else { + panic!("a chunk entry applies exclusive"); + }; + apply.await.outcome + } + + fn holds(outcome: &EntryOutcome) -> bool { + matches!( + outcome, + EntryOutcome::Applied { + durable: false, + result: None + } + ) + } + + /// This node applied a stream's first chunk, left the group, and + /// mounted it again on the group's old log. The next chunk and the + /// final entry refuse here while every other replica installs the + /// redo. Each holds the group's durable floor instead of passing. + #[tokio::test] + async fn a_final_entry_of_a_stream_this_replica_dropped_holds_the_apply() { + let dir = tempfile::tempdir().expect("tempdir"); + let wal = Arc::new(WalManager::open_for_testing(&dir.path().join("wal")).expect("wal")); + let (dispatcher, _sides) = Dispatcher::new(1, 64); + let state = SharedState::new(dispatcher, wal).expect("shared state"); + let tracker = Arc::new(ProposeTracker::new()); + let ctx = ApplyContext { + state: &state, + tracker: &tracker, + }; + + let first = apply_chunk(ctx, pos(GROUP, 10), 0).await; + assert!(matches!(first, EntryOutcome::Applied { durable: true, .. })); + state + .redo_chunks + .drop_unhosted_groups(|_| false, state.credentials.catalog()) + .expect("leave the group"); + assert!(state.redo_chunks.owes_snapshot(GROUP)); + + let second = apply_chunk(ctx, pos(GROUP, 11), 1).await; + assert!(holds(&second), "the dropped stream's next chunk holds"); + + let Prepared::Concluded(last) = + prepare_chunked_final(ctx, pos(GROUP, 12), target(), final_entry(), Vec::new()) + else { + panic!("a final entry with no stream concludes at prepare"); + }; + assert!(holds(&last), "the dropped stream's final entry holds"); + } + + /// In a group that owes no snapshot, every replica refuses a final entry + /// with no stream alike, and the refusal passes the entry. + #[tokio::test] + async fn a_final_entry_with_no_stream_passes_in_a_group_that_owes_nothing() { + let dir = tempfile::tempdir().expect("tempdir"); + let wal = Arc::new(WalManager::open_for_testing(&dir.path().join("wal")).expect("wal")); + let (dispatcher, _sides) = Dispatcher::new(1, 64); + let state = SharedState::new(dispatcher, wal).expect("shared state"); + let tracker = Arc::new(ProposeTracker::new()); + let ctx = ApplyContext { + state: &state, + tracker: &tracker, + }; + + let Prepared::Concluded(outcome) = + prepare_chunked_final(ctx, pos(GROUP, 12), target(), final_entry(), Vec::new()) + else { + panic!("a final entry with no stream concludes at prepare"); + }; + assert!(matches!( + outcome, + EntryOutcome::Applied { + durable: true, + result: Some(_) + } + )); + } +} diff --git a/nodedb/src/control/distributed_applier/apply_loop/snapshot_gate.rs b/nodedb/src/control/distributed_applier/apply_loop/snapshot_gate.rs index 32c8e1327..d86ad0bb7 100644 --- a/nodedb/src/control/distributed_applier/apply_loop/snapshot_gate.rs +++ b/nodedb/src/control/distributed_applier/apply_loop/snapshot_gate.rs @@ -30,6 +30,14 @@ pub(super) enum EntryAdmission { Installing, } +/// An entry's admission and the gate state it read. +pub(super) struct Admission { + pub decision: EntryAdmission, + /// The highest snapshot index an install adopted for the group, read + /// under the gate. `0` with no gates, or while an install holds the gate. + pub installed_through: u64, +} + /// Admit entry `log_index` of `group_id`. `gates` is `None` before /// `start_raft` installs them, when no snapshot installs. `covered_through` /// is the cut of the last snapshot this node installed for the group. @@ -38,21 +46,37 @@ pub(super) fn admit_entry( group_id: u64, log_index: u64, covered_through: u64, -) -> EntryAdmission { - let permit = match gates { - None => None, +) -> Admission { + let (permit, installed_through) = match gates { + None => (None, 0), Some(gates) => match gates.try_apply(group_id) { - None => return EntryAdmission::Installing, - Some(permit) if log_index <= permit.installed_through() => { - return EntryAdmission::Covered; + None => { + return Admission { + decision: EntryAdmission::Installing, + installed_through: 0, + }; + } + Some(permit) => { + let installed_through = permit.installed_through(); + if log_index <= installed_through { + return Admission { + decision: EntryAdmission::Covered, + installed_through, + }; + } + (Some(permit), installed_through) } - Some(permit) => Some(permit), }, }; - if log_index <= covered_through { - return EntryAdmission::CoveredByCut; + let decision = if log_index <= covered_through { + EntryAdmission::CoveredByCut + } else { + EntryAdmission::Start(permit) + }; + Admission { + decision, + installed_through, } - EntryAdmission::Start(permit) } /// `work`, holding `permit` until it completes. @@ -79,7 +103,7 @@ mod tests { #[test] fn a_node_without_raft_starts_every_entry() { assert!(matches!( - admit_entry(None, 1, 7, 0), + admit_entry(None, 1, 7, 0).decision, EntryAdmission::Start(None) )); } @@ -91,32 +115,32 @@ mod tests { let gates = Arc::new(GroupApplyGates::new()); gates.mount(1); assert!(matches!( - admit_entry(Some(&gates), 1, 5, 0), + admit_entry(Some(&gates), 1, 5, 0).decision, EntryAdmission::Start(Some(_)) )); let mut install = gates.install(1).await; assert!(matches!( - admit_entry(Some(&gates), 1, 5, 0), + admit_entry(Some(&gates), 1, 5, 0).decision, EntryAdmission::Installing )); install.adopted(5); drop(install); assert!(matches!( - admit_entry(Some(&gates), 1, 4, 0), + admit_entry(Some(&gates), 1, 4, 0).decision, EntryAdmission::Covered )); assert!(matches!( - admit_entry(Some(&gates), 1, 5, 0), + admit_entry(Some(&gates), 1, 5, 0).decision, EntryAdmission::Covered )); assert!(matches!( - admit_entry(Some(&gates), 1, 6, 0), + admit_entry(Some(&gates), 1, 6, 0).decision, EntryAdmission::Start(Some(_)) )); assert!(matches!( - admit_entry(Some(&gates), 2, 5, 0), + admit_entry(Some(&gates), 2, 5, 0).decision, EntryAdmission::Start(Some(_)) )); } @@ -132,34 +156,47 @@ mod tests { gates.install(1).await.adopted(9); let cut = 12; assert!(matches!( - admit_entry(Some(&gates), 1, 9, cut), + admit_entry(Some(&gates), 1, 9, cut).decision, EntryAdmission::Covered )); for log_index in 10..=cut { assert!( matches!( - admit_entry(Some(&gates), 1, log_index, cut), + admit_entry(Some(&gates), 1, log_index, cut).decision, EntryAdmission::CoveredByCut ), "entry {log_index} is in the installed state and must not apply again" ); } assert!(matches!( - admit_entry(Some(&gates), 1, 13, cut), + admit_entry(Some(&gates), 1, 13, cut).decision, EntryAdmission::Start(Some(_)) )); assert!(matches!( - admit_entry(None, 1, 11, cut), + admit_entry(None, 1, 11, cut).decision, EntryAdmission::CoveredByCut )); } + /// Every admission after an install reports the adopted index, so the + /// lane covers it before any later entry starts. + #[tokio::test] + async fn an_admission_reports_the_installed_index() { + let gates = Arc::new(GroupApplyGates::new()); + gates.mount(1); + assert_eq!(admit_entry(Some(&gates), 1, 3, 0).installed_through, 0); + gates.install(1).await.adopted(7); + assert_eq!(admit_entry(Some(&gates), 1, 6, 0).installed_through, 7); + assert_eq!(admit_entry(Some(&gates), 1, 9, 12).installed_through, 7); + assert_eq!(admit_entry(None, 1, 9, 0).installed_through, 0); + } + /// An install cannot start while a started write still holds its permit. #[tokio::test] async fn an_install_waits_for_a_started_write() { let gates = Arc::new(GroupApplyGates::new()); gates.mount(1); - let EntryAdmission::Start(permit) = admit_entry(Some(&gates), 1, 6, 0) else { + let EntryAdmission::Start(permit) = admit_entry(Some(&gates), 1, 6, 0).decision else { panic!("open gate must start the entry"); }; let (release, released) = tokio::sync::oneshot::channel::<()>(); @@ -182,7 +219,7 @@ mod tests { write.await.expect("write task"); install.await.expect("install task"); assert!(matches!( - admit_entry(Some(&gates), 1, 6, 0), + admit_entry(Some(&gates), 1, 6, 0).decision, EntryAdmission::Covered )); } diff --git a/nodedb/src/control/distributed_applier/apply_loop/start.rs b/nodedb/src/control/distributed_applier/apply_loop/start.rs index 87b8f7ff3..f7563d87c 100644 --- a/nodedb/src/control/distributed_applier/apply_loop/start.rs +++ b/nodedb/src/control/distributed_applier/apply_loop/start.rs @@ -14,10 +14,13 @@ use crate::control::array_sync::raft_apply::{ }; use nodedb_raft::message::LogEntry; +use crate::control::backup::cut_order::OrderedCut; use crate::control::wal_replication::{ReplicatedEntry, ReplicatedWrite}; use crate::types::{DatabaseId, TenantId}; -use super::calvin_read_result::{CalvinReadResultFields, forward_calvin_read_result}; +use super::calvin_read_result::{ + CalvinReadResultFields, forward_calvin_read_result, forward_calvin_read_timeout, +}; use super::context::{ApplyContext, ApplyFuture, EnqueueFuture, FinishedApply}; use super::group_watch::GroupWatch; use super::lane::QueuedEntry; @@ -221,6 +224,9 @@ fn prepare_replicated<'a>( ReplicatedWrite::TransactionRedo { .. } => { prepare_transaction_redo_entry(ctx, pos, &replicated, scope.incarnations) } + ReplicatedWrite::RedoChunk { .. } | ReplicatedWrite::RedoAbandon { .. } => { + super::redo_chunk::prepare_stream_entry(ctx, pos, log_term, replicated) + } ReplicatedWrite::SurrogateBind { ref identities } => { Prepared::Concluded(super::surrogate_bind::apply_surrogate_bind( ctx, @@ -256,42 +262,49 @@ fn prepare_replicated<'a>( // Every entry after the barrier records above the cut, on this // life and on every later one. watch.raise_cut(pos.group_id, pos.log_index, hlc); - let barrier = CutBarrierPoint { - hlc, - restore_point, + super::cut_barrier::prepare_cut_barrier( + ctx, + pos, log_term, - }; - match capture { - Some(request) => prepare_capture_barrier(ctx, pos, barrier, request), - None => prepare_plain_barrier(ctx, pos, barrier), - } + OrderedCut { + hlc, + restore_point, + capture, + }, + ) } ReplicatedWrite::CalvinReadResult { epoch, position, passive_vshard, - tenant_id, + tenant_id: _, ref values, } => { - forward_calvin_read_result( - ctx.tracker, - ctx.calvin_read_result_senders, + // The event's stored row is its durable effect: the entry + // extends the applied prefix once the row holds it. + Prepared::Concluded(forward_calvin_read_result( + ctx, pos, CalvinReadResultFields { target_vshard: replicated.vshard_id, epoch, position, passive_vshard, - tenant_id, values, }, - ); - // A read result is forwarded to an in-memory Calvin scheduler and - // writes nothing durable, so it neither advances the prefix nor - // breaks it. The epoch it belongs to does not survive a restart, - // so a re-delivery cannot usefully replay it. - Prepared::Concluded(EntryOutcome::Skipped) + )) } + ReplicatedWrite::CalvinReadTimeout { + epoch, + position, + tenant_id: _, + } => Prepared::Concluded(forward_calvin_read_timeout( + ctx, + pos, + replicated.vshard_id, + epoch, + position, + )), _ => prepare_generic_entry(ctx, pos, entry, scope, false), } } @@ -355,138 +368,3 @@ fn prepare_array_op<'a>( } })) } - -/// Where a cut barrier cuts its group. -#[derive(Clone, Copy)] -struct CutBarrierPoint { - hlc: u64, - restore_point: u64, - log_term: u64, -} - -/// Prepare a backup capture's cut barrier. The lane starts a barrier with -/// nothing else of its group in flight, and starts no later entry until it -/// finishes: the floor is durable, and the capture holds every entry at or -/// below it and none above. -fn prepare_capture_barrier<'a>( - ctx: ApplyContext<'a>, - pos: AppliedPosition, - barrier: CutBarrierPoint, - request: nodedb_physical::physical_plan::CutCaptureRequest, -) -> Prepared<'a> { - let AppliedPosition { - group_id, - log_index, - applied_key, - .. - } = pos; - let CutBarrierPoint { - hlc, - restore_point, - log_term, - } = barrier; - Prepared::Exclusive(Box::pin(async move { - persist_floor_durably(ctx, group_id, log_index, hlc).await; - record_restore_point(ctx, group_id, restore_point, hlc, log_index, log_term); - crate::control::backup::cut_capture::apply::capture_at_barrier( - ctx.state, group_id, &request, - ) - .await; - finish_barrier(ctx, group_id, log_index, applied_key) - })) -} - -/// Prepare a cut barrier with no capture. A failed floor write holds the -/// group until the floor is durable. -fn prepare_plain_barrier<'a>( - ctx: ApplyContext<'a>, - pos: AppliedPosition, - barrier: CutBarrierPoint, -) -> Prepared<'a> { - let AppliedPosition { - group_id, - log_index, - applied_key, - .. - } = pos; - let CutBarrierPoint { - hlc, - restore_point, - log_term, - } = barrier; - let persisted = - crate::control::pitr::restore_point::persist_cut_floor(ctx.state, group_id, log_index, hlc); - if persisted.is_ok() { - record_restore_point(ctx, group_id, restore_point, hlc, log_index, log_term); - return Prepared::Barrier; - } - Prepared::Exclusive(Box::pin(async move { - persist_floor_durably(ctx, group_id, log_index, hlc).await; - record_restore_point(ctx, group_id, restore_point, hlc, log_index, log_term); - finish_barrier(ctx, group_id, log_index, applied_key) - })) -} - -/// Record the group's place at the cluster restore point `restore_point` a -/// cut barrier takes. `0` takes none. -fn record_restore_point( - ctx: ApplyContext<'_>, - group_id: u64, - restore_point: u64, - hlc: u64, - log_index: u64, - log_term: u64, -) { - if restore_point == 0 { - return; - } - crate::control::pitr::restore_point::record_group_point( - ctx.state, - nodedb_wal::record::RestorePointPayload { - id: restore_point, - hlc, - group_id, - applied_index: log_index, - term: log_term, - next_epoch: 0, - epoch_system_ms: 0, - vshards: Vec::new(), - }, - ); -} - -/// Persist the floor of the barrier at `log_index`, retrying until it holds. -/// A floor write that keeps failing wedges the node until it succeeds. -async fn persist_floor_durably(ctx: ApplyContext<'_>, group_id: u64, log_index: u64, hlc: u64) { - crate::control::pitr::restore_point::persist_until_durable( - &ctx.state.metadata_apply_wedge, - group_id, - log_index, - || { - crate::control::pitr::restore_point::persist_cut_floor( - ctx.state, group_id, log_index, hlc, - ) - }, - ) - .await; -} - -/// Resolve a barrier's waiter once every step of the barrier is durable. -fn finish_barrier( - ctx: ApplyContext<'_>, - group_id: u64, - log_index: u64, - applied_key: u64, -) -> FinishedApply { - ctx.tracker.complete( - group_id, - log_index, - applied_key, - Ok(crate::control::distributed_applier::AppliedWrite::unversioned(Vec::new())), - ); - FinishedApply { - group_id, - log_index, - outcome: EntryOutcome::Skipped, - } -} diff --git a/nodedb/src/control/distributed_applier/apply_loop/transaction_redo.rs b/nodedb/src/control/distributed_applier/apply_loop/transaction_redo.rs index da0299849..34e59d9de 100644 --- a/nodedb/src/control/distributed_applier/apply_loop/transaction_redo.rs +++ b/nodedb/src/control/distributed_applier/apply_loop/transaction_redo.rs @@ -9,6 +9,13 @@ //! entry re-delivered after a restart is recognised by the proposal ledger and //! skipped before it reaches here. //! +//! A chunked body assembles from the `RedoChunk` entries of its stream +//! (see [`super::redo_chunk`]) and then applies like an inline one. +//! +//! A committed Calvin slice's entry claims its position before it applies +//! and reports its install to the vShard's scheduler (see +//! [`super::calvin_redo`]). +//! //! A refusal the Data Plane proves applied nothing (a constraint verdict) is //! final: every replica reaches it at the same log position against the same //! state, and the funnel cancels the record in the WAL before it returns. The @@ -20,14 +27,16 @@ use crate::bridge::envelope::Status; use crate::control::array_sync::raft_apply::AppliedPosition; use crate::control::distributed_applier::propose_tracker::{AppliedWrite, ProposeTracker}; use crate::control::server::dispatch_utils::{ChangeFeedOwner, SubmitOutcome, refusal_is_final}; -use crate::control::wal_replication::decode::transaction_redo_payload; +use crate::control::wal_replication::decode::{DecodedTransactionRedo, decode_transaction_redo}; +use crate::control::wal_replication::transaction_redo::chunks::OpenStream; use crate::control::wal_replication::transaction_redo::{ - RedoTarget, enqueue_transaction_redo, record_cross_shard_key, + RedoTarget, TransactionRedoPayload, enqueue_transaction_redo, record_cross_shard_key, }; use crate::control::wal_replication::{CollectionIncarnation, ReplicatedEntry}; use crate::types::{DatabaseId, TenantId, VShardId}; -use super::context::{ApplyContext, FinishedApply, Started, StartedEntry}; +use super::calvin_redo::{CalvinClaim, ClaimOutcome, claim_inline}; +use super::context::{ApplyContext, EnqueueFuture, FinishedApply, Started, StartedEntry}; use super::helpers::committed_response_result; use super::proposal_gate::{EntryOutcome, ledger_outcome}; use super::start::Prepared; @@ -35,7 +44,8 @@ use super::start::Prepared; /// Prepare one committed `TransactionRedo` entry. Its enqueue appends the /// record and hands it to its core. The apply that follows resolves the /// propose waiter. Its outcome says whether the entry's effect is durable on -/// this node, which is what the group's applied prefix records. +/// this node, which is what the group's applied prefix records. A chunked +/// body assembles from its stream first. /// /// `incarnations` are the entry's collection incarnations, moved out of the /// decoded entry. @@ -45,27 +55,178 @@ pub(super) fn prepare_transaction_redo_entry<'a>( entry: &ReplicatedEntry, incarnations: Vec, ) -> Prepared<'a> { - let ApplyContext { state, tracker, .. } = ctx; - let payload = match transaction_redo_payload(&entry.write) { - Ok(payload) => payload, - Err(error) => { - tracker.complete(pos.group_id, pos.log_index, pos.applied_key, Err(error)); - // The entry's own bytes are malformed; a re-delivery decodes the - // same bytes and fails the same way, so it holds the floor rather - // than skipping a committed transaction. - return Prepared::Concluded(EntryOutcome::Applied { - durable: false, - result: None, - }); - } + let decoded = match decode_transaction_redo(&entry.write) { + Ok(decoded) => decoded, + Err(error) => return conclude_held(ctx, pos, error, None), }; let target = RedoTarget { tenant_id: TenantId::new(entry.tenant_id), database_id: DatabaseId::new(entry.database_id), vshard_id: VShardId::new(entry.vshard_id), }; - Prepared::Enqueue(Box::pin(async move { + match decoded { + DecodedTransactionRedo::Inline(payload) => { + let claim = claim_inline( + ctx.state, + payload.calvin.as_ref(), + payload.redo.calvin_stamp.as_ref(), + ); + let calvin = match claim { + ClaimOutcome::NotCalvin => None, + ClaimOutcome::Claimed(claim) => Some(claim), + ClaimOutcome::Refused => return conclude_installed_copy(ctx, pos, None), + ClaimOutcome::Malformed(error) => return conclude_held(ctx, pos, error, None), + }; + Prepared::Enqueue(enqueue_payload( + ctx, + pos, + target, + RedoApply { + payload: *payload, + incarnations, + stream: None, + calvin, + }, + )) + } + DecodedTransactionRedo::Chunked(chunked) => { + super::redo_chunk::prepare_chunked_final(ctx, pos, target, chunked, incarnations) + } + } +} + +/// Conclude a copy of a Calvin position that another copy installed, or +/// installs now. It applies nothing, and its waiter learns the write +/// committed. `stream` is the chunk stream the copy took: it drops. +pub(super) fn conclude_installed_copy<'a>( + ctx: ApplyContext<'a>, + pos: AppliedPosition, + stream: Option, +) -> Prepared<'a> { + ctx.tracker.complete( + pos.group_id, + pos.log_index, + pos.applied_key, + Ok(AppliedWrite::unversioned(Vec::new())), + ); + drop(stream); + Prepared::Concluded(EntryOutcome::Applied { + durable: true, + result: None, + }) +} + +/// One committed redo on its way to its core. +pub(super) struct RedoApply { + pub payload: TransactionRedoPayload, + /// The entry's collection incarnations, moved out of the decoded entry. + pub incarnations: Vec, + /// The chunk stream the payload assembled from. + pub stream: Option, + /// The Calvin position the entry claimed. + pub calvin: Option, +} + +/// Conclude an entry this replica cannot apply as the other replicas do: +/// its own bytes are malformed, or its stream here is not the one the log +/// built. It holds the floor rather than skipping a committed transaction. +/// A taken stream stays on disk for it. +pub(super) fn conclude_held<'a>( + ctx: ApplyContext<'a>, + pos: AppliedPosition, + error: crate::Error, + stream: Option, +) -> Prepared<'a> { + Prepared::Concluded(held_outcome(ctx, pos, error, stream)) +} + +/// [`conclude_held`] for an entry that claimed a Calvin position: the claim +/// releases and the vShard's scheduler hears the install did not apply. +pub(super) fn conclude_held_claimed<'a>( + ctx: ApplyContext<'a>, + pos: AppliedPosition, + error: crate::Error, + stream: Option, + claim: CalvinClaim, +) -> Prepared<'a> { + let event = claim.not_applied(error.to_string()); + let outcome = held_outcome(ctx, pos, error, stream); + report_then_conclude(ctx, pos, claim, event, outcome) +} + +/// Report `event` to the claim's scheduler, then conclude the entry with +/// `outcome`. The report can wait for room in the scheduler's inbox, so the +/// entry runs as its own apply. +pub(super) fn report_then_conclude<'a>( + ctx: ApplyContext<'a>, + pos: AppliedPosition, + claim: CalvinClaim, + event: crate::control::cluster::calvin::scheduler::CalvinApplyEvent, + outcome: EntryOutcome, +) -> Prepared<'a> { + Prepared::Exclusive(Box::pin(async move { + claim.report(ctx.state, event).await; + FinishedApply { + group_id: pos.group_id, + log_index: pos.log_index, + outcome, + } + })) +} + +/// Resolve the waiter of an entry this replica holds, and settle its +/// stream. +fn held_outcome( + ctx: ApplyContext<'_>, + pos: AppliedPosition, + error: crate::Error, + stream: Option, +) -> EntryOutcome { + ctx.tracker + .complete(pos.group_id, pos.log_index, pos.applied_key, Err(error)); + let outcome = EntryOutcome::Applied { + durable: false, + result: None, + }; + settle_stream(ctx, stream, &outcome); + outcome +} + +/// Release a taken stream once its final entry concluded. A durable outcome +/// drops it. Any other keeps its records on disk, so the next boot rebuilds +/// the stream for the entry's re-apply. +fn settle_stream(ctx: ApplyContext<'_>, stream: Option, outcome: &EntryOutcome) { + let Some(stream) = stream else { + return; + }; + if matches!(outcome, EntryOutcome::Applied { durable: false, .. }) { + ctx.state.redo_chunks.park(stream); + } +} + +/// Route the redo to its collections, append it, and enqueue it on its +/// core. +pub(super) fn enqueue_payload<'a>( + ctx: ApplyContext<'a>, + pos: AppliedPosition, + target: RedoTarget, + apply: RedoApply, +) -> EnqueueFuture<'a> { + let ApplyContext { state, tracker, .. } = ctx; + let RedoApply { + payload, + incarnations, + stream, + calvin, + } = apply; + Box::pin(async move { let collection = payload.collections.first().cloned(); + // A Calvin slice with no primary write writes only derived rows: it + // raises no tenant write mark. + let user_write = payload + .calvin + .as_ref() + .is_none_or(|meta| meta.primary_write); // A redo for a collection incarnation this node no longer holds has // nothing to mutate. The gates stay held until the redo is enqueued. let routed = super::collection_route::route( @@ -78,6 +239,10 @@ pub(super) fn prepare_transaction_redo_entry<'a>( let _gates = match routed { Ok(super::collection_route::CollectionRoute::Apply(gates)) => gates, Ok(super::collection_route::CollectionRoute::Superseded) => { + if let Some(claim) = &calvin { + let event = claim.superseded(state); + claim.report(state, event).await; + } tracker.complete( pos.group_id, pos.log_index, @@ -86,17 +251,25 @@ pub(super) fn prepare_transaction_redo_entry<'a>( crate::bridge::envelope::ErrorCode::NotFound, )), ); - return StartedEntry::concluded(EntryOutcome::Applied { + let outcome = EntryOutcome::Applied { durable: true, result: None, - }); + }; + settle_stream(ctx, stream, &outcome); + return StartedEntry::concluded(outcome); } Err(error) => { + if let Some(claim) = &calvin { + let event = claim.not_applied(error.to_string()); + claim.report(state, event).await; + } tracker.complete(pos.group_id, pos.log_index, pos.applied_key, Err(error)); - return StartedEntry::concluded(EntryOutcome::Applied { + let outcome = EntryOutcome::Applied { durable: false, result: None, - }); + }; + settle_stream(ctx, stream, &outcome); + return StartedEntry::concluded(outcome); } }; let enqueued = enqueue_transaction_redo( @@ -120,21 +293,39 @@ pub(super) fn prepare_transaction_redo_entry<'a>( if let Ok(outcome) = &submitted { record_cross_shard_key(state, &payload, outcome); } + // The ledger holds the position before the entry concludes, + // so before the group's applied index passes it. + if let Some(claim) = &calvin { + let event = claim.settle(state, &submitted); + claim + .report_install(state, event, &payload.collections) + .await; + } + let outcome = conclude_transaction_redo(tracker, pos, submitted); + settle_stream(ctx, stream, &outcome); FinishedApply { group_id: pos.group_id, log_index: pos.log_index, - outcome: conclude_transaction_redo(tracker, pos, submitted), + outcome, } })), - Err(error) => Started::Concluded(conclude_transaction_redo(tracker, pos, Err(error))), + Err(error) => { + if let Some(claim) = &calvin { + let event = claim.not_applied(error.to_string()); + claim.report(state, event).await; + } + let outcome = conclude_transaction_redo(tracker, pos, Err(error)); + settle_stream(ctx, stream, &outcome); + Started::Concluded(outcome) + } }; // A committed transaction's redo carries the rows it wrote. StartedEntry { started, collection, - user_write: true, + user_write, } - })) + }) } /// Resolve a transaction redo's waiter from what the funnel returned. diff --git a/nodedb/src/control/distributed_applier/apply_loop/write_dispatch.rs b/nodedb/src/control/distributed_applier/apply_loop/write_dispatch.rs index 967287026..564621127 100644 --- a/nodedb/src/control/distributed_applier/apply_loop/write_dispatch.rs +++ b/nodedb/src/control/distributed_applier/apply_loop/write_dispatch.rs @@ -303,12 +303,10 @@ fn conclude_generic_entry( // entry that failed to apply must surface to the propose waiter as a // failure, not as an empty success. let result = match submitted { - // The response carries this replica's post-write - // `coll_write_lsn` for the written collection, which the - // proposer needs as its read-your-writes floor: the version is - // minted here (the funnel's WAL append) and never travels on the - // wire, so the propose waiter is the only place it can be - // handed back. + // The response carries the versions the write stamped: the + // entry's log position on the written vShard. The proposer needs + // them as its read-your-writes floor, and the propose waiter is + // where they are handed back. Ok(resp) if resp.status == Status::Ok => Ok(AppliedWrite::from_response(&resp)), Ok(resp) => committed_response_result(&resp), Err(e) => { diff --git a/nodedb/src/control/distributed_applier/proposal_ledger.rs b/nodedb/src/control/distributed_applier/proposal_ledger.rs index 515ff9a55..5e0cb5139 100644 --- a/nodedb/src/control/distributed_applier/proposal_ledger.rs +++ b/nodedb/src/control/distributed_applier/proposal_ledger.rs @@ -131,13 +131,16 @@ impl ProposalLedger { #[cfg(test)] mod tests { use super::*; - use crate::types::{DatabaseId, Lsn, TenantId, VShardId}; + use crate::types::{DatabaseId, TenantId, VShardId}; use crate::wal::manager::NO_APPLY_KEY; fn applied(payload: &[u8]) -> AppliedWrite { AppliedWrite { payload: payload.to_vec(), - write_version: Lsn::new(9), + write_versions: crate::types::ReadVersions::single( + VShardId::new(1), + nodedb_types::WriteVersion::logged(0, 9), + ), } } diff --git a/nodedb/src/control/distributed_applier/propose_tracker/applied_write.rs b/nodedb/src/control/distributed_applier/propose_tracker/applied_write.rs index 667d6c89d..2b59c07e8 100644 --- a/nodedb/src/control/distributed_applier/propose_tracker/applied_write.rs +++ b/nodedb/src/control/distributed_applier/propose_tracker/applied_write.rs @@ -3,53 +3,48 @@ //! What a committed entry produced, as its propose waiter receives it. use crate::bridge::envelope::Response; -use crate::types::Lsn; +use crate::types::ReadVersions; /// What a committed entry produced on the replica that applied it. /// -/// Carries the write's per-collection version alongside the payload because the -/// proposer has no other way to learn it: the version is minted inside the apply -/// path (the write funnel's WAL append), never on the wire, and the propose -/// tracker resolves on the very node that applied locally — so the version this -/// carries is that node's own, which is exactly what shard-local OCC validates -/// against. +/// Carries the write's collection versions alongside the payload because the +/// proposer has no other way to learn them: the versions are recorded inside +/// the apply path, never sent back on the wire, and the propose tracker +/// resolves on the very node that applied locally. #[derive(Debug, Clone)] pub struct AppliedWrite { /// The Data Plane's response payload, verbatim. pub payload: Vec, - /// The written collection's `coll_write_lsn` AFTER this write, in the local - /// WAL-LSN domain — the one domain OCC's read validator compares in. It is - /// NOT the Raft log index: the log index is a per-group counter that shares - /// no scale with the WAL LSNs every other feed of that map records, and - /// mixing the two silently breaks both directions of the comparison. - pub write_version: Lsn, + /// The written collection's version on the write's vShard AFTER this + /// write: the log position of the entry, the same on every replica. + pub write_versions: ReadVersions, } impl AppliedWrite { /// Take both fields off the Data Plane's response to a committed write. /// - /// `Response::read_version_lsn` is stamped by the core loop from the written - /// collection's `coll_write_lsn`, read AFTER the handler recorded this - /// write's LSN into the version index — so on a write response it is the - /// post-write version. It is `Lsn::ZERO` for a plan that maps to no single - /// user collection (see [`AppliedWrite::write_version`]). + /// `Response::read_versions` is stamped by the core loop from the written + /// collection's version, read AFTER the handler recorded this write into + /// the version index, so on a write response it is the post-write + /// version. It is empty for a plan that maps to no user collection (see + /// [`AppliedWrite::unversioned`]). pub fn from_response(response: &Response) -> Self { Self { payload: response.payload.to_vec(), - write_version: response.read_version_lsn, + write_versions: response.read_versions.clone(), } } - /// An applied entry that publishes no per-collection write-version: it wrote - /// no Data-Plane collection state (a decode skip, a forwarded read result, a - /// schema snapshot), or it was deduplicated before reaching the funnel. There - /// is no version to floor a later read at, and `Lsn::ZERO` is the read-set - /// capture's "no own-write floor" value — never a fabricated stand-in for a - /// version that exists but was not read back. + /// An applied entry that publishes no collection version: it wrote no + /// Data-Plane collection state (a decode skip, a forwarded read result, a + /// schema snapshot), or it was deduplicated before reaching the funnel. + /// There is no version to floor a later read at, and empty versions are + /// the read-set capture's "no own-write floor" value: never a fabricated + /// stand-in for a version that exists but was not read back. pub fn unversioned(payload: Vec) -> Self { Self { payload, - write_version: Lsn::ZERO, + write_versions: ReadVersions::new(), } } } diff --git a/nodedb/src/control/distributed_applier/propose_tracker/tracker.rs b/nodedb/src/control/distributed_applier/propose_tracker/tracker.rs index 7f543d0c8..1a92bfad6 100644 --- a/nodedb/src/control/distributed_applier/propose_tracker/tracker.rs +++ b/nodedb/src/control/distributed_applier/propose_tracker/tracker.rs @@ -325,28 +325,28 @@ impl ProposeTracker { mod tests { use super::*; use crate::control::distributed_applier::propose_tracker::AppliedWrite; - use crate::types::Lsn; #[test] fn propose_tracker_register_and_complete() { let tracker = ProposeTracker::new(); let mut rx = tracker.register(1, 5, 0xdead_beef); - // Waiter must receive both payload and coll_write_lsn — its only channel - // for a version minted on the apply path. + // The waiter receives the payload and the versions the apply stamped. + let version = nodedb_types::WriteVersion::logged(2, 5); + let vshard = crate::types::VShardId::new(3); assert!(tracker.complete( 1, 5, 0xdead_beef, Ok(AppliedWrite { payload: b"result".to_vec(), - write_version: Lsn::new(137), + write_versions: crate::types::ReadVersions::single(vshard, version), }), )); let result = rx.try_recv().unwrap().unwrap(); assert_eq!(result.payload, b"result"); - assert_eq!(result.write_version, Lsn::new(137)); + assert_eq!(result.write_versions.of(vshard), Some(version)); } #[test] diff --git a/nodedb/src/control/exec_receiver/backup_cut.rs b/nodedb/src/control/exec_receiver/backup_cut.rs index 88dbac18d..d0c157109 100644 --- a/nodedb/src/control/exec_receiver/backup_cut.rs +++ b/nodedb/src/control/exec_receiver/backup_cut.rs @@ -35,7 +35,7 @@ pub(super) async fn take_backup_cut( else { return Ok(plan); }; - crate::control::backup::cut::cut_at(state, tenant_id, watermark) + crate::control::backup::cut::cut_at(state, watermark) .await .map_err(execution_error_to_typed)?; Ok(PhysicalPlan::Meta(MetaOp::CreateTenantSnapshot { @@ -65,7 +65,6 @@ pub(super) async fn answer_capture_plan( plan: &PhysicalPlan, ) -> Option { let PhysicalPlan::Meta(MetaOp::CreateTenantSnapshot { - tenant_id, cut_watermark, cut_capture: Some(request), .. @@ -79,12 +78,11 @@ pub(super) async fn answer_capture_plan( message: "a cut capture request carries no cut watermark".into(), })); }; - let answer = crate::control::backup::cut_capture::collect::cut_and_reply( - state, *tenant_id, watermark, request, - ) - .await; + let answer = + crate::control::backup::cut_capture::collect::cut_and_reply(state, watermark, request) + .await; Some(match answer { - Ok(payload) => ExecuteResponse::ok(vec![payload], 0, 0), + Ok(payload) => ExecuteResponse::ok(vec![payload], 0, Vec::new()), Err(error) => ExecuteResponse::err(execution_error_to_typed(error)), }) } diff --git a/nodedb/src/control/exec_receiver/executor.rs b/nodedb/src/control/exec_receiver/executor.rs index df46cd15c..05ef7fed3 100644 --- a/nodedb/src/control/exec_receiver/executor.rs +++ b/nodedb/src/control/exec_receiver/executor.rs @@ -249,12 +249,17 @@ impl LocalPlanExecutor { &self.state, proposer, entry, + // The coordinator's remaining budget bounds this hop. + tokio::time::Instant::now() + deadline, ) .await { // Replicated writes carry no read watermark → 0: it floors a // session's later reads, and this RPC seam has no session. - Ok((payload, _write_version)) => ExecuteResponse::ok(vec![payload], 0, 0), + // The write's versions go back to the coordinator. + Ok((payload, write_versions)) => { + ExecuteResponse::ok(vec![payload], 0, write_versions.to_wire()) + } // A replicated write's apply verdict is a Data-Plane // verdict: carry its code, never flatten to internal. Err(e) => ExecuteResponse::err(execution_error_to_typed(e)), @@ -285,10 +290,20 @@ impl LocalPlanExecutor { ) .await { + // A read that found no row keeps its verdict and the versions it + // observed, so the coordinator answers and validates it as a + // local miss. + Ok(Ok(result)) if result.not_found => ExecuteResponse::refused_with_versions( + TypedClusterError::DataPlane { + code: crate::bridge::envelope::ErrorCode::NotFound.into(), + }, + result.watermark_lsn.as_u64(), + result.read_versions.to_wire(), + ), Ok(Ok(result)) => ExecuteResponse::ok( vec![result.payload], result.watermark_lsn.as_u64(), - result.read_version_lsn.as_u64(), + result.read_versions.to_wire(), ), Ok(Err(e)) => ExecuteResponse::err(execution_error_to_typed(e)), Err(_) => ExecuteResponse::err(TypedClusterError::DeadlineExceeded { @@ -369,7 +384,7 @@ impl LocalPlanExecutor { .send_chunk( b.payload, b.watermark_lsn.as_u64(), - b.read_version_lsn.as_u64(), + b.read_versions.to_wire(), ) .await { diff --git a/nodedb/src/control/exec_receiver/metadata_applied.rs b/nodedb/src/control/exec_receiver/metadata_applied.rs index 79b0c1d1f..56e2d99ea 100644 --- a/nodedb/src/control/exec_receiver/metadata_applied.rs +++ b/nodedb/src/control/exec_receiver/metadata_applied.rs @@ -37,7 +37,7 @@ pub(super) async fn answer_applied_plan( } }; Some(if reached { - ExecuteResponse::ok(vec![Vec::new()], 0, 0) + ExecuteResponse::ok(vec![Vec::new()], 0, Vec::new()) } else { ExecuteResponse::err(TypedClusterError::Internal { code: PLAN_DECODE_FAILED, diff --git a/nodedb/src/control/exec_receiver/stream_events.rs b/nodedb/src/control/exec_receiver/stream_events.rs index fea2f8028..0e66dab34 100644 --- a/nodedb/src/control/exec_receiver/stream_events.rs +++ b/nodedb/src/control/exec_receiver/stream_events.rs @@ -128,7 +128,7 @@ fn answer_consume_stream( ); match reply { Ok(reply) => match zerompk::to_msgpack_vec(&reply) { - Ok(payload) => ExecuteResponse::ok(vec![payload], 0, 0), + Ok(payload) => ExecuteResponse::ok(vec![payload], 0, Vec::new()), Err(error) => ExecuteResponse::err(TypedClusterError::Internal { code: PLAN_DECODE_FAILED, message: format!("CDC response encoding failed: {error}"), diff --git a/nodedb/src/control/exec_receiver/surrogate_binds.rs b/nodedb/src/control/exec_receiver/surrogate_binds.rs index f65562c54..29c75bbe4 100644 --- a/nodedb/src/control/exec_receiver/surrogate_binds.rs +++ b/nodedb/src/control/exec_receiver/surrogate_binds.rs @@ -35,7 +35,7 @@ pub(super) fn answer_binds_plan( let encoded = local_binds(state, *tenant_id, *database_id, &vshards, collections).and_then(encode_binds); Some(match encoded { - Ok(payload) => ExecuteResponse::ok(vec![payload], 0, 0), + Ok(payload) => ExecuteResponse::ok(vec![payload], 0, Vec::new()), Err(error) => ExecuteResponse::err(execution_error_to_typed(error)), }) } @@ -57,7 +57,7 @@ pub(super) fn answer_holders_plan( }; let encoded = local_holders(state, *tenant_id, *database_id, entries).and_then(encode_holders); Some(match encoded { - Ok(payload) => ExecuteResponse::ok(vec![payload], 0, 0), + Ok(payload) => ExecuteResponse::ok(vec![payload], 0, Vec::new()), Err(error) => ExecuteResponse::err(execution_error_to_typed(error)), }) } diff --git a/nodedb/src/control/exec_receiver/tenant_marks.rs b/nodedb/src/control/exec_receiver/tenant_marks.rs index 82855c4fa..e1003021c 100644 --- a/nodedb/src/control/exec_receiver/tenant_marks.rs +++ b/nodedb/src/control/exec_receiver/tenant_marks.rs @@ -63,7 +63,7 @@ pub(super) async fn answer_tenant_marks( Err(error) => return ExecuteResponse::err(execution_error_to_typed(error)), }; match encode_marks(&marks) { - Ok(payload) => ExecuteResponse::ok(vec![payload], 0, 0), + Ok(payload) => ExecuteResponse::ok(vec![payload], 0, Vec::new()), Err(error) => ExecuteResponse::err(execution_error_to_typed(error)), } } diff --git a/nodedb/src/control/fail_gate.rs b/nodedb/src/control/fail_gate.rs index d8ed2319c..264a84e1a 100644 --- a/nodedb/src/control/fail_gate.rs +++ b/nodedb/src/control/fail_gate.rs @@ -14,19 +14,19 @@ use std::time::Duration; use crate::bridge::envelope::PhysicalPlan; -use crate::fail_point::{FailAction, lookup}; +use crate::fail_point::{FailAction, FailScope, lookup}; use crate::types::Lsn; /// How often a parked task checks for its release file. const GATE_POLL: Duration = Duration::from_millis(20); -/// Park until the file armed for `name` with `wait_file()` exists. No-op -/// when nothing is armed for `name`. +/// Park until the file armed for `name` in `scope` with `wait_file()` +/// exists. No-op when nothing is armed for `name` in `scope`. /// /// On arrival the gate creates `.parked`, so a test can wait until the /// task has reached the gate before it acts. -pub(crate) async fn wait(name: &str) { - let Some(FailAction::WaitForFile(path)) = lookup(name) else { +pub(crate) async fn wait(scope: FailScope, name: &str) { + let Some(FailAction::WaitForFile(path)) = lookup(scope, name) else { return; }; let mut parked = path.clone().into_os_string(); @@ -39,61 +39,119 @@ pub(crate) async fn wait(name: &str) { } } -/// The write funnel's gate: after a logged write's WAL record is appended and -/// before any core holds the request. Named per collection, -/// `funnel::before_dispatch::`, and only a write carrying a WAL -/// LSN reaches it. A committed redo parks on the gate of each collection it -/// writes. A replicated write parks inside its apply entry's enqueue, so -/// every later entry of its Raft group waits for it. Only a write of another -/// group applies while it is parked. +/// The write funnel's gate on node `node_id`: after a logged write's WAL +/// record is appended and before any core holds the request. Named per +/// collection, `funnel::before_dispatch::`, and only a write +/// carrying a WAL LSN reaches it. A committed redo parks on the gate of each +/// collection it writes. A replicated write parks inside its apply entry's +/// enqueue, so every later entry of its Raft group waits for it. Only a write +/// of another group applies while it is parked. /// -/// `funnel::before_dispatch::node::` parks the write only on -/// node `N`. An in-process cluster test shares one fail-point registry across -/// its nodes, so it names the node to hold one replica's apply. +/// A test arms the gate for one node to hold one replica's apply. pub(crate) async fn before_dispatch(node_id: u64, plan: &PhysicalPlan, wal_lsn: Option) { if wal_lsn.is_none() { return; } for collection in plan.named_collections() { - wait(&format!("funnel::before_dispatch::{collection}")).await; - wait(&format!( - "funnel::before_dispatch::node{node_id}::{collection}" - )) + wait( + FailScope::Node(node_id), + &format!("funnel::before_dispatch::{collection}"), + ) .await; } } -/// The Calvin submit gate: after a transaction's collection incarnations are -/// stamped and before the sequencer admits it. Named per collection, -/// `calvin::after_stamp::`. A test parks a transaction here, then -/// changes a collection it names before the transaction reaches a replica. -pub(crate) async fn after_calvin_stamp(tx_class: &nodedb_cluster::calvin::types::TxClass) { +/// The Calvin submit gate of node `node_id`: after a transaction's collection +/// incarnations are stamped and before the sequencer admits it. Named per +/// collection, `calvin::after_stamp::`. A test parks a +/// transaction here, then changes a collection it names before the +/// transaction reaches a replica. +pub(crate) async fn after_calvin_stamp( + node_id: u64, + tx_class: &nodedb_cluster::calvin::types::TxClass, +) { for named in &tx_class.incarnations { - wait(&format!("calvin::after_stamp::{}", named.collection)).await; + wait( + FailScope::Node(node_id), + &format!("calvin::after_stamp::{}", named.collection), + ) + .await; } } -/// The Calvin scheduler's flush gate: after a committed transaction's redo -/// record is appended and before its flush reaches a core. Named per -/// collection, `calvin::before_flush::`. +/// The Calvin scheduler's redo gate on node `node_id`: after a committed +/// slice's redo is resolved and before the data-group leader proposes it. +/// Named per collection, `calvin::before_redo_propose::`. /// /// The scheduler cannot park on a timer, so the gate answers without waiting: -/// `true` while the file armed for one of the flush's collections is absent. -/// The scheduler then keeps the flush in its re-send queue and asks again on -/// its next pass. -pub(crate) fn holds_flush(plan: &PhysicalPlan) -> bool { - plan.named_collections().iter().any(|collection| { - matches!( - lookup(&format!("calvin::before_flush::{collection}")), - Some(FailAction::WaitForFile(path)) if !path.exists() +/// `true` while the file armed for one of `collections` is absent. The +/// scheduler then keeps the slice unproposed and asks again on its next stall +/// tick. On its first hold the gate creates `.parked`, so a test can +/// wait until the slice is held. +pub(crate) fn holds_redo_propose(node_id: u64, collections: &[String]) -> bool { + collections.iter().any(|collection| { + holds( + FailScope::Node(node_id), + &format!("calvin::before_redo_propose::{collection}"), ) }) } +/// The Calvin install gate on node `node_id`: after a stamped redo's install +/// is durable and before its `RedoApplied` event reaches the vShard's +/// scheduler. Named per collection, +/// `calvin::before_redo_applied_push::`. +/// +/// The apply loop cannot park here without holding the entry back. Returns +/// the name of the gate that holds one of `collections`: its armed file is +/// absent. The caller hands the event to its own task, which waits on +/// [`released`] in the same scope. On its first hold the gate creates +/// `.parked`. +pub(crate) fn holds_redo_applied_push(node_id: u64, collections: &[String]) -> Option { + collections + .iter() + .map(|collection| format!("calvin::before_redo_applied_push::{collection}")) + .find(|name| holds(FailScope::Node(node_id), name)) +} + +/// Resolves once gate `name` in `scope` stops holding: its armed file +/// exists, or nothing is armed for it. +pub(crate) async fn released(scope: FailScope, name: &str) { + while armed_and_absent(scope, name).is_some() { + tokio::time::sleep(GATE_POLL).await; + } +} + +/// The file armed for `name` in `scope` with `wait_file()`, while it +/// is absent. +fn armed_and_absent(scope: FailScope, name: &str) -> Option { + let Some(FailAction::WaitForFile(path)) = lookup(scope, name) else { + return None; + }; + (!path.exists()).then_some(path) +} + +/// Whether gate `name` holds in `scope` now: its armed file is absent. On +/// its first hold it creates `.parked`. +fn holds(scope: FailScope, name: &str) -> bool { + let Some(path) = armed_and_absent(scope, name) else { + return false; + }; + let mut parked = path.into_os_string(); + parked.push(".parked"); + let parked = std::path::PathBuf::from(parked); + if !parked.exists() + && let Err(e) = std::fs::write(&parked, name) + { + tracing::warn!(gate = %name, error = %e, "fail gate parked marker not created"); + } + true +} + /// The committed-message delivery gates of node `node_id`: -/// `publish::before_delivery::node` before a delivery pass sends anything, -/// and `publish::before_cursor_commit::node` after a partition's messages -/// are sent and before its cursor is committed. `point` names the gate. +/// `publish::before_delivery` before a delivery pass sends anything, and +/// `publish::before_cursor_commit` after a partition's messages are sent and +/// before its cursor is committed. `point` names the gate. /// /// Returns `false` when the node shuts down while parked: the pass then stops /// where it is, as a node killed at that point does. @@ -102,13 +160,13 @@ pub(crate) async fn publish_delivery( point: &str, shutdown: tokio::sync::watch::Receiver, ) -> bool { - delivery_gate(&format!("publish::{point}::node{node_id}"), shutdown).await + delivery_gate(node_id, &format!("publish::{point}"), shutdown).await } /// The trigger action lane's firing gates of node `node_id`: -/// `trigger::before_firing::node` before a firing pass fires anything, -/// and `trigger::before_cursor_commit::node` after a partition's actions -/// fired and before its cursor is committed. `point` names the gate. +/// `trigger::before_firing` before a firing pass fires anything, and +/// `trigger::before_cursor_commit` after a partition's actions fired and +/// before its cursor is committed. `point` names the gate. /// /// Returns `false` when the node shuts down while parked. pub(crate) async fn action_firing( @@ -116,42 +174,35 @@ pub(crate) async fn action_firing( point: &str, shutdown: tokio::sync::watch::Receiver, ) -> bool { - delivery_gate(&format!("trigger::{point}::node{node_id}"), shutdown).await + delivery_gate(node_id, &format!("trigger::{point}"), shutdown).await } -/// Park on gate `name` until it is released or the node shuts down. -async fn delivery_gate(name: &str, mut shutdown: tokio::sync::watch::Receiver) -> bool { +/// Park on gate `name` of node `node_id` until it is released or the node +/// shuts down. +async fn delivery_gate( + node_id: u64, + name: &str, + mut shutdown: tokio::sync::watch::Receiver, +) -> bool { if *shutdown.borrow() { return false; } tokio::select! { - () = wait(name) => true, + () = wait(FailScope::Node(node_id), name) => true, _ = shutdown.changed() => false, } } -/// The metadata apply gate of node `node_id`, named by -/// [`crate::control::cluster::metadata_applier::metadata_apply_hold_point`]. +/// The metadata apply gate of node `node_id`, named +/// [`crate::control::cluster::metadata_applier::METADATA_APPLY_HOLD_POINT`]. /// /// The metadata applier runs on the Raft loop and cannot park, so the gate /// answers without waiting: `true` while the file armed for it is absent. /// On its first hold the gate creates `.parked`, so a test can wait /// until the node's apply is held. pub(crate) fn holds_metadata_apply(node_id: u64) -> bool { - let name = crate::control::cluster::metadata_applier::metadata_apply_hold_point(node_id); - let Some(FailAction::WaitForFile(path)) = lookup(&name) else { - return false; - }; - if path.exists() { - return false; - } - let mut parked = path.into_os_string(); - parked.push(".parked"); - let parked = std::path::PathBuf::from(parked); - if !parked.exists() - && let Err(e) = std::fs::write(&parked, &name) - { - tracing::warn!(gate = %name, error = %e, "fail gate parked marker not created"); - } - true + holds( + FailScope::Node(node_id), + crate::control::cluster::metadata_applier::METADATA_APPLY_HOLD_POINT, + ) } diff --git a/nodedb/src/control/gateway/cluster_error.rs b/nodedb/src/control/gateway/cluster_error.rs index 66c2c7210..32c462e10 100644 --- a/nodedb/src/control/gateway/cluster_error.rs +++ b/nodedb/src/control/gateway/cluster_error.rs @@ -42,9 +42,15 @@ pub(super) fn map_typed_cluster_error(err: TypedClusterError, vshard_id: u64) -> descriptor: collection, } } - TypedClusterError::DeadlineExceeded { .. } => Error::DeadlineExceeded { - request_id: crate::types::RequestId::new(0), - }, + TypedClusterError::DeadlineExceeded { elapsed_ms } => { + tracing::warn!( + elapsed_ms, + "gateway: a remote node reported the request's deadline exceeded" + ); + Error::DeadlineExceeded { + request_id: crate::types::RequestId::new(0), + } + } // Remote Data-Plane verdict: keep the code so the client sees the // SQLSTATE local execution renders, not a generic internal error. TypedClusterError::DataPlane { code } => Error::DataPlane(code.into()), diff --git a/nodedb/src/control/gateway/core.rs b/nodedb/src/control/gateway/core.rs index 1a4913e2a..64aaca526 100644 --- a/nodedb/src/control/gateway/core.rs +++ b/nodedb/src/control/gateway/core.rs @@ -139,17 +139,16 @@ impl Gateway { /// Returns one `Vec` payload per vShard result. For point operations /// the returned Vec has exactly one element. /// - /// Thin wrapper over [`Gateway::execute_with_watermarks`] that discards the - /// per-shard read watermarks — for the ~5 existing callers that only need - /// payloads. + /// Thin wrapper over [`Gateway::execute_outcome`] that keeps only the + /// payloads — for the callers that need nothing else. pub async fn execute( &self, ctx: &QueryContext, checked: CloneCheckedTask, ) -> Result>, Error> { - self.execute_with_watermarks(ctx, checked) + self.execute_outcome(ctx, checked) .await - .map(|(payloads, _watermarks, _read_version)| payloads) + .map(|outcome| outcome.payloads) } /// Execute a trusted Control-Plane plan that has no external caller. @@ -163,38 +162,35 @@ impl Gateway { ctx: &QueryContext, plan: PhysicalPlan, ) -> Result>, Error> { - self.execute_internal_with_watermarks(ctx, plan) + self.execute_internal_outcome(ctx, plan) .await - .map(|(payloads, _watermarks, _read_version)| payloads) + .map(|outcome| outcome.payloads) } - pub(crate) async fn execute_internal_with_watermarks( + /// [`Gateway::execute_outcome`] for a trusted Control-Plane plan. + pub(crate) async fn execute_internal_outcome( &self, ctx: &QueryContext, plan: PhysicalPlan, - ) -> Result<(Vec>, Vec<(VShardId, Lsn)>, Lsn), Error> { - self.execute_plan_outcome(ctx, plan) - .await - .map(GatewayOutcome::into_parts) + ) -> Result { + self.execute_plan_outcome(ctx, plan).await } - /// Execute a pre-planned `PhysicalPlan`, returning both the raw payloads and - /// the per-shard read watermarks observed across every dispatched route. + /// Execute a pre-planned `PhysicalPlan`, returning the raw payloads, the + /// per-shard read watermarks, the read versions observed across every + /// dispatched route, and a single route's `NotFound` verdict. /// - /// Each `(vshard, watermark_lsn)` entry is one participating shard's real - /// committed LSN (local SPSC response watermark or the remote's - /// `ExecuteResponse.watermark_lsn`). The cross-node gather consumer folds - /// these into the transaction read-set so a remote-homed read records the - /// remote's actual LSN. - pub async fn execute_with_watermarks( + /// Each `(vshard, watermark_lsn)` entry is one participating shard's WAL + /// position (local SPSC response watermark or the remote's + /// `ExecuteResponse.watermark_lsn`). The read versions are data-group log + /// positions, so the transaction read-set validates them on any replica. + pub async fn execute_outcome( &self, ctx: &QueryContext, checked: CloneCheckedTask, - ) -> Result<(Vec>, Vec<(VShardId, Lsn)>, Lsn), Error> { + ) -> Result { let (plan, _lease) = authorized_plan_for_context(ctx, checked)?; - self.execute_plan_outcome(ctx, plan) - .await - .map(GatewayOutcome::into_parts) + self.execute_plan_outcome(ctx, plan).await } /// Execute an authorized plan and keep every route's result detail. @@ -260,10 +256,8 @@ impl Gateway { let max_total_bytes = shared.tuning.network.max_query_result_bytes as usize; let mut all_payloads: Vec> = Vec::new(); let mut all_shard_watermarks: Vec<(VShardId, Lsn)> = Vec::new(); - // Max-fold of the per-collection read-version across routes: a read - // targets one collection homed to one shard, so non-owning routes - // contribute `Lsn::ZERO` and the owning route's value survives. - let mut max_read_version = Lsn::ZERO; + // The read versions every route observed, one per vShard. + let mut read_versions = crate::types::ReadVersions::new(); let mut accumulated_bytes: usize = 0; for route in routes { @@ -326,9 +320,7 @@ impl Gateway { // read produces one read-set entry per shard. all_shard_watermarks.extend(outcome.shard_watermarks); not_found = single_route && outcome.not_found; - if outcome.read_version_lsn > max_read_version { - max_read_version = outcome.read_version_lsn; - } + read_versions.merge(&outcome.read_versions); for p in outcome.payloads { accumulated_bytes = accumulated_bytes.saturating_add(p.len()); @@ -355,7 +347,7 @@ impl Gateway { Ok(GatewayOutcome { payloads, shard_watermarks: all_shard_watermarks, - read_version_lsn: max_read_version, + read_versions, not_found, }) } diff --git a/nodedb/src/control/gateway/dispatch_local.rs b/nodedb/src/control/gateway/dispatch_local.rs index 53c4b43e3..fc26c5325 100644 --- a/nodedb/src/control/gateway/dispatch_local.rs +++ b/nodedb/src/control/gateway/dispatch_local.rs @@ -6,7 +6,7 @@ use std::sync::Arc; use crate::Error; -use crate::bridge::envelope::{ErrorCode, PhysicalPlan, Response, Status}; +use crate::bridge::envelope::{PhysicalPlan, Response}; use crate::control::local_dispatch::reject_data_plane_error; use crate::control::server::dispatch_utils::{ AutocommitWrite, dispatch_autocommit_write, dispatch_routed_read_to_data_plane, @@ -94,15 +94,19 @@ pub(super) async fn dispatch_local( )? { let proposer = shared.async_raft_proposer()?; - let (payload, write_version) = - crate::control::wal_replication::propose_replicated_entry(shared, proposer, entry) - .await?; + let (payload, write_versions) = crate::control::wal_replication::propose_replicated_entry( + shared, + proposer, + entry, + crate::control::wal_replication::statement_propose_deadline(shared), + ) + .await?; return Ok(DispatchOutcome { payloads: vec![payload], - // A write carries no read watermark (Lsn::ZERO); its post-write - // `coll_write_lsn` is surfaced via `read_version_lsn` instead. + // A write carries no read watermark (Lsn::ZERO). Its write + // versions travel in `read_versions`. shard_watermarks: vec![(vshard_id, Lsn::ZERO)], - read_version_lsn: write_version, + read_versions: write_versions, not_found: false, }); } @@ -125,8 +129,8 @@ pub(super) async fn dispatch_local( Ok(DispatchOutcome { payloads: vec![resp.payload.to_vec()], shard_watermarks: vec![(vshard_id, resp.watermark_lsn)], - read_version_lsn: resp.read_version_lsn, - not_found: is_not_found(&resp), + read_versions: resp.read_versions.clone(), + not_found: crate::control::local_dispatch::is_not_found(&resp), }) } @@ -199,11 +203,3 @@ async fn dispatch_local_plan(local: LocalPlan<'_>) -> Result { ) .await } - -/// Whether the core refused the task with `ErrorCode::NotFound`. -/// -/// `reject_data_plane_error` passes this refusal as an empty success. -/// The flag keeps the verdict for a caller that needs it. -fn is_not_found(resp: &Response) -> bool { - resp.status == Status::Error && resp.error_code.as_deref() == Some(&ErrorCode::NotFound) -} diff --git a/nodedb/src/control/gateway/dispatch_remote.rs b/nodedb/src/control/gateway/dispatch_remote.rs index 50fd74339..94b6de798 100644 --- a/nodedb/src/control/gateway/dispatch_remote.rs +++ b/nodedb/src/control/gateway/dispatch_remote.rs @@ -2,21 +2,18 @@ //! Remote dispatch: send a plan to a remote node via `ExecuteRequest` RPC. //! -//! Split out of `dispatcher.rs` to keep that file under the project's -//! 500-line limit. Holds the one-shot (`dispatch_remote`) and streaming -//! (`dispatch_remote_stream`) remote dispatch paths used by -//! [`super::dispatcher::dispatch_route`] and -//! [`super::dispatcher::dispatch_route_stream`] respectively. +//! Holds the one-shot path, [`dispatch_remote`], used by +//! [`super::dispatcher::dispatch_route`]. The streaming path lives in +//! [`super::dispatch_remote_stream`]. use std::sync::Arc; -use futures::StreamExt; use nodedb_cluster::ClusterError; use nodedb_cluster::rpc_codec::{ExecuteRequest, RaftRpc}; use tracing::debug; use crate::Error; -use crate::control::server::result_stream::{ResultStream, RowBatch}; +use crate::control::security::identity::{Permission, required_permission}; use crate::control::state::SharedState; use crate::types::{DatabaseId, Lsn, TenantId, TraceId, TxnId, VShardId}; use nodedb_physical::physical_plan::wire as plan_wire; @@ -105,10 +102,10 @@ pub(super) async fn dispatch_remote( payloads: vec![resp.payload.to_vec()], shard_watermarks, // Coordinator-gathered at the exchange root: the gather folded - // the per-collection read-version across the responding shards - // and stamped it on this response, so carry it through. This - // route did not observe a version of its own to report. - read_version_lsn: resp.read_version_lsn, + // the read versions across the responding shards and stamped + // them on this response, so carry them through. This route did + // not observe a version of its own to report. + read_versions: resp.read_versions, not_found: false, }); } @@ -121,18 +118,22 @@ pub(super) async fn dispatch_remote( return Ok(DispatchOutcome { payloads: vec![merged], shard_watermarks: vec![(VShardId::new(vshard_id as u32), lsn)], - // A materialized stream reports no per-collection read-version: - // its frames carry per-batch watermarks only. `ZERO` is honest - // here rather than lossy — the streaming branch is gated on + // A materialized stream reports no read versions: its frames + // carry per-batch watermarks only. Empty is honest here rather + // than lossy: the streaming branch is gated on // `txn_id.is_none()` (`resolve/exchange.rs`), so a stream never // serves an in-transaction read and no read-set entry consumes // this value. - read_version_lsn: Lsn::ZERO, + read_versions: crate::types::ReadVersions::new(), not_found: false, }); } }; + // A read can be sent again after a lost answer. Any other plan can have + // run on the target, so a lost answer is an unknown outcome. + let resend_safe = required_permission(&plan) == Permission::Read; + // Encode the plan. let plan_bytes = plan_wire::encode(&plan).map_err(|e| Error::Internal { detail: format!("gateway: plan encode failed: {e}"), @@ -169,40 +170,55 @@ pub(super) async fn dispatch_remote( "gateway: dispatching ExecuteRequest to remote node" ); - let resp_rpc = transport.send_rpc(node_id, req).await.map_err(|e| { - // Transport failure means the target node is unreachable — - // we do NOT know who the new leader is. Use leader_node = 0 - // so the retry loop does NOT re-entrench the unreachable node - // as leader in the routing table. The next retry will route - // locally (leader == 0 → local) and let the local Raft state - // resolve to the actual leader. - Error::NotLeader { - vshard_id: VShardId::new((vshard_id % VShardId::COUNT as u64) as u32), - leader_node: 0, - leader_addr: format!("node-{node_id} (transport error: {e})"), - leader_term: 0, + // The remote handler works within `deadline_ms`: a write can wait at the + // leader's gate or in a Calvin queue for that long. The reply wait follows + // that budget, never the transport's fixed RPC timeout. + let reply_wait = nodedb_cluster::rpc_codec::reply_wait(deadline_ms); + let resp_rpc = match tokio::time::timeout( + reply_wait, + transport.send_rpc_with_read_timeout(node_id, req, reply_wait), + ) + .await + { + Ok(answer) => answer.map_err(|e| send_error(e, resend_safe, node_id, vshard_id))?, + Err(_) => { + return Err(send_error( + ClusterError::ShardTimeout { + vshard_id: (vshard_id % VShardId::COUNT as u64) as u32, + elapsed_ms: u64::try_from(reply_wait.as_millis()).unwrap_or(u64::MAX), + }, + resend_safe, + node_id, + vshard_id, + )); } - })?; + }; match resp_rpc { RaftRpc::ExecuteResponse(resp) => { - if let Some(err) = resp.error { - Err(map_typed_cluster_error(err, vshard_id)) - } else { - // Key the remote's read watermark to the collection's owning - // vShard this route routed to (mirroring `dispatch_local`), NOT - // the `vshard_id % COUNT` retry hint — the read-set validator - // expects the true owning vShard as the entry key. - Ok(DispatchOutcome { - shard_watermarks: vec![( - VShardId::new(vshard_id as u32), - Lsn::new(resp.watermark_lsn), - )], - payloads: resp.payloads, - read_version_lsn: Lsn::new(resp.read_version_lsn), - not_found: false, - }) - } + // A read that found no row reads back like a local miss: an + // empty answer that keeps the `NotFound` verdict and the versions + // the read observed. Every other error crosses as its typed error. + let not_found = match resp.error { + None => false, + Some(err) => match map_typed_cluster_error(err, vshard_id) { + Error::DataPlane(crate::bridge::envelope::ErrorCode::NotFound) => true, + error => return Err(error), + }, + }; + // Key the remote's read watermark to the collection's owning + // vShard this route routed to (mirroring `dispatch_local`), NOT + // the `vshard_id % COUNT` retry hint — the read-set validator + // expects the true owning vShard as the entry key. + Ok(DispatchOutcome { + shard_watermarks: vec![( + VShardId::new(vshard_id as u32), + Lsn::new(resp.watermark_lsn), + )], + payloads: resp.payloads, + read_versions: crate::types::ReadVersions::from_wire(&resp.read_versions), + not_found, + }) } other => Err(Error::Internal { detail: format!("gateway: unexpected RPC response variant: {other:?}"), @@ -210,230 +226,12 @@ pub(super) async fn dispatch_remote( } } -/// Remote streaming dispatch via the multi-frame `ExecuteStreamRequest` RPC. -/// -/// Returns a [`ResultStream`] that yields the remote shard's row batches as -/// they arrive over QUIC, interleaved by the caller's `select_all` with any -/// local routes. -/// -/// ## Retry-vs-stream split (critical) -/// -/// Leader resolution and the FIRST frame are obtained EAGERLY here: the bidi -/// stream is opened and the first stream item is pulled inside this function. -/// A terminal error that arrives BEFORE any row (`NotLeader`, -/// `DescriptorMismatch`, transport failure on open) is mapped via -/// [`map_typed_cluster_error`] to a retryable [`Error`] and propagated to the -/// gateway's existing not-leader retry loop. Once at least one chunk has been -/// observed, any subsequent error is TERMINAL — it is surfaced as a stream -/// `Err` and never retried (re-running the plan duplicates the rows -/// already streamed to the client). -/// -/// The returned stream re-emits the buffered first batch followed by the rest. -pub(super) async fn dispatch_remote_stream( - args: RemoteDispatchArgs<'_>, -) -> Result { - let RemoteDispatchArgs { - plan, - shared, - node_id, - vshard_id, - tenant_id, - database_id, - trace_id, - deadline_ms, - version_set, - txn_id, - linearizable, - read_groups, - } = args; - let transport = shared.cluster_transport.as_ref().ok_or(Error::Internal { - detail: "gateway: cluster transport not available for remote stream dispatch".into(), - })?; - - // Resolve Exchange nodes before shipping (symmetric with `dispatch_remote`). - // No session-transaction context crosses this boundary yet, so `None`. - let scope = crate::control::server::exchange::ReadScope { - database_id, - tenant_id, - trace_id, - txn_id: None, - linearizable, - }; - let plan = match Box::pin(crate::control::server::exchange::resolve_exchange_in_plan( - shared, plan, scope, - )) - .await? - { - crate::control::server::exchange::Resolved::Plan(p) => *p, - // A streamable child whose Exchange resolved at the coordinator into a - // ready response/stream — re-emit it as a single-batch / forwarded - // stream. These do not occur for the streamable-scan plans routed here, - // but handle exhaustively and behaviour-preservingly. - crate::control::server::exchange::Resolved::Gathered( - resp, - _shard_watermarks, - _shuffle_reads, - ) => { - let batch = RowBatch { - payload: resp.payload.to_vec(), - watermark_lsn: resp.watermark_lsn, - read_version_lsn: resp.read_version_lsn, - }; - return Ok(Box::pin(futures::stream::once(async move { Ok(batch) }))); - } - crate::control::server::exchange::Resolved::Stream(s) => return Ok(s), - }; - - let plan_bytes = plan_wire::encode(&plan).map_err(|e| Error::Internal { - detail: format!("gateway: plan encode failed: {e}"), - })?; - - let descriptor_versions: Vec = version_set - .iter() - .map( - |(name, version)| nodedb_cluster::rpc_codec::DescriptorVersionEntry { - collection: name.clone(), - version: *version, - }, - ) - .collect(); - - let scoped_vshard = scoped_vshard(&plan, vshard_id)?; - let req = RaftRpc::ExecuteStreamRequest(ExecuteRequest { - plan_bytes, - tenant_id: tenant_id.as_u64(), - database_id: database_id.as_u64(), - deadline_remaining_ms: deadline_ms, - trace_id: trace_id.0, - descriptor_versions, - txn_id, - vshard_id: scoped_vshard, - read_groups, - }); - - debug!( - node_id, - vshard_id, - tenant_id = tenant_id.as_u64(), - "gateway: dispatching ExecuteStreamRequest to remote node" - ); - - // Open the stream eagerly. A failure to even open / send the request is a - // pre-row condition: map it like a transport failure in `dispatch_remote` - // so the retry loop routes elsewhere on the next attempt. - let stream = transport - .send_rpc_stream(node_id, req) - .await - .map_err(|e| Error::NotLeader { - vshard_id: VShardId::new((vshard_id % VShardId::COUNT as u64) as u32), - leader_node: 0, - leader_addr: format!("node-{node_id} (stream open error: {e})"), - leader_term: 0, - })?; - // The `async_stream` body is `!Unpin`; pin it on the heap so we can pull - // the eager first frame and then keep the tail around for `.chain`. - let mut stream = Box::pin(stream); - - // Eagerly pull the FIRST frame so a pre-row terminal error is catchable and - // retryable. Any error here is a pre-row error. - let first = match stream.next().await { - Some(Ok((payload, lsn))) => RowBatch { - payload, - watermark_lsn: Lsn::new(lsn), - // The `ExecuteStream` wire chunk carries only the watermark; no - // per-collection read version is threaded on this remote path. - read_version_lsn: Lsn::ZERO, - }, - Some(Err(e)) => return Err(map_stream_cluster_error(e, vshard_id)), - // Clean EOF with zero rows: a valid empty result. Return an empty stream. - None => return Ok(Box::pin(futures::stream::empty())), - }; - - // Build the result stream: re-emit the buffered first batch, then forward - // the rest. Errors past the first frame are TERMINAL — surfaced as stream - // `Err`, never retried. - let rest = stream.map(move |item| match item { - Ok((payload, lsn)) => Ok(RowBatch { - payload, - watermark_lsn: Lsn::new(lsn), - read_version_lsn: Lsn::ZERO, - }), - Err(e) => Err(Error::Dispatch { - detail: format!("remote stream terminal error: {e}"), - }), - }); - - let head = futures::stream::once(async move { Ok(first) }); - Ok(Box::pin(head.chain(rest))) -} - -/// Map a pre-row [`nodedb_cluster::ClusterError`] from a streaming dispatch to -/// an [`Error`]. -/// -/// A typed error (`StreamTerminal`, `ShardExecution`) maps through the same -/// [`map_typed_cluster_error`] used by the one-shot path, so the gateway retry -/// loop handles it identically. A Data-Plane verdict keeps its code. Any other -/// cluster error becomes a transport-style `NotLeader` (leader_node = 0), so -/// the next attempt re-resolves routing rather than re-entrenching an -/// unreachable node. -fn map_stream_cluster_error(err: ClusterError, vshard_id: u64) -> Error { - match err { - ClusterError::StreamTerminal { error, .. } | ClusterError::ShardExecution { error, .. } => { - map_typed_cluster_error(*error, vshard_id) - } - // A verdict from a shard that answered. Retrying it on another route - // repeats it, so it keeps its SQLSTATE. - ClusterError::DataPlane { code } => Error::DataPlane(code.into()), - other @ (ClusterError::Raft(_) - | ClusterError::VShardNotMapped { .. } - | ClusterError::GroupNotFound { .. } - | ClusterError::LearnerNotCaughtUp { .. } - | ClusterError::MigrationInProgress { .. } - | ClusterError::MigrationPauseBudgetExceeded { .. } - | ClusterError::NodeUnreachable { .. } - | ClusterError::GhostNotFound { .. } - | ClusterError::Transport { .. } - | ClusterError::ShardTimeout { .. } - | ClusterError::Storage { .. } - | ClusterError::Codec { .. } - | ClusterError::UnsupportedWireVersion { .. } - | ClusterError::CircuitOpen { .. } - | ClusterError::JoinGroupDisappeared { .. } - | ClusterError::JoinCommitTimeout { .. } - | ClusterError::ReadIndexNotLeader { .. } - | ClusterError::ReadIndexTimeout { .. } - | ClusterError::Config { .. } - | ClusterError::MigrationCheckpoint(_) - | ClusterError::MigrationRecovery(_) - | ClusterError::WrongOwner { .. } - | ClusterError::Calvin(_) - | ClusterError::SnapshotCrcMismatch { .. } - | ClusterError::SnapshotOffsetRegression { .. } - | ClusterError::PartialSnapshotCorrupt { .. } - | ClusterError::PartialSnapshotCleanupFailed { .. } - | ClusterError::SnapshotApplyFailed { .. } - | ClusterError::Mirror(_) - | ClusterError::BspBarrier(_) - | ClusterError::VectorGather(_) - | ClusterError::SpatialGather(_) - | ClusterError::Bm25Gather(_) - | ClusterError::TsGather(_) - | ClusterError::ShufflePush(_) - | ClusterError::RemoteUntyped { .. }) => Error::NotLeader { - vshard_id: VShardId::new((vshard_id % VShardId::COUNT as u64) as u32), - leader_node: 0, - leader_addr: format!("stream dispatch error: {other}"), - leader_term: 0, - }, - } -} - /// The vShard a remote receiver must run `plan` on, or `None` when the plan /// fans across the receiver's local cores. /// /// A transaction meta-op names no collection, so only the route's own vShard /// can pick the core that holds the transaction's staging overlay. -fn scoped_vshard( +pub(super) fn scoped_vshard( plan: &nodedb_physical::physical_plan::PhysicalPlan, vshard_id: u64, ) -> Result, Error> { @@ -451,11 +249,79 @@ fn scoped_vshard( Ok(Some(VShardId::new(raw))) } +/// The error of a failed `ExecuteRequest` send. +/// +/// A request the target never received, and a lost answer to a read, retry as +/// `NotLeader` with `leader_node = 0`. The unreachable node is then not kept as +/// leader, and the next try routes locally to the real leader. A lost answer +/// to any other plan is an unknown outcome: the plan can have run, and a +/// second send would run it again. +fn send_error(e: ClusterError, resend_safe: bool, node_id: u64, vshard_id: u64) -> Error { + let outcome_unknown = matches!( + e, + ClusterError::Unanswered { .. } | ClusterError::ShardTimeout { .. } + ); + if outcome_unknown && !resend_safe { + tracing::warn!( + node_id, + vshard_id, + error = %e, + "gateway: a remote execute reached its deadline with no answer" + ); + return Error::DeadlineExceeded { + request_id: crate::types::RequestId::new(0), + }; + } + Error::NotLeader { + vshard_id: VShardId::new((vshard_id % VShardId::COUNT as u64) as u32), + leader_node: 0, + leader_addr: format!("node-{node_id} (transport error: {e})"), + leader_term: 0, + } +} + #[cfg(test)] mod tests { use super::*; use nodedb_physical::physical_plan::{MetaOp, PhysicalPlan}; + fn unanswered() -> ClusterError { + ClusterError::Unanswered { + node_id: 2, + detail: "stream reset".into(), + } + } + + /// A write whose answer was lost can have applied: it is never sent again. + #[test] + fn a_lost_answer_to_a_write_is_an_unknown_outcome() { + assert!(matches!( + send_error(unanswered(), false, 2, 7), + Error::DeadlineExceeded { .. } + )); + } + + /// A read whose answer was lost retries through the leader lookup. + #[test] + fn a_lost_answer_to_a_read_retries() { + assert!(matches!( + send_error(unanswered(), true, 2, 7), + Error::NotLeader { leader_node: 0, .. } + )); + } + + /// A request that never reached the target retries, read or write. + #[test] + fn an_unsent_request_retries() { + let unsent = ClusterError::Transport { + detail: "connect refused".into(), + }; + assert!(matches!( + send_error(unsent, false, 2, 7), + Error::NotLeader { leader_node: 0, .. } + )); + } + #[test] fn a_transaction_meta_op_carries_its_route_vshard() { let plan = PhysicalPlan::Meta(MetaOp::DropTxnOverlay { diff --git a/nodedb/src/control/gateway/dispatch_remote_stream.rs b/nodedb/src/control/gateway/dispatch_remote_stream.rs new file mode 100644 index 000000000..3dd04f63a --- /dev/null +++ b/nodedb/src/control/gateway/dispatch_remote_stream.rs @@ -0,0 +1,237 @@ +// SPDX-License-Identifier: BUSL-1.1 + +//! Remote streaming dispatch: run a plan on a remote node via the +//! multi-frame `ExecuteStreamRequest` RPC. Used by +//! [`super::dispatcher::dispatch_route_stream`]. + +use futures::StreamExt; +use nodedb_cluster::ClusterError; +use nodedb_cluster::rpc_codec::{ExecuteRequest, RaftRpc}; +use nodedb_physical::physical_plan::wire as plan_wire; +use tracing::debug; + +use crate::Error; +use crate::control::server::result_stream::{ResultStream, RowBatch}; +use crate::types::{Lsn, VShardId}; + +use super::cluster_error::map_typed_cluster_error; +use super::dispatch_remote::{RemoteDispatchArgs, scoped_vshard}; + +/// Remote streaming dispatch via the multi-frame `ExecuteStreamRequest` RPC. +/// +/// Returns a [`ResultStream`] that yields the remote shard's row batches as +/// they arrive over QUIC, interleaved by the caller's `select_all` with any +/// local routes. +/// +/// ## Retry-vs-stream split (critical) +/// +/// Leader resolution and the FIRST frame are obtained EAGERLY here: the bidi +/// stream is opened and the first stream item is pulled inside this function. +/// A terminal error that arrives BEFORE any row (`NotLeader`, +/// `DescriptorMismatch`, transport failure on open) is mapped via +/// [`map_typed_cluster_error`] to a retryable [`Error`] and propagated to the +/// gateway's existing not-leader retry loop. Once at least one chunk has been +/// observed, any subsequent error is TERMINAL — it is surfaced as a stream +/// `Err` and never retried (re-running the plan duplicates the rows +/// already streamed to the client). +/// +/// The returned stream re-emits the buffered first batch followed by the rest. +pub(super) async fn dispatch_remote_stream( + args: RemoteDispatchArgs<'_>, +) -> Result { + let RemoteDispatchArgs { + plan, + shared, + node_id, + vshard_id, + tenant_id, + database_id, + trace_id, + deadline_ms, + version_set, + txn_id, + linearizable, + read_groups, + } = args; + let transport = shared.cluster_transport.as_ref().ok_or(Error::Internal { + detail: "gateway: cluster transport not available for remote stream dispatch".into(), + })?; + + // Resolve Exchange nodes before shipping (symmetric with `dispatch_remote`). + // No session-transaction context crosses this boundary yet, so `None`. + let scope = crate::control::server::exchange::ReadScope { + database_id, + tenant_id, + trace_id, + txn_id: None, + linearizable, + }; + let plan = match Box::pin(crate::control::server::exchange::resolve_exchange_in_plan( + shared, plan, scope, + )) + .await? + { + crate::control::server::exchange::Resolved::Plan(p) => *p, + // A streamable child whose Exchange resolved at the coordinator into a + // ready response/stream — re-emit it as a single-batch / forwarded + // stream. These do not occur for the streamable-scan plans routed here, + // but handle exhaustively and behaviour-preservingly. + crate::control::server::exchange::Resolved::Gathered( + resp, + _shard_watermarks, + _shuffle_reads, + ) => { + let batch = RowBatch { + payload: resp.payload.to_vec(), + watermark_lsn: resp.watermark_lsn, + read_versions: resp.read_versions, + }; + return Ok(Box::pin(futures::stream::once(async move { Ok(batch) }))); + } + crate::control::server::exchange::Resolved::Stream(s) => return Ok(s), + }; + + let plan_bytes = plan_wire::encode(&plan).map_err(|e| Error::Internal { + detail: format!("gateway: plan encode failed: {e}"), + })?; + + let descriptor_versions: Vec = version_set + .iter() + .map( + |(name, version)| nodedb_cluster::rpc_codec::DescriptorVersionEntry { + collection: name.clone(), + version: *version, + }, + ) + .collect(); + + let scoped_vshard = scoped_vshard(&plan, vshard_id)?; + let req = RaftRpc::ExecuteStreamRequest(ExecuteRequest { + plan_bytes, + tenant_id: tenant_id.as_u64(), + database_id: database_id.as_u64(), + deadline_remaining_ms: deadline_ms, + trace_id: trace_id.0, + descriptor_versions, + txn_id, + vshard_id: scoped_vshard, + read_groups, + }); + + debug!( + node_id, + vshard_id, + tenant_id = tenant_id.as_u64(), + "gateway: dispatching ExecuteStreamRequest to remote node" + ); + + // Open the stream eagerly. A failure to even open / send the request is a + // pre-row condition: map it like a transport failure in `dispatch_remote` + // so the retry loop routes elsewhere on the next attempt. + let stream = transport + .send_rpc_stream(node_id, req) + .await + .map_err(|e| Error::NotLeader { + vshard_id: VShardId::new((vshard_id % VShardId::COUNT as u64) as u32), + leader_node: 0, + leader_addr: format!("node-{node_id} (stream open error: {e})"), + leader_term: 0, + })?; + // The `async_stream` body is `!Unpin`; pin it on the heap so we can pull + // the eager first frame and then keep the tail around for `.chain`. + let mut stream = Box::pin(stream); + + // Eagerly pull the FIRST frame so a pre-row terminal error is catchable and + // retryable. Any error here is a pre-row error. + let first = match stream.next().await { + Some(Ok((payload, lsn))) => RowBatch { + payload, + watermark_lsn: Lsn::new(lsn), + // The `ExecuteStream` wire chunk carries only the watermark; no + // read version is threaded on this remote path. + read_versions: crate::types::ReadVersions::new(), + }, + Some(Err(e)) => return Err(map_stream_cluster_error(e, vshard_id)), + // Clean EOF with zero rows: a valid empty result. Return an empty stream. + None => return Ok(Box::pin(futures::stream::empty())), + }; + + // Build the result stream: re-emit the buffered first batch, then forward + // the rest. Errors past the first frame are TERMINAL — surfaced as stream + // `Err`, never retried. + let rest = stream.map(move |item| match item { + Ok((payload, lsn)) => Ok(RowBatch { + payload, + watermark_lsn: Lsn::new(lsn), + read_versions: crate::types::ReadVersions::new(), + }), + Err(e) => Err(Error::Dispatch { + detail: format!("remote stream terminal error: {e}"), + }), + }); + + let head = futures::stream::once(async move { Ok(first) }); + Ok(Box::pin(head.chain(rest))) +} + +/// Map a pre-row [`nodedb_cluster::ClusterError`] from a streaming dispatch to +/// an [`Error`]. +/// +/// A typed error (`StreamTerminal`, `ShardExecution`) maps through the same +/// [`map_typed_cluster_error`] used by the one-shot path, so the gateway retry +/// loop handles it identically. A Data-Plane verdict keeps its code. Any other +/// cluster error becomes a transport-style `NotLeader` (leader_node = 0), so +/// the next attempt re-resolves routing rather than re-entrenching an +/// unreachable node. +fn map_stream_cluster_error(err: ClusterError, vshard_id: u64) -> Error { + match err { + ClusterError::StreamTerminal { error, .. } | ClusterError::ShardExecution { error, .. } => { + map_typed_cluster_error(*error, vshard_id) + } + // A verdict from a shard that answered. Retrying it on another route + // repeats it, so it keeps its SQLSTATE. + ClusterError::DataPlane { code } => Error::DataPlane(code.into()), + other @ (ClusterError::Raft(_) + | ClusterError::VShardNotMapped { .. } + | ClusterError::GroupNotFound { .. } + | ClusterError::LearnerNotCaughtUp { .. } + | ClusterError::MigrationInProgress { .. } + | ClusterError::MigrationPauseBudgetExceeded { .. } + | ClusterError::NodeUnreachable { .. } + | ClusterError::GhostNotFound { .. } + | ClusterError::Transport { .. } + | ClusterError::ShardTimeout { .. } + | ClusterError::Unanswered { .. } + | ClusterError::Storage { .. } + | ClusterError::Codec { .. } + | ClusterError::UnsupportedWireVersion { .. } + | ClusterError::CircuitOpen { .. } + | ClusterError::JoinGroupDisappeared { .. } + | ClusterError::JoinCommitTimeout { .. } + | ClusterError::ReadIndexNotLeader { .. } + | ClusterError::ReadIndexTimeout { .. } + | ClusterError::Config { .. } + | ClusterError::MigrationCheckpoint(_) + | ClusterError::MigrationRecovery(_) + | ClusterError::WrongOwner { .. } + | ClusterError::Calvin(_) + | ClusterError::SnapshotCrcMismatch { .. } + | ClusterError::SnapshotOffsetRegression { .. } + | ClusterError::PartialSnapshotCorrupt { .. } + | ClusterError::PartialSnapshotCleanupFailed { .. } + | ClusterError::SnapshotApplyFailed { .. } + | ClusterError::Mirror(_) + | ClusterError::BspBarrier(_) + | ClusterError::VectorGather(_) + | ClusterError::SpatialGather(_) + | ClusterError::Bm25Gather(_) + | ClusterError::TsGather(_) + | ClusterError::ShufflePush(_) + | ClusterError::RemoteUntyped { .. }) => Error::NotLeader { + vshard_id: VShardId::new((vshard_id % VShardId::COUNT as u64) as u32), + leader_node: 0, + leader_addr: format!("stream dispatch error: {other}"), + leader_term: 0, + }, + } +} diff --git a/nodedb/src/control/gateway/dispatcher.rs b/nodedb/src/control/gateway/dispatcher.rs index 552b01918..79e396855 100644 --- a/nodedb/src/control/gateway/dispatcher.rs +++ b/nodedb/src/control/gateway/dispatcher.rs @@ -12,12 +12,12 @@ use std::sync::Arc; use crate::Error; use crate::bridge::envelope::PhysicalPlan; use crate::control::server::result_stream::ResultStream; -use crate::control::server::shared::session::served_reads; use crate::control::state::SharedState; use crate::types::{DatabaseId, Lsn, TenantId, TraceId, TxnId, VShardId}; use super::dispatch_local::{LocalContext, dispatch_local}; -use super::dispatch_remote::{RemoteDispatchArgs, dispatch_remote, dispatch_remote_stream}; +use super::dispatch_remote::{RemoteDispatchArgs, dispatch_remote}; +use super::dispatch_remote_stream::dispatch_remote_stream; use super::read_leg::{confirm_local_read, linearizable_read_groups}; use super::route::{RouteDecision, TaskRoute}; use super::version_check::check_local_descriptor_versions; @@ -28,16 +28,14 @@ use super::version_set::GatewayVersionSet; /// /// `shard_watermarks` is one `(vshard, watermark_lsn)` per contributing shard /// — local SPSC watermark, or remote `ExecuteResponse.watermark_lsn` keyed to -/// the owning vShard — accumulated so an in-transaction read gets one -/// read-set entry per shard at its true committed LSN. +/// the owning vShard. A read-set entry takes its version from +/// `read_versions`, never from a watermark. pub struct DispatchOutcome { pub payloads: Vec>, pub shard_watermarks: Vec<(VShardId, Lsn)>, - /// This route's scanned collection's read-version LSN (`coll_write_lsn` - /// at read time), `Lsn::ZERO` for writes. Max-folded across routes — a - /// read targets one collection, so one non-zero value survives — for - /// cross-shard OCC read validation. - pub read_version_lsn: Lsn, + /// The versions this route observed, one per vShard. Folded across + /// routes for OCC read validation. + pub read_versions: crate::types::ReadVersions, /// The owning core refused the task with `ErrorCode::NotFound`. /// /// A fan-out reads it as a shard that holds no slice. A single-route @@ -79,11 +77,10 @@ pub(crate) async fn dispatch_route( } = params; reject_unadmitted_crdt_apply(&route.plan)?; let read_groups = linearizable_read_groups(shared, &route, linearizable)?; - let route_vshard = route.vshard_id; match route.decision { RouteDecision::Local => { confirm_local_read(shared, database_id, &route.plan, &read_groups, deadline_ms).await?; - let outcome = dispatch_local( + dispatch_local( route, LocalContext { shared, @@ -94,13 +91,10 @@ pub(crate) async fn dispatch_route( version_set, }, ) - .await?; - // The read's versions are this node's WAL positions. - served_reads::note(route_vshard, shared.node_id); - Ok(outcome) + .await } RouteDecision::Remote { node_id, vshard_id } => { - let outcome = dispatch_remote(RemoteDispatchArgs { + dispatch_remote(RemoteDispatchArgs { plan: route.plan, shared, node_id, @@ -114,10 +108,7 @@ pub(crate) async fn dispatch_route( linearizable, read_groups, }) - .await?; - // The read's versions are the remote node's WAL positions. - served_reads::note(route_vshard, node_id); - Ok(outcome) + .await } RouteDecision::Broadcast { .. } => { // Split into individual Local/Remote routes by the router before diff --git a/nodedb/src/control/gateway/error_map/class_parity/code_samples.rs b/nodedb/src/control/gateway/error_map/class_parity/code_samples.rs index 5b67f3f64..aa3a526bb 100644 --- a/nodedb/src/control/gateway/error_map/class_parity/code_samples.rs +++ b/nodedb/src/control/gateway/error_map/class_parity/code_samples.rs @@ -8,7 +8,7 @@ use nodedb_types::sync::wire::SyncProvenance; use crate::bridge::envelope::{CounterFault, ErrorCode, SyncHold}; /// The number of `ErrorCode` variants [`variant_index`] numbers. -pub(super) const VARIANT_COUNT: usize = 52; +pub(super) const VARIANT_COUNT: usize = 53; /// A dense index per variant. Exhaustive, so a new variant fails to compile /// here until it gets an index, and [`every_variant_has_a_sample`] then fails @@ -67,6 +67,7 @@ pub(super) fn variant_index(code: &ErrorCode) -> usize { ErrorCode::DatetimeFieldOverflow { .. } => 49, ErrorCode::UndefinedObject { .. } => 50, ErrorCode::ObjectNotInPrerequisiteState { .. } => 51, + ErrorCode::CoreFailStopped { .. } => 52, } } @@ -88,6 +89,10 @@ pub(super) fn samples() -> Vec { ErrorCode::DeadlineExceeded, ErrorCode::RejectedPrevalidation { reason: text() }, ErrorCode::RetryableRefusal { reason: text() }, + ErrorCode::CoreFailStopped { + core_id: 0, + detail: text(), + }, ErrorCode::SyncRejected { violation: ViolationType::PermissionDenied, applied_seq: 1, diff --git a/nodedb/src/control/gateway/error_map/system_dispatch_refusal.rs b/nodedb/src/control/gateway/error_map/system_dispatch_refusal.rs index f63dcf8b0..578828d75 100644 --- a/nodedb/src/control/gateway/error_map/system_dispatch_refusal.rs +++ b/nodedb/src/control/gateway/error_map/system_dispatch_refusal.rs @@ -85,8 +85,8 @@ async fn refuse_once( payload: Payload::empty(), watermark_lsn: Lsn::ZERO, error_code: code.clone().map(Box::new), - read_set_valid: None, - read_version_lsn: Lsn::ZERO, + stage_vote: None, + read_versions: crate::types::ReadVersions::new(), write_set: Vec::new(), }, }) diff --git a/nodedb/src/control/gateway/mod.rs b/nodedb/src/control/gateway/mod.rs index 873416326..eeecaef3b 100644 --- a/nodedb/src/control/gateway/mod.rs +++ b/nodedb/src/control/gateway/mod.rs @@ -6,6 +6,7 @@ pub mod colocation_guard; pub mod core; pub mod dispatch_local; pub mod dispatch_remote; +pub mod dispatch_remote_stream; pub mod dispatcher; pub mod error_map; pub mod fuser; diff --git a/nodedb/src/control/gateway/outcome.rs b/nodedb/src/control/gateway/outcome.rs index 1a29714c8..1b196a9ad 100644 --- a/nodedb/src/control/gateway/outcome.rs +++ b/nodedb/src/control/gateway/outcome.rs @@ -5,18 +5,18 @@ //! A transport that renders a Data-Plane `Response` gets the same shape from //! the gateway that local SPSC dispatch gives it. The shape keeps the error //! status and code of a `NotFound` verdict, the read watermark, and the -//! read-version LSN. +//! read versions. use crate::Error; use crate::bridge::envelope::{ErrorCode, Payload, Response, Status}; use crate::control::server::shared::clone_write::CloneCheckedTask; -use crate::types::{Lsn, RequestId, VShardId}; +use crate::types::{Lsn, ReadVersions, RequestId, VShardId}; use super::core::{Gateway, QueryContext, authorized_plan_for_context}; -/// A gateway execution's payloads, per-shard read watermarks, and -/// read-version LSN, as [`GatewayOutcome::into_parts`] returns them. -pub type GatewayParts = (Vec>, Vec<(VShardId, Lsn)>, Lsn); +/// A gateway execution's payloads, per-shard read watermarks, and read +/// versions, as [`GatewayOutcome::into_parts`] returns them. +pub type GatewayParts = (Vec>, Vec<(VShardId, Lsn)>, ReadVersions); /// Everything one gateway execution observed across its routes. pub struct GatewayOutcome { @@ -24,8 +24,8 @@ pub struct GatewayOutcome { pub payloads: Vec>, /// One `(vshard, watermark_lsn)` per participating shard. pub shard_watermarks: Vec<(VShardId, Lsn)>, - /// Max-folded per-collection read-version LSN. - pub read_version_lsn: Lsn, + /// The read versions every route observed, one per vShard. + pub read_versions: ReadVersions, /// A single-route plan's owning core refused it with `ErrorCode::NotFound`. /// /// Always `false` for a fan-out: there it means a shard holds no slice. @@ -33,9 +33,9 @@ pub struct GatewayOutcome { } impl GatewayOutcome { - /// Payloads, per-shard watermarks, and read-version LSN. + /// Payloads, per-shard watermarks, and read versions. pub fn into_parts(self) -> GatewayParts { - (self.payloads, self.shard_watermarks, self.read_version_lsn) + (self.payloads, self.shard_watermarks, self.read_versions) } /// The response local SPSC dispatch returns for the same task. @@ -47,7 +47,7 @@ impl GatewayOutcome { .max() .unwrap_or(Lsn::ZERO); if self.not_found { - return not_found_response(watermark_lsn, self.read_version_lsn); + return not_found_response(watermark_lsn, self.read_versions); } let payload = self .payloads @@ -55,48 +55,34 @@ impl GatewayOutcome { .next() .map(Payload::from_vec) .unwrap_or_else(Payload::empty); - response( - Status::Ok, - None, - payload, - watermark_lsn, - self.read_version_lsn, - ) + response(Status::Ok, None, payload, watermark_lsn, self.read_versions) } } impl Gateway { /// Execute one authorized task and return its Data-Plane response shape. /// - /// A `NotFound` verdict returns as an error-status response, the same as - /// local SPSC dispatch returns it. The caller records a phantom read from - /// it and renders the verdict. Every other error returns as its typed - /// `Err`. + /// A `NotFound` verdict, local or remote, returns as an error-status + /// response carrying the versions the read observed, the same as local + /// SPSC dispatch returns it. The caller records a phantom read from it + /// and renders the verdict. Every other error returns as its typed `Err`. pub async fn execute_response( &self, ctx: &QueryContext, checked: CloneCheckedTask, ) -> Result { let (plan, _lease) = authorized_plan_for_context(ctx, checked)?; - match self.execute_plan_outcome(ctx, plan).await { - Ok(outcome) => Ok(outcome.into_response()), - // A remote leaseholder returns its `NotFound` verdict as a typed - // error. It gets the same shape as a local one. - Err(Error::DataPlane(ErrorCode::NotFound)) => { - Ok(not_found_response(Lsn::ZERO, Lsn::ZERO)) - } - Err(error) => Err(error), - } + Ok(self.execute_plan_outcome(ctx, plan).await?.into_response()) } } -fn not_found_response(watermark_lsn: Lsn, read_version_lsn: Lsn) -> Response { +fn not_found_response(watermark_lsn: Lsn, read_versions: ReadVersions) -> Response { response( Status::Error, Some(ErrorCode::NotFound), Payload::empty(), watermark_lsn, - read_version_lsn, + read_versions, ) } @@ -105,7 +91,7 @@ fn response( error_code: Option, payload: Payload, watermark_lsn: Lsn, - read_version_lsn: Lsn, + read_versions: ReadVersions, ) -> Response { Response { request_id: RequestId::new(0), @@ -115,8 +101,8 @@ fn response( payload, watermark_lsn, error_code: error_code.map(Box::new), - read_set_valid: None, - read_version_lsn, + stage_vote: None, + read_versions, write_set: Vec::new(), } } @@ -125,6 +111,10 @@ fn response( mod tests { use super::*; + fn read_versions() -> ReadVersions { + ReadVersions::single(VShardId::new(3), nodedb_types::WriteVersion::logged(1, 5)) + } + fn outcome(not_found: bool) -> GatewayOutcome { GatewayOutcome { payloads: vec![vec![0x90]], @@ -132,7 +122,7 @@ mod tests { (VShardId::new(3), Lsn::new(7)), (VShardId::new(4), Lsn::new(9)), ], - read_version_lsn: Lsn::new(5), + read_versions: read_versions(), not_found, } } @@ -143,16 +133,16 @@ mod tests { assert_eq!(resp.status, Status::Error); assert_eq!(resp.error_code.as_deref(), Some(&ErrorCode::NotFound)); assert_eq!(resp.watermark_lsn, Lsn::new(9)); - assert_eq!(resp.read_version_lsn, Lsn::new(5)); + assert_eq!(resp.read_versions, read_versions()); } #[test] - fn a_success_keeps_its_payload_and_lsns() { + fn a_success_keeps_its_payload_watermark_and_read_versions() { let resp = outcome(false).into_response(); assert_eq!(resp.status, Status::Ok); assert!(resp.error_code.is_none()); assert_eq!(resp.payload.to_vec(), vec![0x90u8]); assert_eq!(resp.watermark_lsn, Lsn::new(9)); - assert_eq!(resp.read_version_lsn, Lsn::new(5)); + assert_eq!(resp.read_versions, read_versions()); } } diff --git a/nodedb/src/control/gateway/router.rs b/nodedb/src/control/gateway/router.rs index 6df80e2f8..c028f2d6b 100644 --- a/nodedb/src/control/gateway/router.rs +++ b/nodedb/src/control/gateway/router.rs @@ -534,6 +534,7 @@ mod tests { collections: vec![], sum_targets: vec![], origin: nodedb_physical::physical_plan::RedoOrigin::Commit, + calvin: None, }), ] { for table in [None, Some(single_node_table())] { diff --git a/nodedb/src/control/insert_select/expand_staged.rs b/nodedb/src/control/insert_select/expand_staged.rs index 3d1c33613..0f3a10998 100644 --- a/nodedb/src/control/insert_select/expand_staged.rs +++ b/nodedb/src/control/insert_select/expand_staged.rs @@ -30,8 +30,9 @@ use nodedb_physical::physical_task::{PhysicalTask, PostSetOp}; /// /// `task.txn_id` must be set to the active transaction so the scan folds /// earlier-staged rows. Emitted ops carry the same `txn_id` and a recomputed -/// target vShard (as the MERGE / `UPDATE ... FROM` expanders do). The caller -/// stages + buffers each returned op. +/// target vShard (as the MERGE / `UPDATE ... FROM` expanders do). The ops carry +/// no sum or period-lock resolution: the caller runs the statement pass on +/// them, then stages + buffers each one. pub(crate) async fn resolve_and_emit_insert_select_ops( state: &SharedState, tenant_id: TenantId, @@ -76,35 +77,6 @@ pub(crate) async fn resolve_and_emit_insert_select_ops( )? .vshard(); - // Resolve materialized-sum targets: these ops stage directly, bypassing - // statement-level resolution, so without this a bound target collection - // folds against an empty resolution. - let sum_bodies: Vec<&[u8]> = rows.iter().map(|(_, value, _)| value.as_slice()).collect(); - let mut resolved_sum_targets = - crate::control::planner::materialized_sum::resolve_sum_targets_for_bodies( - state, - &sum_bodies, - target_collection.as_str(), - tenant_id, - task.database_id, - crate::types::TraceId::ZERO, - ) - .await?; - // Period-lock target for the same rows, in the SAME slot — a target - // collection under a period lock reads its reference row's surrogate off - // `resolved_sum_targets` exactly like a materialized-sum fold does. - resolved_sum_targets.extend( - crate::control::planner::period_lock::resolve_period_lock_targets_for_bodies( - state, - &sum_bodies, - target_collection.as_str(), - tenant_id, - task.database_id, - crate::types::TraceId::ZERO, - ) - .await?, - ); - let mut out: Vec = Vec::with_capacity(rows.len()); for (document_id, value, surrogate) in rows { out.push(PhysicalTask { @@ -121,7 +93,9 @@ pub(crate) async fn resolve_and_emit_insert_select_ops( // orchestrator's paged batch insert. returning: None, rls_filters: Vec::new(), - resolved_sum_targets: resolved_sum_targets.clone(), + // Filled by the caller's sum and period-lock pass, the one + // a plain INSERT takes. + resolved_sum_targets: Vec::new(), deferred_sum_targets: Vec::new(), }), post_set_op: PostSetOp::None, diff --git a/nodedb/src/control/insert_select/orchestrator.rs b/nodedb/src/control/insert_select/orchestrator.rs index 0ea9ca185..28034f6a6 100644 --- a/nodedb/src/control/insert_select/orchestrator.rs +++ b/nodedb/src/control/insert_select/orchestrator.rs @@ -222,8 +222,8 @@ pub(crate) async fn run_insert_select( payload: Payload::from_vec(payload), watermark_lsn: max_lsn, error_code: None, - read_set_valid: None, - read_version_lsn: crate::types::Lsn::ZERO, + stage_vote: None, + read_versions: crate::types::ReadVersions::new(), write_set: Vec::new(), }) } diff --git a/nodedb/src/control/local_dispatch/collect.rs b/nodedb/src/control/local_dispatch/collect.rs index 67ff384eb..28f74425a 100644 --- a/nodedb/src/control/local_dispatch/collect.rs +++ b/nodedb/src/control/local_dispatch/collect.rs @@ -169,8 +169,8 @@ mod collect_budget_tests { payload: Payload::from_vec(array_payload(n)), watermark_lsn: Lsn::ZERO, error_code: None, - read_set_valid: None, - read_version_lsn: crate::types::Lsn::ZERO, + stage_vote: None, + read_versions: crate::types::ReadVersions::new(), write_set: Vec::new(), } } @@ -184,8 +184,8 @@ mod collect_budget_tests { payload: Payload::from_vec(array_payload(n)), watermark_lsn: Lsn::ZERO, error_code: None, - read_set_valid: None, - read_version_lsn: crate::types::Lsn::ZERO, + stage_vote: None, + read_versions: crate::types::ReadVersions::new(), write_set: Vec::new(), } } @@ -200,8 +200,8 @@ mod collect_budget_tests { payload: Payload::from_vec(vec![0u8; bytes]), watermark_lsn: Lsn::ZERO, error_code: None, - read_set_valid: None, - read_version_lsn: crate::types::Lsn::ZERO, + stage_vote: None, + read_versions: crate::types::ReadVersions::new(), write_set: Vec::new(), } } @@ -215,8 +215,8 @@ mod collect_budget_tests { payload: Payload::from_vec(vec![0u8; bytes]), watermark_lsn: Lsn::ZERO, error_code: None, - read_set_valid: None, - read_version_lsn: crate::types::Lsn::ZERO, + stage_vote: None, + read_versions: crate::types::ReadVersions::new(), write_set: Vec::new(), } } diff --git a/nodedb/src/control/local_dispatch/error_status.rs b/nodedb/src/control/local_dispatch/error_status.rs index b2fa8c0c6..824c21edb 100644 --- a/nodedb/src/control/local_dispatch/error_status.rs +++ b/nodedb/src/control/local_dispatch/error_status.rs @@ -8,6 +8,13 @@ use crate::bridge::envelope::{ErrorCode, Response, Status}; +/// Whether the core refused the task with `ErrorCode::NotFound`: a read +/// that found no row. [`reject_data_plane_error`] passes it as an empty +/// success, and the flag keeps the verdict for a caller that needs it. +pub(crate) fn is_not_found(resp: &Response) -> bool { + resp.status == Status::Error && resp.error_code.as_deref() == Some(&ErrorCode::NotFound) +} + /// Reject a Data-Plane error response, keeping its typed code. /// /// `NotFound` passes: it means the shard holds no slice of the collection, @@ -55,8 +62,8 @@ mod tests { payload: Payload::empty(), watermark_lsn: Lsn::ZERO, error_code: Some(Box::new(code)), - read_set_valid: None, - read_version_lsn: Lsn::ZERO, + stage_vote: None, + read_versions: crate::types::ReadVersions::new(), write_set: Vec::new(), } } diff --git a/nodedb/src/control/local_dispatch/local_read.rs b/nodedb/src/control/local_dispatch/local_read.rs index ee1f232df..8c833977c 100644 --- a/nodedb/src/control/local_dispatch/local_read.rs +++ b/nodedb/src/control/local_dispatch/local_read.rs @@ -82,6 +82,7 @@ pub(crate) async fn dispatch_local_read( wal_lsn: None, resolved_now_ms: None, commit_hlc: None, + entry_version: None, admission: Admission::Exempt(ExemptReason::Read), }; diff --git a/nodedb/src/control/local_dispatch/mod.rs b/nodedb/src/control/local_dispatch/mod.rs index 24bd2ac16..f9219b8d4 100644 --- a/nodedb/src/control/local_dispatch/mod.rs +++ b/nodedb/src/control/local_dispatch/mod.rs @@ -11,5 +11,5 @@ mod local_read; pub(crate) use collect::{ DeadlineCollect, DispatchCollectError, collect_bounded_response, collect_under_deadline, }; -pub(crate) use error_status::reject_data_plane_error; +pub(crate) use error_status::{is_not_found, reject_data_plane_error}; pub(crate) use local_read::{LocalRead, dispatch_local_read}; diff --git a/nodedb/src/control/maintenance/clone_materializer/chained.rs b/nodedb/src/control/maintenance/clone_materializer/chained.rs index 30758a4ca..1a3286d82 100644 --- a/nodedb/src/control/maintenance/clone_materializer/chained.rs +++ b/nodedb/src/control/maintenance/clone_materializer/chained.rs @@ -22,9 +22,9 @@ use crate::control::planner::sql_plan_convert::convert::db_qualified; use crate::control::security::catalog::{StoredCollection, SystemCatalog}; use crate::control::state::SharedState; use crate::control::surrogate::CarriedIdentity; -use crate::control::wal_replication::encode::transaction_redo_entry; -use crate::control::wal_replication::propose_replicated_entry; -use crate::control::wal_replication::transaction_redo::{RedoTarget, TransactionRedoPayload}; +use crate::control::wal_replication::transaction_redo::{ + RedoTarget, TransactionRedoPayload, propose_transaction_redo, +}; use crate::event::EventSource; use crate::types::hash_chain::CHAIN_SEQ_FIELD; use crate::types::{HomedRecord, RecordHomes}; @@ -249,6 +249,7 @@ fn relink_payload( // a copy fires no AFTER trigger. event_source: EventSource::Restore, origin: RedoOrigin::Restore, + calvin: None, }) } @@ -258,13 +259,7 @@ async fn commit_relink( target: RedoTarget, payload: &TransactionRedoPayload, ) -> crate::Result<()> { - let proposer = state.async_raft_proposer()?; - let entry = transaction_redo_entry( - target.tenant_id, - target.database_id, - target.vshard_id, - payload, - ); - propose_replicated_entry(state, proposer, entry).await?; + let deadline = crate::control::wal_replication::statement_propose_deadline(state); + propose_transaction_redo(state, target, payload, deadline).await?; Ok(()) } diff --git a/nodedb/src/control/maintenance/clone_materializer/dispatch.rs b/nodedb/src/control/maintenance/clone_materializer/dispatch.rs index c5a4b634d..43040674f 100644 --- a/nodedb/src/control/maintenance/clone_materializer/dispatch.rs +++ b/nodedb/src/control/maintenance/clone_materializer/dispatch.rs @@ -178,6 +178,7 @@ pub(crate) async fn dispatch_on_this_node( wal_lsn: None, resolved_now_ms: None, commit_hlc: None, + entry_version: None, admission: crate::bridge::envelope::Admission::Exempt( crate::bridge::envelope::ExemptReason::AlreadyOrdered, ), diff --git a/nodedb/src/control/metadata_proposer/catalog.rs b/nodedb/src/control/metadata_proposer/catalog.rs index f964a878a..d71f2408c 100644 --- a/nodedb/src/control/metadata_proposer/catalog.rs +++ b/nodedb/src/control/metadata_proposer/catalog.rs @@ -233,7 +233,7 @@ pub(super) async fn propose_prepared( .await?; match outcome { WaitOutcome::Reached - if shared.metadata_ddl_applied_token.load(Ordering::Acquire) == token => + if shared.metadata_ddl.applied_token.load(Ordering::Acquire) == token => { Ok(log_index) } diff --git a/nodedb/src/control/metadata_proposer/ddl_owner.rs b/nodedb/src/control/metadata_proposer/ddl_owner.rs index 70de3f1a2..9c4c83e57 100644 --- a/nodedb/src/control/metadata_proposer/ddl_owner.rs +++ b/nodedb/src/control/metadata_proposer/ddl_owner.rs @@ -80,7 +80,8 @@ pub(crate) fn reclaim_cause(shared: &SharedState, owner: &DdlPrepareOwner) -> Op /// Whether `token` owns the preparation lease on this replica. pub(crate) fn owns_ddl_lease(shared: &SharedState, token: u64) -> bool { shared - .metadata_ddl_owner + .metadata_ddl + .owner .lock() .unwrap_or_else(|p| p.into_inner()) .is_some_and(|owner| owner.token == token) @@ -89,7 +90,8 @@ pub(crate) fn owns_ddl_lease(shared: &SharedState, token: u64) -> bool { /// The current owner on this replica, if the lease is held. pub(crate) fn current_owner(shared: &SharedState) -> Option { *shared - .metadata_ddl_owner + .metadata_ddl + .owner .lock() .unwrap_or_else(|p| p.into_inner()) } diff --git a/nodedb/src/control/metadata_proposer/ddl_prepare.rs b/nodedb/src/control/metadata_proposer/ddl_prepare.rs index 9fefdee02..2e03fcc77 100644 --- a/nodedb/src/control/metadata_proposer/ddl_prepare.rs +++ b/nodedb/src/control/metadata_proposer/ddl_prepare.rs @@ -35,7 +35,8 @@ const DDL_PREPARE_POLL: Duration = Duration::from_millis(10); /// A fresh preparation-lease token for this node. fn next_token(shared: &SharedState) -> u64 { let sequence = shared - .metadata_ddl_token_seq + .metadata_ddl + .token_seq .fetch_add(1, Ordering::Relaxed); shared.node_id.wrapping_mul(0x9e37_79b9_7f4a_7c15) ^ wall_now_ns().rotate_left(17) ^ sequence } @@ -214,7 +215,7 @@ pub(crate) async fn acquire_ddl_prepare_lease_async<'a>( pub(crate) async fn lock_ddl_preparation_async( shared: &SharedState, ) -> tokio::sync::MutexGuard<'_, ()> { - shared.metadata_ddl_lock.lock().await + shared.metadata_ddl.lock.lock().await } #[cfg(test)] @@ -223,7 +224,8 @@ mod tests { fn owner(shared: &SharedState) -> Option { let current = *shared - .metadata_ddl_owner + .metadata_ddl + .owner .lock() .unwrap_or_else(|p| p.into_inner()); current.map(|owner| owner.token) diff --git a/nodedb/src/control/metadata_proposer/handle.rs b/nodedb/src/control/metadata_proposer/handle.rs index bf060605e..175799028 100644 --- a/nodedb/src/control/metadata_proposer/handle.rs +++ b/nodedb/src/control/metadata_proposer/handle.rs @@ -136,6 +136,7 @@ fn metadata_propose_error(error: ClusterError) -> Error { | ClusterError::GhostNotFound { .. } | ClusterError::Transport { .. } | ClusterError::ShardTimeout { .. } + | ClusterError::Unanswered { .. } | ClusterError::Storage { .. } | ClusterError::Codec { .. } | ClusterError::UnsupportedWireVersion { .. } diff --git a/nodedb/src/control/orchestrated_write.rs b/nodedb/src/control/orchestrated_write.rs index e15400aa3..dd4568dbf 100644 --- a/nodedb/src/control/orchestrated_write.rs +++ b/nodedb/src/control/orchestrated_write.rs @@ -49,18 +49,20 @@ pub(crate) async fn apply_orchestrated_write( } })?; let request_id = RequestId::new(state.request_id_counter.fetch_add(1, Ordering::Relaxed)); - match propose_replicated_entry(state, proposer, entry).await { - Ok((payload, write_version)) => { + let deadline = crate::control::wal_replication::statement_propose_deadline(state); + match propose_replicated_entry(state, proposer, entry, deadline).await { + Ok((payload, write_versions)) => { let response = Response { request_id, status: Status::Ok, attempt: 1, partial: false, payload: payload.into(), - watermark_lsn: write_version, + // A write carries no read watermark. + watermark_lsn: crate::types::Lsn::ZERO, error_code: None, - read_set_valid: None, - read_version_lsn: write_version, + stage_vote: None, + read_versions: write_versions, write_set: Vec::new(), }; Ok(response) @@ -80,8 +82,8 @@ fn data_plane_verdict(request_id: RequestId, code: ErrorCode) -> Response { payload: crate::bridge::envelope::Payload::from_vec(Vec::new()), watermark_lsn: crate::types::Lsn::ZERO, error_code: Some(Box::new(code)), - read_set_valid: None, - read_version_lsn: crate::types::Lsn::ZERO, + stage_vote: None, + read_versions: crate::types::ReadVersions::new(), write_set: Vec::new(), } } diff --git a/nodedb/src/control/pitr/restore_point/create.rs b/nodedb/src/control/pitr/restore_point/create.rs index e3c7cc772..7bdf73d59 100644 --- a/nodedb/src/control/pitr/restore_point/create.rs +++ b/nodedb/src/control/pitr/restore_point/create.rs @@ -30,7 +30,11 @@ pub async fn create_restore_point( // it: a test orders a metadata entry stamped at or above the watermark // before the point. #[cfg(feature = "failpoints")] - crate::control::fail_gate::wait("restore_point::after_watermark").await; + crate::control::fail_gate::wait( + crate::fail_point::FailScope::Node(state.node_id), + "restore_point::after_watermark", + ) + .await; let created_at_ms = std::time::SystemTime::now() .duration_since(std::time::UNIX_EPOCH) .map(|d| u64::try_from(d.as_millis()).unwrap_or(u64::MAX)) diff --git a/nodedb/src/control/pitr/restore_point/record.rs b/nodedb/src/control/pitr/restore_point/record.rs index cbffeb47f..db4cf11c5 100644 --- a/nodedb/src/control/pitr/restore_point/record.rs +++ b/nodedb/src/control/pitr/restore_point/record.rs @@ -7,8 +7,11 @@ //! data group it hosts, plus the Calvin sequencer, at the point's watermark. //! Every replica of a group applies the cut barrier at the same log index and //! records it, so the group's place at the point is that index on every node. -//! A group can hold more than one barrier for a point: every replica proposes -//! its own. The lowest index is the group's place. +//! The group's leader places the barrier (see +//! `crate::control::backup::cut_order`). A group can still hold more than one +//! barrier for a point: a leader proposes it again when a later term +//! overwrote its proposal, and the overwritten copy can commit too. The +//! lowest index is the group's place. use std::sync::{Arc, Weak}; @@ -21,12 +24,6 @@ use tracing::{error, info, warn}; use crate::control::state::SharedState; use crate::wal::manager::NO_APPLY_KEY; -/// Tenant the restore point's cut barriers are framed under. A barrier -/// orders every entry of its group, whatever tenant writes it. No step reads -/// the tenant of a barrier: the proposal stamps and forwards opaque bytes, -/// the apply's barrier arms ignore it, and a barrier raises no tenant mark. -const CUT_FRAME_TENANT: u64 = 0; - /// Record one group's place at a restore point in this node's WAL, durably, /// and move this node's clock past the point's watermark. A data group's /// record lists the vShards it homes now. A failure is logged: a cluster @@ -107,7 +104,7 @@ pub fn sequencer_hook(shared: Weak) -> RestorePointHook { /// point's records is then sealed, so the archiver uploads it. pub fn spawn_node_cut(state: Arc, id: u64, hlc: u64) { tokio::spawn(async move { - match crate::control::backup::cut::cut_at_point(&state, CUT_FRAME_TENANT, hlc, id).await { + match crate::control::backup::cut::cut_at_point(&state, hlc, id).await { Ok(()) => info!(restore_point = id, "restore point cut taken on this node"), Err(e) => warn!( restore_point = id, diff --git a/nodedb/src/control/planner/calvin/abort_error.rs b/nodedb/src/control/planner/calvin/abort_error.rs index 6eef969e8..9ff3e12ec 100644 --- a/nodedb/src/control/planner/calvin/abort_error.rs +++ b/nodedb/src/control/planner/calvin/abort_error.rs @@ -13,6 +13,14 @@ use crate::Error; pub fn calvin_abort_error(reason: AbortReason) -> Error { match reason { AbortReason::ParticipantError => Error::CalvinParticipantError, + // A participant rejected the plans: they do not decode or route, or + // hold no work for it. A resubmit of the same plans fails the same + // way, so this is an internal error (`XX000`), never a retry class. + AbortReason::PlanRejected => Error::Internal { + detail: "cross-shard transaction aborted: a participant vShard rejected the \ + transaction's plans" + .to_owned(), + }, // Planned against a collection a purge and a same-name create replaced. AbortReason::CollectionSuperseded => Error::RetryableSchemaChanged { descriptor: "a collection the transaction writes was dropped and recreated".into(), @@ -39,6 +47,14 @@ mod tests { )); } + #[test] + fn a_rejected_plan_is_an_internal_error() { + assert!(matches!( + calvin_abort_error(AbortReason::PlanRejected), + Error::Internal { .. } + )); + } + #[test] fn a_superseded_collection_asks_for_a_retry() { assert!(matches!( diff --git a/nodedb/src/control/planner/calvin/dependent_recon.rs b/nodedb/src/control/planner/calvin/dependent_recon.rs index 27fb732fd..8bbbb277b 100644 --- a/nodedb/src/control/planner/calvin/dependent_recon.rs +++ b/nodedb/src/control/planner/calvin/dependent_recon.rs @@ -23,7 +23,7 @@ use crate::Error; use crate::control::cluster::calvin::executor::ollp::error::OllpError; use crate::control::planner::calvin::preexec::PreexecScan; use crate::control::planner::calvin::{ - DependentOutcome, DependentRetryArgs, build_single_vshard_dependent_tx_class, + DependentOutcome, DependentRetryArgs, build_single_vshard_predicted_tx_class, is_dependent_predicate, predicate_class_for_filters, run_dependent_with_retry, submit_calvin_routed_assign, }; @@ -134,7 +134,7 @@ pub fn is_edge_recon_plan(plan: &nodedb_physical::physical_plan::PhysicalPlan) - /// `submit_calvin_routed_assign`) that mirrors the doc write together with /// the derived EdgeDelete/EdgePut tasks, ATOMICALLY. A delete adds one /// `NodeEdgeGuard` per node and one `EdgeDelete` per incident edge. -/// 4. On a POST-EXEC predicate-drift mismatch or a guard's drift abort, +/// 4. On a `PredictionDrift` abort verdict, from a participant or a guard, /// re-scans (FRESH reconnaissance) and resubmits, via /// [`run_dependent_with_retry`]. /// @@ -146,7 +146,7 @@ pub fn is_edge_recon_plan(plan: &nodedb_physical::physical_plan::PhysicalPlan) - /// [`plan_needs_implicit_edge_recon`]) so it does not have to be re-derived. /// /// The `TxClass` this builds accepts a write set on one vShard -/// ([`build_single_vshard_dependent_tx_class`]). A delete whose row, node +/// ([`build_single_vshard_predicted_tx_class`]). A delete whose row, node /// guard and edges all home on one vShard is a legitimate one-vShard /// transaction, and so is a contended single-collection predicate write /// routed here by the write-admission gate. @@ -213,7 +213,7 @@ async fn dispatch_dependent_edge_recon_inner( // OLLP path: the coordinator owns the retry loop. `run_dependent_with_retry` // submits + awaits the assignment/completion via the local registry and, on - // a post-exec predicate-drift mismatch, runs a FRESH pre-execution scan + // a `PredictionDrift` abort verdict, runs a FRESH pre-execution scan // (`rescan`) before resubmitting with the fresh prediction. let dep_task = tasks .iter() @@ -384,7 +384,7 @@ async fn dispatch_dependent_edge_recon_inner( t }) .collect(); - let tx_class = build_single_vshard_dependent_tx_class( + let tx_class = build_single_vshard_predicted_tx_class( &modified_tasks, tenant_id, dep_collection, @@ -406,7 +406,7 @@ async fn dispatch_dependent_edge_recon_inner( } }; - // `rescan`: FRESH reconnaissance on each post-exec mismatch or drift. + // `rescan`: FRESH reconnaissance on each drift abort. let rescan = || { reconnoitre( state, diff --git a/nodedb/src/control/planner/calvin/dispatch.rs b/nodedb/src/control/planner/calvin/dispatch.rs index 71841cdc3..fc8583c3e 100644 --- a/nodedb/src/control/planner/calvin/dispatch.rs +++ b/nodedb/src/control/planner/calvin/dispatch.rs @@ -214,7 +214,7 @@ mod tests { use crate::control::server::shared::session::read_set::{ EngineTag, ReadKey, ReadOrigin, ReadSetEntry, }; - use crate::types::{DatabaseId, Lsn, TenantId, VShardId}; + use crate::types::{DatabaseId, TenantId, VShardId}; use nodedb_physical::physical_plan::{ColumnarOp, CrdtOp, DocumentOp, PhysicalPlan}; use nodedb_physical::physical_task::{PhysicalTask, PostSetOp}; use nodedb_types::QualifiedCollection; @@ -489,13 +489,11 @@ mod tests { // them (`session::commit::run_commit`). // // WHY this must stay `MultiShard`: only the `MultiShard` branch of COMMIT - // flushes through the Calvin barrier (`run_commit_calvin`), which validates - // B's read slice on B's OWNING node using the real per-shard `read_lsn`. If a - // foreign read failed to widen the class, COMMIT will take the `SingleShard` - // branch and run only the local-WAL `si_conflict_abort`, which never sees a - // stale read on the remote owner — silently committing a non-serializable - // cross-node transaction. This test guarantees a future refactor of - // `read_vshards_of` / `classify_dispatch` cannot reopen that hole. + // commits through the Calvin barrier (`run_commit_calvin`), which + // validates B's read slice on B's participants inside the barrier, in + // the same order as every write to B. This test guarantees a future + // refactor of `read_vshards_of` / `classify_dispatch` keeps a foreign + // read inside the barrier. let (write_coll, read_coll) = two_distinct_vshard_collections(); let write_vshard = CollectionKey::from_bare(DatabaseId::DEFAULT, &write_coll) .vshard() @@ -512,11 +510,9 @@ mod tests { tenant_id: TenantId::new(1), collection: read_coll.clone(), key: ReadKey::Predicate, - read_lsn: Lsn::new(1), - read_version_lsn: Lsn::ZERO, + read_version: nodedb_types::WriteVersion::logged(0, 1), origin: ReadOrigin::Session, home: None, - home_node: 0, }; // The homing step under test: a foreign-collection read must home to a @@ -538,8 +534,8 @@ mod tests { } other => panic!( "expected MultiShard for write-on-{write_vshard} + foreign-read-on-{read_vshard}, \ - got {other:?} (a SingleShard here would route COMMIT to local-WAL \ - si_conflict_abort and reopen the cross-node serializability hole)" + got {other:?} (a SingleShard here validates the read outside the \ + Calvin barrier)" ), } } diff --git a/nodedb/src/control/planner/calvin/dispatch_multi.rs b/nodedb/src/control/planner/calvin/dispatch_multi.rs index f9e9a2337..166e846ba 100644 --- a/nodedb/src/control/planner/calvin/dispatch_multi.rs +++ b/nodedb/src/control/planner/calvin/dispatch_multi.rs @@ -208,7 +208,7 @@ pub(crate) async fn dispatch_tasks_to_calvin( mod tests { use super::*; use crate::control::server::shared::session::read_set::{EngineTag, ReadKey, ReadOrigin}; - use crate::types::{DatabaseId, KeyRepr, Lsn, VShardId}; + use crate::types::{DatabaseId, KeyRepr, VShardId}; use nodedb_cluster::calvin::types::TxnIdWire; use nodedb_physical::physical_plan::{DocumentOp, GraphOp, PhysicalPlan}; use nodedb_physical::physical_task::PostSetOp; @@ -244,11 +244,9 @@ mod tests { key: ReadKey::Point { repr: KeyRepr::Surrogate(1), }, - read_lsn: Lsn::new(1), - read_version_lsn: Lsn::new(1), + read_version: nodedb_types::WriteVersion::logged(0, 1), origin: ReadOrigin::Session, home: None, - home_node: 0, } } diff --git a/nodedb/src/control/planner/calvin/edge_sequencing.rs b/nodedb/src/control/planner/calvin/edge_sequencing.rs index bd1d5b23c..a2fb73234 100644 --- a/nodedb/src/control/planner/calvin/edge_sequencing.rs +++ b/nodedb/src/control/planner/calvin/edge_sequencing.rs @@ -8,10 +8,9 @@ //! version applied below it. So on every replica, and under any clock skew, //! a TRUNCATE hides exactly the versions sequenced before it. //! -//! A data-group Raft entry cannot give that. Each replica stamps it from its -//! own clock, and each replica applies it at its own point among the Calvin -//! flushes. It also takes no Calvin lock, so a node delete's guard does not -//! order against it. +//! A plain data-group Raft entry cannot give that. It carries no Calvin +//! ordinal, so its edge versions order against no TRUNCATE cut. It also +//! takes no Calvin lock, so a node delete's guard does not order against it. //! //! The Raft propose seams (`propose_replicated_entry`, `propose_sync_write`) //! hand every edge write here. A session COMMIT that buffered an edge write diff --git a/nodedb/src/control/planner/calvin/mod.rs b/nodedb/src/control/planner/calvin/mod.rs index a37011a80..b0feb9ac1 100644 --- a/nodedb/src/control/planner/calvin/mod.rs +++ b/nodedb/src/control/planner/calvin/mod.rs @@ -16,6 +16,7 @@ pub mod explain; pub mod node_delete_txn; pub mod predicate; pub mod preexec; +pub mod read_dependent; pub mod reservation; pub mod retry_loop; pub mod submit; @@ -46,7 +47,7 @@ pub use submit::{ submit_calvin_routed, submit_calvin_routed_assign, submit_calvin_routed_write, }; pub use tx_class::{ - build_dependent_tx_class, build_single_vshard_dependent_tx_class, build_single_vshard_tx_class, - build_static_tx_class, + PassiveReads, build_predicted_tx_class, build_read_dependent_tx_class, + build_single_vshard_predicted_tx_class, build_single_vshard_tx_class, build_static_tx_class, }; pub use types::{DispatchClass, DispatchOutcome, TxnDispatchPosition}; diff --git a/nodedb/src/control/planner/calvin/read_dependent/item_move.rs b/nodedb/src/control/planner/calvin/read_dependent/item_move.rs new file mode 100644 index 000000000..05ef4a5e0 --- /dev/null +++ b/nodedb/src/control/planner/calvin/read_dependent/item_move.rs @@ -0,0 +1,247 @@ +// SPDX-License-Identifier: BUSL-1.1 + +//! `TRANSFER_ITEM` across two vShards, as one read-dependent Calvin +//! transaction. +//! +//! The item leaves the source collection's vShard and arrives at the +//! destination collection's vShard with the bytes it held at the source. +//! The destination write depends on a row of another vShard, so: +//! +//! 1. The coordinator reads the item at its source owner. +//! 2. It plans a delete of the item at the source and a put of the same +//! bytes at the destination, and submits them as one transaction whose +//! passive read is the item, with the bytes it read as the expected value. +//! 3. Under the transaction's locks the source vShard reads the item again +//! and broadcasts it to both active vShards through their data-group logs. +//! Each votes `PredictionDrift` when the item moved, and the coordinator +//! reads again and resubmits. An item gone at the source ends the move +//! with `NotFound`. +//! +//! Both halves commit, or neither does. + +use std::collections::BTreeMap; + +use nodedb_cluster::calvin::types::{EngineKeySet, PassiveReadKeyId, SortedVec}; +use nodedb_physical::physical_plan::{KvOp, PhysicalPlan}; +use nodedb_physical::physical_task::{PhysicalTask, PostSetOp}; +use nodedb_types::{CollectionKey, QualifiedCollection, RlsWriteCheck, Surrogate}; + +use super::recon::read_kv_row; +use crate::Error; +use crate::bridge::envelope::ErrorCode; +use crate::control::cluster::calvin::executor::ollp::error::OllpError; +use crate::control::planner::calvin::dependent_recon_finish::finish_committed; +use crate::control::planner::calvin::{ + DependentOutcome, DependentRetryArgs, PassiveReads, build_read_dependent_tx_class, + predicate_class, run_dependent_with_retry, submit_calvin_routed_assign, +}; +use crate::control::planner::rls_injection::inject_rls_for_single_plan; +use crate::control::security::audit::ArcAuditEmitter; +use crate::control::security::identity::AuthenticatedIdentity; +use crate::control::security::request_scope::RequestAuthScope; +use crate::control::server::shared::authorization::authorize_task_set; +use crate::control::state::SharedState; +use crate::types::{DatabaseId, TenantId}; + +/// One item move between collections on two vShards. +pub struct CrossShardItemMove { + pub tenant_id: TenantId, + pub database_id: DatabaseId, + pub source_collection: QualifiedCollection, + pub dest_collection: QualifiedCollection, + /// The item's key at the source. + pub item_key: Vec, + /// The item's key at the destination. + pub dest_key: Vec, + /// The moved row's identity at the destination. + pub surrogate: Surrogate, + /// The source collection's compiled write policy, decided against the + /// row the move removes. + pub source_rls_write_check: RlsWriteCheck, +} + +/// Move the item of `mv` across its two vShards as one read-dependent +/// Calvin transaction, for `identity`. +/// +/// Returns once both halves committed. An item absent at the source ends +/// the move with `ErrorCode::NotFound`, as a same-vShard move does. A write +/// policy of the destination that refuses the item's bytes refuses the move +/// before anything is submitted. +pub async fn move_item_across_shards( + state: &SharedState, + identity: &AuthenticatedIdentity, + mv: CrossShardItemMove, +) -> crate::Result<()> { + let registry = state + .calvin_completion_registry + .get() + .ok_or(Error::SequencerUnavailable)?; + let orchestrator = state + .ollp_orchestrator + .get() + .ok_or(Error::SequencerUnavailable)?; + let source_vshard = + CollectionKey::from_qualified_str(mv.database_id, mv.source_collection.as_str())?.vshard(); + let dest_vshard = + CollectionKey::from_qualified_str(mv.database_id, mv.dest_collection.as_str())?.vshard(); + + let delete_task = PhysicalTask { + tenant_id: mv.tenant_id, + vshard_id: source_vshard, + database_id: mv.database_id, + plan: PhysicalPlan::Kv(KvOp::Delete { + collection: mv.source_collection.clone(), + keys: vec![mv.item_key.clone()], + rls_write_check: mv.source_rls_write_check.clone(), + returning: None, + rls_filters: Vec::new(), + provenance: None, + }), + post_set_op: PostSetOp::None, + txn_id: None, + }; + let put_task = |value: Vec| PhysicalTask { + tenant_id: mv.tenant_id, + vshard_id: dest_vshard, + database_id: mv.database_id, + plan: PhysicalPlan::Kv(KvOp::Put { + collection: mv.dest_collection.clone(), + key: mv.dest_key.clone(), + value, + ttl_ms: 0, + surrogate: mv.surrogate, + returning: None, + rls_filters: Vec::new(), + provenance: None, + }), + post_set_op: PostSetOp::None, + txn_id: None, + }; + let passive_keys = BTreeMap::from([( + source_vshard.as_u32(), + vec![EngineKeySet::Kv { + collection: mv.source_collection.as_str().to_owned(), + keys: SortedVec::new(vec![mv.item_key.clone()]), + }], + )]); + let item_row = PassiveReadKeyId::kv(mv.source_collection.clone(), mv.item_key.clone()); + let class_hash = predicate_class( + &format!("TRANSFER_ITEM TO {}", mv.dest_collection.as_str()), + mv.source_collection.as_str(), + ); + let tenant_id = mv.tenant_id; + let source = &mv; + // Copy: it captures only references. + let read_item = move || async move { + read_kv_row( + state, + source.tenant_id, + source.database_id, + &source.source_collection, + &source.item_key, + ) + .await? + .ok_or(Error::DataPlane(ErrorCode::NotFound)) + }; + + let submit = |item: &Vec| { + let item = item.clone(); + let delete_task = delete_task.clone(); + let passive_keys = passive_keys.clone(); + let item_row = item_row.clone(); + let put = put_task(item.clone()); + async move { + let put = admit_destination_image(state, identity, put) + .map_err(|e| OllpError::Terminal(Box::new(e)))?; + // Each attempt's tasks pass the authorization boundary as the + // capability they are submitted under. + let emitter = ArcAuditEmitter(std::sync::Arc::clone(&state.audit)); + let tasks: Vec = authorize_task_set( + identity, + &[delete_task, put], + &state.permissions, + &state.roles, + &emitter, + ) + .map_err(|e| OllpError::Terminal(Box::new(e.into())))? + .into_tasks() + .into_iter() + .map(|task| task.into_physical_task()) + .collect(); + let reads = PassiveReads { + keys: passive_keys, + expected: BTreeMap::from([(item_row, Some(item))]), + }; + orchestrator + .submit_with_retry_via( + class_hash, + tenant_id, + || { + build_read_dependent_tx_class(&tasks, tenant_id, reads.clone()) + .map(Some) + .map_err(|e| OllpError::Terminal(Box::new(e))) + }, + |tx_class| async move { + submit_calvin_routed_assign(state, tx_class) + .await + .map_err(|e| OllpError::Retryable(Box::new(e))) + }, + ) + .await + } + }; + + let initial_item = read_item().await?; + let outcome = run_dependent_with_retry(DependentRetryArgs { + registry, + orchestrator, + predicate_class_hash: class_hash, + timeout: std::time::Duration::from_secs(state.tuning.network.default_deadline_secs), + ollp_max_retries: u32::from(orchestrator.ollp_max_retries()), + initial_predicted: initial_item, + submit, + rescan: read_item, + }) + .await?; + let (txn_id, ack_results) = match outcome { + DependentOutcome::Committed { + txn_id, + ack_results, + } => (txn_id, ack_results), + DependentOutcome::NoOp => { + return Err(Error::Internal { + detail: "a cross-shard TRANSFER_ITEM submitted no transaction".to_owned(), + }); + } + }; + // The applied responses of the two writes are counts the move does not + // report. Draining them keeps the sidecar from holding them. The drain + // reads only the plans' collections and kinds. + finish_committed( + state, + &[delete_task, put_task(Vec::new())], + txn_id, + &ack_results, + ) + .await?; + Ok(()) +} + +/// Run the destination's write policy over the item's bytes in `put`: the +/// injection pass admits the image, or refuses the move. +fn admit_destination_image( + state: &SharedState, + identity: &AuthenticatedIdentity, + mut put: PhysicalTask, +) -> crate::Result { + let scope = RequestAuthScope::for_database(identity, state.auth_stores(), put.database_id); + inject_rls_for_single_plan( + put.tenant_id.as_u64(), + put.database_id, + &mut put.plan, + &state.rls, + state.credentials.catalog(), + scope.auth(), + )?; + Ok(put) +} diff --git a/nodedb/src/control/planner/calvin/read_dependent/mod.rs b/nodedb/src/control/planner/calvin/read_dependent/mod.rs new file mode 100644 index 000000000..afbea4442 --- /dev/null +++ b/nodedb/src/control/planner/calvin/read_dependent/mod.rs @@ -0,0 +1,9 @@ +// SPDX-License-Identifier: BUSL-1.1 + +//! Read-dependent Calvin transactions: cross-shard writes whose values +//! depend on rows of another vShard. + +pub mod item_move; +mod recon; + +pub use item_move::{CrossShardItemMove, move_item_across_shards}; diff --git a/nodedb/src/control/planner/calvin/read_dependent/recon.rs b/nodedb/src/control/planner/calvin/read_dependent/recon.rs new file mode 100644 index 000000000..837af4310 --- /dev/null +++ b/nodedb/src/control/planner/calvin/read_dependent/recon.rs @@ -0,0 +1,47 @@ +// SPDX-License-Identifier: BUSL-1.1 + +//! The coordinator's read of a row a read-dependent transaction depends on. +//! +//! The read reaches the row's vShard owner through the gateway, as a +//! `SELECT` does, and returns the stored bytes a passive read of the same +//! row returns under the transaction's locks. The active participants +//! compare the two, so the read carries no row filter: both sides see the +//! row as stored. + +use nodedb_physical::physical_plan::{KvOp, PhysicalPlan}; +use nodedb_types::QualifiedCollection; + +use crate::control::state::SharedState; +use crate::types::{DatabaseId, TenantId, TraceId}; + +/// The stored bytes of key-value row `key` of `collection`, read on the +/// collection's vShard owner. `None` when the row is absent. +pub(super) async fn read_kv_row( + state: &SharedState, + tenant_id: TenantId, + database_id: DatabaseId, + collection: &QualifiedCollection, + key: &[u8], +) -> crate::Result>> { + let plan = PhysicalPlan::Kv(KvOp::Get { + collection: collection.clone(), + key: key.to_vec(), + rls_filters: Vec::new(), + surrogate_ceiling: None, + }); + let gateway = state.installed_gateway()?; + let ctx = crate::control::gateway::core::QueryContext { + tenant_id, + trace_id: TraceId::ZERO, + database_id, + txn_id: None, + linearizable: true, + }; + let payloads = gateway.execute_internal(&ctx, plan).await?; + // A KV get routes to the collection's one vShard. An absent row answers + // with an empty payload: a stored row is a msgpack image, never empty. + Ok(payloads + .into_iter() + .next() + .filter(|bytes| !bytes.is_empty())) +} diff --git a/nodedb/src/control/planner/calvin/reservation.rs b/nodedb/src/control/planner/calvin/reservation.rs index 191d01ed4..e37c6b174 100644 --- a/nodedb/src/control/planner/calvin/reservation.rs +++ b/nodedb/src/control/planner/calvin/reservation.rs @@ -22,8 +22,8 @@ use std::time::Duration; use nodedb_cluster::calvin::SEQUENCER_GROUP_ID; use nodedb_cluster::calvin::types::{LockKeyWire, ReleaseReason, TxnIdWire}; use nodedb_cluster::{ - RaftRpc, ReleaseReservationRequest, ReleaseReservationResponse, ReserveReadRequest, - ReserveReadResponse, + ClusterError, RaftRpc, ReleaseReservationRequest, ReleaseReservationResponse, + ReserveReadRequest, ReserveReadResponse, }; use crate::Error; @@ -86,8 +86,6 @@ pub(crate) async fn submit_reserve_read( vshard: u32, owner: Option, ) -> crate::Result { - let local_timeout = Duration::from_secs(state.tuning.network.default_deadline_secs); - // Every running server has a cluster transport, the synthesized one-node // cluster included. Without one, `start_raft` never ran here. let Some(transport) = state.cluster_transport.as_ref() else { @@ -116,6 +114,7 @@ pub(crate) async fn submit_reserve_read( // Leader is self: submit locally (a self-RPC will be a pointless extra // hop and the local inbox is the one that gets the assignment). if leader == state.node_id { + let local_timeout = super::submit::budget::statement_budget(state)?; return submit_local_reserve_read(state, key, vshard, owner, local_timeout).await; } @@ -138,12 +137,8 @@ pub(crate) async fn submit_reserve_read( }) .transpose()?; - let deadline_remaining_ms = state - .tuning - .network - .default_deadline_secs - .saturating_mul(1000) - .max(1); + // The leader works on the reservation only as long as the statement waits. + let deadline_remaining_ms = super::submit::budget::statement_budget_ms(state)?; let req = ReserveReadRequest { lock_key_bytes, vshard, @@ -176,6 +171,11 @@ pub(crate) async fn submit_reserve_read( Ok(other) => Err(Error::Internal { detail: format!("reserve-read: unexpected reply from node {leader}: {other:?}"), }), + // The request went out and its answer was lost. The leader can have + // granted it, so its outcome is unknown. Lease GC reaps a lost grant. + Err(ClusterError::Unanswered { .. }) => Err(Error::DeadlineExceeded { + request_id: crate::types::RequestId::new(0), + }), Err(e) => Err(Error::Internal { detail: format!("reserve-read RPC to sequencer leader node {leader} failed: {e}"), }), @@ -282,6 +282,9 @@ pub(crate) async fn release_reservation( } }; + // A release runs on the abort path too, often after the statement's + // deadline passed. It frees a lock other statements wait on, so it takes + // the node default, never what is left of the statement. let deadline_remaining_ms = state .tuning .network diff --git a/nodedb/src/control/planner/calvin/retry_loop.rs b/nodedb/src/control/planner/calvin/retry_loop.rs index 0bea546d1..fbe58d25c 100644 --- a/nodedb/src/control/planner/calvin/retry_loop.rs +++ b/nodedb/src/control/planner/calvin/retry_loop.rs @@ -4,10 +4,10 @@ //! //! The retry loop for dependent-read (OLLP) Calvin transactions is owned by the //! coordinator (the pgwire handler), not the per-vshard scheduler. On a -//! post-exec predicate-drift mismatch the loop runs a FRESH pre-execution +//! `PredictionDrift` abort verdict the loop runs a FRESH pre-execution //! reconnaissance before resubmitting — a stale prediction can never converge -//! under predicate drift. The scheduler's only job on mismatch is to release the -//! aborted attempt's locks and signal the completion registry so this loop wakes. +//! under predicate drift. A participant that finds drift votes abort, drops +//! its slice at the verdict, and acks, which wakes this loop. use nodedb_cluster::calvin::{AbortReason, AttemptOutcome, CalvinCompletionRegistry, TxnId}; @@ -53,9 +53,9 @@ pub enum DependentOutcome { /// per attempt. /// /// This is the single owner of the submit → await-assignment → await-completion -/// → (mismatch ? re-scan : done) loop for dependent-read Calvin transactions. -/// On a POST-EXEC predicate-drift mismatch (the executor released the aborted -/// attempt's locks and the scheduler signalled the registry), the loop runs the +/// → (drift ? re-scan : done) loop for dependent-read Calvin transactions. +/// On a `PredictionDrift` abort verdict (every participant dropped the attempt +/// and released its locks before it acked), the loop runs the /// injected `rescan` closure to produce a FRESH prediction and resubmits — a /// stale prediction can never converge under predicate drift. On a PRE-ADMISSION /// failure (`OllpError` from the circuit-breaker / sequencer / tenant budget), @@ -156,31 +156,21 @@ where // for a reason a fresh reconnaissance cannot change, so surface it // to the client immediately instead of burning retries. The // verdict's reason picks the error: a stale read-set is SQLSTATE - // 40001, a participant error is not. `PredictionDrift` and - // `PartsLost` retry, in the arm below: a multi-part transaction - // whose parts a leader change lost staged nothing anywhere. + // 40001, a participant error is not, and rejected plans are an + // internal error. `PredictionDrift` and `PartsLost` retry, in the + // arm below: a multi-part transaction whose parts a leader change + // lost staged nothing anywhere. AttemptOutcome::Aborted { reason } if reason != AbortReason::PredictionDrift && reason != AbortReason::PartsLost => { return Err(calvin_abort_error(reason)); } - // Terminal, NON-retryable: the scheduler rejected the transaction's - // local plan routing and broadcast `TxnRoutingFailed`. A fresh - // reconnaissance can never fix a routing rejection, so surface it - // to the caller immediately instead of burning retries. - AttemptOutcome::Failed { detail } => { - return Err(Error::Internal { - detail: format!("calvin transaction routing failed: {detail}"), - }); - } - AttemptOutcome::Mismatch | AttemptOutcome::Aborted { .. } => { - // POST-EXEC predicate drift: an OLLP mismatch, or an abort - // verdict because a participant found state other than the - // reconnaissance predicted and wrote nothing. The scheduler - // already released the aborted attempt's locks before - // signalling the registry, so a FRESH reconnaissance is safe — - // and necessary, since the stale prediction can never converge - // under drift. + AttemptOutcome::Aborted { .. } => { + // A participant found state other than the reconnaissance + // predicted and wrote nothing, or the parts were lost. Every + // participant released the attempt's locks before it acked, + // so a FRESH reconnaissance is safe — and necessary, since the + // stale prediction can never converge under drift. if retry >= ollp_max_retries { return Err(Error::OllpExhausted { retries: ollp_max_retries.min(u8::MAX as u32) as u8, @@ -206,18 +196,18 @@ mod tests { //! returns a `RoutedAssignment` carrying a deterministic `(epoch, position)` — //! exactly the `(epoch, position)` the loop feeds to //! `register_completion_report(TxnId::new(epoch, position))`. The closure also forwards - //! that same `TxnId` to the fake over an mpsc channel; the fake then either calls - //! `note_ollp_mismatch(txn)` (first K submissions → `Mismatch`) or - //! `note_completion_ack(txn, 1)` (submission K+1, 1 participant → fires - //! `Completed`). The `rescan` closure increments a counter and returns a fresh - //! prediction vec. + //! that same `TxnId` to the fake over an mpsc channel; the fake stores a + //! verdict, `Abort(PredictionDrift)` for the first K submissions and + //! `Commit` after, then calls `note_completion_ack(txn, 1)` (1 participant → + //! fires the outcome). The `rescan` closure increments a counter and returns a + //! fresh prediction vec. //! //! The routed `submit` returns the assignment itself (the leader //! assigns and replies with `(epoch, position)`), so the loop never calls //! `register_submission` and the fake needs no `note_assigned`. The //! `(epoch, position)` source is deterministic on the test side: `epoch = //! inbox_seq`, `position = 0`. Both the `RoutedAssignment` returned by `submit` - //! AND the fake's `note_completion_ack` / `note_ollp_mismatch` use that same + //! AND the fake's `note_verdict` / `note_completion_ack` use that same //! `TxnId`, so they always match. //! //! Determinism: a current-thread runtime plus a bounded fake channel with enough @@ -230,6 +220,7 @@ mod tests { use std::sync::atomic::{AtomicU32, AtomicU64, Ordering}; use crate::control::cluster::calvin::executor::ollp::config::OllpConfig; + use nodedb_cluster::calvin::VerdictOutcome; use nodedb_cluster::calvin::sequencer::error::SequencerError; /// Build an orchestrator with zero backoff so the retry loop runs instantly. @@ -242,24 +233,26 @@ mod tests { } /// Spawn the fake scheduler. It reads `TxnId` events (the same `TxnId` the loop - /// registers for completion) and, for the first `mismatch_count` events, signals - /// an OLLP mismatch; on the next event it acks completion with a single - /// participant (which fires `Completed`). The loop never calls - /// `register_submission`, so `note_assigned` is not needed. + /// registers for completion). For the first `drift_count` events it stores an + /// `Abort(PredictionDrift)` verdict, and a `Commit` verdict after. Each event + /// then acks with a single participant, which fires the outcome. The loop + /// never calls `register_submission`, so `note_assigned` is not needed. fn spawn_fake_scheduler( registry: Arc, mut rx: tokio::sync::mpsc::Receiver, - mismatch_count: u32, + drift_count: u32, ) -> tokio::task::JoinHandle<()> { tokio::spawn(async move { let mut seen: u32 = 0; while let Some(txn) = rx.recv().await { - if seen < mismatch_count { - registry.note_ollp_mismatch(txn); + let verdict = if seen < drift_count { + VerdictOutcome::Abort(AbortReason::PredictionDrift) } else { - // 1 participant so a single ack completes the attempt. - registry.note_completion_ack(txn, 1); - } + VerdictOutcome::Commit + }; + registry.note_verdict(txn, verdict); + // 1 participant so a single ack completes the attempt. + registry.note_completion_ack(txn, 1); seen += 1; } }) @@ -278,7 +271,7 @@ mod tests { } #[test] - fn converges_after_two_mismatches() { + fn converges_after_two_drift_aborts() { let rt = tokio::runtime::Builder::new_current_thread() .enable_all() .build() @@ -337,12 +330,12 @@ mod tests { assert_eq!( submit_calls.load(Ordering::SeqCst), 3, - "two mismatches + one success → three submits" + "two drift aborts + one success → three submits" ); assert_eq!( rescan_calls.load(Ordering::SeqCst), 2, - "fresh re-scan runs once per mismatch" + "fresh re-scan runs once per drift abort" ); drop(tx); let _ = fake.await; @@ -350,7 +343,7 @@ mod tests { } #[test] - fn exhausts_on_persistent_mismatch() { + fn exhausts_on_persistent_drift() { let rt = tokio::runtime::Builder::new_current_thread() .enable_all() .build() @@ -359,7 +352,7 @@ mod tests { let registry = CalvinCompletionRegistry::new_detached(); let orchestrator = zero_backoff_orchestrator(); - // Mismatch on every attempt (large count → never acks). + // Drift on every attempt (large count → never commits). let (tx, rx) = tokio::sync::mpsc::channel::(16); let fake = spawn_fake_scheduler(Arc::clone(®istry), rx, u32::MAX); @@ -425,8 +418,8 @@ mod tests { let registry = CalvinCompletionRegistry::new_detached(); let orchestrator = zero_backoff_orchestrator(); - // No mismatch path needed: the first two submits fail pre-admission, the - // third admits and the fake immediately acks completion. + // No drift path needed: the first two submits fail pre-admission, the + // third admits and the fake immediately commits it. let (tx, rx) = tokio::sync::mpsc::channel::(16); let fake = spawn_fake_scheduler(Arc::clone(®istry), rx, 0); @@ -652,13 +645,13 @@ mod tests { tokio::spawn(async move { let mut first = true; while let Some(txn) = rx.recv().await { - if first { - registry.note_verdict( - txn, - nodedb_cluster::calvin::VerdictOutcome::Abort(reason), - ); - first = false; - } + let verdict = if first { + VerdictOutcome::Abort(reason) + } else { + VerdictOutcome::Commit + }; + first = false; + registry.note_verdict(txn, verdict); registry.note_completion_ack(txn, 1); } }) @@ -721,6 +714,17 @@ mod tests { assert_eq!(rescans, 1, "the drift abort reads again once"); } + #[test] + fn a_plan_rejected_abort_is_terminal() { + let (result, submits, rescans) = run_with_first_abort(AbortReason::PlanRejected); + assert!( + matches!(result, Err(Error::Internal { .. })), + "expected an internal error, got {result:?}" + ); + assert_eq!(submits, 1); + assert_eq!(rescans, 0); + } + #[test] fn a_participant_error_abort_is_terminal() { let (result, submits, rescans) = run_with_first_abort(AbortReason::ParticipantError); diff --git a/nodedb/src/control/planner/calvin/submit/assign.rs b/nodedb/src/control/planner/calvin/submit/assign.rs index 4459347ad..f2781ac71 100644 --- a/nodedb/src/control/planner/calvin/submit/assign.rs +++ b/nodedb/src/control/planner/calvin/submit/assign.rs @@ -10,7 +10,7 @@ use std::time::Duration; use nodedb_cluster::calvin::SEQUENCER_GROUP_ID; use nodedb_cluster::calvin::types::TxClass; -use nodedb_cluster::{RaftRpc, SubmitCalvinInboxRequest, SubmitCalvinInboxResponse}; +use nodedb_cluster::{ClusterError, RaftRpc, SubmitCalvinInboxRequest, SubmitCalvinInboxResponse}; use super::stream::{PartStream, StreamTarget, stream_parts}; use crate::Error; @@ -49,6 +49,7 @@ pub(crate) async fn submit_local_assign( ) -> crate::Result { super::local::raise_metadata_floor(state, &mut tx_class); super::local::stamp_incarnations(state, &mut tx_class)?; + super::unique_claims::stamp_unique_claims(state, &mut tx_class)?; let stream = super::parts::split_into_parts(state, &mut tx_class)?; assign_prepared(state, tx_class, stream, timeout).await } @@ -119,8 +120,8 @@ pub async fn submit_calvin_routed_assign( ) -> crate::Result { super::local::raise_metadata_floor(state, &mut tx_class); super::local::stamp_incarnations(state, &mut tx_class)?; + super::unique_claims::stamp_unique_claims(state, &mut tx_class)?; let stream = super::parts::split_into_parts(state, &mut tx_class)?; - let local_timeout = Duration::from_secs(state.tuning.network.default_deadline_secs); // Every running server has a cluster transport, the synthesized one-node // cluster included. Without one, `start_raft` never ran here. @@ -151,6 +152,7 @@ pub async fn submit_calvin_routed_assign( // Leader is self: submit-and-assign locally (a self-RPC will be a pointless // extra hop and the local registry is the one that gets the assignment). if leader == state.node_id { + let local_timeout = super::budget::statement_budget(state)?; return assign_prepared(state, tx_class, stream, local_timeout).await; } @@ -165,12 +167,8 @@ pub async fn submit_calvin_routed_assign( detail: format!("failed to encode TxClass for routed Calvin inbox submit: {e}"), })?; - let deadline_remaining_ms = state - .tuning - .network - .default_deadline_secs - .saturating_mul(1000) - .max(1); + // The leader works on the transaction only as long as the statement waits. + let deadline_remaining_ms = super::budget::statement_budget_ms(state)?; let req = SubmitCalvinInboxRequest { tx_class_bytes, deadline_remaining_ms, @@ -211,6 +209,19 @@ pub async fn submit_calvin_routed_assign( detail: format!("calvin-inbox: unexpected reply from node {leader}: {other:?}"), }); } + // The submit went out and its answer was lost. The leader can have + // assigned it, so its outcome is unknown. + Err(error @ ClusterError::Unanswered { .. }) => { + tracing::warn!( + leader, + %error, + "calvin-inbox submit reached its deadline with no answer from the sequencer \ + leader" + ); + return Err(Error::DeadlineExceeded { + request_id: crate::types::RequestId::new(0), + }); + } Err(e) => { return Err(Error::Internal { detail: format!("calvin-inbox RPC to sequencer leader node {leader} failed: {e}"), diff --git a/nodedb/src/control/planner/calvin/submit/budget.rs b/nodedb/src/control/planner/calvin/submit/budget.rs new file mode 100644 index 000000000..7d50b8dae --- /dev/null +++ b/nodedb/src/control/planner/calvin/submit/budget.rs @@ -0,0 +1,71 @@ +// SPDX-License-Identifier: BUSL-1.1 + +//! The budget a Calvin request carries to the sequencer leader. +//! +//! A Calvin submit belongs to the statement that issued it. The sequencer +//! leader works on it only as long as the statement waits. The budget is +//! what is left of the statement's deadline, never the node default. + +use std::time::Duration; + +use crate::bridge::envelope::ErrorCode; +use crate::control::state::SharedState; +use crate::control::wal_replication::statement_propose_deadline; + +/// The milliseconds left to `deadline`, for a request about to go out. +/// +/// A spent budget refuses the request before it goes out. Nothing was +/// submitted, so the refusal is definite. +pub(crate) fn budget_ms_until(deadline: tokio::time::Instant) -> crate::Result { + match nodedb_cluster::rpc_codec::remaining_budget_ms(deadline) { + 0 => Err(crate::Error::DataPlane(ErrorCode::ExpiredBeforeExecution)), + ms => Ok(ms), + } +} + +/// The milliseconds left on the running statement, or on the node default +/// outside a statement. +pub(crate) fn statement_budget_ms(state: &SharedState) -> crate::Result { + budget_ms_until(statement_propose_deadline(state)) +} + +/// [`statement_budget_ms`] as a wait bound for the local half of the request. +pub(crate) fn statement_budget(state: &SharedState) -> crate::Result { + statement_budget_ms(state).map(Duration::from_millis) +} + +#[cfg(test)] +mod tests { + use super::*; + + use crate::control::server::shared::session::{conn_scope, deadline, statement_deadline}; + + /// A statement with 100 ms left forwards at most 100 ms, under a node + /// default of 30 s. + #[tokio::test] + async fn the_budget_ends_within_the_statement_deadline() { + conn_scope::scoped(async { + let _statement = deadline::enter(Some(Duration::from_millis(100)), 30); + let at = tokio::time::Instant::from_std(statement_deadline(30)); + let budget = budget_ms_until(at).expect("a live budget"); + assert!( + budget <= 100, + "budget {budget} ms exceeds the statement's 100 ms" + ); + assert!(budget > 0); + }) + .await; + } + + /// A spent statement sends nothing. Its refusal is definite. + #[tokio::test] + async fn a_spent_statement_is_refused_before_it_goes_out() { + let error = + budget_ms_until(tokio::time::Instant::now()).expect_err("a spent budget is refused"); + let crate::Error::DataPlane(code) = &error else { + panic!("expected a Data-Plane verdict, got {error:?}"); + }; + assert_eq!(code, &ErrorCode::ExpiredBeforeExecution); + assert!(crate::control::server::dispatch_utils::write_definitely_not_applied(code)); + } +} diff --git a/nodedb/src/control/planner/calvin/submit/local.rs b/nodedb/src/control/planner/calvin/submit/local.rs index 27574e6d3..043dac370 100644 --- a/nodedb/src/control/planner/calvin/submit/local.rs +++ b/nodedb/src/control/planner/calvin/submit/local.rs @@ -38,8 +38,8 @@ pub(super) fn synthetic_returning_response(payload_bytes: Vec) -> Response { payload: Payload::from_vec(payload_bytes), watermark_lsn: Lsn::ZERO, error_code: None, - read_set_valid: None, - read_version_lsn: crate::types::Lsn::ZERO, + stage_vote: None, + read_versions: crate::types::ReadVersions::new(), write_set: Vec::new(), } } @@ -153,6 +153,7 @@ pub async fn submit_and_await_calvin_with_timeout( ) -> crate::Result> { raise_metadata_floor(state, &mut tx_class); stamp_incarnations(state, &mut tx_class)?; + super::unique_claims::stamp_unique_claims(state, &mut tx_class)?; let stream = super::parts::split_into_parts(state, &mut tx_class)?; submit_prepared_and_await(state, tx_class, stream, timeout).await } @@ -168,7 +169,7 @@ pub(crate) async fn submit_prepared_and_await( timeout: Duration, ) -> crate::Result> { #[cfg(feature = "failpoints")] - crate::control::fail_gate::after_calvin_stamp(&tx_class).await; + crate::control::fail_gate::after_calvin_stamp(state.node_id, &tx_class).await; let fold = ReplyFold::of_tx_class(&tx_class)?; let inbox = state .sequencer_inbox @@ -236,34 +237,11 @@ pub(crate) async fn submit_prepared_and_await( .map_err(|_| Error::Internal { detail: "Calvin completion channel closed".to_owned(), })?; - let outcome = report.outcome; - // Terminal, NON-retryable: the scheduler rejected the transaction's local - // plan routing and broadcast `TxnRoutingFailed`. Surface it immediately — - // falling through to the RETURNING-drain below will silently report - // `Ok(None)` for a transaction that never applied. - if let AttemptOutcome::Failed { detail } = &outcome { - return Err(Error::Internal { - detail: format!("calvin transaction routing failed: {detail}"), - }); - } - // Terminal, NON-retryable: the global cross-shard verdict was ABORT and the - // writes were dropped. This is a fall-through chain, NOT a match — without - // this explicit check `Aborted` will fall through to the RETURNING drain - // below and silently return `Ok(None)`, reporting COMMIT SUCCESS for a - // transaction that never applied. The verdict's reason picks the error the - // client retries on. - if let AttemptOutcome::Aborted { reason } = &outcome { - return Err(calvin_abort_error(*reason)); - } - // The static (non-dependent) Calvin path never produces an OLLP mismatch — - // `note_ollp_mismatch` only fires on the dependent-predicate retry path — so - // this branch is unreachable at runtime today. It is kept as a typed error - // (never a panic) so any future mismatch signal on this channel surfaces - // deterministically instead of crashing. - if outcome == AttemptOutcome::Mismatch { - return Err(Error::Internal { - detail: "OLLP mismatch outcome on non-dependent Calvin path".to_owned(), - }); + // The global cross-shard verdict was ABORT and the writes were dropped. + // The verdict's reason picks the error the client sees. + match report.outcome { + AttemptOutcome::Completed => {} + AttemptOutcome::Aborted { reason } => return Err(calvin_abort_error(reason)), } if binds_authorization { crate::control::security::auth_lease::calvin_write_barrier(state).await?; diff --git a/nodedb/src/control/planner/calvin/submit/mod.rs b/nodedb/src/control/planner/calvin/submit/mod.rs index 1deb2959d..20bec2a59 100644 --- a/nodedb/src/control/planner/calvin/submit/mod.rs +++ b/nodedb/src/control/planner/calvin/submit/mod.rs @@ -31,11 +31,13 @@ //! schedulers; this module never does storage I/O or io_uring directly. pub mod assign; +pub mod budget; pub mod edge_slices; pub mod local; pub mod parts; pub mod routed; pub mod stream; +pub mod unique_claims; pub(crate) use assign::submit_local_assign; pub use assign::{RoutedAssignment, submit_calvin_routed_assign}; diff --git a/nodedb/src/control/planner/calvin/submit/routed.rs b/nodedb/src/control/planner/calvin/submit/routed.rs index 18ba17b47..7e71b6e80 100644 --- a/nodedb/src/control/planner/calvin/submit/routed.rs +++ b/nodedb/src/control/planner/calvin/submit/routed.rs @@ -11,7 +11,9 @@ use std::time::Duration; use nodedb_cluster::calvin::SEQUENCER_GROUP_ID; use nodedb_cluster::calvin::types::TxClass; -use nodedb_cluster::{RaftRpc, SubmitCalvinTxnRequest, SubmitCalvinTxnResponse, TypedClusterError}; +use nodedb_cluster::{ + ClusterError, RaftRpc, SubmitCalvinTxnRequest, SubmitCalvinTxnResponse, TypedClusterError, +}; use crate::Error; use crate::bridge::envelope::Response; @@ -70,6 +72,7 @@ pub async fn submit_calvin_routed( ) -> crate::Result> { super::local::raise_metadata_floor(state, &mut tx_class); super::local::stamp_incarnations(state, &mut tx_class)?; + super::unique_claims::stamp_unique_claims(state, &mut tx_class)?; let stream = super::parts::split_into_parts(state, &mut tx_class)?; // Every running server has a cluster transport, the synthesized one-node // cluster included. Without one, `start_raft` never ran here. @@ -119,7 +122,7 @@ pub async fn submit_calvin_routed( // Leader is self: submit-and-await locally (a self-RPC will be a pointless // extra hop and the local registry is the one that completes). if leader == state.node_id { - let timeout = Duration::from_secs(state.tuning.network.default_deadline_secs); + let timeout = super::budget::statement_budget(state)?; return submit_prepared_and_await(state, tx_class, stream, timeout).await; } @@ -134,12 +137,8 @@ pub async fn submit_calvin_routed( detail: format!("failed to encode TxClass for routed Calvin submit: {e}"), })?; - let deadline_remaining_ms = state - .tuning - .network - .default_deadline_secs - .saturating_mul(1000) - .max(1); + // The leader works on the transaction only as long as the statement waits. + let deadline_remaining_ms = super::budget::statement_budget_ms(state)?; let req = SubmitCalvinTxnRequest { tx_class_bytes, deadline_remaining_ms, @@ -203,6 +202,18 @@ pub async fn submit_calvin_routed( Ok(other) => Err(Error::Internal { detail: format!("calvin-submit: unexpected reply from node {leader}: {other:?}"), }), + // The submit went out and its answer was lost. The leader can have + // sequenced it, so its outcome is unknown. + Err(error @ ClusterError::Unanswered { .. }) => { + tracing::warn!( + leader, + %error, + "calvin-submit reached its deadline with no answer from the sequencer leader" + ); + Err(Error::DeadlineExceeded { + request_id: crate::types::RequestId::new(0), + }) + } Err(e) => Err(Error::Internal { detail: format!("calvin-submit RPC to sequencer leader node {leader} failed: {e}"), }), diff --git a/nodedb/src/control/planner/calvin/submit/unique_claims.rs b/nodedb/src/control/planner/calvin/submit/unique_claims.rs new file mode 100644 index 000000000..c49d12ee9 --- /dev/null +++ b/nodedb/src/control/planner/calvin/submit/unique_claims.rs @@ -0,0 +1,532 @@ +// SPDX-License-Identifier: BUSL-1.1 + +//! UNIQUE value claims of a Calvin transaction's writes. +//! +//! A document write that sets a UNIQUE-indexed value locks +//! `(collection, index, value)` exclusively. Two transactions that claim one +//! value then serialize, so the Data Plane's UNIQUE judgment of the second +//! one sees the first one's row on every replica. +//! +//! A write whose claimed value the plan does not carry (an expression +//! update, a body this module cannot decode) locks its whole collection +//! instead. That over-locks, and never under-locks. + +#![deny(clippy::wildcard_enum_match_arm)] + +use std::collections::{BTreeMap, BTreeSet}; + +use nodedb_cluster::calvin::types::{EngineKeySet, SortedVec, TxClass}; +use nodedb_physical::physical_plan::{DocumentOp, PhysicalPlan, UpdateValue}; + +use crate::Error; +use crate::control::state::SharedState; +use crate::engine::document::store::extract_index_values; + +/// One UNIQUE index of a collection, as its claims are extracted. +#[derive(Debug, Clone, PartialEq, Eq)] +pub(crate) struct UniquePath { + /// Index name. + pub name: String, + /// Indexed path, without a trailing `[]`. + pub path: String, + /// Whether the path indexes each array element. + pub is_array: bool, + /// Whether values are lowercased before the index judges them. + pub case_insensitive: bool, +} + +/// Add the UNIQUE claims of `tx_class`'s plans to its write set, read from +/// this node's catalog. +/// +/// A class already split into parts was stamped by its coordinator, so it is +/// left as it is. A forwarded class is stamped again with the same result: +/// a key set already in the write set is not added twice. +pub(crate) fn stamp_unique_claims( + state: &SharedState, + tx_class: &mut TxClass, +) -> crate::Result<()> { + if tx_class.is_multi_part() { + return Ok(()); + } + let plans = + nodedb_physical::physical_plan::wire::decode_batch(&tx_class.plans).map_err(|e| { + Error::Serialization { + format: "msgpack".into(), + detail: format!("calvin unique claims: plan decode: {e}"), + } + })?; + let sets = catalog_unique_claim_sets( + state, + tx_class.database_id, + tx_class.tenant_id.as_u64(), + &plans, + )?; + for set in sets { + if !tx_class.write_set.0.contains(&set) { + tx_class.write_set.0.push(set); + } + } + Ok(()) +} + +/// The UNIQUE claim key sets of `plans`, with each collection's UNIQUE +/// indexes read from this node's catalog. A Calvin transaction and an +/// autocommit write at the write-admission gate both claim through it. +pub(crate) fn catalog_unique_claim_sets( + state: &SharedState, + database_id: nodedb_types::DatabaseId, + tenant_id: u64, + plans: &[PhysicalPlan], +) -> crate::Result> { + let catalog = state.credentials.catalog(); + unique_claim_sets(plans, |collection| { + let key = nodedb_types::CollectionKey::from_qualified_str(database_id, collection) + .unwrap_or_else(|_| nodedb_types::CollectionKey::from_bare(database_id, collection)); + let stored = catalog.get_collection(key.database_id(), tenant_id, key.name())?; + Ok(stored.map_or_else(Vec::new, |stored| { + stored + .indexes + .iter() + .filter(|index| index.unique) + .map(|index| UniquePath { + name: index.name.clone(), + path: index + .field + .strip_suffix("[]") + .unwrap_or(&index.field) + .to_owned(), + is_array: index.field.ends_with("[]"), + case_insensitive: index.case_insensitive, + }) + .collect() + })) + }) +} + +/// The UNIQUE claim key sets of `plans`. `unique_paths` names the UNIQUE +/// indexes of a collection, as the plans name it. +pub(crate) fn unique_claim_sets( + plans: &[PhysicalPlan], + mut unique_paths: impl FnMut(&str) -> crate::Result>, +) -> crate::Result> { + let mut claims = Claims::default(); + let mut paths_of: BTreeMap> = BTreeMap::new(); + for plan in plans { + let op = match plan { + PhysicalPlan::Document(op) => op, + PhysicalPlan::Kv(_) + | PhysicalPlan::Vector(_) + | PhysicalPlan::Graph(_) + | PhysicalPlan::Crdt(_) + | PhysicalPlan::Columnar(_) + | PhysicalPlan::Timeseries(_) + | PhysicalPlan::Array(_) + | PhysicalPlan::Meta(_) + | PhysicalPlan::Text(_) + | PhysicalPlan::Spatial(_) + | PhysicalPlan::Query(_) + | PhysicalPlan::ClusterArray(_) + | PhysicalPlan::ClusterEvent(_) => continue, + }; + let Some(collection) = claiming_collection(op) else { + continue; + }; + if !paths_of.contains_key(collection) { + let paths = unique_paths(collection)?; + paths_of.insert(collection.to_owned(), paths); + } + let Some(paths) = paths_of.get(collection) else { + continue; + }; + if paths.is_empty() { + continue; + } + claims.add_op(collection, op, paths); + } + Ok(claims.into_key_sets()) +} + +/// The collection of a document op that can claim a UNIQUE value. `None` +/// for a delete, a read, a DDL op, a write that locks its collection whole, +/// and a write no Calvin transaction sequences. +fn claiming_collection(op: &DocumentOp) -> Option<&str> { + match op { + DocumentOp::PointPut { collection, .. } + | DocumentOp::PointInsert { collection, .. } + | DocumentOp::Upsert { collection, .. } + | DocumentOp::BatchInsert { collection, .. } + | DocumentOp::PointUpdate { collection, .. } + | DocumentOp::ApplyBalanceDelta { collection, .. } => Some(collection.as_str()), + DocumentOp::PointDelete { .. } + | DocumentOp::InsertSelect { .. } + | DocumentOp::BulkUpdate { .. } + | DocumentOp::BulkDelete { .. } + | DocumentOp::Truncate { .. } + | DocumentOp::Merge { .. } + | DocumentOp::UpdateFromJoin { .. } + | DocumentOp::ResolvedWrite { .. } + | DocumentOp::ResolveWrite(_) + | DocumentOp::PointGet { .. } + | DocumentOp::Scan { .. } + | DocumentOp::RangeScan { .. } + | DocumentOp::IndexLookup { .. } + | DocumentOp::IndexedFetch { .. } + | DocumentOp::EstimateCount { .. } + | DocumentOp::MaterializeScan { .. } + | DocumentOp::Register { .. } + | DocumentOp::DropIndex { .. } + | DocumentOp::BackfillIndex { .. } => None, + } +} + +/// Claimed values by collection and index, and collections locked whole +/// because a claim is unknown. Ordered maps keep the key sets identical on +/// every node. +#[derive(Default)] +struct Claims { + values: BTreeMap<(String, String), BTreeSet>>, + whole: BTreeSet, +} + +impl Claims { + /// Add the claims of `op` on `collection` against its UNIQUE `paths`. + fn add_op(&mut self, collection: &str, op: &DocumentOp, paths: &[UniquePath]) { + match op { + DocumentOp::PointPut { value, .. } | DocumentOp::PointInsert { value, .. } => { + self.add_body(collection, value, paths); + } + DocumentOp::Upsert { + value, + on_conflict_updates, + .. + } => { + self.add_body(collection, value, paths); + self.add_updates(collection, on_conflict_updates, paths); + } + DocumentOp::BatchInsert { documents, .. } => { + for (_, body) in documents { + self.add_body(collection, body, paths); + } + } + DocumentOp::PointUpdate { updates, .. } => { + self.add_updates(collection, updates, paths); + } + // A delta moves a numeric column by an amount the plan does not + // resolve to a value. + DocumentOp::ApplyBalanceDelta { column, .. } => { + if paths.iter().any(|path| touches(path, column)) { + self.whole.insert(collection.to_owned()); + } + } + DocumentOp::PointDelete { .. } + | DocumentOp::InsertSelect { .. } + | DocumentOp::BulkUpdate { .. } + | DocumentOp::BulkDelete { .. } + | DocumentOp::Truncate { .. } + | DocumentOp::Merge { .. } + | DocumentOp::UpdateFromJoin { .. } + | DocumentOp::ResolvedWrite { .. } + | DocumentOp::ResolveWrite(_) + | DocumentOp::PointGet { .. } + | DocumentOp::Scan { .. } + | DocumentOp::RangeScan { .. } + | DocumentOp::IndexLookup { .. } + | DocumentOp::IndexedFetch { .. } + | DocumentOp::EstimateCount { .. } + | DocumentOp::MaterializeScan { .. } + | DocumentOp::Register { .. } + | DocumentOp::DropIndex { .. } + | DocumentOp::BackfillIndex { .. } => {} + } + } + + /// Claim every UNIQUE value of a submitted document body. + fn add_body(&mut self, collection: &str, body: &[u8], paths: &[UniquePath]) { + let Some(doc) = decode_body(body) else { + self.whole.insert(collection.to_owned()); + return; + }; + for path in paths { + self.claim(collection, path, &doc); + } + } + + /// Claim the UNIQUE values field updates assign. An update that touches + /// an indexed path without a literal for exactly that path claims an + /// unknown value. + fn add_updates( + &mut self, + collection: &str, + updates: &[(String, UpdateValue)], + paths: &[UniquePath], + ) { + for (field, value) in updates { + for path in paths.iter().filter(|path| touches(path, field)) { + let literal = match value { + UpdateValue::Literal(bytes) if normalize(&path.path) == field.as_str() => { + nodedb_types::json_from_msgpack(bytes).ok() + } + UpdateValue::Literal(_) | UpdateValue::Expr(_) => None, + }; + match literal { + Some(literal) => { + let doc = nest(field, literal); + self.claim(collection, path, &doc); + } + None => { + self.whole.insert(collection.to_owned()); + } + } + } + } + } + + /// Claim the values of `path` in `doc`, canonicalized as the index + /// judges them. + fn claim(&mut self, collection: &str, path: &UniquePath, doc: &serde_json::Value) { + let values = extract_index_values(doc, &path.path, path.is_array); + if values.is_empty() { + return; + } + let claimed = self + .values + .entry((collection.to_owned(), path.name.clone())) + .or_default(); + for value in values { + let value = if path.case_insensitive { + value.to_lowercase() + } else { + value + }; + claimed.insert(value.into_bytes()); + } + } + + /// One `Unique` key set per collection and index, then one `Collection` + /// key set per collection with an unknown claim. + fn into_key_sets(self) -> Vec { + let mut sets: Vec = self + .values + .into_iter() + .map(|((collection, index), values)| EngineKeySet::Unique { + collection, + index, + values: SortedVec::new(values.into_iter().collect()), + }) + .collect(); + sets.extend( + self.whole + .into_iter() + .map(|collection| EngineKeySet::Collection { + collection, + vshards: SortedVec::new(Vec::new()), + }), + ); + sets + } +} + +/// `path` without a leading `$.` or `$`. +fn normalize(path: &str) -> &str { + path.strip_prefix("$.") + .or_else(|| path.strip_prefix('$')) + .unwrap_or(path) +} + +/// Whether assigning `field` can change the value at the indexed `path`: +/// the two name one field, or one lies inside the other. +fn touches(path: &UniquePath, field: &str) -> bool { + let indexed = normalize(&path.path); + indexed == field + || indexed + .strip_prefix(field) + .is_some_and(|rest| rest.starts_with('.')) + || field + .strip_prefix(indexed) + .is_some_and(|rest| rest.starts_with('.')) +} + +/// A document holding `value` at the dotted `field`. +fn nest(field: &str, value: serde_json::Value) -> serde_json::Value { + field.rsplit('.').fold(value, |inner, segment| { + let mut object = serde_json::Map::new(); + object.insert(segment.to_owned(), inner); + serde_json::Value::Object(object) + }) +} + +/// Decode a submitted document body: a MessagePack map or JSON. `None` for +/// any other form, such as a tagged value array or a strict Binary Tuple. +fn decode_body(body: &[u8]) -> Option { + let first = *body.first()?; + let msgpack_map = matches!(first, 0x80..=0x8f | 0xde | 0xdf); + let msgpack_array = matches!(first, 0x90..=0x9f | 0xdc | 0xdd); + if msgpack_map { + return nodedb_types::json_from_msgpack(body).ok(); + } + if msgpack_array { + return None; + } + sonic_rs::from_slice::(body) + .ok() + .filter(serde_json::Value::is_object) +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::control::cluster::calvin::scheduler::{ + AcquireOutcome, LockKey, LockManager, LockMode, TxnId, + }; + use nodedb_types::{DatabaseId, QualifiedCollection, Surrogate}; + + const USERS: &str = "users"; + + fn email_index() -> UniquePath { + UniquePath { + name: "users_email".to_owned(), + path: "$.email".to_owned(), + is_array: false, + case_insensitive: true, + } + } + + fn paths(collection: &str) -> crate::Result> { + Ok(if collection == USERS { + vec![email_index()] + } else { + Vec::new() + }) + } + + fn insert(id: &str, surrogate: u32, email: &str) -> PhysicalPlan { + let body = nodedb_types::json_to_msgpack(&serde_json::json!({ "email": email })) + .expect("encode body"); + PhysicalPlan::Document(DocumentOp::PointInsert { + collection: QualifiedCollection::new(DatabaseId::DEFAULT, USERS), + document_id: id.to_owned(), + value: body, + if_absent: false, + surrogate: Surrogate::new(surrogate), + returning: None, + rls_filters: Vec::new(), + resolved_sum_targets: Vec::new(), + deferred_sum_targets: Vec::new(), + }) + } + + fn update(id: &str, value: UpdateValue) -> PhysicalPlan { + PhysicalPlan::Document(DocumentOp::PointUpdate { + collection: QualifiedCollection::new(DatabaseId::DEFAULT, USERS), + document_id: id.to_owned(), + surrogate: Some(Surrogate::new(3)), + pk_bytes: id.as_bytes().to_vec(), + updates: vec![("email".to_owned(), value)], + returning: None, + rls_filters: Vec::new(), + rls_write_check: nodedb_types::RlsWriteCheck::pending_injection(), + resolved_sum_targets: Vec::new(), + declared_primary_key: None, + }) + } + + /// The scheduler's lock request for the claims of `plans`, as write keys. + fn claim_locks(plans: &[PhysicalPlan]) -> BTreeMap { + let sets = unique_claim_sets(plans, paths).expect("claims"); + let tx_class = TxClass::new_single_vshard( + nodedb_cluster::calvin::types::ReadWriteSet::new(Vec::new()), + nodedb_cluster::calvin::types::ReadWriteSet::new(sets), + Vec::new(), + nodedb_types::TenantId::new(1), + None, + nodedb_cluster::calvin::types::VersionedReadSet::default(), + ) + .expect("tx class"); + crate::control::cluster::calvin::scheduler::driver::helpers::expand_rw_set( + &nodedb_cluster::calvin::types::SequencedTxn { + epoch: 1, + position: 0, + tx_class, + epoch_system_ms: 1_700_000_000_000, + epoch_vshard_txn_count: 2, + lock_owner: None, + }, + ) + } + + #[test] + fn an_insert_claims_its_unique_value_canonicalized() { + let sets = unique_claim_sets(&[insert("a", 1, "A@x.io")], paths).expect("claims"); + assert_eq!( + sets, + vec![EngineKeySet::Unique { + collection: USERS.to_owned(), + index: "users_email".to_owned(), + values: SortedVec::new(vec![b"a@x.io".to_vec()]), + }] + ); + } + + #[test] + fn two_inserts_of_one_unique_value_serialize() { + let (first, second) = (TxnId::new(1, 0), TxnId::new(1, 1)); + let mut table = LockManager::new(); + assert_eq!( + table.acquire(first, claim_locks(&[insert("a", 1, "a@x.io")])), + AcquireOutcome::Ready + ); + assert_eq!( + table.acquire(second, claim_locks(&[insert("b", 2, "A@X.IO")])), + AcquireOutcome::Blocked, + "a second claim of one value waits for the first" + ); + let third = TxnId::new(1, 2); + assert_eq!( + table.acquire(third, claim_locks(&[insert("c", 4, "c@x.io")])), + AcquireOutcome::Ready, + "a claim of another value runs beside them" + ); + assert_eq!(table.release(first), vec![second]); + } + + #[test] + fn a_literal_update_claims_and_an_expression_update_locks_the_collection() { + let literal = nodedb_types::json_to_msgpack(&serde_json::json!("New@x.io")).expect("lit"); + let sets = + unique_claim_sets(&[update("a", UpdateValue::Literal(literal))], paths).expect("ok"); + assert_eq!( + sets, + vec![EngineKeySet::Unique { + collection: USERS.to_owned(), + index: "users_email".to_owned(), + values: SortedVec::new(vec![b"new@x.io".to_vec()]), + }] + ); + + let expr = UpdateValue::Expr(nodedb_query::expr::SqlExpr::Column("other".to_owned())); + let sets = unique_claim_sets(&[update("a", expr)], paths).expect("ok"); + assert_eq!( + sets, + vec![EngineKeySet::Collection { + collection: USERS.to_owned(), + vshards: SortedVec::new(Vec::new()), + }] + ); + } + + #[test] + fn a_collection_without_unique_indexes_claims_nothing() { + let plan = PhysicalPlan::Document(DocumentOp::PointInsert { + collection: QualifiedCollection::new(DatabaseId::DEFAULT, "plain"), + document_id: "a".to_owned(), + value: Vec::new(), + if_absent: false, + surrogate: Surrogate::new(1), + returning: None, + rls_filters: Vec::new(), + resolved_sum_targets: Vec::new(), + deferred_sum_targets: Vec::new(), + }); + assert!(unique_claim_sets(&[plan], paths).expect("ok").is_empty()); + } +} diff --git a/nodedb/src/control/planner/calvin/tx_class/mod.rs b/nodedb/src/control/planner/calvin/tx_class/mod.rs index a355244f9..1aa9dc862 100644 --- a/nodedb/src/control/planner/calvin/tx_class/mod.rs +++ b/nodedb/src/control/planner/calvin/tx_class/mod.rs @@ -6,20 +6,25 @@ //! task slice: the per-engine write set (`EngineKeySet` — document / vector //! surrogates, KV raw keys, graph-edge identity + routing homes, array tile //! vShards) plus the -//! msgpack-encoded plans. Four builders, split by shape (static vs -//! dependent-read) and participant floor (strict multi-vshard vs the +//! msgpack-encoded plans. The builders split by shape (static, predicted, +//! read-dependent) and participant floor (strict multi-vshard vs the //! single-vshard opt-in): //! //! - [`build_static_tx_class`] / [`build_single_vshard_tx_class`] — every //! write key is known upfront. -//! - [`build_dependent_tx_class`] / [`build_single_vshard_dependent_tx_class`] +//! - [`build_predicted_tx_class`] / [`build_single_vshard_predicted_tx_class`] //! — the OLLP collection's write set comes from reconnaissance-predicted //! surrogates; all other tasks use static extraction. +//! - [`build_read_dependent_tx_class`] — a cross-shard write whose values +//! depend on rows of another vShard. Its passive vShards broadcast those +//! rows to its active vShards through their data-group logs. -pub mod dependent_builder; +pub mod predicted_builder; +pub mod read_dependent_builder; pub mod shared; pub mod static_builder; pub mod write_keys; -pub use dependent_builder::{build_dependent_tx_class, build_single_vshard_dependent_tx_class}; +pub use predicted_builder::{build_predicted_tx_class, build_single_vshard_predicted_tx_class}; +pub use read_dependent_builder::{PassiveReads, build_read_dependent_tx_class}; pub use static_builder::{build_single_vshard_tx_class, build_static_tx_class}; diff --git a/nodedb/src/control/planner/calvin/tx_class/dependent_builder.rs b/nodedb/src/control/planner/calvin/tx_class/predicted_builder.rs similarity index 59% rename from nodedb/src/control/planner/calvin/tx_class/dependent_builder.rs rename to nodedb/src/control/planner/calvin/tx_class/predicted_builder.rs index ad8130841..46f86d2c2 100644 --- a/nodedb/src/control/planner/calvin/tx_class/dependent_builder.rs +++ b/nodedb/src/control/planner/calvin/tx_class/predicted_builder.rs @@ -1,15 +1,14 @@ // SPDX-License-Identifier: BUSL-1.1 -//! `TxClass` construction for a dependent-read (OLLP) transaction: the OLLP +//! `TxClass` construction for a predicted (OLLP) transaction: the OLLP //! collection's write set comes from reconnaissance-predicted surrogates; all //! other tasks use static surrogate extraction. //! -//! Despite the "dependent" naming (shared with the OLLP reconnaissance -//! terminology), the `TxClass` built here carries `dependent_reads: None` — -//! it is NOT a Calvin dependent-read-barrier transaction (see -//! [`TxClass::new_dependent`]; that is a distinct mechanism for passive -//! vshards to broadcast reads to active participants before they write). This -//! builder is write-set-identity construction only, exactly like +//! The `TxClass` built here carries `dependent_reads: None`. Its participant +//! verifies the prediction against its own rows at stage time. A transaction +//! whose writes depend on rows of another vShard is built by +//! [`super::read_dependent_builder`] instead. This builder is +//! write-set-identity construction only, exactly like //! `build_static_tx_class`, sourced from a pre-exec-scan surrogate //! prediction instead of statically-known plan fields. @@ -21,9 +20,11 @@ use nodedb_physical::physical_task::PhysicalTask; use nodedb_types::{DatabaseId, TenantId}; use super::shared::{read_set_from, versioned_reads_from}; -use super::write_keys::task_write_keys; +use super::write_keys::{WriteKeys, add_plan_write_keys, task_write_keys}; +use crate::control::planner::calvin::is_dependent_predicate; +use crate::control::planner::calvin::write_class::is_write_plan; -/// Build a **multi-vshard** `TxClass` for a dependent-read (OLLP) transaction. +/// Build a **multi-vshard** `TxClass` for a predicted (OLLP) transaction. /// /// For `BulkUpdate`/`BulkDelete` plans that have `ollp_predicted_surrogates` /// set, the OLLP collection's write set is built from `predicted_surrogates`. @@ -33,25 +34,25 @@ use super::write_keys::task_write_keys; /// produce a valid multi-vshard `TxClass`. A write set that collapses to a /// single vshard is rejected (`SingleVshardTxn`) — for the legitimate /// contended single-collection predicate write, use -/// [`build_single_vshard_dependent_tx_class`]. +/// [`build_single_vshard_predicted_tx_class`]. /// /// `reads` is the neutral session read-set, projected onto the `TxClass`'s /// routing/identity `read_set` (collection-homed) so read shards are enumerated -/// as participants, and onto `versioned_reads` (LSN-versioned OCC validation +/// as participants, and onto `versioned_reads` (write-versioned OCC validation /// set) for commit-time optimistic-concurrency validation. Autocommit paths /// pass an empty slice. /// /// Returns `Ok(None)` when the batch turns out to write nothing at all — the /// predicate matched no rows and no other task in the batch carries a write. /// Returns `Err` if encoding fails or the resulting TxClass is invalid. -pub fn build_dependent_tx_class( +pub fn build_predicted_tx_class( tasks: &[PhysicalTask], tenant_id: TenantId, collection: &str, predicted_surrogates: &[u32], reads: &[ReadSetEntry], ) -> crate::Result> { - build_dependent_tx_class_impl( + build_predicted_tx_class_impl( tasks, tenant_id, collection, @@ -61,7 +62,7 @@ pub fn build_dependent_tx_class( ) } -/// Build a `TxClass` for a dependent-read (OLLP) transaction that is permitted +/// Build a `TxClass` for a predicted (OLLP) transaction that is permitted /// to resolve to a **single vshard**. /// /// Used only by the contended single-collection predicate write routing path @@ -69,19 +70,19 @@ pub fn build_dependent_tx_class( /// gate returned `RouteToCalvin` because a pending commit holds a key in the /// predicate's range, so the write must sequence through the deterministic /// scheduler to serialize on the SAME shared per-vShard `LockManager` the -/// holder is on. Identical extraction to [`build_dependent_tx_class`]; only +/// holder is on. Identical extraction to [`build_predicted_tx_class`]; only /// the participant floor differs (via [`TxClass::new_single_vshard`] — the /// SAME opt-in the point-write path uses, since this `TxClass` shape is a /// plain write-set-identity construction like the static builder, not a /// Calvin dependent-read-barrier transaction). -pub fn build_single_vshard_dependent_tx_class( +pub fn build_single_vshard_predicted_tx_class( tasks: &[PhysicalTask], tenant_id: TenantId, collection: &str, predicted_surrogates: &[u32], reads: &[ReadSetEntry], ) -> crate::Result> { - build_dependent_tx_class_impl( + build_predicted_tx_class_impl( tasks, tenant_id, collection, @@ -91,12 +92,12 @@ pub fn build_single_vshard_dependent_tx_class( ) } -/// Shared body for the dependent builders. `allow_single_vshard` selects +/// Shared body for the predicted builders. `allow_single_vshard` selects /// between [`TxClass::new`] (multi-vshard, `>=2` floor) and /// [`TxClass::new_single_vshard`] (single-vshard opt-in) — the same pair /// `build_static_tx_class_impl` selects between; this builder only differs in /// how the write set's surrogate identity is sourced. -fn build_dependent_tx_class_impl( +fn build_predicted_tx_class_impl( tasks: &[PhysicalTask], tenant_id: TenantId, collection: &str, @@ -115,22 +116,23 @@ fn build_dependent_tx_class_impl( }); } + // A predicate that matched no rows, in a batch whose other tasks write + // nothing either, decides an empty statement: there is no state change to + // sequence, so no Calvin entry is proposed. The caller returns the + // statement's zero-row result instead of retrying. + if predicted_surrogates.is_empty() && !others_write(tasks, collection)? { + return Ok(None); + } + // Every other task's keys, extracted as `build_static_tx_class` extracts - // them. The OLLP collection's rows are the ones reconnaissance - // predicted, in place of the collection key its predicate write locks. + // them. The OLLP collection's rows are the ones reconnaissance predicted. + // Its predicate write keeps the whole-collection lock, so an insert that + // would join the predicate's match set waits for it. let mut keys = task_write_keys(tasks)?; keys.drop_documents(collection); keys.rows(collection, predicted_surrogates.iter().copied()); let write_set = ReadWriteSet::new(keys.into_key_sets()); - // A predicate that matched no rows, in a batch whose other tasks write - // nothing either, decides an empty write set: there is no state change to - // sequence, so no Calvin entry is proposed. `TxClass` rejects an empty - // write set, and that rejection is not a retryable condition — the caller - // returns the statement's zero-row result instead of retrying. - if write_set.is_empty() { - return Ok(None); - } // Populate the routing/identity read_set from the session read-set (a txn // that writes shard A but reads shard B enumerates B as a participant). An // empty `reads` slice yields an empty read_set. @@ -139,10 +141,10 @@ fn build_dependent_tx_class_impl( let plans: Vec<&PhysicalPlan> = tasks.iter().map(|t| &t.plan).collect(); let plans_bytes = zerompk::to_msgpack_vec(&plans).map_err(|e| Error::Serialization { format: "msgpack".to_owned(), - detail: format!("failed to encode PhysicalPlan vec for Calvin dependent TxClass: {e}"), + detail: format!("failed to encode PhysicalPlan vec for Calvin predicted TxClass: {e}"), })?; - // versioned_reads carries the LSN-versioned OCC validation set, populated + // versioned_reads carries the write-versioned OCC validation set, populated // from the same session read-set the routing `read_set` above was built // from. let versioned_reads = versioned_reads_from(reads); @@ -169,10 +171,24 @@ fn build_dependent_tx_class_impl( ) }; result.map(Some).map_err(|e| Error::BadRequest { - detail: format!("invalid dependent TxClass: {e}"), + detail: format!("invalid predicted TxClass: {e}"), }) } +/// Whether any task besides the OLLP predicate write on `collection` writes +/// a key. +fn others_write(tasks: &[PhysicalTask], collection: &str) -> crate::Result { + let mut keys = WriteKeys::default(); + let others = tasks.iter().filter(|task| { + is_write_plan(&task.plan) + && !(is_dependent_predicate(&task.plan) && task.plan.collection() == Some(collection)) + }); + for task in others { + add_plan_write_keys(&mut keys, &task.plan)?; + } + Ok(keys.into_key_sets().iter().any(|set| !set.is_empty())) +} + #[cfg(test)] mod tests { use super::*; @@ -211,16 +227,16 @@ mod tests { .as_u32(); // Strict builder rejects the single-vshard write set. - let strict = build_dependent_tx_class(&tasks, TenantId::new(1), "users", &[7, 8], &[]); + let strict = build_predicted_tx_class(&tasks, TenantId::new(1), "users", &[7, 8], &[]); assert!( matches!(strict, Err(crate::Error::BadRequest { .. })), - "strict dependent builder must reject single-vshard write set" + "strict predicted builder must reject single-vshard write set" ); // Single-vshard builder accepts it, with exactly one participating vshard. let tx = - build_single_vshard_dependent_tx_class(&tasks, TenantId::new(1), "users", &[7, 8], &[]) - .expect("single-vshard dependent TxClass accepted") + build_single_vshard_predicted_tx_class(&tasks, TenantId::new(1), "users", &[7, 8], &[]) + .expect("single-vshard predicted TxClass accepted") .expect("non-empty write set builds a TxClass"); assert_eq!(tx.participating_vshards().len(), 1); assert_eq!(tx.participating_vshards()[0].as_u32(), want_vshard); @@ -233,8 +249,82 @@ mod tests { // error the retry loop will mistake for predicate drift. let tasks = vec![bulk_delete_task("users")]; let built = - build_single_vshard_dependent_tx_class(&tasks, TenantId::new(1), "users", &[], &[]) + build_single_vshard_predicted_tx_class(&tasks, TenantId::new(1), "users", &[], &[]) .expect("zero-match predicate is not an error"); assert!(built.is_none(), "zero-match predicate builds no TxClass"); } + + /// The lock request the scheduler takes for `tx_class` at `position`. + fn locks( + tx_class: TxClass, + position: u32, + ) -> std::collections::BTreeMap< + crate::control::cluster::calvin::scheduler::LockKey, + crate::control::cluster::calvin::scheduler::LockMode, + > { + crate::control::cluster::calvin::scheduler::driver::helpers::expand_rw_set( + &nodedb_cluster::calvin::types::SequencedTxn { + epoch: 1, + position, + tx_class, + epoch_system_ms: 1_700_000_000_000, + epoch_vshard_txn_count: 2, + lock_owner: None, + }, + ) + } + + /// An OLLP predicate write locks its predicted rows and its whole + /// collection. An insert of a row the prediction never saw (a phantom) + /// waits for it, so the predicate cannot miss the row on one replica and + /// match it on another. + #[test] + fn an_ollp_predicate_write_blocks_a_phantom_insert() { + use crate::control::cluster::calvin::scheduler::lock_manager::TxnId; + use crate::control::cluster::calvin::scheduler::{AcquireOutcome, LockManager}; + + let predicate = build_single_vshard_predicted_tx_class( + &[bulk_delete_task("users")], + TenantId::new(1), + "users", + &[7], + &[], + ) + .expect("predicted TxClass") + .expect("non-empty write set"); + let insert_task = PhysicalTask { + tenant_id: TenantId::new(1), + vshard_id: VShardId::new(0), + database_id: DatabaseId::DEFAULT, + plan: PhysicalPlan::Document(DocumentOp::PointInsert { + collection: nodedb_types::QualifiedCollection::new(DatabaseId::DEFAULT, "users"), + document_id: "phantom".to_owned(), + value: Vec::new(), + if_absent: false, + surrogate: nodedb_types::Surrogate::new(99), + returning: None, + rls_filters: Vec::new(), + resolved_sum_targets: Vec::new(), + deferred_sum_targets: Vec::new(), + }), + post_set_op: nodedb_physical::physical_task::PostSetOp::None, + txn_id: None, + }; + let insert = + super::super::build_single_vshard_tx_class(&[insert_task], TenantId::new(1), &[]) + .expect("insert TxClass"); + + let (first, second) = (TxnId::new(1, 0), TxnId::new(1, 1)); + let mut table = LockManager::new(); + assert_eq!( + table.acquire(first, locks(predicate, 0)), + AcquireOutcome::Ready + ); + assert_eq!( + table.acquire(second, locks(insert, 1)), + AcquireOutcome::Blocked, + "a phantom insert waits for the predicate write" + ); + assert_eq!(table.release(first), vec![second]); + } } diff --git a/nodedb/src/control/planner/calvin/tx_class/read_dependent_builder.rs b/nodedb/src/control/planner/calvin/tx_class/read_dependent_builder.rs new file mode 100644 index 000000000..0db6b3899 --- /dev/null +++ b/nodedb/src/control/planner/calvin/tx_class/read_dependent_builder.rs @@ -0,0 +1,230 @@ +// SPDX-License-Identifier: BUSL-1.1 + +//! `TxClass` construction for a read-dependent transaction: a cross-shard +//! write whose values depend on rows of another vShard. +//! +//! The coordinator reads the rows first and plans the writes from what it +//! read. The class carries a [`DependentReadSpec`]: the rows each passive +//! vShard reads again under the transaction's locks, and the values the +//! coordinator read. Each active vShard waits at its dependent-read barrier +//! for every passive vShard's values, through its own data-group log, and +//! votes `PredictionDrift` when one differs. The coordinator then reads +//! again and resubmits. +//! +//! Every passive key set also enters the read set, so the passive vShard +//! locks the rows it reads shared until the transaction's verdict. + +use std::collections::BTreeMap; + +use nodedb_cluster::calvin::types::{ + DependentReadSpec, EngineKeySet, PassiveReadKey, PassiveReadKeyId, ReadWriteSet, TxClass, + VersionedReadSet, +}; +use nodedb_physical::physical_plan::PhysicalPlan; +use nodedb_physical::physical_task::PhysicalTask; +use nodedb_types::{DatabaseId, TenantId}; + +use super::write_keys::task_write_keys; +use crate::Error; + +/// The rows a read-dependent transaction reads on other vShards, and the +/// values its coordinator read for them. +#[derive(Debug, Clone, Default)] +pub struct PassiveReads { + /// Each passive vShard and the key sets it reads. + pub keys: BTreeMap>, + /// The stored bytes the coordinator read for each row, `None` for an + /// absent row. + pub expected: BTreeMap>>, +} + +/// Build a multi-vShard `TxClass` for `tasks`, whose values depend on +/// `reads`. +/// +/// The write set comes from the tasks' static write keys, as +/// [`super::build_static_tx_class`] derives it. The write set spans two or +/// more vShards. A `PassiveReads` with no key set is refused: the class +/// would carry a barrier that waits for nothing. +pub fn build_read_dependent_tx_class( + tasks: &[PhysicalTask], + tenant_id: TenantId, + reads: PassiveReads, +) -> crate::Result { + let database_id = tasks + .first() + .map_or(DatabaseId::DEFAULT, |task| task.database_id); + if tasks.iter().any(|task| task.database_id != database_id) { + return Err(Error::BadRequest { + detail: "Calvin transaction spans multiple databases".to_owned(), + }); + } + if reads.keys.values().all(Vec::is_empty) { + return Err(Error::Internal { + detail: "a read-dependent Calvin transaction names no row it reads".to_owned(), + }); + } + + let write_set = ReadWriteSet::new(task_write_keys(tasks)?.into_key_sets()); + let read_set = ReadWriteSet::new(reads.keys.values().flatten().cloned().collect()); + let passive_reads = reads + .keys + .into_iter() + .map(|(vshard, key_sets)| { + let keys = key_sets + .into_iter() + .map(|engine_key| PassiveReadKey { engine_key }) + .collect(); + (vshard, keys) + }) + .collect(); + let spec = DependentReadSpec { + passive_reads, + expected: reads.expected, + }; + + let plans: Vec<&PhysicalPlan> = tasks.iter().map(|t| &t.plan).collect(); + let plans_bytes = zerompk::to_msgpack_vec(&plans).map_err(|e| Error::Serialization { + format: "msgpack".to_owned(), + detail: format!("failed to encode PhysicalPlan vec for Calvin read-dependent TxClass: {e}"), + })?; + + TxClass::new_dependent( + read_set, + write_set, + plans_bytes, + tenant_id, + database_id, + spec, + VersionedReadSet::default(), + ) + .map_err(|e| Error::BadRequest { + detail: format!("invalid read-dependent TxClass: {e}"), + }) +} + +#[cfg(test)] +mod tests { + use nodedb_cluster::calvin::types::SortedVec; + use nodedb_physical::physical_plan::KvOp; + use nodedb_types::{QualifiedCollection, RlsWriteCheck, Surrogate}; + + use super::*; + use crate::types::VShardId; + + /// Two collection names whose default-database vShards differ. + fn two_distinct_collections() -> (String, String) { + let first = "read_dep_0".to_owned(); + let second = (1u32..4096) + .map(|i| format!("read_dep_{i}")) + .find(|name| vshard_of(name) != vshard_of(&first)) + .expect("a collection on another vShard"); + (first, second) + } + + fn kv_task(plan: KvOp, collection: &str) -> PhysicalTask { + PhysicalTask { + tenant_id: TenantId::new(1), + vshard_id: nodedb_types::CollectionKey::from_bare(DatabaseId::DEFAULT, collection) + .vshard(), + database_id: DatabaseId::DEFAULT, + plan: PhysicalPlan::Kv(plan), + post_set_op: nodedb_physical::physical_task::PostSetOp::None, + txn_id: None, + } + } + + fn vshard_of(collection: &str) -> u32 { + nodedb_types::CollectionKey::from_bare(DatabaseId::DEFAULT, collection) + .vshard() + .as_u32() + } + + /// The class carries the passive reads and the coordinator's values, + /// locks the passive rows shared, and spans the write vShards. + #[test] + fn a_read_dependent_class_carries_its_passive_reads() { + let (source, dest) = two_distinct_collections(); + let tasks = vec![ + kv_task( + KvOp::Delete { + collection: QualifiedCollection::new(DatabaseId::DEFAULT, &source), + keys: vec![b"alice:sword".to_vec()], + rls_write_check: RlsWriteCheck::pending_injection(), + returning: None, + rls_filters: Vec::new(), + provenance: None, + }, + &source, + ), + kv_task( + KvOp::Put { + collection: QualifiedCollection::new(DatabaseId::DEFAULT, &dest), + key: b"bob:sword".to_vec(), + value: b"item".to_vec(), + ttl_ms: 0, + surrogate: Surrogate::new(3), + returning: None, + rls_filters: Vec::new(), + provenance: None, + }, + &dest, + ), + ]; + let row = PassiveReadKeyId::kv( + QualifiedCollection::new(DatabaseId::DEFAULT, &source), + b"alice:sword".to_vec(), + ); + let reads = PassiveReads { + keys: BTreeMap::from([( + vshard_of(&source), + vec![EngineKeySet::Kv { + collection: source.clone(), + keys: SortedVec::new(vec![b"alice:sword".to_vec()]), + }], + )]), + expected: BTreeMap::from([(row.clone(), Some(b"item".to_vec()))]), + }; + + let class = + build_read_dependent_tx_class(&tasks, TenantId::new(1), reads).expect("valid class"); + + let spec = class.dependent_reads.as_ref().expect("dependent reads"); + assert_eq!( + spec.passive_reads.keys().copied().collect::>(), + vec![vshard_of(&source)] + ); + assert_eq!(spec.expected.get(&row), Some(&Some(b"item".to_vec()))); + assert_eq!(class.read_set.0.len(), 1); + let mut active = class.active_vshards().expect("active vShards"); + active.sort_unstable(); + let mut expected_active = vec![vshard_of(&source), vshard_of(&dest)]; + expected_active.sort_unstable(); + assert_eq!(active, expected_active); + assert!( + class + .participating_vshards() + .contains(&VShardId::new(vshard_of(&source))) + ); + } + + /// A class that names no passive row is refused. + #[test] + fn a_class_with_no_passive_row_is_refused() { + let (source, _) = two_distinct_collections(); + let tasks = vec![kv_task( + KvOp::Delete { + collection: QualifiedCollection::new(DatabaseId::DEFAULT, &source), + keys: vec![b"k".to_vec()], + rls_write_check: RlsWriteCheck::pending_injection(), + returning: None, + rls_filters: Vec::new(), + provenance: None, + }, + &source, + )]; + assert!( + build_read_dependent_tx_class(&tasks, TenantId::new(1), PassiveReads::default()) + .is_err() + ); + } +} diff --git a/nodedb/src/control/planner/calvin/tx_class/shared.rs b/nodedb/src/control/planner/calvin/tx_class/shared.rs index ea1f1f4ef..5cdd4efa4 100644 --- a/nodedb/src/control/planner/calvin/tx_class/shared.rs +++ b/nodedb/src/control/planner/calvin/tx_class/shared.rs @@ -4,20 +4,22 @@ //! dependent `TxClass` builders. The write keys live in //! [`super::write_keys`]. +use std::collections::{BTreeMap, BTreeSet}; + use crate::control::server::shared::session::read_set::{ReadKey, ReadSetEntry}; use nodedb_cluster::calvin::types::{ EngineKeySet, EngineTag, ReadKeyIdent, ReadWriteSet, SortedVec, VersionedReadEntry, VersionedReadSet, }; +use nodedb_types::KeyRepr; -/// Map the neutral session read-set into the replicated, LSN-versioned +/// Map the neutral session read-set into the replicated, versioned /// [`VersionedReadSet`] carried on the `TxClass`. /// /// Each [`ReadSetEntry`] becomes one [`VersionedReadEntry`], preserving -/// engine, collection, `read_version_lsn` (the collection's write floor — -/// the sound OCC comparand, not the core-global `read_lsn`), and the -/// point/predicate distinction. Own-overlay exclusion already happened at -/// capture time, so this is a faithful 1:1 projection. +/// engine, collection, read version, home, and the point/predicate +/// distinction. Own-overlay exclusion already happened at capture time, so +/// this is a faithful 1:1 projection. pub(super) fn versioned_reads_from(reads: &[ReadSetEntry]) -> VersionedReadSet { VersionedReadSet::new( reads @@ -38,98 +40,180 @@ pub(super) fn versioned_reads_from(reads: &[ReadSetEntry]) -> VersionedReadSet { hi: hi.clone(), }, }, - read_lsn: entry.read_version_lsn, + read_version: entry.read_version, home_vshard: entry.home.map(|home| home.as_u32()), - served_by: entry.home_node, }) .collect(), ) } -/// Build the routing/identity `read_set` for a Calvin `TxClass` from the -/// neutral session read-set — the key-IDENTITY set for participant/routing, -/// not the LSN-versioned OCC set ([`versioned_reads_from`]). +/// Build the routing and lock `read_set` for a Calvin `TxClass` from the +/// neutral session read-set. This is the key-IDENTITY set for participants +/// and locks, not the versioned OCC set ([`versioned_reads_from`]). +/// +/// Each read maps to the keys the scheduler locks `Shared`: +/// - A point read locks its row: a surrogate, a KV key, or an edge's two +/// node lock pairs. +/// - A predicate, index-equality, or index-range read locks its whole +/// collection. /// -/// A [`ReadSetEntry`] carries only `(engine, collection)`, no key identity, -/// so an unhomed entry maps to a COLLECTION-homed [`EngineKeySet`] with an -/// empty key vector. That over-approximates participants: more validation, -/// never a dropped participant. A homed entry (one vShard of a cross-shard -/// graph read) maps to an `EngineKeySet::Edge` with no edges and its home in -/// `home_vshards`, so the vShard it read participates and validates it. +/// An unhomed read participates on its collection's vShard. A homed read +/// (one vShard of a cross-shard graph read) participates on its home only. +/// A homed read with no collection name enlists its home and locks nothing. pub(super) fn read_set_from(reads: &[ReadSetEntry]) -> ReadWriteSet { - use std::collections::BTreeSet; - - // Dedup by (engine-variant, collection): many reads on one collection - // collapse to a single keyset, keeping the Raft-log read_set compact. - // Vector/KV keep their engine variant; every other engine routes by - // collection name via a Document keyset. - let mut vector_colls: BTreeSet = BTreeSet::new(); - let mut kv_colls: BTreeSet = BTreeSet::new(); - let mut doc_colls: BTreeSet = BTreeSet::new(); - // Homed reads (cross-shard graph reads) participate on their home vShards, - // not on the collection's vShard. They key no edge, so they lock nothing. - let mut homed: std::collections::BTreeMap> = - std::collections::BTreeMap::new(); + let mut locks = ReadLocks::default(); for entry in reads { - if let Some(home) = entry.home { - homed - .entry(entry.collection.clone()) - .or_default() - .insert(home.as_u32()); - continue; + match entry.home { + Some(home) => locks.homed(entry, home.as_u32()), + None if entry.collection.is_empty() => {} + None => locks.unhomed(entry), } - if entry.collection.is_empty() { - continue; + } + ReadWriteSet::new(locks.into_key_sets()) +} + +/// The edge reads of one collection. +#[derive(Default)] +struct EdgeReads { + /// The node lock pairs the reads touch. + pairs: Vec<(u32, u32)>, + /// The vShards the collection is read on. + homes: Vec, +} + +/// The read lock keys of a session read-set, by engine and collection. +/// Every map is ordered, so the key sets are identical on every node. +#[derive(Default)] +struct ReadLocks { + documents: BTreeMap>, + vectors: BTreeMap>, + kv: BTreeMap>>, + edges: BTreeMap, + /// Collection to the vShards it is read whole on. Empty: its own vShard. + whole: BTreeMap>, +} + +impl ReadLocks { + /// A read homed on `home`: it participates there only. + fn homed(&mut self, entry: &ReadSetEntry, home: u32) { + let collection = entry.collection.clone(); + if collection.is_empty() { + self.edges.entry(collection).or_default().homes.push(home); + return; } - match entry.engine { - EngineTag::Vector => { - vector_colls.insert(entry.collection.clone()); - } - EngineTag::Kv => { - kv_colls.insert(entry.collection.clone()); + match &entry.key { + ReadKey::Point { repr } => { + let edges = self.edges.entry(collection).or_default(); + edges.homes.push(home); + match repr { + KeyRepr::Edge { src, dst, .. } => { + edges + .pairs + .extend([node_lock_pair(src), node_lock_pair(dst)]); + } + // A row key routes by its collection, not by this home, + // so a homed row read only enlists its home. + KeyRepr::Surrogate(_) | KeyRepr::KvKey(_) => {} + } } - EngineTag::Document - | EngineTag::Graph - | EngineTag::Text - | EngineTag::Columnar - | EngineTag::Timeseries - | EngineTag::Spatial - | EngineTag::Crdt - | EngineTag::Query - | EngineTag::Meta - | EngineTag::Array - | EngineTag::ClusterArray => { - doc_colls.insert(entry.collection.clone()); + ReadKey::Predicate | ReadKey::IndexEq { .. } | ReadKey::IndexRange { .. } => { + self.whole.entry(collection).or_default().insert(home); } } } - let mut sets: Vec = Vec::new(); - for collection in vector_colls { - sets.push(EngineKeySet::Vector { - collection, - surrogates: SortedVec::new(vec![]), - }); - } - for collection in kv_colls { - sets.push(EngineKeySet::Kv { - collection, - keys: SortedVec::new(vec![]), - }); - } - for collection in doc_colls { - sets.push(EngineKeySet::Document { - collection, - surrogates: SortedVec::new(vec![]), - }); + + /// A read of a named collection on the collection's own vShard. + fn unhomed(&mut self, entry: &ReadSetEntry) { + let collection = entry.collection.clone(); + match &entry.key { + ReadKey::Point { + repr: KeyRepr::Surrogate(surrogate), + } => { + let rows = match entry.engine { + EngineTag::Vector => &mut self.vectors, + EngineTag::Document + | EngineTag::Graph + | EngineTag::Kv + | EngineTag::Text + | EngineTag::Columnar + | EngineTag::Timeseries + | EngineTag::Spatial + | EngineTag::Crdt + | EngineTag::Query + | EngineTag::Meta + | EngineTag::Array + | EngineTag::ClusterArray => &mut self.documents, + }; + rows.entry(collection).or_default().push(*surrogate); + } + ReadKey::Point { + repr: KeyRepr::KvKey(key), + } => self.kv.entry(collection).or_default().push(key.to_vec()), + // An edge locks by node pairs, which route by key home. The + // collection's own vShard is enlisted by an empty document set. + ReadKey::Point { + repr: KeyRepr::Edge { src, dst, .. }, + } => { + self.edges + .entry(collection.clone()) + .or_default() + .pairs + .extend([node_lock_pair(src), node_lock_pair(dst)]); + self.documents.entry(collection).or_default(); + } + ReadKey::Predicate | ReadKey::IndexEq { .. } | ReadKey::IndexRange { .. } => { + self.whole.entry(collection).or_default(); + } + } } - for (collection, homes) in homed { - sets.push(EngineKeySet::Edge { - collection, - edges: SortedVec::new(vec![]), - home_vshards: SortedVec::new(homes.into_iter().collect()), - }); + + /// The key sets, one per engine and collection, ordered by collection. + fn into_key_sets(self) -> Vec { + let mut sets: Vec = Vec::new(); + sets.extend( + self.documents + .into_iter() + .map(|(collection, rows)| EngineKeySet::Document { + collection, + surrogates: SortedVec::new(rows), + }), + ); + sets.extend( + self.vectors + .into_iter() + .map(|(collection, rows)| EngineKeySet::Vector { + collection, + surrogates: SortedVec::new(rows), + }), + ); + sets.extend( + self.kv + .into_iter() + .map(|(collection, keys)| EngineKeySet::Kv { + collection, + keys: SortedVec::new(keys), + }), + ); + sets.extend( + self.edges + .into_iter() + .map(|(collection, reads)| EngineKeySet::Edge { + collection, + edges: SortedVec::new(reads.pairs), + home_vshards: SortedVec::new(reads.homes), + }), + ); + sets.extend( + self.whole + .into_iter() + .map(|(collection, vshards)| EngineKeySet::Collection { + collection, + vshards: SortedVec::new(vshards.into_iter().collect()), + }), + ); + sets.sort_by(|a, b| a.collection().cmp(b.collection())); + sets } - ReadWriteSet::new(sets) } /// The lock pair of node `node` within one edge collection. @@ -150,11 +234,12 @@ pub(crate) fn node_lock_pair(node: &str) -> (u32, u32) { #[cfg(test)] mod lockstep_tests { use super::*; - use crate::control::cluster::calvin::scheduler::lock_manager::LockKey; + use crate::control::cluster::calvin::scheduler::driver::helpers::expand_rw_set; + use crate::control::cluster::calvin::scheduler::lock_manager::{LockKey, LockMode}; use crate::control::planner::calvin::tx_class::static_builder::build_single_vshard_tx_class; use crate::control::server::shared::write_admission::lock_keys::plan_lock_keys; use crate::types::{DatabaseId, TenantId, VShardId}; - use nodedb_cluster::calvin::types::EngineKeySet; + use nodedb_cluster::calvin::types::{SequencedTxn, TxClass}; use nodedb_physical::physical_plan::{DocumentOp, GraphOp, KvOp, PhysicalPlan}; use nodedb_physical::physical_task::{PhysicalTask, PostSetOp}; use nodedb_types::Surrogate; @@ -172,64 +257,22 @@ mod lockstep_tests { } } - /// Mirrors the scheduler's `expand_rw_set` `EngineKeySet` → `LockKey` - /// mapping (fixed translation). What's under test is whether the - /// extractor threads the same `(collection, key/surrogate)` as the - /// gate's `plan_lock_keys`, not this mapping. - fn scheduler_lock_keys(sets: &[EngineKeySet]) -> BTreeSet { - let mut keys = BTreeSet::new(); - for ks in sets { - match ks { - EngineKeySet::Document { - collection, - surrogates, - } - | EngineKeySet::Vector { - collection, - surrogates, - } => { - let coll: Arc = Arc::from(collection.as_str()); - for &surrogate in surrogates.iter() { - keys.insert(LockKey::Surrogate { - collection: Arc::clone(&coll), - surrogate, - }); - } - } - EngineKeySet::Kv { - collection, - keys: kv_keys, - } => { - let coll: Arc = Arc::from(collection.as_str()); - for k in kv_keys.iter() { - keys.insert(LockKey::Kv { - collection: Arc::clone(&coll), - key: Arc::from(k.as_slice()), - }); - } - } - EngineKeySet::Edge { - collection, edges, .. - } => { - let coll: Arc = Arc::from(collection.as_str()); - for &(src, dst) in edges.iter() { - keys.insert(LockKey::Edge { - collection: Arc::clone(&coll), - src, - dst, - }); - } - } - EngineKeySet::Array { collection, .. } => { - keys.insert(LockKey::Surrogate { - collection: Arc::from(collection.as_str()), - surrogate: - crate::control::planner::calvin::tx_class::write_keys::COLLECTION_KEY, - }); - } - } - } - keys + /// The row keys the scheduler locks `Exclusive` for `tx`, from its real + /// `expand_rw_set`. The collection's `Intent` key is left out: the gate + /// fences row keys only. + fn scheduler_lock_keys(tx: TxClass) -> BTreeSet { + expand_rw_set(&SequencedTxn { + epoch: 1, + position: 0, + tx_class: tx, + epoch_system_ms: 1_700_000_000_000, + epoch_vshard_txn_count: 1, + lock_owner: None, + }) + .into_iter() + .filter(|(_, mode)| *mode == LockMode::Exclusive) + .map(|(key, _)| key) + .collect() } fn assert_gate_matches_scheduler(plan: PhysicalPlan) { @@ -238,7 +281,7 @@ mod lockstep_tests { plan_lock_keys(&t.plan).expect("op must be fast-path eligible for this test"); let tx = build_single_vshard_tx_class(&[t], TenantId::new(1), &[]) .expect("valid single-vshard TxClass"); - let scheduler_keys = scheduler_lock_keys(&tx.write_set.0); + let scheduler_keys = scheduler_lock_keys(tx); assert_eq!( gate_keys, scheduler_keys, "gate and scheduler must lock the identical key set" diff --git a/nodedb/src/control/planner/calvin/tx_class/static_builder.rs b/nodedb/src/control/planner/calvin/tx_class/static_builder.rs index 6b18326de..00178eb3f 100644 --- a/nodedb/src/control/planner/calvin/tx_class/static_builder.rs +++ b/nodedb/src/control/planner/calvin/tx_class/static_builder.rs @@ -26,7 +26,7 @@ use super::write_keys::task_write_keys; /// `reads` is the neutral session read-set captured during the transaction; it /// is projected onto the `TxClass`'s routing/identity `read_set` (collection- /// homed) so a txn that writes shard A but reads shard B enumerates B as a -/// participant, and onto `versioned_reads` (LSN-versioned OCC validation set) +/// participant, and onto `versioned_reads` (write-versioned OCC validation set) /// for commit-time optimistic-concurrency validation. Autocommit and /// pure-write paths pass an empty slice, yielding an empty read_set and /// versioned_reads. @@ -102,8 +102,8 @@ fn build_static_tx_class_impl( // must NOT enter the OCC read set. The txn's own staged write advances that // collection's write floor, so validating the earlier read against it will // flag it stale and abort the commit — a false serialization conflict. This - // mirrors the written-collection exclusion the single-shard - // `si_conflict_abort` path already applies. + // mirrors the written-collection exclusion the non-Calvin commit check + // (`session::commit::read_validation`) applies. // // A PLAN-DERIVATION read is exempt, and the exemption is what makes the // derivation guard exist at all. Such a read observed COMMITTED base state @@ -144,7 +144,7 @@ fn build_static_tx_class_impl( detail: format!("failed to encode PhysicalPlan vec for Calvin TxClass: {e}"), })?; - // versioned_reads carries the LSN-versioned OCC validation set, populated + // versioned_reads carries the write-versioned OCC validation set, populated // from the same session read-set the routing `read_set` above was built // from. let versioned_reads = versioned_reads_from(&owned_reads); @@ -179,7 +179,7 @@ fn build_static_tx_class_impl( mod tests { use super::*; use crate::control::server::shared::session::read_set::{EngineTag, ReadKey, ReadOrigin}; - use crate::types::{DatabaseId, KeyRepr, Lsn, VShardId}; + use crate::types::{DatabaseId, KeyRepr, VShardId}; use nodedb_physical::physical_plan::DocumentOp; use nodedb_types::Surrogate; @@ -222,14 +222,14 @@ mod tests { } } - fn read_entry(collection: &str, key: ReadKey, read_lsn: u64) -> ReadSetEntry { - origin_read_entry(collection, key, read_lsn, ReadOrigin::Session) + fn read_entry(collection: &str, key: ReadKey, read_index: u64) -> ReadSetEntry { + origin_read_entry(collection, key, read_index, ReadOrigin::Session) } fn origin_read_entry( collection: &str, key: ReadKey, - read_lsn: u64, + read_index: u64, origin: ReadOrigin, ) -> ReadSetEntry { ReadSetEntry { @@ -238,13 +238,9 @@ mod tests { tenant_id: TenantId::new(1), collection: collection.to_owned(), key, - read_lsn: Lsn::new(read_lsn), - // The per-collection read-version is the OCC comparand - // `versioned_reads_from` propagates; give it the same synthetic LSN. - read_version_lsn: Lsn::new(read_lsn), + read_version: nodedb_types::WriteVersion::logged(0, read_index), origin, home: None, - home_node: 0, } } @@ -274,8 +270,8 @@ mod tests { let (col_a, col_b) = two_distinct_collections(); let tasks = vec![point_insert_task(&col_a, 1), point_insert_task(&col_b, 2)]; - // Synthetic read-set: one point read (surrogate identity) at LSN 7 and - // one collection-scoped predicate read at LSN 11, on two collections + // Synthetic read-set: one point read (surrogate identity) at index 7 and + // one collection-scoped predicate read at index 11, on two collections // distinct from the write collections. let reads = vec![ read_entry( @@ -291,7 +287,7 @@ mod tests { let tx = build_static_tx_class(&tasks, TenantId::new(1), &reads) .expect("valid multi-vShard TxClass"); - // versioned_reads (the LSN-versioned OCC validation set) carries the + // versioned_reads (the versioned OCC validation set) carries the // same session reads, 1:1, for commit-time validation. assert_eq!( tx.versioned_reads.len(), @@ -300,7 +296,7 @@ mod tests { ); for (entry, read) in tx.versioned_reads.iter().zip(reads.iter()) { assert_eq!(entry.collection, read.collection); - assert_eq!(entry.read_lsn, read.read_version_lsn); + assert_eq!(entry.read_version, read.read_version); let expected_key = match &read.key { ReadKey::Point { repr } => { nodedb_cluster::calvin::types::ReadKeyIdent::Point(repr.clone()) @@ -361,6 +357,88 @@ mod tests { } } + /// The lock request the scheduler takes for `tx_class` at `position`. + fn locks( + tx_class: TxClass, + position: u32, + ) -> std::collections::BTreeMap< + crate::control::cluster::calvin::scheduler::LockKey, + crate::control::cluster::calvin::scheduler::LockMode, + > { + crate::control::cluster::calvin::scheduler::driver::helpers::expand_rw_set( + &nodedb_cluster::calvin::types::SequencedTxn { + epoch: 1, + position, + tx_class, + epoch_system_ms: 1_700_000_000_000, + epoch_vshard_txn_count: 2, + lock_owner: None, + }, + ) + } + + /// A predicate read locks its whole collection `Shared`. A later row + /// writer of that collection takes `Intent` on it, so it waits until the + /// reader releases. A point read locks only its row. + #[test] + fn a_predicate_read_blocks_a_later_writer_until_released() { + use crate::control::cluster::calvin::scheduler::LockMode; + use crate::control::cluster::calvin::scheduler::lock_manager::TxnId; + use crate::control::cluster::calvin::scheduler::{AcquireOutcome, LockKey, LockManager}; + + let (col_a, col_b) = two_distinct_collections(); + let tasks = vec![point_insert_task(&col_a, 1), point_insert_task(&col_b, 2)]; + let reads = vec![ + read_entry("scan_col", ReadKey::Predicate, 11), + read_entry( + "read_col", + ReadKey::Point { + repr: KeyRepr::Surrogate(9), + }, + 10, + ), + ]; + let reader = build_static_tx_class(&tasks, TenantId::new(1), &reads) + .expect("valid multi-vShard TxClass"); + let reader_locks = locks(reader, 0); + assert_eq!( + reader_locks.get(&LockKey::Collection { + collection: std::sync::Arc::from("scan_col"), + }), + Some(&LockMode::Shared) + ); + assert_eq!( + reader_locks.get(&LockKey::Surrogate { + collection: std::sync::Arc::from("read_col"), + surrogate: 9, + }), + Some(&LockMode::Shared) + ); + assert!( + !reader_locks.contains_key(&LockKey::Collection { + collection: std::sync::Arc::from("read_col"), + }), + "a point read locks nothing collection-wide" + ); + + let writer = build_single_vshard_tx_class( + &[point_insert_task("scan_col", 5)], + TenantId::new(1), + &[], + ) + .expect("writer TxClass"); + + let (first, second) = (TxnId::new(1, 0), TxnId::new(1, 1)); + let mut table = LockManager::new(); + assert_eq!(table.acquire(first, reader_locks), AcquireOutcome::Ready); + assert_eq!( + table.acquire(second, locks(writer, 1)), + AcquireOutcome::Blocked, + "a row writer waits for the predicate reader of its collection" + ); + assert_eq!(table.release(first), vec![second]); + } + #[test] fn empty_read_set_yields_empty_read_and_versioned_reads() { let (col_a, col_b) = two_distinct_collections(); @@ -381,7 +459,7 @@ mod tests { /// A PLAN-DERIVATION read of a collection the transaction WRITES survives /// the own-write exclusion, reaching BOTH the routing read_set and the - /// LSN-versioned `versioned_reads` the Data Plane's + /// write-versioned `versioned_reads` the Data Plane's /// `read_set_still_current` check consumes. /// /// This is the whole of the cross-shard materialized-sum guard: the delta @@ -412,7 +490,10 @@ mod tests { "a derivation read must reach versioned_reads — that set IS the guard" ); assert_eq!(tx.versioned_reads.0[0].collection, col_a); - assert_eq!(tx.versioned_reads.0[0].read_lsn, Lsn::new(42)); + assert_eq!( + tx.versioned_reads.0[0].read_version, + nodedb_types::WriteVersion::logged(0, 42) + ); assert_eq!( tx.versioned_reads.0[0].key, nodedb_cluster::calvin::types::ReadKeyIdent::Point(KeyRepr::Surrogate(11)) diff --git a/nodedb/src/control/planner/calvin/tx_class/write_keys/columnar.rs b/nodedb/src/control/planner/calvin/tx_class/write_keys/columnar.rs index f815bdb73..5030ef1e5 100644 --- a/nodedb/src/control/planner/calvin/tx_class/write_keys/columnar.rs +++ b/nodedb/src/control/planner/calvin/tx_class/write_keys/columnar.rs @@ -12,8 +12,8 @@ use super::set::WriteKeys; /// Add the write keys of columnar op `op` to `keys`. pub(super) fn add_columnar_keys(keys: &mut WriteKeys, op: &ColumnarOp) -> crate::Result<()> { match op { - // Every row it inserts, and the collection key a multi-row write - // holds. + // Every row it inserts, and the whole collection, so inserts append + // to the collection in sequencer order. ColumnarOp::Insert { collection, surrogates, diff --git a/nodedb/src/control/planner/calvin/tx_class/write_keys/crdt.rs b/nodedb/src/control/planner/calvin/tx_class/write_keys/crdt.rs index 743e5805a..9c23a4e7e 100644 --- a/nodedb/src/control/planner/calvin/tx_class/write_keys/crdt.rs +++ b/nodedb/src/control/planner/calvin/tx_class/write_keys/crdt.rs @@ -5,6 +5,13 @@ //! A CRDT document is keyed by its document id, bound or not: a delete //! planned before a concurrent upsert bound the document locks the key the //! upsert locks. A write of the whole collection locks the collection key. +//! +//! A constraint install or drop changes no row. It replaces the validator +//! rules a CRDT delta passes at its log apply, and each delta carries the +//! constraint version it was admitted against: the apply fences a delta whose +//! version the replica has not installed yet. So the install needs no +//! exclusion from row writers, and it takes its collection `Intent` like one. +//! It still orders against a writer of the whole collection. #![deny(clippy::wildcard_enum_match_arm)] @@ -56,9 +63,10 @@ pub(super) fn add_keys(keys: &mut WriteKeys, op: &CrdtOp) -> crate::Result<()> { document_id, .. } => keys.row_id(collection.as_str(), document_id), - CrdtOp::ImportSnapshot { collection, .. } - | CrdtOp::SetConstraints { collection, .. } - | CrdtOp::DropConstraints { collection, .. } => keys.whole_collection(collection.as_str()), + CrdtOp::ImportSnapshot { collection, .. } => keys.whole_collection(collection.as_str()), + CrdtOp::SetConstraints { collection, .. } | CrdtOp::DropConstraints { collection, .. } => { + keys.rows(collection.as_str(), []) + } CrdtOp::Read { .. } | CrdtOp::ReadConstraints { .. } | CrdtOp::SetPolicy { .. } @@ -73,3 +81,45 @@ pub(super) fn add_keys(keys: &mut WriteKeys, op: &CrdtOp) -> crate::Result<()> { } Ok(()) } + +#[cfg(test)] +mod tests { + use nodedb_types::{DatabaseId, QualifiedCollection}; + + use super::*; + use crate::control::cluster::calvin::scheduler::driver::helpers::expand_write_key_sets; + use crate::control::cluster::calvin::scheduler::lock::{LockKey, LockMode}; + + fn locks(op: &CrdtOp) -> Vec<(LockKey, LockMode)> { + let mut keys = WriteKeys::default(); + add_keys(&mut keys, op).expect("a write"); + expand_write_key_sets(&keys.into_key_sets()) + .into_iter() + .collect() + } + + fn collection_key() -> LockKey { + LockKey::Collection { + collection: std::sync::Arc::from("docs"), + } + } + + /// A constraint install takes its collection `Intent` alone, so it runs + /// beside a row writer of the collection. + #[test] + fn a_constraint_install_takes_its_collection_intent() { + let collection = QualifiedCollection::new(DatabaseId::DEFAULT, "docs"); + let set = CrdtOp::SetConstraints { + collection: collection.clone(), + constraint_version: 1, + constraints: Vec::new(), + }; + let drop = CrdtOp::DropConstraints { + collection, + constraint_version: 2, + }; + for op in [set, drop] { + assert_eq!(locks(&op), vec![(collection_key(), LockMode::Intent)]); + } + } +} diff --git a/nodedb/src/control/planner/calvin/tx_class/write_keys/document.rs b/nodedb/src/control/planner/calvin/tx_class/write_keys/document.rs index ebdd76bc1..ee6bd7c7f 100644 --- a/nodedb/src/control/planner/calvin/tx_class/write_keys/document.rs +++ b/nodedb/src/control/planner/calvin/tx_class/write_keys/document.rs @@ -43,10 +43,9 @@ pub(super) fn add_keys(keys: &mut WriteKeys, op: &DocumentOp) -> crate::Result<( surrogate, .. } => keys.rows(collection.as_str(), [surrogate.as_u32()]), - // A key unbound in its database names no row yet: it takes the - // collection key and its row id. The row id orders it after an - // insert that binds the key, so its rebind at dispatch finds that - // binding on every replica. + // A key unbound in its database names no row yet: it locks only its + // row id. The row id orders it after an insert that binds the key, so + // its rebind at dispatch finds that binding on every replica. DocumentOp::PointDelete { collection, document_id, @@ -59,14 +58,12 @@ pub(super) fn add_keys(keys: &mut WriteKeys, op: &DocumentOp) -> crate::Result<( surrogate, .. } => { - match surrogate { - Some(surrogate) => keys.rows(collection.as_str(), [surrogate.as_u32()]), - None => keys.whole_collection(collection.as_str()), + if let Some(surrogate) = surrogate { + keys.rows(collection.as_str(), [surrogate.as_u32()]); } keys.row_id(collection.as_str(), document_id); } - // Every row it inserts, by surrogate and row id, and the collection - // key a multi-row write holds. + // Every row it inserts, by surrogate and row id. DocumentOp::BatchInsert { collection, documents, @@ -77,7 +74,6 @@ pub(super) fn add_keys(keys: &mut WriteKeys, op: &DocumentOp) -> crate::Result<( for (document_id, _) in documents { keys.row_id(collection.as_str(), document_id); } - keys.whole_collection(collection.as_str()); } DocumentOp::InsertSelect { target_collection, .. diff --git a/nodedb/src/control/planner/calvin/tx_class/write_keys/graph.rs b/nodedb/src/control/planner/calvin/tx_class/write_keys/graph.rs index 2c66de50d..58c935872 100644 --- a/nodedb/src/control/planner/calvin/tx_class/write_keys/graph.rs +++ b/nodedb/src/control/planner/calvin/tx_class/write_keys/graph.rs @@ -61,12 +61,13 @@ pub(super) fn add_keys(keys: &mut WriteKeys, op: &GraphOp) -> crate::Result<()> } => keys.node(collection.as_str(), node_id), // A TRUNCATE's edge share locks no edge: it records a cut at its // transaction's ordinal, which hides every version applied below it - // whatever order this vShard flushes the edge writes in. + // whatever order this vShard installs the edge writes in. GraphOp::TruncateEdges { collection, vshard } => keys.home(collection.as_str(), *vshard), // A CRDT delete's presence guard locks every document it names, - // stored or not, by the key every writer of that document locks, - // and the collection key every collection-wide write locks. So it - // runs after every lower writer of those documents flushed. + // stored or not, by the key every writer of that document locks. Its + // `Intent` on the collection orders it against every collection-wide + // write. So it runs after every lower writer of those documents + // installed. GraphOp::NodePresenceGuard { collection, vshard, @@ -78,7 +79,6 @@ pub(super) fn add_keys(keys: &mut WriteKeys, op: &GraphOp) -> crate::Result<()> collection.as_str(), present.iter().chain(absent.iter()).map(|id| row_id_key(id)), ); - keys.whole_collection(collection.as_str()); } GraphOp::ResolveEdgeDelete(_) | GraphOp::Hop { .. } diff --git a/nodedb/src/control/planner/calvin/tx_class/write_keys/kv.rs b/nodedb/src/control/planner/calvin/tx_class/write_keys/kv.rs index 9e5974052..ff0227a25 100644 --- a/nodedb/src/control/planner/calvin/tx_class/write_keys/kv.rs +++ b/nodedb/src/control/planner/calvin/tx_class/write_keys/kv.rs @@ -61,16 +61,12 @@ pub(super) fn add_keys(keys: &mut WriteKeys, op: &KvOp) -> crate::Result<()> { dest_key, .. } => keys.kv_keys(collection.as_str(), [source_key.clone(), dest_key.clone()]), - // Every key it writes, and the collection key a multi-row write - // holds. + // Every key it writes. KvOp::BatchPut { collection, entries, .. - } => { - keys.kv_keys(collection.as_str(), entries.iter().map(|(k, _)| k.clone())); - keys.whole_collection(collection.as_str()); - } + } => keys.kv_keys(collection.as_str(), entries.iter().map(|(k, _)| k.clone())), KvOp::Truncate { collection, .. } | KvOp::PredicateUpdate { collection, .. } | KvOp::PredicateDelete { collection, .. } => keys.whole_collection(collection.as_str()), diff --git a/nodedb/src/control/planner/calvin/tx_class/write_keys/mod.rs b/nodedb/src/control/planner/calvin/tx_class/write_keys/mod.rs index 21df48d04..0d79c6654 100644 --- a/nodedb/src/control/planner/calvin/tx_class/write_keys/mod.rs +++ b/nodedb/src/control/planner/calvin/tx_class/write_keys/mod.rs @@ -14,4 +14,4 @@ pub mod set; pub mod vector; pub use plan::{add_plan_write_keys, task_write_keys}; -pub use set::{COLLECTION_KEY, WriteKeys, row_id_key}; +pub use set::{WriteKeys, row_id_key}; diff --git a/nodedb/src/control/planner/calvin/tx_class/write_keys/plan.rs b/nodedb/src/control/planner/calvin/tx_class/write_keys/plan.rs index e02396acd..e1236fb40 100644 --- a/nodedb/src/control/planner/calvin/tx_class/write_keys/plan.rs +++ b/nodedb/src/control/planner/calvin/tx_class/write_keys/plan.rs @@ -24,9 +24,9 @@ pub fn task_write_keys(tasks: &[PhysicalTask]) -> crate::Result { /// Add the Calvin write keys of `plan` to `keys`. /// /// Every write op of every engine is matched by name. A write with row -/// identity locks its rows. A write without one locks -/// [`super::COLLECTION_KEY`] of its collection. A graph write locks its -/// edges and nodes on their key homes. +/// identity locks its rows, and the scheduler adds an `Intent` lock on their +/// collection. A write without one locks its whole collection. A graph write +/// locks its edges and nodes on their key homes. /// /// A write no single transaction can sequence, and a plan that writes /// nothing, is refused with a typed error. The caller never builds a @@ -75,7 +75,7 @@ pub(super) fn unsequenced(reason: &str) -> crate::Error { /// guard is sound only when the higher transaction waits for the lower one. #[cfg(test)] mod tests { - use std::collections::BTreeSet; + use std::collections::BTreeMap; use nodedb_cluster::calvin::types::SequencedTxn; use nodedb_physical::physical_plan::{ @@ -86,7 +86,9 @@ mod tests { use super::*; use crate::control::cluster::calvin::scheduler::driver::helpers::expand_rw_set; - use crate::control::cluster::calvin::scheduler::{AcquireOutcome, LockKey, LockManager, TxnId}; + use crate::control::cluster::calvin::scheduler::{ + AcquireOutcome, LockKey, LockManager, LockMode, TxnId, + }; use crate::control::planner::calvin::tx_class::build_single_vshard_tx_class; use crate::types::{DatabaseId, RecordHomes, TenantId, VShardId}; @@ -110,7 +112,7 @@ mod tests { } /// The lock keys the scheduler takes for a transaction of `plans`. - fn locks(plans: Vec, position: u32) -> BTreeSet { + fn locks(plans: Vec, position: u32) -> BTreeMap { let tasks: Vec = plans.into_iter().map(task).collect(); let tx_class = build_single_vshard_tx_class(&tasks, TenantId::new(1), &[]) .expect("a valid transaction class"); @@ -199,7 +201,7 @@ mod tests { /// An upsert binds and stores `x` at a lower position. A delete planned /// while `x` was unbound names it absent and carries an unbound delete. - /// Its guard must run after the upsert flushed, see `x` stored, and + /// Its guard must run after the upsert installed, see `x` stored, and /// retry; before, it ran first and the delete was lost. #[test] fn a_crdt_delete_of_an_unbound_document_waits_for_its_upsert() { @@ -299,9 +301,9 @@ mod tests { /// An insert binds and stores `x` at a lower position. A delete or /// update planned while `x` was unbound must wait for it: its rebind at - /// dispatch then finds the binding on every replica. Before, it locked - /// only the collection key and can dispatch first on one replica and - /// second on another. + /// dispatch then finds the binding on every replica. Its row id lock + /// orders it, so it never dispatches first on one replica and second on + /// another. #[test] fn an_unbound_document_delete_or_update_waits_for_an_insert_of_its_id() { assert!(higher_waits( @@ -324,6 +326,45 @@ mod tests { assert!(higher_waits(vec![batch], vec![doc_delete("x", None)])); } + fn doc_truncate() -> PhysicalPlan { + PhysicalPlan::Document(DocumentOp::Truncate { + collection: qualified(DOCS), + restart_identity: false, + resolved_sum_targets: Vec::new(), + declared_primary_key: None, + }) + } + + /// A truncate locks its collection `Exclusive`, and a point write takes + /// `Intent` on it. So they serialize in sequencer order, whichever comes + /// first. + #[test] + fn a_truncate_and_a_point_write_serialize_in_sequence_order() { + assert!(higher_waits( + vec![doc_truncate()], + vec![doc_insert("x", 17)] + )); + assert!(higher_waits( + vec![doc_insert("x", 17)], + vec![doc_truncate()] + )); + assert!(higher_waits( + vec![doc_delete("x", None)], + vec![doc_truncate()] + )); + let collection_key = LockKey::Collection { + collection: std::sync::Arc::from(DOCS), + }; + assert_eq!( + locks(vec![doc_truncate()], 0).get(&collection_key), + Some(&LockMode::Exclusive) + ); + assert_eq!( + locks(vec![doc_insert("x", 17)], 0).get(&collection_key), + Some(&LockMode::Intent) + ); + } + /// Every point writer of one document id serializes, bound or not, and /// writers of distinct ids do not wait on each other. #[test] diff --git a/nodedb/src/control/planner/calvin/tx_class/write_keys/set.rs b/nodedb/src/control/planner/calvin/tx_class/write_keys/set.rs index bed524f56..4e1125ae5 100644 --- a/nodedb/src/control/planner/calvin/tx_class/write_keys/set.rs +++ b/nodedb/src/control/planner/calvin/tx_class/write_keys/set.rs @@ -2,18 +2,13 @@ //! The accumulated Calvin write keys of a task slice. -use std::collections::BTreeMap; +use std::collections::{BTreeMap, BTreeSet}; use nodedb_cluster::calvin::types::{EngineKeySet, SortedVec}; use super::super::shared::node_lock_pair; use crate::types::{RecordHomes, VShardId}; -/// The row surrogate a write with no single row identity locks: a -/// predicate, bulk, truncate or multi-row write, or a point write whose key -/// is unbound. Every such write of a collection serializes on it. -pub const COLLECTION_KEY: u32 = 0; - /// The lock key of the row `document_id` names, bound or not: the bytes its /// surrogate binds under (`bind_plan`). /// @@ -43,6 +38,8 @@ pub struct WriteKeys { edges: BTreeMap, /// Array collection to the vShards its written cells live on. arrays: BTreeMap>, + /// Collections a write covers whole. + whole: BTreeSet, } impl WriteKeys { @@ -55,9 +52,11 @@ impl WriteKeys { .extend(surrogates); } - /// Lock [`COLLECTION_KEY`] of `collection`. + /// Lock `collection` whole, exclusively: a write with no row identity, + /// such as a predicate, bulk, or truncate write. It orders against every + /// row writer and every reader of the collection. pub fn whole_collection(&mut self, collection: &str) { - self.rows(collection, [COLLECTION_KEY]); + self.whole.insert(collection.to_owned()); } /// Lock the vector rows `surrogates` of `collection`. An empty list @@ -172,6 +171,14 @@ impl WriteKeys { vshards: SortedVec::new(vshards), }), ); + sets.extend( + self.whole + .into_iter() + .map(|collection| EngineKeySet::Collection { + collection, + vshards: SortedVec::new(Vec::new()), + }), + ); sets.sort_by(|a, b| a.collection().cmp(b.collection())); sets } diff --git a/nodedb/src/control/planner/calvin/tx_class/write_keys/vector.rs b/nodedb/src/control/planner/calvin/tx_class/write_keys/vector.rs index e3fb7ccde..c3ea562f5 100644 --- a/nodedb/src/control/planner/calvin/tx_class/write_keys/vector.rs +++ b/nodedb/src/control/planner/calvin/tx_class/write_keys/vector.rs @@ -46,8 +46,8 @@ pub(super) fn add_keys(keys: &mut WriteKeys, op: &VectorOp) -> crate::Result<()> document_surrogate, .. } => keys.vector_rows(collection.as_str(), [document_surrogate.as_u32()]), - // A key unbound in its database names no row: it takes the - // collection key. + // A key unbound in its database names no row: it locks the whole + // collection. VectorOp::DeleteBySurrogate { collection, surrogate, diff --git a/nodedb/src/control/planner/materialized_sum/mod.rs b/nodedb/src/control/planner/materialized_sum/mod.rs index c4ea272cd..825a2410c 100644 --- a/nodedb/src/control/planner/materialized_sum/mod.rs +++ b/nodedb/src/control/planner/materialized_sum/mod.rs @@ -29,6 +29,7 @@ pub use cross_shard::append_cross_shard_balance_tasks; pub use extract::join_value_from_body; pub use index::MaterializedSumIndex; pub use resolve::{ - resolve_materialized_sum_targets, resolve_sum_targets_for_bodies, source_drives_bindings, + drives_cross_shard_sum, resolve_materialized_sum_targets, resolve_sum_targets_for_bodies, + source_drives_bindings, }; pub use resolve_target::ResolvedTargets; diff --git a/nodedb/src/control/planner/materialized_sum/predicate.rs b/nodedb/src/control/planner/materialized_sum/predicate.rs index 4b448843c..979077aa3 100644 --- a/nodedb/src/control/planner/materialized_sum/predicate.rs +++ b/nodedb/src/control/planner/materialized_sum/predicate.rs @@ -58,10 +58,16 @@ struct PredicateScope { /// collection drives a binding — so the caller knows the op is accounted for and /// does not also run the body-driven pass over it. `Ok(None)` means `op` is not /// predicate-driven and the caller still owns it. +/// +/// `txn_id` is the source task's own transaction, which each shipped balance +/// task inherits. `read_txn` is the open transaction the statement runs in: +/// the recon scan reads through its staging overlay, as the staged write +/// itself matches rows. pub(super) async fn resolve_predicate_sum_targets( state: &SharedState, op: &mut DocumentOp, txn_id: Option, + read_txn: Option, tenant_id: TenantId, database_id: DatabaseId, trace_id: TraceId, @@ -79,6 +85,7 @@ pub(super) async fn resolve_predicate_sum_targets( state, tenant_id, database_id, + read_txn, &scope.collection, // Cloned rather than moved: `scope` is still needed below to fold the // images from this same scan, and a filter blob is negligible beside the @@ -107,8 +114,7 @@ pub(super) async fn resolve_predicate_sum_targets( // observation this settlement rests on is the whole collection: a row // that JOINS the match set after the scan has to invalidate it too. source_row: None, - read_version_lsn: read.read_version_lsn, - served_by: read.served_by, + read_version: read.read_version, }; let settlement = settle_cross_shard_images(&bindings, &input, &resolved, txn_id, tenant_id, database_id)?; diff --git a/nodedb/src/control/planner/materialized_sum/recon.rs b/nodedb/src/control/planner/materialized_sum/recon.rs index 4e741e9ca..51b6dc170 100644 --- a/nodedb/src/control/planner/materialized_sum/recon.rs +++ b/nodedb/src/control/planner/materialized_sum/recon.rs @@ -21,6 +21,16 @@ //! vShard returns nothing, which will silently under-resolve and leave the //! write with no target to address. //! +//! # Transaction view +//! +//! Inside a transaction block the read carries the transaction's id. The Data +//! Plane then reads the transaction's staging overlay over committed base, as +//! an in-transaction `SELECT` does. A statement's write applies on top of the +//! rows the transaction's earlier statements staged, so its images are read +//! there too. A base-only read misses a row the transaction inserted and +//! returns the base value of a row it rewrote. The delta settled from such an +//! image is wrong by that row's staged contribution. +//! //! # Plane discipline //! //! Runs on the coordinator's Control Plane (Tokio). The scan goes through the @@ -29,7 +39,7 @@ use nodedb_types::{Surrogate, TenantId}; use crate::control::state::SharedState; -use crate::types::{DatabaseId, Lsn, TraceId}; +use crate::types::{DatabaseId, TraceId, TxnId}; use nodedb_physical::physical_plan::{DocumentOp, PhysicalPlan}; /// What a plan-time reconnaissance read observed, and the version it observed @@ -43,13 +53,10 @@ use nodedb_physical::physical_plan::{DocumentOp, PhysicalPlan}; pub(crate) struct ReconRead { /// The decoded rows. pub rows: T, - /// The source collection's write floor at read time — the comparand - /// cross-shard OCC validation checks the read against. - pub read_version_lsn: Lsn, - /// The node that served the read. `read_version_lsn` is a position in - /// that node's WAL, so the commit vote compares it only on that node. - /// `0` when no one node is known to have served it. - pub served_by: u64, + /// The source collection's version on its vShard at read time — the + /// comparand OCC validation checks the read against. Every replica + /// numbers versions alike, so any replica can have served the read. + pub read_version: nodedb_types::WriteVersion, } /// Scan `collection` for the rows `filters` matches, returning each row's full @@ -63,10 +70,14 @@ pub(crate) struct ReconRead { /// /// Empty `filters` means "no WHERE clause" — every row, which is what `TRUNCATE` /// needs. +/// +/// `txn_id` is the open transaction the statement runs in, `None` outside a +/// transaction block. See the module's transaction view. pub(in crate::control::planner) async fn recon_scan_rows( state: &SharedState, tenant_id: TenantId, database_id: DatabaseId, + txn_id: Option, collection: &str, filters: Vec, ) -> crate::Result>> { @@ -85,15 +96,14 @@ pub(in crate::control::planner) async fn recon_scan_rows( prefilter: None, }); - let read = execute_read(state, tenant_id, database_id, collection, scan_plan).await?; + let read = execute_read(state, tenant_id, database_id, txn_id, collection, scan_plan).await?; let mut rows = Vec::new(); for payload in &read.rows { rows.extend(decode_rows(payload.as_slice())); } Ok(ReconRead { rows, - read_version_lsn: read.read_version_lsn, - served_by: read.served_by, + read_version: read.read_version, }) } @@ -111,10 +121,15 @@ pub(in crate::control::planner) async fn recon_scan_rows( /// /// Identity is the surrogate, exactly as on the write path — `document_id` is /// the user-facing primary key and carries no storage addressing. +/// +/// `txn_id` is the open transaction the statement runs in, `None` outside a +/// transaction block. A row this transaction staged reads as staged, and a row +/// it staged a delete of reads as absent. pub(crate) async fn recon_point_row( state: &SharedState, tenant_id: TenantId, database_id: DatabaseId, + txn_id: Option, collection: &str, document_id: &str, surrogate: Surrogate, @@ -133,7 +148,7 @@ pub(crate) async fn recon_point_row( valid_at_ms: None, }); - let read = execute_read(state, tenant_id, database_id, collection, get_plan).await?; + let read = execute_read(state, tenant_id, database_id, txn_id, collection, get_plan).await?; // A point get answers with the row's normalized MessagePack body, and with // an EMPTY payload when the row is absent. Ok(ReconRead { @@ -142,8 +157,7 @@ pub(crate) async fn recon_point_row( .iter() .find(|payload| !payload.is_empty()) .and_then(|payload| nodedb_types::json_from_msgpack(payload.as_slice()).ok()), - read_version_lsn: read.read_version_lsn, - served_by: read.served_by, + read_version: read.read_version, }) } @@ -154,39 +168,43 @@ pub(crate) async fn recon_point_row( /// write with no target to address — so every plan-time read routes through /// the gateway. /// -/// The gateway notes the node that served each vShard it read. The read -/// validates on `collection`'s vShard, so that vShard's note names the node -/// whose WAL numbers `read_version_lsn`. +/// The read validates on `collection`'s vShard, so its version is the one +/// the read reported for that vShard. A vShard the read reported no version +/// of had no write the serving core knew of, so the read observed it at +/// `WriteVersion::ZERO`. +/// +/// The version is the collection's COMMITTED version whether or not `txn_id` +/// is set: a staged write moves no write version. So an entry stamped with it +/// still detects a concurrent commit to the rows read, and never this +/// transaction's own staged writes. async fn execute_read( state: &SharedState, tenant_id: TenantId, database_id: DatabaseId, + txn_id: Option, collection: &str, plan: PhysicalPlan, ) -> crate::Result>>> { let validation_vshard = - nodedb_types::CollectionKey::from_qualified_str(database_id, collection)? - .vshard() - .as_u32(); + nodedb_types::CollectionKey::from_qualified_str(database_id, collection)?.vshard(); let gateway = state.installed_gateway()?; let gw_ctx = crate::control::gateway::core::QueryContext { tenant_id, trace_id: TraceId::ZERO, database_id, - txn_id: None, + // The transaction's staging overlay over committed base, as an + // in-transaction read sees it. + txn_id, linearizable: true, }; // A shard verdict keeps its own typed error. - let (payloads, _watermarks, read_version_lsn) = gateway - .execute_internal_with_watermarks(&gw_ctx, plan) - .await?; + let outcome = gateway.execute_internal_outcome(&gw_ctx, plan).await?; Ok(ReconRead { - rows: payloads, - read_version_lsn, - served_by: crate::control::server::shared::session::read_set::serving_node( - state, - validation_vshard, - ), + read_version: outcome + .read_versions + .of(validation_vshard) + .unwrap_or_default(), + rows: outcome.payloads, }) } diff --git a/nodedb/src/control/planner/materialized_sum/resolve.rs b/nodedb/src/control/planner/materialized_sum/resolve.rs index 81eb0f526..2ed6bb5ee 100644 --- a/nodedb/src/control/planner/materialized_sum/resolve.rs +++ b/nodedb/src/control/planner/materialized_sum/resolve.rs @@ -20,7 +20,7 @@ use super::stored::stored_row_scope; use crate::control::server::shared::session::read_set::ReadSetEntry; use crate::control::server::surrogate_exchange::lookup_surrogate_routed; use crate::control::state::SharedState; -use crate::types::{DatabaseId, TenantId, TraceId}; +use crate::types::{DatabaseId, TenantId, TraceId, TxnId}; /// Resolve the materialized-sum target rows for every document write in /// `tasks`, storing the result in that op's `resolved_sum_targets`. @@ -43,11 +43,18 @@ use crate::types::{DatabaseId, TenantId, TraceId}; /// The returned [`ReadSetEntry`]s cover the folded images; the caller must /// union them into the dispatch read-set so Calvin OCC aborts the /// statement on concurrent source writes, before any row moves. +/// +/// `read_txn` is the open transaction the statement runs in, `None` outside a +/// transaction block. Every source read goes through its staging overlay, so +/// each image is the row as this transaction sees it. A planned task's own +/// `txn_id` does not say this: the planner leaves it unset, and the staging +/// gate stamps it only after this pass. pub async fn resolve_materialized_sum_targets( state: &SharedState, tasks: &mut Vec, tenant_id: TenantId, database_id: DatabaseId, + read_txn: Option, trace_id: TraceId, ) -> crate::Result> { let schema_version = state.schema_version.current(); @@ -67,6 +74,7 @@ pub async fn resolve_materialized_sum_targets( state, op, txn_id, + read_txn, tenant_id, database_id, trace_id, @@ -81,7 +89,7 @@ pub async fn resolve_materialized_sum_targets( // end before the resolution is written back through `&mut op`. let outcome = { let carried = value_carrying(op); - let stored = stored_row_scope(op); + let stored = stored_row_scope(op, read_txn); // The two classifications name the same collection when both apply; // an op that is neither carries no join value at all and is skipped. let Some(collection) = stored @@ -135,8 +143,7 @@ pub async fn resolve_materialized_sum_targets( source_collection: collection, images: &images.images, source_row: Some(scope.surrogate), - read_version_lsn: images.read_version_lsn, - served_by: images.served_by, + read_version: images.read_version, }; let settlement = settle_cross_shard_images( &bindings, @@ -280,12 +287,16 @@ fn set_resolved(op: &mut DocumentOp, resolved: Vec) { /// Resolve the materialized-sum targets for a set of row BODIES of one source /// collection, for a caller that holds the bodies itself rather than a plan. /// -/// This is the seam the Control-Plane orchestrators use. `INSERT ... SELECT`, -/// `MERGE` and `UPDATE ... FROM` all resolve their rows on the Control Plane and -/// re-issue concrete work (a `BatchInsert` page, an APPLY pass, a write pass) -/// through `dispatch_local`, which never passes through +/// This is the seam the autocommit Control-Plane orchestrators use. +/// `INSERT ... SELECT`, `MERGE` and `UPDATE ... FROM` resolve their rows on the +/// Control Plane and re-issue concrete work (a `BatchInsert` page, an APPLY +/// pass, a write pass) that never passes through /// [`resolve_materialized_sum_targets`]. Without this they will ship an empty -/// resolution and the Data-Plane fold will have no target to address. +/// resolution and the Data-Plane fold will have no target to address. Every +/// binding they resolve is co-resident: a statement that moves a cross-shard +/// sum runs in an implicit transaction on every protocol and never reaches +/// them +/// ([`statement_needs_implicit_txn`](crate::control::server::shared::txn_route::statement_needs_implicit_txn)). /// /// `source_collection` is the db-qualified name as it appears on the plan. /// Returns an empty vec — and issues no lookup at all — when the collection @@ -341,6 +352,28 @@ pub fn source_drives_bindings( ) } +/// Whether `source_collection` drives a materialized sum whose target does not +/// share its vShard. `source_collection` is the db-qualified plan name. +/// +/// Such a balance cannot ride a write applied on the source's vShard alone. +/// It ships on an `ApplyBalanceDelta` task homed on the target, so the write +/// that moves it must run where Calvin can commit the pair together. +pub fn drives_cross_shard_sum( + state: &SharedState, + source_collection: &str, + tenant_id: TenantId, + database_id: DatabaseId, +) -> crate::Result { + let Some(bindings) = source_drives_bindings(state, source_collection, tenant_id, database_id)? + else { + return Ok(false); + }; + let source = nodedb_types::CollectionKey::from_qualified_str(database_id, source_collection)?; + Ok(bindings.iter().any(|binding| { + !crate::query::sum_target_is_co_resident(source, &binding.target_collection) + })) +} + /// Resolve one already-extracted join VALUE to its target row's surrogate. /// /// `lookup_surrogate_routed`, never `assign_surrogate_routed`: a join value that @@ -545,7 +578,7 @@ mod tests { .expect("bind target row"); let mut tasks = vec![insert_task("entries", body("acc-1"))]; - resolve_materialized_sum_targets(&state, &mut tasks, TENANT, DB, TraceId::ZERO) + resolve_materialized_sum_targets(&state, &mut tasks, TENANT, DB, None, TraceId::ZERO) .await .expect("resolution succeeds"); @@ -596,7 +629,7 @@ mod tests { ); let mut tasks = vec![insert_task("entries", body("acc-1"))]; - resolve_materialized_sum_targets(&state, &mut tasks, TENANT, DB, TraceId::ZERO) + resolve_materialized_sum_targets(&state, &mut tasks, TENANT, DB, None, TraceId::ZERO) .await .expect("resolution succeeds"); @@ -630,9 +663,10 @@ mod tests { declare_binding(&state); let mut tasks = vec![insert_task("entries", body("acc-missing"))]; - let error = resolve_materialized_sum_targets(&state, &mut tasks, TENANT, DB, TraceId::ZERO) - .await - .expect_err("a missing target must fail the statement"); + let error = + resolve_materialized_sum_targets(&state, &mut tasks, TENANT, DB, None, TraceId::ZERO) + .await + .expect_err("a missing target must fail the statement"); match error { crate::Error::MaterializedSumTargetNotFound { @@ -656,7 +690,7 @@ mod tests { declare_binding(&state); let mut tasks = vec![insert_task("unrelated", body("acc-1"))]; - resolve_materialized_sum_targets(&state, &mut tasks, TENANT, DB, TraceId::ZERO) + resolve_materialized_sum_targets(&state, &mut tasks, TENANT, DB, None, TraceId::ZERO) .await .expect("resolution succeeds"); @@ -714,7 +748,7 @@ mod tests { post_set_op: PostSetOp::None, txn_id: None, }]; - resolve_materialized_sum_targets(&state, &mut tasks, TENANT, DB, TraceId::ZERO) + resolve_materialized_sum_targets(&state, &mut tasks, TENANT, DB, None, TraceId::ZERO) .await .expect("resolution succeeds"); @@ -733,4 +767,132 @@ mod tests { other => panic!("plan shape changed: {other:?}"), } } + + /// The fixture pair this module's cross-shard tests rest on. + #[test] + fn the_fixture_target_is_cross_shard() { + assert_ne!( + nodedb_types::CollectionKey::from_bare(DB, "entries").vshard(), + nodedb_types::CollectionKey::from_bare(DB, "accounts").vshard(), + "'accounts' must not share 'entries'' vShard" + ); + } + + /// A staged `INSERT ... SELECT` emits one `PointInsert` per copied row, + /// in the open transaction. The pass the expander runs on them ships each + /// cross-shard balance on a task homed on the target, in that same + /// transaction. The redo's fold table then names no cross-shard target, + /// so the install never folds one on the source's core. + #[tokio::test] + async fn expanded_inserts_ship_cross_shard_balances_and_fold_none() { + let (state, _directory) = test_state(); + declare_binding(&state); + state + .surrogate_assigner + .assign( + nodedb_types::CollectionKey::from_bare(DB, "accounts"), + TENANT, + b"acc-1", + ) + .await + .expect("bind target row"); + let txn = Some(crate::types::TxnId::new(41)); + let mut tasks: Vec = (0..2) + .map(|_| { + let mut task = insert_task("entries", body("acc-1")); + task.txn_id = txn; + task + }) + .collect(); + + let reads = crate::control::server::shared::plan_admission::append_sum_and_period_targets( + &state, + &mut tasks, + TENANT, + DB, + txn, + TraceId::ZERO, + ) + .await + .expect("the pass succeeds"); + assert!(reads.is_empty(), "an insert reads no image"); + + let balances: Vec<&PhysicalTask> = tasks + .iter() + .filter(|task| { + matches!( + task.plan, + PhysicalPlan::Document(DocumentOp::ApplyBalanceDelta { .. }) + ) + }) + .collect(); + assert_eq!(balances.len(), 2, "one balance per copied row"); + for balance in balances { + assert_eq!( + balance.vshard_id, + crate::query::sum_target_vshard(DB, "accounts") + ); + assert_eq!(balance.txn_id, txn, "the balance joins the transaction"); + match &balance.plan { + PhysicalPlan::Document(DocumentOp::ApplyBalanceDelta { delta, .. }) => { + assert_eq!(delta, "25"); + } + other => panic!("plan shape changed: {other:?}"), + } + } + + let plans: Vec = tasks.iter().map(|task| task.plan.clone()).collect(); + assert!( + crate::control::wal_replication::transaction_redo::sum_targets::redo_sum_targets( + &plans + ) + .is_empty(), + "the source core folds no cross-shard target" + ); + } + + /// A source that drives a cross-shard target is reported. A collection + /// that drives nothing is not. + #[test] + fn a_cross_shard_source_is_reported() { + let (state, _directory) = test_state(); + declare_binding(&state); + + assert!(drives_cross_shard_sum(&state, "entries", TENANT, DB).expect("index probe")); + assert!(!drives_cross_shard_sum(&state, "unrelated", TENANT, DB).expect("index probe")); + } + + /// A source whose every target shares its vShard rides the source write, + /// so it reports no cross-shard sum. + #[test] + fn a_co_resident_source_is_not_cross_shard() { + let (state, _directory) = test_state(); + let (source, target) = ("uo_entries", "uo_accounts"); + assert_eq!( + nodedb_types::CollectionKey::from_bare(DB, source).vshard(), + nodedb_types::CollectionKey::from_bare(DB, target).vshard(), + "the pair must share a vShard" + ); + let catalog = state.credentials.catalog(); + let mut stored = StoredCollection::stamped_for_test(TENANT.as_u64(), target, "tester"); + stored.materialized_sums.push(MaterializedSumDef { + target_collection: target.to_string(), + target_column: "balance".to_string(), + source_collection: source.to_string(), + join_column: "account_id".to_string(), + value_expr: nodedb_query::expr::SqlExpr::Column("amount".to_string()), + }); + catalog + .put_collection(DB, &stored) + .expect("persist target collection"); + catalog + .put_collection( + DB, + &StoredCollection::stamped_for_test(TENANT.as_u64(), source, "tester"), + ) + .expect("persist source collection"); + state.materialized_sum_index.invalidate(); + + assert!(!drives_cross_shard_sum(&state, source, TENANT, DB).expect("index probe")); + } } diff --git a/nodedb/src/control/planner/materialized_sum/settle.rs b/nodedb/src/control/planner/materialized_sum/settle.rs index c6c744cde..6ecb7d014 100644 --- a/nodedb/src/control/planner/materialized_sum/settle.rs +++ b/nodedb/src/control/planner/materialized_sum/settle.rs @@ -40,13 +40,15 @@ //! the presence of a resolved surrogate is what says "nobody else is applying //! this one". //! -//! That leaves the Control-Plane orchestrators — `MERGE`, `UPDATE ... FROM`, -//! `INSERT ... SELECT` and the staged-transaction expanders — untouched: they -//! resolve their rows through +//! The staged-transaction expanders for `MERGE`, `UPDATE ... FROM` and +//! `INSERT ... SELECT` run this same pass on the point writes they emit. The +//! autocommit orchestrators resolve their rows through //! [`resolve_sum_targets_for_bodies`](super::resolve::resolve_sum_targets_for_bodies) -//! and dispatch through `dispatch_local` without ever reaching this pass, so -//! every join value they resolve stays resolved and the Data Plane keeps -//! applying their deltas exactly as before. +//! and apply on the source's vShard alone, so they fold co-resident targets +//! only. A source with a cross-shard target never reaches them: every +//! protocol runs such a statement through the expanders in an implicit +//! transaction +//! ([`statement_needs_implicit_txn`](crate::control::server::shared::txn_route::statement_needs_implicit_txn)). //! //! # The images are only as good as the version they were read at //! @@ -73,7 +75,7 @@ use crate::control::server::shared::session::read_set::{ }; use crate::engine::document::store::StorageKey; use crate::query::{sum_target_is_co_resident, sum_target_vshard}; -use crate::types::{DatabaseId, KeyRepr, Lsn, TenantId}; +use crate::types::{DatabaseId, KeyRepr, TenantId}; /// One source write's plan-time images, ready to settle. pub(super) struct SettleInput<'a> { @@ -89,11 +91,8 @@ pub(super) struct SettleInput<'a> { /// that row. `None` for a predicate-shaped write, whose observation is the /// whole collection and whose entry is collection-scoped accordingly. pub source_row: Option, - /// Version the images were read at. - pub read_version_lsn: Lsn, - /// The node that served the image read. `read_version_lsn` is a position - /// in its WAL, so only that node's commit vote can find the read current. - pub served_by: u64, + /// Version the images were read at, on the source collection's vShard. + pub read_version: nodedb_types::WriteVersion, } /// What settling produced. @@ -272,17 +271,23 @@ fn image_read_entry( }, None => ReadKey::Predicate, }, - read_lsn: input.read_version_lsn, - read_version_lsn: input.read_version_lsn, - // A DERIVATION read, never a read-your-own-write. The image was read - // from committed base state before this transaction existed, and the - // delta shipped to the target rests entirely on it; the statement writes - // the source collection too, so an entry marked `Session` here will be + read_version: input.read_version, + // A DERIVATION read, never a read-your-own-write. The delta shipped to + // the target rests entirely on the image. The statement writes the + // source collection too, so an entry marked `Session` here will be // dropped by the own-write exclusion and the fold will never be // validated against a concurrent writer. + // + // Inside a transaction the image is read through its staging overlay. + // `read_version` is still the committed version, because a + // staged write moves no write version. So the entry aborts the commit + // when another transaction commits a write to the row first. It never + // aborts on this transaction's own staged writes, which the image + // already includes. For a row that exists only in the overlay, the + // entry asserts that the base still holds no committed write to that + // surrogate. A concurrent commit of that row invalidates it. origin: ReadOrigin::PlanDerivation, home: None, - home_node: input.served_by, } } @@ -397,8 +402,7 @@ mod tests { source_collection: &source, images: &images, source_row: Some(Surrogate::new(11)), - read_version_lsn: Lsn::new(42), - served_by: 7, + read_version: nodedb_types::WriteVersion::logged(3, 42), }; let settlement = settle_cross_shard_images( &[binding(&target)], @@ -433,8 +437,7 @@ mod tests { source_collection: &source, images: &images, source_row: Some(Surrogate::new(11)), - read_version_lsn: Lsn::new(42), - served_by: 7, + read_version: nodedb_types::WriteVersion::logged(3, 42), }; let settlement = settle_cross_shard_images( &[binding(&target)], @@ -474,8 +477,7 @@ mod tests { source_collection: &source, images: &images, source_row: Some(Surrogate::new(11)), - read_version_lsn: Lsn::new(42), - served_by: 7, + read_version: nodedb_types::WriteVersion::logged(3, 42), }; let settlement = settle_cross_shard_images( &[binding(&target)], @@ -499,8 +501,7 @@ mod tests { source_collection: &source, images: &images, source_row: Some(Surrogate::new(11)), - read_version_lsn: Lsn::new(42), - served_by: 7, + read_version: nodedb_types::WriteVersion::logged(3, 42), }; let settlement = settle_cross_shard_images( &[binding(&target)], @@ -528,8 +529,7 @@ mod tests { source_collection: &source, images: &images, source_row: Some(Surrogate::new(11)), - read_version_lsn: Lsn::new(42), - served_by: 7, + read_version: nodedb_types::WriteVersion::logged(3, 42), }; // A binding whose target IS the source collection is co-resident by // construction, whatever the hash function does. @@ -558,8 +558,7 @@ mod tests { source_collection: &source, images: &images, source_row: Some(Surrogate::new(11)), - read_version_lsn: Lsn::new(42), - served_by: 7, + read_version: nodedb_types::WriteVersion::logged(3, 42), }; let settlement = settle_cross_shard_images( &[binding(&target)], @@ -578,11 +577,10 @@ mod tests { repr: KeyRepr::Surrogate(11) } ); - assert_eq!(settlement.reads[0].read_version_lsn, Lsn::new(42)); assert_eq!( - settlement.reads[0].home_node, 7, - "the entry names the node that served the image read; `0` makes \ - every commit vote treat the read as changed" + settlement.reads[0].read_version, + nodedb_types::WriteVersion::logged(3, 42), + "the entry carries the version the image read observed" ); } @@ -597,8 +595,7 @@ mod tests { source_collection: &source, images: &images, source_row: None, - read_version_lsn: Lsn::new(42), - served_by: 7, + read_version: nodedb_types::WriteVersion::logged(3, 42), }; let settlement = settle_cross_shard_images( &[binding(&target)], diff --git a/nodedb/src/control/planner/materialized_sum/stored.rs b/nodedb/src/control/planner/materialized_sum/stored.rs index 138abac09..142566460 100644 --- a/nodedb/src/control/planner/materialized_sum/stored.rs +++ b/nodedb/src/control/planner/materialized_sum/stored.rs @@ -24,12 +24,13 @@ use std::sync::Arc; use nodedb_physical::physical_plan::{DocumentOp, MaterializedSumBinding, UpdateValue}; use nodedb_types::Surrogate; +use nodedb_types::id::TxnId; use super::recon::recon_point_row; use super::resolve::lookup_join_value; use super::resolve_target::ResolvedTargets; use crate::control::state::SharedState; -use crate::types::{DatabaseId, Lsn, TenantId, TraceId}; +use crate::types::{DatabaseId, TenantId, TraceId}; /// How a point-shaped write's POST-image is formed. /// @@ -69,6 +70,10 @@ pub(super) struct StoredRowScope<'a> { pub updates: &'a [(String, UpdateValue)], /// How this shape's post-image is formed. pub post_image: PostImage<'a>, + /// The open transaction the write runs in, `None` outside a transaction + /// block. The stored row is read through its staging overlay, so the + /// pre-image is the row as this transaction sees it. + pub txn_id: Option, } /// What reading the stored row produced: the write's pre-/post-image pairs @@ -77,10 +82,8 @@ pub(super) struct StoredRowScope<'a> { pub(super) struct StoredImages { /// One pair per row this write touches — at most one, for a point shape. pub images: Vec<(Option, Option)>, - /// The source collection's write floor at read time. - pub read_version_lsn: Lsn, - /// The node that served the read, whose WAL numbers `read_version_lsn`. - pub served_by: u64, + /// The source collection's version on its vShard at read time. + pub read_version: nodedb_types::WriteVersion, } /// The stored row an op rewrites or removes, or `None` for every op that @@ -89,7 +92,13 @@ pub(super) struct StoredImages { /// `Upsert` (a join-column rewrite debits the leaving target, named only by /// the stored image). `PointInsert`/`BatchInsert` are absent: rows are new /// by construction, so there's no pre-image to read. -pub(super) fn stored_row_scope(op: &DocumentOp) -> Option> { +/// +/// `txn_id` is the open transaction the write runs in, `None` outside a +/// transaction block. +pub(super) fn stored_row_scope( + op: &DocumentOp, + txn_id: Option, +) -> Option> { match op { // A key unbound in its database names no stored row. DocumentOp::PointUpdate { @@ -104,6 +113,7 @@ pub(super) fn stored_row_scope(op: &DocumentOp) -> Option> { surrogate, updates: updates.as_slice(), post_image: PostImage::Assigned, + txn_id, }), // A delete assigns nothing: the row's pre-image join value is the whole // of what it owes. @@ -118,6 +128,7 @@ pub(super) fn stored_row_scope(op: &DocumentOp) -> Option> { surrogate, updates: &[], post_image: PostImage::Removed, + txn_id, }), // A put replaces the row wholesale, so its post-image is the submitted // body — already resolved from the body — and the stored row is needed @@ -134,6 +145,7 @@ pub(super) fn stored_row_scope(op: &DocumentOp) -> Option> { surrogate: *surrogate, updates: &[], post_image: PostImage::Body(value.as_slice()), + txn_id, }), // On the conflict branch an upsert's post-image is the stored row with // `on_conflict_updates` applied, so those assignments decide the target @@ -152,6 +164,7 @@ pub(super) fn stored_row_scope(op: &DocumentOp) -> Option> { surrogate: *surrogate, updates: on_conflict_updates.as_slice(), post_image: PostImage::BodyOrAssigned(value.as_slice()), + txn_id, }), DocumentOp::PointInsert { .. } | DocumentOp::BatchInsert { .. } @@ -201,6 +214,7 @@ pub(super) async fn extend_with_stored_row( state, tenant_id, database_id, + scope.txn_id, scope.collection, scope.document_id, scope.surrogate, @@ -209,8 +223,7 @@ pub(super) async fn extend_with_stored_row( let outcome = StoredImages { images: images_of(scope, read.rows.as_ref())?, - read_version_lsn: read.read_version_lsn, - served_by: read.served_by, + read_version: read.read_version, }; if read.rows.is_none() { @@ -395,7 +408,7 @@ mod tests { #[test] fn every_point_shape_names_the_stored_row_it_rewrites() { for op in [point_delete(), point_update(), point_put(), upsert()] { - let scope = stored_row_scope(&op) + let scope = stored_row_scope(&op, None) .unwrap_or_else(|| panic!("a point write must name its stored row: {op:?}")); assert_eq!(scope.collection, "entries"); assert_eq!(scope.document_id, "e1"); @@ -410,7 +423,7 @@ mod tests { /// read and no round trip to spend learning that. #[test] fn an_insert_names_no_stored_row() { - assert!(stored_row_scope(&point_insert()).is_none()); + assert!(stored_row_scope(&point_insert(), None).is_none()); } /// The assignments travel on the scope, so a write that rewrites the join @@ -420,7 +433,7 @@ mod tests { #[test] fn assignments_that_rewrite_the_join_column_travel_on_the_scope() { for op in [point_update(), upsert()] { - let scope = stored_row_scope(&op).expect("a point write names its stored row"); + let scope = stored_row_scope(&op, None).expect("a point write names its stored row"); assert_eq!( scope.updates.len(), 1, @@ -435,7 +448,7 @@ mod tests { #[test] fn the_shapes_without_assignments_carry_none() { for op in [point_delete(), point_put()] { - let scope = stored_row_scope(&op).expect("a point write names its stored row"); + let scope = stored_row_scope(&op, None).expect("a point write names its stored row"); assert!(scope.updates.is_empty(), "{op:?}"); } } @@ -486,7 +499,7 @@ mod tests { body("acc-1", 60), vec![("amount".to_string(), excluded("amount"))], ); - let scope = stored_row_scope(&op).expect("an upsert names its stored row"); + let scope = stored_row_scope(&op, None).expect("an upsert names its stored row"); let stored = serde_json::json!({"account_id": "acc-1", "amount": 25}); let images = images_of(&scope, Some(&stored)).expect("images"); @@ -501,7 +514,7 @@ mod tests { #[test] fn an_upsert_without_conflict_assignments_merges_the_body() { let op = upsert_with(body("acc-1", 60), Vec::new()); - let scope = stored_row_scope(&op).expect("an upsert names its stored row"); + let scope = stored_row_scope(&op, None).expect("an upsert names its stored row"); let stored = serde_json::json!({"account_id": "acc-1", "amount": 25, "memo": "kept"}); let images = images_of(&scope, Some(&stored)).expect("images"); @@ -529,7 +542,7 @@ mod tests { ("amount".to_string(), excluded("amount")), ], ); - let scope = stored_row_scope(&op).expect("an upsert names its stored row"); + let scope = stored_row_scope(&op, None).expect("an upsert names its stored row"); let stored = serde_json::json!({"account_id": "acc-1", "amount": 25}); let images = images_of(&scope, Some(&stored)).expect("images"); @@ -552,7 +565,7 @@ mod tests { body("acc-1", 60), vec![("amount".to_string(), excluded("amount"))], ); - let scope = stored_row_scope(&op).expect("an upsert names its stored row"); + let scope = stored_row_scope(&op, None).expect("an upsert names its stored row"); let images = images_of(&scope, None).expect("images"); assert_eq!(images.len(), 1); @@ -575,7 +588,7 @@ mod tests { rls_filters: Vec::new(), resolved_sum_targets: Vec::new(), }; - let scope = stored_row_scope(&op).expect("a put names its stored row"); + let scope = stored_row_scope(&op, None).expect("a put names its stored row"); let stored = serde_json::json!({"account_id": "acc-1", "amount": 25}); let images = images_of(&scope, Some(&stored)).expect("images"); diff --git a/nodedb/src/control/planner/period_lock/lookup.rs b/nodedb/src/control/planner/period_lock/lookup.rs index e9d207c24..205c26c6a 100644 --- a/nodedb/src/control/planner/period_lock/lookup.rs +++ b/nodedb/src/control/planner/period_lock/lookup.rs @@ -4,7 +4,7 @@ use crate::control::server::surrogate_exchange::lookup_surrogate_routed; use crate::control::state::SharedState; -use crate::types::{DatabaseId, TenantId, TraceId}; +use crate::types::{DatabaseId, TenantId, TraceId, TxnId}; /// Request scope shared by every per-row period-lock resolution call within /// one [`resolve_period_lock_targets`](super::resolve::resolve_period_lock_targets) @@ -14,6 +14,11 @@ pub(super) struct PeriodLockScope<'a> { pub state: &'a SharedState, pub tenant_id: TenantId, pub database_id: DatabaseId, + /// The open transaction the statement runs in, `None` outside a + /// transaction block. Every stored-row read goes through its staging + /// overlay, so the period a write gates on is the row as this transaction + /// sees it. + pub read_txn: Option, pub trace_id: TraceId, } diff --git a/nodedb/src/control/planner/period_lock/point_update.rs b/nodedb/src/control/planner/period_lock/point_update.rs index e64fb8c1f..7cbb8245c 100644 --- a/nodedb/src/control/planner/period_lock/point_update.rs +++ b/nodedb/src/control/planner/period_lock/point_update.rs @@ -29,6 +29,7 @@ pub(super) async fn resolve_update_period_values( scope.state, scope.tenant_id, scope.database_id, + scope.read_txn, collection, document_id, surrogate, diff --git a/nodedb/src/control/planner/period_lock/predicate.rs b/nodedb/src/control/planner/period_lock/predicate.rs index ccf79a68d..2855b9bdf 100644 --- a/nodedb/src/control/planner/period_lock/predicate.rs +++ b/nodedb/src/control/planner/period_lock/predicate.rs @@ -40,6 +40,7 @@ pub(super) async fn resolve_predicate_period_values( scope.state, scope.tenant_id, scope.database_id, + scope.read_txn, collection, filters, ) diff --git a/nodedb/src/control/planner/period_lock/resolve.rs b/nodedb/src/control/planner/period_lock/resolve.rs index bcc7f2856..d63f3b6ca 100644 --- a/nodedb/src/control/planner/period_lock/resolve.rs +++ b/nodedb/src/control/planner/period_lock/resolve.rs @@ -53,7 +53,7 @@ use super::point_update::resolve_update_period_values; use super::predicate::{PeriodLockEffect, resolve_predicate_period_values}; use super::singular::{resolve_batch_period_values, singular_period_value}; use crate::control::state::SharedState; -use crate::types::{DatabaseId, TenantId, TraceId}; +use crate::types::{DatabaseId, TenantId, TraceId, TxnId}; /// Resolve the period-lock reference row for every write in `tasks` that a /// period lock gates, appending the entry to the op's `resolved_sum_targets` @@ -63,11 +63,15 @@ use crate::types::{DatabaseId, TenantId, TraceId}; /// value) adds no entry: `check_period_lock` treats an absent entry as an /// unknown period and refuses the write, exactly as it treats a genuinely /// closed one. +/// +/// `read_txn` is the open transaction the statement runs in, `None` outside a +/// transaction block. A stored row is read through its staging overlay. pub async fn resolve_period_lock_targets( state: &SharedState, tasks: &mut [PhysicalTask], tenant_id: TenantId, database_id: DatabaseId, + read_txn: Option, trace_id: TraceId, ) -> crate::Result<()> { let catalog = state.credentials.catalog(); @@ -75,6 +79,7 @@ pub async fn resolve_period_lock_targets( state, tenant_id, database_id, + read_txn, trace_id, }; for task in tasks.iter_mut() { @@ -180,9 +185,7 @@ pub async fn resolve_period_lock_targets( continue; } - let Some(period_key) = - singular_period_value(state, op, &collection, &def, tenant_id, database_id).await? - else { + let Some(period_key) = singular_period_value(&scope, op, &collection, &def).await? else { continue; }; diff --git a/nodedb/src/control/planner/period_lock/singular.rs b/nodedb/src/control/planner/period_lock/singular.rs index d75baa496..b6e635321 100644 --- a/nodedb/src/control/planner/period_lock/singular.rs +++ b/nodedb/src/control/planner/period_lock/singular.rs @@ -5,7 +5,7 @@ use nodedb_physical::physical_plan::{DocumentOp, ResolvedSumTarget}; -use super::lookup::lookup_period_surrogate; +use super::lookup::{PeriodLockScope, lookup_period_surrogate}; use crate::control::planner::materialized_sum::recon::recon_point_row; use crate::control::planner::materialized_sum::{ResolvedTargets, join_value_from_body}; use crate::control::security::catalog::PeriodLockDef; @@ -21,12 +21,10 @@ use crate::types::{DatabaseId, TenantId, TraceId}; /// off its submitted body. `PointDelete` carries no body, so its value comes /// off the row it is about to remove. pub(super) async fn singular_period_value( - state: &SharedState, + scope: &PeriodLockScope<'_>, op: &DocumentOp, collection: &str, def: &PeriodLockDef, - tenant_id: TenantId, - database_id: DatabaseId, ) -> crate::Result> { match op { DocumentOp::PointPut { value, .. } @@ -42,9 +40,10 @@ pub(super) async fn singular_period_value( return Ok(None); }; let read = recon_point_row( - state, - tenant_id, - database_id, + scope.state, + scope.tenant_id, + scope.database_id, + scope.read_txn, collection, document_id, *surrogate, diff --git a/nodedb/src/control/planner/procedural/executor/core/dispatch.rs b/nodedb/src/control/planner/procedural/executor/core/dispatch.rs index 8759385a2..f11e5325a 100644 --- a/nodedb/src/control/planner/procedural/executor/core/dispatch.rs +++ b/nodedb/src/control/planner/procedural/executor/core/dispatch.rs @@ -129,6 +129,9 @@ impl<'a> StatementExecutor<'a> { // locks) join the statement as they do for a client statement. The // plan never carries a Calvin OLLP prediction or a resolved write: // the planner emits neither, so every task takes the staging gate. + // They read their source rows through the transaction this statement + // stages into, so they see its earlier statements. + let read_txn = self.read_txn().await?; let (tasks, planned, lease_scope, sum_target_reads) = crate::control::server::shared::retry::retry_on_schema_change( &self.state.lease_drain, @@ -149,6 +152,7 @@ impl<'a> StatementExecutor<'a> { &mut tasks, self.tenant_id, self.database_id, + read_txn, crate::types::TraceId::ZERO, ) .await?; @@ -199,6 +203,20 @@ impl<'a> StatementExecutor<'a> { .map_err(crate::Error::from) } + /// The id of the transaction the next statement stages into: the open + /// one, else the joined statement's. `None` when neither exists yet, so + /// the transaction the statement opens has staged nothing. + async fn read_txn(&self) -> crate::Result> { + let txn = self.txn.lock().await; + if let Some(open) = txn.as_ref() { + let ctx = open.txn_ctx()?; + return Ok(ctx.sessions.tx_id(ctx.session_id)); + } + Ok(self + .joined + .and_then(|ctx| ctx.sessions.tx_id(ctx.session_id))) + } + /// The open transaction, begun now when none is open. A joined body /// joins its statement's transaction instead. async fn open_txn<'g>( @@ -494,8 +512,8 @@ mod origination_tests { payload: Payload::from_vec(payload), watermark_lsn: Lsn::ZERO, error_code, - read_set_valid: None, - read_version_lsn: Lsn::ZERO, + stage_vote: None, + read_versions: crate::types::ReadVersions::new(), write_set: Vec::new(), }; side.response_tx diff --git a/nodedb/src/control/planner/rls_injection/meta.rs b/nodedb/src/control/planner/rls_injection/meta.rs index 1506be6b1..dad359032 100644 --- a/nodedb/src/control/planner/rls_injection/meta.rs +++ b/nodedb/src/control/planner/rls_injection/meta.rs @@ -57,7 +57,6 @@ pub(super) fn inject_meta(ctx: &RlsCtx<'_>, op: &mut MetaOp) -> crate::Result<() MetaOp::TransactionBatch { plans, .. } | MetaOp::CalvinExecuteStatic { plans, .. } | MetaOp::CalvinExecuteActive { plans, .. } - | MetaOp::RecordCalvinWriteVersions { plans, .. } | MetaOp::ResolveTxn { plans, .. } => { for plan in plans.iter_mut() { walk(ctx, plan)?; @@ -107,7 +106,6 @@ pub(super) fn inject_meta(ctx: &RlsCtx<'_>, op: &mut MetaOp) -> crate::Result<() | MetaOp::DropTxnOverlay { .. } | MetaOp::MarkSavepoint { .. } | MetaOp::RollbackToSavepoint { .. } - | MetaOp::CalvinFlush { .. } | MetaOp::CalvinDrop { .. } | MetaOp::CalvinResolve { .. } | MetaOp::ApplyTransactionRedo { .. } diff --git a/nodedb/src/control/planner/rls_injection/permission_tree/meta.rs b/nodedb/src/control/planner/rls_injection/permission_tree/meta.rs index 9d52694a0..116f32989 100644 --- a/nodedb/src/control/planner/rls_injection/permission_tree/meta.rs +++ b/nodedb/src/control/planner/rls_injection/permission_tree/meta.rs @@ -57,7 +57,6 @@ pub(super) fn apply_meta(ctx: &PermCtx<'_>, op: &mut MetaOp) -> crate::Result<() MetaOp::TransactionBatch { plans, .. } | MetaOp::CalvinExecuteStatic { plans, .. } | MetaOp::CalvinExecuteActive { plans, .. } - | MetaOp::RecordCalvinWriteVersions { plans, .. } | MetaOp::ResolveTxn { plans, .. } => { for plan in plans.iter_mut() { walk(ctx, plan)?; @@ -106,7 +105,6 @@ pub(super) fn apply_meta(ctx: &PermCtx<'_>, op: &mut MetaOp) -> crate::Result<() | MetaOp::DropTxnOverlay { .. } | MetaOp::MarkSavepoint { .. } | MetaOp::RollbackToSavepoint { .. } - | MetaOp::CalvinFlush { .. } | MetaOp::CalvinDrop { .. } | MetaOp::CalvinResolve { .. } | MetaOp::ApplyTransactionRedo { .. } diff --git a/nodedb/src/control/planner/sql_plan_convert/array_convert/dml.rs b/nodedb/src/control/planner/sql_plan_convert/array_convert/dml.rs index 46bf76048..6e1538742 100644 --- a/nodedb/src/control/planner/sql_plan_convert/array_convert/dml.rs +++ b/nodedb/src/control/planner/sql_plan_convert/array_convert/dml.rs @@ -2,8 +2,8 @@ //! `INSERT INTO ARRAY` / `DELETE FROM ARRAY` lowering to `PhysicalTask`. -use nodedb_array::coord::encode::encode_hilbert_prefix; use nodedb_array::schema::ArraySchema; +use nodedb_array::tile::cell_tile_prefix; use nodedb_array::types::ArrayId; use nodedb_sql::types_array::{ArrayCoordLiteral, ArrayInsertRow}; @@ -84,8 +84,10 @@ pub(in super::super) fn convert_insert_array( detail: format!("array coord pk encode: {e}"), })?; let surrogate = ctx.surrogate_for_pk(ctx.collection_key(name), &pk_bytes)?; + // A cell routes by its tile's key, the key a slice's shard + // fan-out and each shard's tile filter use. let hilbert = - encode_hilbert_prefix(&schema, &coord).map_err(|e| crate::Error::PlanError { + cell_tile_prefix(&schema, &coord).map_err(|e| crate::Error::PlanError { detail: format!("INSERT INTO ARRAY {name}: Hilbert prefix: {e}"), })?; let cell = ArrayPutCell { @@ -211,7 +213,7 @@ pub(in super::super) fn convert_delete_array( for row in coords { let typed = coerce_coords(row, &schema)?; let hilbert = - encode_hilbert_prefix(&schema, &typed).map_err(|e| crate::Error::PlanError { + cell_tile_prefix(&schema, &typed).map_err(|e| crate::Error::PlanError { detail: format!("DELETE FROM ARRAY {name}: Hilbert prefix: {e}"), })?; let cell = ArrayDeleteCell { diff --git a/nodedb/src/control/planner/sql_plan_convert/array_fn_convert/slice.rs b/nodedb/src/control/planner/sql_plan_convert/array_fn_convert/slice.rs index 7daa52d12..453669ffc 100644 --- a/nodedb/src/control/planner/sql_plan_convert/array_fn_convert/slice.rs +++ b/nodedb/src/control/planner/sql_plan_convert/array_fn_convert/slice.rs @@ -4,9 +4,10 @@ use nodedb_array::query::slice::{DimRange, Slice}; use nodedb_array::schema::ArraySchema; -use nodedb_array::tile::layout::tile_id_for_cell; +use nodedb_array::schema::DimSpec; +use nodedb_array::tile::{tile_prefix, tiles_per_dim}; use nodedb_array::types::ArrayId; -use nodedb_array::types::coord::value::CoordValue; +use nodedb_array::types::domain::DomainBound; use nodedb_sql::temporal::TemporalScope; use nodedb_sql::types_array::ArraySliceAst; @@ -66,10 +67,8 @@ pub(crate) fn convert_slice( // intercepts it and calls ArrayCoordinator::coord_slice instead // of sending to the local SPSC bridge. // - // Compute a conservative Hilbert bounding range from the slice's - // dim-range corners so the coordinator only fans out to the shards - // whose Hilbert buckets overlap the query range, rather than all - // 1024 shards (the "empty ranges = unbounded" fallback). + // The coordinator fans out only to the shards that own a tile the + // slice covers. An empty list fans out to every shard. let slice_hilbert_ranges = compute_slice_hilbert_ranges(&schema, &slice.dim_ranges, entry.prefix_bits); PhysicalPlan::ClusterArray(ClusterArrayOp::Slice { @@ -105,92 +104,94 @@ pub(crate) fn convert_slice( }]) } -/// Compute a conservative Hilbert bounding range for a slice predicate. -/// -/// For each constrained dimension, uses the lo and hi bounds as the range -/// endpoints. Generates all 2^num_dims corner coordinates of the bounding -/// box and computes the Hilbert prefix for each. Returns a single -/// `(min_prefix, max_prefix)` range that conservatively covers all cells -/// that can match the slice. +/// The most tiles a slice's shard fan-out enumerates. A slice covering more +/// tiles fans out to every shard. +const MAX_FAN_OUT_TILES: u128 = 4096; + +/// The Hilbert ranges a slice's shard fan-out covers: one `(key, key)` pair +/// per routing bucket, each holding the key of one tile in that bucket. /// -/// Falls back to an empty vec (unbounded = contact all shards) if the -/// schema has more than 16 dimensions (2^16 corners will be excessive) -/// or if any Hilbert encoding fails. +/// Every tile whose cells can match the slice is enumerated, so every shard +/// that holds such a cell, or will hold one a later write lands there, is +/// covered. A transaction's read of the slice then validates on each of +/// those shards. A cell routes by its tile's key (`cell_tile_prefix`), the +/// same key enumerated here. /// -/// The result can over-estimate the shard set — the shard-side slice -/// filter is always applied and never produces false positives. +/// Returns an empty vec, which fans out to every shard, when the slice +/// covers more than [`MAX_FAN_OUT_TILES`] tiles, when the schema has no +/// dimension, or when a tile key fails to encode. fn compute_slice_hilbert_ranges( schema: &ArraySchema, dim_ranges: &[Option], - _prefix_bits: u8, + prefix_bits: u8, ) -> Vec<(u64, u64)> { - use nodedb_array::types::domain::DomainBound; - - let ndim = schema.dims.len(); - if ndim == 0 || ndim > 16 { - return Vec::new(); // fallback: unbounded + if schema.dims.is_empty() || prefix_bits == 0 || prefix_bits > 16 { + return Vec::new(); } - - // Build lo/hi bound for each dim. Unconstrained dims use the schema domain. - let mut lo_vals: Vec = Vec::with_capacity(ndim); - let mut hi_vals: Vec = Vec::with_capacity(ndim); - - for (i, dr_opt) in dim_ranges.iter().enumerate() { - let dim = &schema.dims[i]; - let (lo_bound, hi_bound) = if let Some(dr) = dr_opt { - (dr.lo.clone(), dr.hi.clone()) - } else { - (dim.domain.lo.clone(), dim.domain.hi.clone()) - }; - - let lo_cv = match lo_bound { - DomainBound::Int64(v) => CoordValue::Int64(v), - DomainBound::Float64(v) => CoordValue::Float64(v), - DomainBound::TimestampMs(v) => CoordValue::TimestampMs(v), - DomainBound::String(v) => CoordValue::String(v), - }; - let hi_cv = match hi_bound { - DomainBound::Int64(v) => CoordValue::Int64(v), - DomainBound::Float64(v) => CoordValue::Float64(v), - DomainBound::TimestampMs(v) => CoordValue::TimestampMs(v), - DomainBound::String(v) => CoordValue::String(v), - }; - - lo_vals.push(lo_cv); - hi_vals.push(hi_cv); + let spans: Vec<(u64, u64)> = schema + .dims + .iter() + .zip(schema.tile_extents.iter()) + .enumerate() + .map(|(i, (dim, extent))| { + dim_tile_span(dim, *extent, dim_ranges.get(i).and_then(Option::as_ref)) + }) + .collect(); + let tiles: u128 = spans + .iter() + .map(|(lo, hi)| u128::from(hi - lo) + 1) + .fold(1u128, |acc, n| acc.saturating_mul(n)); + if tiles > MAX_FAN_OUT_TILES { + return Vec::new(); } - // Generate all 2^ndim corners and compute their Hilbert prefix. - let num_corners = 1usize << ndim; - let mut min_prefix = u64::MAX; - let mut max_prefix = 0u64; - let mut found_any = false; - - for mask in 0..num_corners { - let corner: Vec = (0..ndim) - .map(|i| { - if (mask >> i) & 1 == 1 { - hi_vals[i].clone() - } else { - lo_vals[i].clone() - } - }) - .collect(); - - // Use the tile prefix (computed from tile indices = coord / tile_extents), - // not the cell prefix — tiles store and the per-shard filter compares - // against tile-level Hilbert keys. - if let Ok(tile_id) = tile_id_for_cell(schema, &corner, 0) { - min_prefix = min_prefix.min(tile_id.hilbert_prefix); - max_prefix = max_prefix.max(tile_id.hilbert_prefix); - found_any = true; + let shift = 64 - u32::from(prefix_bits); + let mut buckets: std::collections::BTreeMap = std::collections::BTreeMap::new(); + let mut indices: Vec = spans.iter().map(|(lo, _)| *lo).collect(); + loop { + let Ok(key) = tile_prefix(schema, &indices) else { + return Vec::new(); + }; + buckets.entry(key >> shift).or_insert(key); + // Advance the odometer over the per-dimension tile spans. + let mut dim = 0; + loop { + if dim == indices.len() { + return buckets.into_values().map(|key| (key, key)).collect(); + } + if indices[dim] < spans[dim].1 { + indices[dim] += 1; + break; + } + indices[dim] = spans[dim].0; + dim += 1; } } +} - if found_any { - vec![(min_prefix, max_prefix)] - } else { - Vec::new() // fallback: unbounded +/// The inclusive tile index span along `dim` that `range` covers, the whole +/// dimension when `range` is `None`. A string dimension tiles by hash, so a +/// range on it covers every tile. A bound outside the domain is clamped to +/// the domain's tiles. +fn dim_tile_span(dim: &DimSpec, extent: u64, range: Option<&DimRange>) -> (u64, u64) { + let last = u64::try_from(tiles_per_dim(dim, extent).saturating_sub(1)).unwrap_or(u64::MAX); + let extent = u128::from(extent.max(1)); + let index = |bound: &DomainBound| -> Option { + let offset: u128 = match (bound, &dim.domain.lo) { + (DomainBound::Int64(v), DomainBound::Int64(lo)) + | (DomainBound::TimestampMs(v), DomainBound::TimestampMs(lo)) => { + ((*v as i128) - (*lo as i128)).max(0) as u128 + } + (DomainBound::Float64(v), DomainBound::Float64(lo)) if v.is_finite() => { + (v - lo).max(0.0) as u128 + } + _ => return None, + }; + Some(u64::try_from(offset / extent).unwrap_or(u64::MAX).min(last)) + }; + match range.map(|r| (index(&r.lo), index(&r.hi))) { + Some((Some(lo), Some(hi))) => (lo.min(hi), lo.max(hi)), + _ => (0, last), } } @@ -265,6 +266,68 @@ mod tests { (ctx, handle) } + /// Every cell a slice can match, or a later write can land in its box, + /// routes to a shard the slice fans out to. + #[test] + fn the_fan_out_covers_the_shard_of_every_cell_in_the_box() { + use nodedb_array::types::coord::value::CoordValue; + let schema = ArraySchemaBuilder::new("grid") + .dim(DimSpec::new( + "chr".to_string(), + DimType::Int64, + Domain::new(DomainBound::Int64(0), DomainBound::Int64(9)), + )) + .dim(DimSpec::new( + "pos".to_string(), + DimType::Int64, + Domain::new(DomainBound::Int64(0), DomainBound::Int64(99)), + )) + .attr(AttrSpec::new("qual".to_string(), AttrType::Float64, true)) + .tile_extents(vec![1, 100]) + .build() + .expect("schema build"); + let box_ranges = vec![ + Some(DimRange::new(DomainBound::Int64(0), DomainBound::Int64(2))), + Some(DimRange::new(DomainBound::Int64(0), DomainBound::Int64(99))), + ]; + let ranges = compute_slice_hilbert_ranges(&schema, &box_ranges, 8); + assert!(!ranges.is_empty(), "a three-tile slice fans out by tile"); + let fanned = nodedb_cluster::distributed_array::array_vshards_for_slice(&ranges, 8, 1024) + .expect("fan-out"); + let mut owners = std::collections::BTreeSet::new(); + for chr in 0..=2 { + for pos in [0, 10, 20, 50, 99] { + let coord = [CoordValue::Int64(chr), CoordValue::Int64(pos)]; + let key = nodedb_array::tile::cell_tile_prefix(&schema, &coord).expect("route"); + let owner = nodedb_cluster::distributed_array::array_vshard_for_tile(key, 8) + .expect("owner"); + assert!( + fanned.contains(&owner), + "cell ({chr}, {pos}) routes to vShard {owner}, outside the fan-out {fanned:?}" + ); + owners.insert(owner); + } + } + assert!(owners.len() > 1, "the slice's tiles spread over shards"); + } + + #[test] + fn a_slice_of_more_tiles_than_the_cap_fans_out_to_every_shard() { + let (_, schema) = make_schema(); + let wide = ArraySchemaBuilder::new("wide") + .dim(DimSpec::new( + "x".to_string(), + DimType::Int64, + Domain::new(DomainBound::Int64(0), DomainBound::Int64(1_000_000)), + )) + .attr(AttrSpec::new("val".to_string(), AttrType::Float64, true)) + .tile_extents(vec![1]) + .build() + .expect("schema build"); + assert!(compute_slice_hilbert_ranges(&wide, &[None], 8).is_empty()); + assert!(!compute_slice_hilbert_ranges(&schema, &[None], 8).is_empty()); + } + #[test] fn single_node_emits_local_array() { let (ctx, _handle) = make_ctx(false); diff --git a/nodedb/src/control/request_tracker.rs b/nodedb/src/control/request_tracker.rs index 7fbafd085..e854257df 100644 --- a/nodedb/src/control/request_tracker.rs +++ b/nodedb/src/control/request_tracker.rs @@ -220,8 +220,8 @@ mod tests { payload: Payload::empty(), watermark_lsn: Lsn::ZERO, error_code: None, - read_set_valid: None, - read_version_lsn: crate::types::Lsn::ZERO, + stage_vote: None, + read_versions: crate::types::ReadVersions::new(), write_set: Vec::new(), } } @@ -235,8 +235,8 @@ mod tests { payload: Payload::from_vec(data.as_bytes().to_vec()), watermark_lsn: Lsn::ZERO, error_code: None, - read_set_valid: None, - read_version_lsn: crate::types::Lsn::ZERO, + stage_vote: None, + read_versions: crate::types::ReadVersions::new(), write_set: Vec::new(), } } diff --git a/nodedb/src/control/security/auth_fence/state.rs b/nodedb/src/control/security/auth_fence/state.rs index fd010136b..51995c51d 100644 --- a/nodedb/src/control/security/auth_fence/state.rs +++ b/nodedb/src/control/security/auth_fence/state.rs @@ -8,7 +8,6 @@ use std::sync::{Arc, Mutex, OnceLock}; use tokio::sync::Notify; -use crate::control::cluster::calvin::scheduler::AppliedMirrors; use crate::control::security::auth_lease::{ CalvinAckCoverage, LeaderLeaseService, LeaseHolder, LeaseTiming, }; @@ -31,8 +30,6 @@ pub struct AuthorizationFence { tree_defs: PendingTreeDefs, /// The permission cache's source collections, readable without its lock. sources: Arc, - /// Which Calvin positions this node's schedulers applied. - calvin_mirrors: AppliedMirrors, /// Sequencer completion acks not yet settled against local schedulers. calvin_acks: CalvinAckCoverage, /// This node's authorization lease. @@ -56,7 +53,6 @@ impl AuthorizationFence { read_index: Mutex::new(HashMap::new()), tree_defs: PendingTreeDefs::default(), sources, - calvin_mirrors: AppliedMirrors::default(), calvin_acks: CalvinAckCoverage::default(), holder: LeaseHolder::default(), timing: OnceLock::new(), @@ -106,11 +102,6 @@ impl AuthorizationFence { &self.sources } - /// Which Calvin positions this node's schedulers applied. - pub fn calvin_mirrors(&self) -> &AppliedMirrors { - &self.calvin_mirrors - } - /// Sequencer completion acks not yet settled against local schedulers. pub fn calvin_acks(&self) -> &CalvinAckCoverage { &self.calvin_acks diff --git a/nodedb/src/control/security/auth_lease/calvin_acks.rs b/nodedb/src/control/security/auth_lease/calvin_acks.rs index 3d6ed6362..26b8b52fe 100644 --- a/nodedb/src/control/security/auth_lease/calvin_acks.rs +++ b/nodedb/src/control/security/auth_lease/calvin_acks.rs @@ -10,15 +10,15 @@ //! its own scheduler applied that transaction too. //! //! The sequencer state machine records each ack it applies. This tracker -//! takes them in log order and settles each against the local scheduler's -//! applied mirror. The first unsettled ack bounds the coverage. +//! takes them in log order and settles each against the vShard's applied +//! ledger on this node. The first unsettled ack bounds the coverage. use std::collections::VecDeque; use std::sync::Mutex; use nodedb_cluster::calvin::{AppliedCompletionAck, CalvinCompletionRegistry}; -use crate::control::cluster::calvin::scheduler::AppliedMirrors; +use crate::control::cluster::calvin::scheduler::CalvinAppliedLedgers; /// Acks taken from the sequencer log and not yet settled. #[derive(Debug, Default)] @@ -36,7 +36,7 @@ impl CalvinAckCoverage { pub fn covered_through( &self, registry: &CalvinCompletionRegistry, - mirrors: &AppliedMirrors, + ledgers: &CalvinAppliedLedgers, applied: u64, replicates: impl Fn(u32) -> bool, ) -> u64 { @@ -44,9 +44,9 @@ impl CalvinAckCoverage { pending.extend(registry.applied_acks.drain()); while let Some(ack) = pending.front() { let settled = !replicates(ack.vshard_id) - || mirrors + || ledgers .get(ack.vshard_id) - .is_some_and(|mirror| mirror.is_applied(ack.txn.epoch, ack.txn.position)); + .is_some_and(|ledger| ledger.is_applied(ack.txn.epoch, ack.txn.position)); if !settled { break; } @@ -80,8 +80,8 @@ mod tests { fn an_ack_the_local_replica_has_not_applied_bounds_the_coverage() { let registry = CalvinCompletionRegistry::new_detached(); registry.applied_acks.enable(); - let mirrors = AppliedMirrors::default(); - let mirror = mirrors.register(7, NOT_YET_APPLIED_EPOCH, &BTreeSet::new()); + let ledgers = CalvinAppliedLedgers::default(); + let ledger = ledgers.install(7, NOT_YET_APPLIED_EPOCH, BTreeSet::new()); let coverage = CalvinAckCoverage::default(); registry.applied_acks.record(ack(4, 1, 7)); @@ -89,38 +89,38 @@ mod tests { // vShard 7 is replicated here and its scheduler has not applied epoch 1. let replicates = |vshard: u32| vshard == 7; assert_eq!( - coverage.covered_through(®istry, &mirrors, 8, replicates), + coverage.covered_through(®istry, &ledgers, 8, replicates), 3 ); - mirror.mark(1, 0); + ledger.mark_applied(1, 0); // The ack for vShard 9 settles at once: it is not replicated here. assert_eq!( - coverage.covered_through(®istry, &mirrors, 8, replicates), + coverage.covered_through(®istry, &ledgers, 8, replicates), 8 ); } #[test] - fn a_replicated_vshard_without_a_scheduler_yet_is_unsettled() { + fn a_replicated_vshard_without_a_ledger_is_unsettled() { let registry = CalvinCompletionRegistry::new_detached(); registry.applied_acks.enable(); - let mirrors = AppliedMirrors::default(); + let ledgers = CalvinAppliedLedgers::default(); let coverage = CalvinAckCoverage::default(); registry.applied_acks.record(ack(2, 1, 5)); assert_eq!( - coverage.covered_through(®istry, &mirrors, 3, |_| true), + coverage.covered_through(®istry, &ledgers, 3, |_| true), 1 ); } - /// A restart loses the in-memory mirror and, after a checkpoint, the WAL - /// markers of transactions applied before it. The mirror rebuilt from the + /// A restart loses the in-memory ledger and, after a checkpoint, the WAL + /// markers of transactions applied before it. The ledger rebuilt from the /// state the checkpoint saved settles every ack the sequencer log replays, /// so the coverage reaches the sequencer log's applied index. #[test] - fn a_mirror_rebuilt_from_saved_state_settles_replayed_acks() { - use crate::control::cluster::calvin::scheduler::recover_applied; + fn a_ledger_rebuilt_from_saved_state_settles_replayed_acks() { + use crate::control::cluster::calvin::scheduler::recover_all_applied; use crate::control::security::catalog::SystemCatalog; use crate::control::security::catalog::calvin_applied::StoredCalvinApplied; @@ -138,9 +138,10 @@ mod tests { }]) .expect("save"); - let recovered = recover_applied(&wal, &catalog, 7, None).expect("recover"); - let mirrors = AppliedMirrors::default(); - mirrors.register(7, recovered.fully_applied_epoch, &recovered.applied_tail); + let mut recovered = recover_all_applied(&wal, &catalog, &|_| None).expect("recover"); + let recovered = recovered.remove(&7).expect("vShard 7 recovered"); + let ledgers = CalvinAppliedLedgers::default(); + ledgers.install(7, recovered.fully_applied_epoch, recovered.applied_tail); let registry = CalvinCompletionRegistry::new_detached(); registry.applied_acks.enable(); @@ -156,9 +157,9 @@ mod tests { }); let coverage = CalvinAckCoverage::default(); assert_eq!( - coverage.covered_through(®istry, &mirrors, 19, |_| true), + coverage.covered_through(®istry, &ledgers, 19, |_| true), 19, - "every replayed ack settles against the rebuilt mirror" + "every replayed ack settles against the rebuilt ledger" ); } } diff --git a/nodedb/src/control/security/auth_lease/coverage.rs b/nodedb/src/control/security/auth_lease/coverage.rs index e0836b587..3722dedbf 100644 --- a/nodedb/src/control/security/auth_lease/coverage.rs +++ b/nodedb/src/control/security/auth_lease/coverage.rs @@ -78,7 +78,7 @@ fn sequencer_coverage(state: &SharedState) -> u64 { let fence = &state.authorization_fence; fence .calvin_acks() - .covered_through(registry, fence.calvin_mirrors(), applied, |vshard_id| { + .covered_through(registry, &state.calvin.applied, applied, |vshard_id| { group_of_vshard(state, vshard_id).is_ok_and(|g| hosts_group(state, g)) }) } diff --git a/nodedb/src/control/security/catalog/bootstrap_tables.rs b/nodedb/src/control/security/catalog/bootstrap_tables.rs index 35b42b2f8..15c0953ee 100644 --- a/nodedb/src/control/security/catalog/bootstrap_tables.rs +++ b/nodedb/src/control/security/catalog/bootstrap_tables.rs @@ -79,8 +79,10 @@ pub(super) const BOOTSTRAP_TABLES: &[BootstrapTable] = bootstrap_tables![ "tenant_group_marks" => super::tenant_group_marks::TENANT_GROUP_MARKS, "tenant_group_restore_marks" => super::tenant_group_marks::TENANT_GROUP_RESTORE_MARKS, "calvin_applied" => super::calvin_applied::CALVIN_APPLIED, + "calvin_barrier_logs" => super::calvin_barrier_logs::CALVIN_BARRIER_LOGS, "calvin_base" => super::calvin_base::CALVIN_BASE, "calvin_sequencer_install" => super::calvin_base::CALVIN_SEQUENCER_INSTALL, + "redo_snapshot_owed" => super::redo_snapshot_owed::REDO_SNAPSHOT_OWED, "l2_cleanup_queue" => L2_CLEANUP_QUEUE, "pending_reclaim" => PENDING_RECLAIM, // ── Metadata-group host state ── diff --git a/nodedb/src/control/security/catalog/calvin_applied.rs b/nodedb/src/control/security/catalog/calvin_applied.rs index 18ca14c20..e003c87c0 100644 --- a/nodedb/src/control/security/catalog/calvin_applied.rs +++ b/nodedb/src/control/security/catalog/calvin_applied.rs @@ -75,30 +75,29 @@ fn decode_tail(bytes: &[u8]) -> crate::Result> { } impl SystemCatalog { - /// The saved applied state of `vshard_id`, if any. - pub fn load_calvin_applied( - &self, - vshard_id: u32, - ) -> crate::Result> { + /// The saved applied state of every vShard. + pub fn load_all_calvin_applied(&self) -> crate::Result> { let read_txn = self .db .begin_read() - .map_err(|e| catalog_err("load_calvin_applied read txn", e))?; + .map_err(|e| catalog_err("load_all_calvin_applied read txn", e))?; let table = read_txn .open_table(CALVIN_APPLIED) .map_err(|e| catalog_err("open calvin_applied", e))?; - let Some(row) = table - .get(vshard_id) - .map_err(|e| catalog_err("get calvin_applied", e))? - else { - return Ok(None); - }; - let (fully_applied_epoch, tail) = row.value(); - Ok(Some(StoredCalvinApplied { - vshard_id, - fully_applied_epoch, - tail: decode_tail(tail)?, - })) + let mut out = Vec::new(); + for row in table + .iter() + .map_err(|e| catalog_err("iter calvin_applied", e))? + { + let (vshard, value) = row.map_err(|e| catalog_err("read calvin_applied", e))?; + let (fully_applied_epoch, tail) = value.value(); + out.push(StoredCalvinApplied { + vshard_id: vshard.value(), + fully_applied_epoch, + tail: decode_tail(tail)?, + }); + } + Ok(out) } /// Replace the saved state of each vShard in `states` with the given one. @@ -198,7 +197,7 @@ mod tests { fn saved_state_reloads_and_only_grows() { let dir = tempfile::tempdir().expect("tempdir"); let catalog = SystemCatalog::open(&dir.path().join("system.redb")).expect("catalog"); - assert_eq!(catalog.load_calvin_applied(3).expect("load"), None); + assert!(catalog.load_all_calvin_applied().expect("load").is_empty()); catalog .save_calvin_applied(vec![state(3, NONE_FULLY_APPLIED, &[(0, 0), (2, 1)])]) @@ -207,8 +206,8 @@ mod tests { .save_calvin_applied(vec![state(3, 1, &[(4, 0)])]) .expect("save"); assert_eq!( - catalog.load_calvin_applied(3).expect("load"), - Some(state(3, 1, &[(2, 1), (4, 0)])) + catalog.load_all_calvin_applied().expect("load"), + vec![state(3, 1, &[(2, 1), (4, 0)])] ); } } diff --git a/nodedb/src/control/security/catalog/calvin_barrier_logs.rs b/nodedb/src/control/security/catalog/calvin_barrier_logs.rs new file mode 100644 index 000000000..9be335917 --- /dev/null +++ b/nodedb/src/control/security/catalog/calvin_barrier_logs.rs @@ -0,0 +1,300 @@ +// SPDX-License-Identifier: BUSL-1.1 + +//! Persistent dependent-read barrier logs backing +//! `_system.calvin_barrier_logs`. +//! +//! The data-group apply loop folds each `CalvinReadResult` and +//! `CalvinReadTimeout` entry into the row of its `(vShard, txn)` before the +//! entry counts as applied. Boot resumes Raft delivery above the durable +//! applied floor, so an entry at or below it is never delivered again. Its +//! row is what a barrier of the txn reads after a restart. +//! +//! Each row holds the encoded barrier log of one txn on one vShard. This +//! module stores the bytes and never decodes them. + +use std::collections::BTreeSet; + +use redb::{ReadableDatabase, ReadableTable, TableDefinition}; + +use super::types::{SystemCatalog, catalog_err}; + +/// Table: `(vshard_id, epoch, position)` -> encoded barrier log. +pub(super) const CALVIN_BARRIER_LOGS: TableDefinition<(u32, u64, u32), &[u8]> = + TableDefinition::new("_system.calvin_barrier_logs"); + +/// One stored barrier log. +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct StoredBarrierLog { + pub vshard_id: u32, + pub epoch: u64, + pub position: u32, + /// The encoded barrier log. + pub log: Vec, +} + +/// The key range of every row of `vshard_id`. +fn vshard_range(vshard_id: u32) -> std::ops::RangeInclusive<(u32, u64, u32)> { + (vshard_id, 0, 0)..=(vshard_id, u64::MAX, u32::MAX) +} + +/// Append every row of `range` to `out`. +fn collect_rows( + out: &mut Vec, + range: redb::Range<'_, (u32, u64, u32), &'static [u8]>, +) -> crate::Result<()> { + for row in range { + let (key, value) = row.map_err(|e| catalog_err("read calvin_barrier_logs", e))?; + let (vshard_id, epoch, position) = key.value(); + out.push(StoredBarrierLog { + vshard_id, + epoch, + position, + log: value.value().to_vec(), + }); + } + Ok(()) +} + +impl SystemCatalog { + /// Replace the row of `(vshard_id, epoch, position)` with what `fold` + /// returns for the row it holds, in one transaction. A fold that + /// returns `None` keeps the row, and nothing commits. + pub fn fold_calvin_barrier_log( + &self, + vshard_id: u32, + epoch: u64, + position: u32, + fold: impl FnOnce(Option<&[u8]>) -> crate::Result>>, + ) -> crate::Result<()> { + let write_txn = self + .db + .begin_write() + .map_err(|e| catalog_err("fold_calvin_barrier_log txn", e))?; + { + let mut table = write_txn + .open_table(CALVIN_BARRIER_LOGS) + .map_err(|e| catalog_err("open calvin_barrier_logs", e))?; + let key = (vshard_id, epoch, position); + let folded = { + let held = table + .get(key) + .map_err(|e| catalog_err("get calvin_barrier_logs", e))?; + fold(held.as_ref().map(|row| row.value()))? + }; + let Some(folded) = folded else { + return Ok(()); + }; + table + .insert(key, folded.as_slice()) + .map_err(|e| catalog_err("insert calvin_barrier_logs", e))?; + } + write_txn + .commit() + .map_err(|e| catalog_err("commit calvin_barrier_logs", e)) + } + + /// The row of `(vshard_id, epoch, position)`, when one is stored. + pub fn load_calvin_barrier_log( + &self, + vshard_id: u32, + epoch: u64, + position: u32, + ) -> crate::Result>> { + let read_txn = self + .db + .begin_read() + .map_err(|e| catalog_err("load_calvin_barrier_log read txn", e))?; + let table = read_txn + .open_table(CALVIN_BARRIER_LOGS) + .map_err(|e| catalog_err("open calvin_barrier_logs", e))?; + let row = table + .get((vshard_id, epoch, position)) + .map_err(|e| catalog_err("get calvin_barrier_logs", e))?; + Ok(row.map(|row| row.value().to_vec())) + } + + /// Every stored row of the vShards in `vshards`, or of every vShard + /// when `vshards` is `None`, in key order. + pub fn load_calvin_barrier_logs( + &self, + vshards: Option<&BTreeSet>, + ) -> crate::Result> { + let read_txn = self + .db + .begin_read() + .map_err(|e| catalog_err("load_calvin_barrier_logs read txn", e))?; + let table = read_txn + .open_table(CALVIN_BARRIER_LOGS) + .map_err(|e| catalog_err("open calvin_barrier_logs", e))?; + let mut out = Vec::new(); + match vshards { + None => collect_rows( + &mut out, + table + .iter() + .map_err(|e| catalog_err("iter calvin_barrier_logs", e))?, + )?, + Some(vshards) => { + for vshard_id in vshards { + collect_rows( + &mut out, + table + .range(vshard_range(*vshard_id)) + .map_err(|e| catalog_err("range calvin_barrier_logs", e))?, + )?; + } + } + } + Ok(out) + } + + /// Remove the rows of `keys`, each `(vshard_id, epoch, position)`, in + /// one transaction. A key with no row changes nothing. + pub fn remove_calvin_barrier_logs(&self, keys: &[(u32, u64, u32)]) -> crate::Result<()> { + if keys.is_empty() { + return Ok(()); + } + let write_txn = self + .db + .begin_write() + .map_err(|e| catalog_err("remove_calvin_barrier_logs txn", e))?; + { + let mut table = write_txn + .open_table(CALVIN_BARRIER_LOGS) + .map_err(|e| catalog_err("open calvin_barrier_logs", e))?; + for key in keys { + table + .remove(*key) + .map_err(|e| catalog_err("remove calvin_barrier_logs", e))?; + } + } + write_txn + .commit() + .map_err(|e| catalog_err("commit calvin_barrier_logs", e)) + } + + /// Replace every row of the vShards in `vshards` with `rows`, in one + /// transaction. A data-group snapshot install brings the rows the + /// builder held for the group's vShards: they replace this node's. + pub fn replace_calvin_barrier_logs( + &self, + vshards: &BTreeSet, + rows: &[StoredBarrierLog], + ) -> crate::Result<()> { + let write_txn = self + .db + .begin_write() + .map_err(|e| catalog_err("replace_calvin_barrier_logs txn", e))?; + { + let mut table = write_txn + .open_table(CALVIN_BARRIER_LOGS) + .map_err(|e| catalog_err("open calvin_barrier_logs", e))?; + for vshard_id in vshards { + table + .retain_in(vshard_range(*vshard_id), |_, _| false) + .map_err(|e| catalog_err("clear calvin_barrier_logs", e))?; + } + for row in rows { + table + .insert((row.vshard_id, row.epoch, row.position), row.log.as_slice()) + .map_err(|e| catalog_err("insert calvin_barrier_logs", e))?; + } + } + write_txn + .commit() + .map_err(|e| catalog_err("commit calvin_barrier_logs", e)) + } +} + +#[cfg(test)] +mod tests { + use super::*; + + fn row(vshard_id: u32, epoch: u64, position: u32, log: &[u8]) -> StoredBarrierLog { + StoredBarrierLog { + vshard_id, + epoch, + position, + log: log.to_vec(), + } + } + + fn put(catalog: &SystemCatalog, row: &StoredBarrierLog) { + let bytes = row.log.clone(); + catalog + .fold_calvin_barrier_log(row.vshard_id, row.epoch, row.position, |_| Ok(Some(bytes))) + .expect("fold"); + } + + /// A fold sees the row it replaces, and the row reloads. + #[test] + fn a_fold_sees_the_held_row() { + let catalog = SystemCatalog::open_in_memory().expect("catalog"); + put(&catalog, &row(3, 7, 1, b"a")); + catalog + .fold_calvin_barrier_log(3, 7, 1, |held| { + assert_eq!(held, Some(b"a".as_slice())); + Ok(Some(b"ab".to_vec())) + }) + .expect("fold"); + assert_eq!( + catalog.load_calvin_barrier_log(3, 7, 1).expect("load"), + Some(b"ab".to_vec()) + ); + assert_eq!( + catalog.load_calvin_barrier_log(3, 7, 2).expect("load"), + None + ); + // A fold that keeps the row commits nothing. + catalog + .fold_calvin_barrier_log(3, 7, 1, |_| Ok(None)) + .expect("fold"); + catalog + .fold_calvin_barrier_log(3, 7, 2, |_| Ok(None)) + .expect("fold"); + assert_eq!( + catalog.load_calvin_barrier_log(3, 7, 1).expect("load"), + Some(b"ab".to_vec()) + ); + assert_eq!( + catalog.load_calvin_barrier_log(3, 7, 2).expect("load"), + None + ); + } + + /// A load of some vShards returns their rows alone, in key order. + #[test] + fn rows_load_per_vshard_in_key_order() { + let catalog = SystemCatalog::open_in_memory().expect("catalog"); + put(&catalog, &row(4, 2, 0, b"y")); + put(&catalog, &row(3, 9, 0, b"x")); + put(&catalog, &row(3, 1, 5, b"w")); + let all = catalog.load_calvin_barrier_logs(None).expect("load"); + assert_eq!( + all, + vec![row(3, 1, 5, b"w"), row(3, 9, 0, b"x"), row(4, 2, 0, b"y")] + ); + let only = catalog + .load_calvin_barrier_logs(Some(&BTreeSet::from([4]))) + .expect("load"); + assert_eq!(only, vec![row(4, 2, 0, b"y")]); + } + + /// A replace drops every row of the named vShards and keeps the rest. + #[test] + fn a_replace_swaps_the_named_vshards_rows() { + let catalog = SystemCatalog::open_in_memory().expect("catalog"); + put(&catalog, &row(3, 1, 0, b"old")); + put(&catalog, &row(4, 1, 0, b"kept")); + catalog + .replace_calvin_barrier_logs(&BTreeSet::from([3]), &[row(3, 2, 0, b"new")]) + .expect("replace"); + let all = catalog.load_calvin_barrier_logs(None).expect("load"); + assert_eq!(all, vec![row(3, 2, 0, b"new"), row(4, 1, 0, b"kept")]); + catalog + .remove_calvin_barrier_logs(&[(3, 2, 0), (9, 9, 9)]) + .expect("remove"); + let all = catalog.load_calvin_barrier_logs(None).expect("load"); + assert_eq!(all, vec![row(4, 1, 0, b"kept")]); + } +} diff --git a/nodedb/src/control/security/catalog/mod.rs b/nodedb/src/control/security/catalog/mod.rs index 0f37cebd1..80ba2fd62 100644 --- a/nodedb/src/control/security/catalog/mod.rs +++ b/nodedb/src/control/security/catalog/mod.rs @@ -9,6 +9,7 @@ pub mod backup_schedule_marks; pub mod blacklist; pub mod bootstrap_tables; pub mod calvin_applied; +pub mod calvin_barrier_logs; pub mod calvin_base; pub mod change_streams; pub mod checkpoint; @@ -59,6 +60,7 @@ pub mod procedure_types; pub mod procedures; pub mod read_only; pub mod redaction; +pub mod redo_snapshot_owed; pub mod replicated_image; pub mod replicated_image_merge; pub mod restore_points; diff --git a/nodedb/src/control/security/catalog/redo_snapshot_owed.rs b/nodedb/src/control/security/catalog/redo_snapshot_owed.rs new file mode 100644 index 000000000..6ae86066a --- /dev/null +++ b/nodedb/src/control/security/catalog/redo_snapshot_owed.rs @@ -0,0 +1,104 @@ +// SPDX-License-Identifier: BUSL-1.1 + +//! The data groups whose replica on this node owes a snapshot install, +//! backing `_system.redo_snapshot_owed`. +//! +//! A node that leaves a data group drops the group's open chunked redo +//! streams. A later remount resumes the group's old log, which can hold a +//! later chunk or final entry of a dropped stream. The row makes the remount +//! catch up by snapshot instead. A snapshot install of the group removes it. +//! +//! Table: `group_id` -> `()`. + +use redb::{ReadableDatabase, ReadableTable, TableDefinition}; + +use super::types::{SystemCatalog, catalog_err}; + +pub(super) const REDO_SNAPSHOT_OWED: TableDefinition = + TableDefinition::new("_system.redo_snapshot_owed"); + +impl SystemCatalog { + /// Every data group whose replica here owes a snapshot install. + pub fn load_redo_snapshot_owed(&self) -> crate::Result> { + let txn = self + .db + .begin_read() + .map_err(|e| catalog_err("redo_snapshot_owed read txn", e))?; + let table = txn + .open_table(REDO_SNAPSHOT_OWED) + .map_err(|e| catalog_err("open redo_snapshot_owed", e))?; + let mut out = Vec::new(); + for row in table + .iter() + .map_err(|e| catalog_err("iterate redo_snapshot_owed", e))? + { + let (group_id, _) = row.map_err(|e| catalog_err("read redo_snapshot_owed", e))?; + out.push(group_id.value()); + } + Ok(out) + } + + /// Record in one transaction that the replicas of `groups` here owe a + /// snapshot install. + pub fn save_redo_snapshot_owed(&self, groups: &[u64]) -> crate::Result<()> { + if groups.is_empty() { + return Ok(()); + } + let txn = self + .db + .begin_write() + .map_err(|e| catalog_err("redo_snapshot_owed write txn", e))?; + { + let mut table = txn + .open_table(REDO_SNAPSHOT_OWED) + .map_err(|e| catalog_err("open redo_snapshot_owed", e))?; + for &group_id in groups { + table + .insert(group_id, ()) + .map_err(|e| catalog_err("insert redo_snapshot_owed", e))?; + } + } + txn.commit() + .map_err(|e| catalog_err("redo_snapshot_owed commit", e)) + } + + /// Remove the row of `group_id`. Removing an absent row succeeds. + pub fn remove_redo_snapshot_owed(&self, group_id: u64) -> crate::Result<()> { + let txn = self + .db + .begin_write() + .map_err(|e| catalog_err("redo_snapshot_owed write txn", e))?; + { + let mut table = txn + .open_table(REDO_SNAPSHOT_OWED) + .map_err(|e| catalog_err("open redo_snapshot_owed", e))?; + table + .remove(group_id) + .map_err(|e| catalog_err("remove redo_snapshot_owed", e))?; + } + txn.commit() + .map_err(|e| catalog_err("redo_snapshot_owed commit", e)) + } +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn owed_groups_save_reload_and_remove() { + let dir = tempfile::tempdir().expect("tempdir"); + let catalog = SystemCatalog::open(&dir.path().join("system.redb")).expect("catalog"); + assert!(catalog.load_redo_snapshot_owed().expect("load").is_empty()); + + catalog.save_redo_snapshot_owed(&[4, 9]).expect("save"); + catalog.save_redo_snapshot_owed(&[4]).expect("save again"); + assert_eq!(catalog.load_redo_snapshot_owed().expect("load"), vec![4, 9]); + + catalog.remove_redo_snapshot_owed(4).expect("remove"); + catalog + .remove_redo_snapshot_owed(5) + .expect("remove an absent row"); + assert_eq!(catalog.load_redo_snapshot_owed().expect("load"), vec![9]); + } +} diff --git a/nodedb/src/control/security/catalog/replicated_image.rs b/nodedb/src/control/security/catalog/replicated_image.rs index c59552a6f..5d611acb1 100644 --- a/nodedb/src/control/security/catalog/replicated_image.rs +++ b/nodedb/src/control/security/catalog/replicated_image.rs @@ -23,6 +23,10 @@ //! - `tenant_group_marks`, `tenant_group_restore_marks`: data-group write //! marks, carried by data-group snapshots. //! - `calvin_applied`: this node's Calvin apply ledger. +//! - `calvin_barrier_logs`: this node's dependent-read barrier logs, +//! carried by data-group snapshots. +//! - `redo_snapshot_owed`: the data-group snapshot installs this node's +//! replicas owe. //! - `l2_cleanup_queue`, `pending_reclaim`, `pending_history_compaction`: //! work this node still owes its own storage. //! - `surrogate_pk_v3`, `surrogate_pk_rev_v3`: PK bindings of stored rows, diff --git a/nodedb/src/control/security/identity/plan_permission.rs b/nodedb/src/control/security/identity/plan_permission.rs index 9e826ab7f..945bcb725 100644 --- a/nodedb/src/control/security/identity/plan_permission.rs +++ b/nodedb/src/control/security/identity/plan_permission.rs @@ -394,8 +394,6 @@ pub fn required_permission(plan: &crate::bridge::envelope::PhysicalPlan) -> Perm MetaOp::CalvinExecuteStatic { .. } | MetaOp::CalvinExecutePassive { .. } | MetaOp::CalvinExecuteActive { .. } - | MetaOp::RecordCalvinWriteVersions { .. } - | MetaOp::CalvinFlush { .. } | MetaOp::CalvinDrop { .. }, ) => Permission::Write, diff --git a/nodedb/src/control/security/permission_tree/event_handler.rs b/nodedb/src/control/security/permission_tree/event_handler.rs index b6a798c81..50c9444b3 100644 --- a/nodedb/src/control/security/permission_tree/event_handler.rs +++ b/nodedb/src/control/security/permission_tree/event_handler.rs @@ -41,7 +41,11 @@ pub async fn apply_ring_events( // is not acknowledged before the step applies it. The gate sits before // the lock, so a reload can still run. #[cfg(feature = "failpoints")] - crate::control::fail_gate::wait("permission_tree::before_apply").await; + crate::control::fail_gate::wait( + crate::fail_point::FailScope::Any, + "permission_tree::before_apply", + ) + .await; { let mut guard = cache.write().await; for event in events { diff --git a/nodedb/src/control/server/broadcast.rs b/nodedb/src/control/server/broadcast.rs index 06b7053f8..36684de33 100644 --- a/nodedb/src/control/server/broadcast.rs +++ b/nodedb/src/control/server/broadcast.rs @@ -120,8 +120,8 @@ pub async fn broadcast_to_all_cores_txn( payload: crate::bridge::envelope::Payload::from_vec(outcome.merged_array), watermark_lsn: outcome.watermark_lsn, error_code: None, - read_set_valid: None, - read_version_lsn: crate::types::Lsn::ZERO, + stage_vote: None, + read_versions: outcome.read_versions, write_set: Vec::new(), }) } @@ -192,6 +192,7 @@ pub(crate) async fn broadcast_count_to_all_cores_until( wal_lsn: None, resolved_now_ms: None, commit_hlc: None, + entry_version: None, admission, }; @@ -272,8 +273,8 @@ pub(crate) async fn broadcast_count_to_all_cores_until( payload: crate::bridge::envelope::Payload::from_vec(payload), watermark_lsn: max_lsn, error_code: None, - read_set_valid: None, - read_version_lsn: crate::types::Lsn::ZERO, + stage_vote: None, + read_versions: crate::types::ReadVersions::new(), write_set: Vec::new(), }) } @@ -323,6 +324,7 @@ pub async fn broadcast_register_to_all_cores( wal_lsn: None, resolved_now_ms: None, commit_hlc: None, + entry_version: None, admission, }; diff --git a/nodedb/src/control/server/dispatch_utils/change_events/mod.rs b/nodedb/src/control/server/dispatch_utils/change_events/mod.rs index 3d30d142e..f968a6adf 100644 --- a/nodedb/src/control/server/dispatch_utils/change_events/mod.rs +++ b/nodedb/src/control/server/dispatch_utils/change_events/mod.rs @@ -8,7 +8,4 @@ mod extract; mod publish; mod redo; -pub(crate) use publish::{ - CalvinApply, PendingChanges, WriteChangeSet, extract_write_change_set, - publish_calvin_change_sets, publish_settled_changes, redo_change_set, -}; +pub(crate) use publish::{PendingChanges, extract_write_change_set, publish_settled_changes}; diff --git a/nodedb/src/control/server/dispatch_utils/change_events/publish.rs b/nodedb/src/control/server/dispatch_utils/change_events/publish.rs index 4a8c4db9e..ff024bcd6 100644 --- a/nodedb/src/control/server/dispatch_utils/change_events/publish.rs +++ b/nodedb/src/control/server/dispatch_utils/change_events/publish.rs @@ -7,9 +7,8 @@ use std::sync::Arc; use crate::bridge::envelope::{PhysicalPlan, Response}; -use crate::control::change_stream::{ChangeEvent, ChangePartition, ChangeRun, PositionedChange}; +use crate::control::change_stream::{ChangeEvent, ChangePartition}; use crate::control::state::SharedState; -use crate::event::cdc::CdcOffset; use crate::types::{DatabaseId, TenantId}; use super::extract::{WriteChangeMeta, extract_write_metadata}; @@ -111,15 +110,6 @@ pub(crate) fn extract_write_change_set(plan: &PhysicalPlan, tenant_id: TenantId) } } -/// The change events of a committed transaction's redo record `redo`: one per -/// row it installs. A Calvin commit publishes these, as the data-group apply -/// of a `TransactionRedo` entry does. -pub(crate) fn redo_change_set(redo: &[u8]) -> WriteChangeSet { - WriteChangeSet { - metas: super::redo::redo_change_meta(redo), - } -} - fn change_events( shared: &SharedState, tenant_id: TenantId, @@ -214,64 +204,5 @@ pub(crate) fn publish_settled_changes( shared.change_stream.settle_group(group_id, first, last); shared .change_stream - .forward(shared, ChangePartition::Group(group_id)); -} - -/// Publish the changes of the Calvin transaction at `(sequencer_epoch, -/// position)` that vShard `vshard` applied, and forward them when this node -/// leads the vShard's data group. -/// -/// Every replica's scheduler applies the vShard's transactions in sequencer -/// order, so every replica publishes them at the same positions. -pub(crate) fn publish_calvin_change_sets( - shared: &Arc, - calvin: CalvinApply, - change_sets: Vec, - lsn: nodedb_types::Lsn, -) { - let CalvinApply { - tenant_id, - database_id, - vshard, - sequencer_epoch, - position, - commit_hlc, - } = calvin; - let base = u64::from(position); - let changes: Vec = change_sets - .into_iter() - .flat_map(|change_set| { - change_events(shared, tenant_id, database_id, change_set, lsn, commit_hlc) - }) - .enumerate() - .map(|(ordinal, event)| PositionedChange { - position: CdcOffset::data_event_in(0, sequencer_epoch, base, ordinal as u64 + 1), - database_id, - event, - }) - .collect(); - if changes.is_empty() { - return; - } - shared.change_stream.publish_run(ChangeRun { - partition: ChangePartition::Calvin(vshard), - after: None, - through: CdcOffset::data_event_in(0, sequencer_epoch, base, u64::MAX).correction(), - changes, - }); - shared - .change_stream - .forward(shared, ChangePartition::Calvin(vshard)); -} - -/// The Calvin transaction a published change set belongs to. -#[derive(Debug, Clone, Copy)] -pub(crate) struct CalvinApply { - pub tenant_id: TenantId, - pub database_id: DatabaseId, - pub vshard: u32, - pub sequencer_epoch: u64, - pub position: u32, - /// The transaction's commit HLC, which every replica derives alike. - pub commit_hlc: u64, + .forward(shared, ChangePartition(group_id)); } diff --git a/nodedb/src/control/server/dispatch_utils/dispatch.rs b/nodedb/src/control/server/dispatch_utils/dispatch.rs index f6583661e..b1faa3f65 100644 --- a/nodedb/src/control/server/dispatch_utils/dispatch.rs +++ b/nodedb/src/control/server/dispatch_utils/dispatch.rs @@ -572,8 +572,8 @@ mod tests { payload: Payload::empty(), watermark_lsn: Lsn::ZERO, error_code: None, - read_set_valid: None, - read_version_lsn: Lsn::ZERO, + stage_vote: None, + read_versions: crate::types::ReadVersions::new(), write_set: Vec::new(), }, }) @@ -751,8 +751,8 @@ mod tests { payload: Payload::empty(), watermark_lsn: Lsn::ZERO, error_code: code.clone().map(Box::new), - read_set_valid: None, - read_version_lsn: Lsn::ZERO, + stage_vote: None, + read_versions: crate::types::ReadVersions::new(), write_set: Vec::new(), }, }) diff --git a/nodedb/src/control/server/dispatch_utils/durability_barrier.rs b/nodedb/src/control/server/dispatch_utils/durability_barrier.rs index 699eccf1e..6f5a94006 100644 --- a/nodedb/src/control/server/dispatch_utils/durability_barrier.rs +++ b/nodedb/src/control/server/dispatch_utils/durability_barrier.rs @@ -169,6 +169,7 @@ mod tests { collections: vec!["c".into()], sum_targets: Vec::new(), origin: nodedb_physical::physical_plan::RedoOrigin::Commit, + calvin: None, }); assert_eq!(funnel_minted_redo_engine(&plan), Some("transaction")); } diff --git a/nodedb/src/control/server/dispatch_utils/durable_write.rs b/nodedb/src/control/server/dispatch_utils/durable_write.rs index b86295ec1..3458f1439 100644 --- a/nodedb/src/control/server/dispatch_utils/durable_write.rs +++ b/nodedb/src/control/server/dispatch_utils/durable_write.rs @@ -215,23 +215,25 @@ async fn propose_if_replicable( return Ok(None); }; let entry = entry.with_event_source(event_source); - let (payload, write_version) = propose_replicated_entry(shared, proposer, entry).await?; + let deadline = crate::control::wal_replication::statement_propose_deadline(shared); + let (payload, write_versions) = + propose_replicated_entry(shared, proposer, entry, deadline).await?; Ok(Some(replicated_write_response( shared, payload, - write_version, + write_versions, ))) } /// The response a committed and applied replicated write answers with. /// -/// `write_version` is the written collection's `coll_write_lsn` after the -/// write. It is the watermark and the read version, so a session can floor a -/// later read at it. +/// `write_versions` are the versions the write stamped. A session floors its +/// later reads of the written vShards at them. A write carries no read +/// watermark. fn replicated_write_response( shared: &SharedState, payload: Vec, - write_version: Lsn, + write_versions: crate::types::ReadVersions, ) -> Response { Response { request_id: RequestId::new(shared.request_id_counter.fetch_add(1, Ordering::Relaxed)), @@ -239,10 +241,10 @@ fn replicated_write_response( attempt: 1, partial: false, payload: payload.into(), - watermark_lsn: write_version, + watermark_lsn: Lsn::ZERO, error_code: None, - read_set_valid: None, - read_version_lsn: write_version, + stage_vote: None, + read_versions: write_versions, write_set: Vec::new(), } } diff --git a/nodedb/src/control/server/dispatch_utils/minted/owned.rs b/nodedb/src/control/server/dispatch_utils/minted/owned.rs index fa40cf37f..daefee3f3 100644 --- a/nodedb/src/control/server/dispatch_utils/minted/owned.rs +++ b/nodedb/src/control/server/dispatch_utils/minted/owned.rs @@ -49,9 +49,10 @@ pub(crate) enum OwnedResponse { /// A response arrived by the deadline. `closed` is the result of closing /// the records from it: a failed cancel returns its error and holds the /// window. A partial response reports `Ok` here, and the task closes the - /// records from the final one. + /// records from the final one. The response is boxed: it is far larger + /// than every other variant. Answered { - response: Response, + response: Box, closed: crate::Result<()>, }, /// The deadline passed first. The task closes the records once the @@ -127,7 +128,7 @@ async fn wait_and_close( match collected { Ok(Ok(response)) if response.partial => { let _ = report.send(OwnedResponse::Answered { - response, + response: Box::new(response), closed: Ok(()), }); forward(resolve_at_final(&wal, owner, final_refusal_key, rx, minted).await); @@ -135,7 +136,10 @@ async fn wait_and_close( Ok(Ok(response)) => { let closed = resolve_on_response(&wal, owner, final_refusal_key, &response, minted).await; - let _ = report.send(OwnedResponse::Answered { response, closed }); + let _ = report.send(OwnedResponse::Answered { + response: Box::new(response), + closed, + }); } Ok(Err(DispatchCollectError::OverBudget { bytes })) => { let _ = report.send(OwnedResponse::OverBudget { bytes }); @@ -198,8 +202,8 @@ mod tests { constraint: "unique".into(), detail: "duplicate key".into(), })), - read_set_valid: None, - read_version_lsn: Lsn::ZERO, + stage_vote: None, + read_versions: crate::types::ReadVersions::new(), write_set: Vec::new(), } } diff --git a/nodedb/src/control/server/dispatch_utils/minted/records.rs b/nodedb/src/control/server/dispatch_utils/minted/records.rs index d0fd26aa8..67d14b737 100644 --- a/nodedb/src/control/server/dispatch_utils/minted/records.rs +++ b/nodedb/src/control/server/dispatch_utils/minted/records.rs @@ -141,11 +141,6 @@ impl MintedRecords { self.sent.store(true, Ordering::Release); } - /// The highest LSN appended under this window, `None` before any append. - pub(crate) fn last_lsn(&self) -> Option { - self.recorded().iter().map(|record| record.lsn).max() - } - /// Every appended LSN, in append order. #[cfg(test)] pub(crate) fn lsns(&self) -> Vec { diff --git a/nodedb/src/control/server/dispatch_utils/minted/resolve.rs b/nodedb/src/control/server/dispatch_utils/minted/resolve.rs index 32d400fcf..88f0f127e 100644 --- a/nodedb/src/control/server/dispatch_utils/minted/resolve.rs +++ b/nodedb/src/control/server/dispatch_utils/minted/resolve.rs @@ -121,8 +121,8 @@ mod tests { payload: Payload::empty(), watermark_lsn: Lsn::ZERO, error_code: code.map(Box::new), - read_set_valid: None, - read_version_lsn: Lsn::ZERO, + stage_vote: None, + read_versions: crate::types::ReadVersions::new(), write_set: Vec::new(), } } diff --git a/nodedb/src/control/server/dispatch_utils/mod.rs b/nodedb/src/control/server/dispatch_utils/mod.rs index 6582a3fe4..ab3c59b3a 100644 --- a/nodedb/src/control/server/dispatch_utils/mod.rs +++ b/nodedb/src/control/server/dispatch_utils/mod.rs @@ -13,10 +13,7 @@ mod types; mod unlogged_dispatch; mod write_abort; -pub(crate) use change_events::{ - CalvinApply, WriteChangeSet, publish_calvin_change_sets, publish_settled_changes, - redo_change_set, -}; +pub(crate) use change_events::publish_settled_changes; pub use dispatch::{dispatch_authorized_autocommit_write, dispatch_authorized_to_data_plane}; pub(crate) use dispatch::{ dispatch_autocommit_write, dispatch_replayed_write_to_data_plane, dispatch_to_data_plane, diff --git a/nodedb/src/control/server/dispatch_utils/owner_read.rs b/nodedb/src/control/server/dispatch_utils/owner_read.rs index c14997490..5050277c6 100644 --- a/nodedb/src/control/server/dispatch_utils/owner_read.rs +++ b/nodedb/src/control/server/dispatch_utils/owner_read.rs @@ -30,7 +30,7 @@ use crate::control::gateway::live_leaders::resolve_live_decision; use crate::control::security::identity::{Permission, required_permission}; use crate::control::server::payload_merge::merge_msgpack_arrays; use crate::control::state::SharedState; -use crate::types::{DatabaseId, Lsn, RequestId, TenantId, TraceId, TxnId, VShardId}; +use crate::types::{DatabaseId, Lsn, ReadVersions, RequestId, TenantId, TraceId, TxnId, VShardId}; /// Where one read runs. pub(crate) struct OwnedReadScope { @@ -95,11 +95,6 @@ async fn route_cluster_read( if scope.linearizable { confirm_linearizable_read(shared, &hosted, deadline).await?; } - // The read's versions are this node's WAL positions. - crate::control::server::shared::session::served_reads::note( - scope.vshard_id.as_u32(), - shared.node_id, - ); Ok(OwnedRead::Local(Box::new(plan))) } ReadPlacement::Owner => { @@ -111,7 +106,7 @@ async fn route_cluster_read( txn_id: scope.txn_id, linearizable: scope.linearizable, }; - owner_response(gateway.execute_internal_with_watermarks(&ctx, plan).await) + owner_response(gateway.execute_internal_outcome(&ctx, plan).await) .map(OwnedRead::Served) } } @@ -151,40 +146,42 @@ pub(crate) fn read_placement( } /// The owner's answer to a read, in the shape a local dispatch returns. +/// +/// A read that found no row answers `NotFound` with the versions it +/// observed, the same as a local miss, so the read-set records the vShard's +/// real version. pub(crate) fn owner_response( - outcome: crate::Result, + outcome: crate::Result, ) -> crate::Result { - match outcome { - Ok((payloads, watermarks, read_version_lsn)) => { - let watermark_lsn = watermarks - .iter() - .map(|(_, lsn)| *lsn) - .max() - .unwrap_or(Lsn::ZERO); - let payload = match payloads.len() { - 0 => Vec::new(), - 1 => payloads.into_iter().next().unwrap_or_default(), - _ => merge_msgpack_arrays(&payloads), - }; - Ok(response( - Status::Ok, - payload, - watermark_lsn, - read_version_lsn, - None, - )) - } - // The local path answers a missing row with a `NotFound` status, not an - // error. The owner's answer keeps that shape. - Err(crate::Error::DataPlane(ErrorCode::NotFound)) => Ok(response( + let outcome = outcome?; + let watermark_lsn = outcome + .shard_watermarks + .iter() + .map(|(_, lsn)| *lsn) + .max() + .unwrap_or(Lsn::ZERO); + if outcome.not_found { + return Ok(response( Status::Error, Vec::new(), - Lsn::ZERO, - Lsn::ZERO, + watermark_lsn, + outcome.read_versions, Some(ErrorCode::NotFound), - )), - Err(error) => Err(error), + )); } + let payloads = outcome.payloads; + let payload = match payloads.len() { + 0 => Vec::new(), + 1 => payloads.into_iter().next().unwrap_or_default(), + _ => merge_msgpack_arrays(&payloads), + }; + Ok(response( + Status::Ok, + payload, + watermark_lsn, + outcome.read_versions, + None, + )) } /// Prepare a read-only pass that must run on this node's cores. @@ -233,7 +230,13 @@ fn leads_vshard(shared: &SharedState, vshard_id: VShardId) -> bool { /// A successful response carrying `payload`, in the shape a local dispatch /// returns. pub(crate) fn ok_payload_response(payload: Payload) -> Response { - response(Status::Ok, payload.to_vec(), Lsn::ZERO, Lsn::ZERO, None) + response( + Status::Ok, + payload.to_vec(), + Lsn::ZERO, + ReadVersions::new(), + None, + ) } /// A `NotFound` refusal, the shape a local dispatch returns for a read that @@ -243,7 +246,7 @@ pub(crate) fn not_found_response() -> Response { Status::Error, Vec::new(), Lsn::ZERO, - Lsn::ZERO, + ReadVersions::new(), Some(ErrorCode::NotFound), ) } @@ -252,7 +255,7 @@ fn response( status: Status, payload: Vec, watermark_lsn: Lsn, - read_version_lsn: Lsn, + read_versions: ReadVersions, error_code: Option, ) -> Response { Response { @@ -263,8 +266,49 @@ fn response( payload: Payload::from_vec(payload), watermark_lsn, error_code: error_code.map(Box::new), - read_set_valid: None, - read_version_lsn, + stage_vote: None, + read_versions, write_set: Vec::new(), } } + +#[cfg(test)] +mod tests { + use super::*; + use crate::control::gateway::outcome::GatewayOutcome; + + fn outcome(not_found: bool) -> GatewayOutcome { + GatewayOutcome { + payloads: Vec::new(), + shard_watermarks: vec![(VShardId::new(3), Lsn::new(7))], + read_versions: ReadVersions::single( + VShardId::new(3), + nodedb_types::WriteVersion::logged(1, 40), + ), + not_found, + } + } + + /// A remote read that found no row answers like a local miss, with the + /// version it observed for the read-set. + #[test] + fn an_owner_miss_keeps_its_verdict_and_its_version() { + let response = owner_response(Ok(outcome(true))).expect("an owner miss answers"); + assert_eq!(response.status, Status::Error); + assert_eq!(response.error_code.as_deref(), Some(&ErrorCode::NotFound)); + assert_eq!( + response.read_versions.of(VShardId::new(3)), + Some(nodedb_types::WriteVersion::logged(1, 40)) + ); + } + + #[test] + fn an_owner_answer_keeps_its_version() { + let response = owner_response(Ok(outcome(false))).expect("an owner answer"); + assert_eq!(response.status, Status::Ok); + assert_eq!( + response.read_versions.of(VShardId::new(3)), + Some(nodedb_types::WriteVersion::logged(1, 40)) + ); + } +} diff --git a/nodedb/src/control/server/dispatch_utils/submit_write/funnel/admission.rs b/nodedb/src/control/server/dispatch_utils/submit_write/funnel/admission.rs index 7315b095e..8203dac78 100644 --- a/nodedb/src/control/server/dispatch_utils/submit_write/funnel/admission.rs +++ b/nodedb/src/control/server/dispatch_utils/submit_write/funnel/admission.rs @@ -5,14 +5,27 @@ use tokio::sync::OwnedMutexGuard; use crate::bridge::envelope::PhysicalPlan; +use crate::control::server::shared::write_admission::gate::admit_routed; use crate::control::server::shared::write_admission::{ - WriteAdmission, WriteAdmissionGuard, WriteTarget, admit, + WriteAdmission, WriteAdmissionGuard, WriteTarget, }; use crate::control::state::SharedState; use crate::types::{DatabaseId, TenantId, VShardId}; use super::super::params::WriteOrdering; +/// The write the admission phase admits. +pub(super) struct AdmissionTarget<'a> { + pub tenant_id: TenantId, + pub database_id: DatabaseId, + pub vshard_id: VShardId, + pub plan: &'a PhysicalPlan, + /// Whether a contended write may run through the scheduler, which + /// applies it as a client write. A write of another event source waits + /// for its keys instead. + pub may_route: bool, +} + /// Outcome of the write-admission phase. pub(super) enum AdmissionOutcome { /// Proceed with the funnel's remaining phases under these guards. @@ -26,25 +39,33 @@ pub(super) enum AdmissionOutcome { } /// Write-admission gate: every write-class plan whose ordering is not already -/// final passes here. An uncontended point write takes the fast path holding -/// its per-vShard deterministic locks; a contended or bulk write is submitted -/// through the deterministic scheduler and its applied response is surfaced -/// here; reads / control ops are `Exempt`. +/// final passes here. An uncontended write takes the fast path holding its +/// per-vShard deterministic locks; a contended write is submitted through the +/// deterministic scheduler and its applied response is surfaced here; reads / +/// control ops are `Exempt`. /// /// Ordering (fast path): the guard is acquired FIRST, then — for a write that /// owns its durability (`AppendHere`) — the WAL append happens after this /// call returns, under the guard, minting the LSN just before the enqueue. /// The guard is released immediately after the enqueue (not across the /// response await). +/// +/// A contended write no Calvin transaction can sequence waits for its keys +/// until `deadline`, holding none meanwhile. pub(super) async fn admit_write( shared: &SharedState, - tenant_id: TenantId, - database_id: DatabaseId, - vshard_id: VShardId, - plan: &PhysicalPlan, + target: AdmissionTarget<'_>, ordering: WriteOrdering, -) -> AdmissionOutcome { - match ordering { + deadline: std::time::Instant, +) -> crate::Result { + let AdmissionTarget { + tenant_id, + database_id, + vshard_id, + plan, + may_route, + } = target; + Ok(match ordering { WriteOrdering::AlreadyOrdered => AdmissionOutcome::Proceed { admission: crate::bridge::envelope::Admission::Exempt( crate::bridge::envelope::ExemptReason::AlreadyOrdered, @@ -52,7 +73,7 @@ pub(super) async fn admit_write( admission_guard: None, order_guard: None, }, - WriteOrdering::Gate => match admit( + WriteOrdering::Gate => match admit_routed( shared, &WriteTarget { tenant_id, @@ -60,7 +81,8 @@ pub(super) async fn admit_write( vshard_id, plan, }, - ) { + may_route, + )? { WriteAdmission::ExemptRead => AdmissionOutcome::Proceed { admission: crate::bridge::envelope::Admission::Exempt( crate::bridge::envelope::ExemptReason::Read, @@ -89,6 +111,14 @@ pub(super) async fn admit_write( } } WriteAdmission::RouteToCalvin => AdmissionOutcome::RouteToCalvin, + WriteAdmission::Wait(wait) => AdmissionOutcome::Proceed { + admission: crate::bridge::envelope::Admission::Admitted, + admission_guard: Some( + wait.acquire(tokio::time::Instant::from_std(deadline)) + .await?, + ), + order_guard: None, + }, }, - } + }) } diff --git a/nodedb/src/control/server/dispatch_utils/submit_write/funnel/answer.rs b/nodedb/src/control/server/dispatch_utils/submit_write/funnel/answer.rs index 01468577c..b5fbd302e 100644 --- a/nodedb/src/control/server/dispatch_utils/submit_write/funnel/answer.rs +++ b/nodedb/src/control/server/dispatch_utils/submit_write/funnel/answer.rs @@ -39,7 +39,7 @@ impl Answer { .await; Ok(match collected { Ok(Ok(response)) => OwnedResponse::Answered { - response, + response: Box::new(response), closed: Ok(()), }, Ok(Err(DispatchCollectError::OverBudget { bytes })) => { diff --git a/nodedb/src/control/server/dispatch_utils/submit_write/funnel/dispatch.rs b/nodedb/src/control/server/dispatch_utils/submit_write/funnel/dispatch.rs index f289df5a6..c7fb27a57 100644 --- a/nodedb/src/control/server/dispatch_utils/submit_write/funnel/dispatch.rs +++ b/nodedb/src/control/server/dispatch_utils/submit_write/funnel/dispatch.rs @@ -43,6 +43,9 @@ pub(super) struct DispatchTarget { pub resolved_now_ms: Option, /// The write's commit HLC, which dates every event it emits. pub commit_hlc: u64, + /// The log position of the data-group entry the write applies, its + /// version on every replica. `None` for a write that applies no entry. + pub entry_version: Option, pub admission: Admission, /// The record group the write journals its write set into. pub journal: Option, @@ -99,6 +102,7 @@ pub(super) async fn dispatch_to_data_plane( wal_lsn: target.wal_lsn, resolved_now_ms: target.resolved_now_ms, commit_hlc: Some(target.commit_hlc), + entry_version: target.entry_version, admission: target.admission, }; diff --git a/nodedb/src/control/server/dispatch_utils/submit_write/funnel/driver.rs b/nodedb/src/control/server/dispatch_utils/submit_write/funnel/driver.rs index 8a4b92400..65cb2f4a1 100644 --- a/nodedb/src/control/server/dispatch_utils/submit_write/funnel/driver.rs +++ b/nodedb/src/control/server/dispatch_utils/submit_write/funnel/driver.rs @@ -16,7 +16,7 @@ use crate::control::server::dispatch_utils::minted::{ }; use crate::control::server::shared::session::statement_deadline; use crate::control::server::shared::write_admission::{ - bare_ok_response, order_row_write, route_write_to_calvin, + bare_ok_response, calvin_route_keeps, order_row_write, route_write_to_calvin, }; use crate::control::server::wal_dispatch; use crate::control::state::SharedState; @@ -25,7 +25,7 @@ use crate::engine::timeseries::resolved_ingest::TsDriftPolicy; use super::super::params::{ ChangeFeedOwner, SubmitOutcome, SubmitWrite, WalDurability, WriteOrdering, }; -use super::admission::{AdmissionOutcome, admit_write}; +use super::admission::{AdmissionOutcome, AdmissionTarget, admit_write}; use super::answer::Answer; use super::dispatch::{DispatchTarget, HeldGuards, dispatch_to_data_plane}; use super::late_parts::LateParts; @@ -196,37 +196,59 @@ pub(crate) async fn enqueue_write( }; // Write-admission gate: every write-class plan whose ordering is not already - // final passes here. On the Calvin route the deterministic scheduler applies - // the write, emits its own WriteEvents, and owns durability (the sequenced - // TxClass plus its own `CalvinApplied` WAL record), so no local WAL append or - // enqueue happens. A plain write with no RETURNING rows yields `None`, + // final passes here. On the Calvin route the deterministic scheduler owns + // the write: the sequenced TxClass, then the stamped redo entry the data + // group applies. No local WAL append or enqueue happens. A plain write with no RETURNING rows yields `None`, // synthesized into a bare `Ok`. - let (admission, admission_guard, order_guard) = - match admit_write(shared, tenant_id, database_id, vshard_id, &plan, ordering).await { - AdmissionOutcome::Proceed { - admission, - admission_guard, - order_guard, - } => (admission, admission_guard, order_guard), - AdmissionOutcome::RouteToCalvin => { - // The scheduler journals the write in its own sequenced record. - // The scheduler applies the write from its own records, so - // the caller's records never apply. - let superseded = caller_minted.map(|minted| { - minted.supersede(std::sync::Arc::clone(&shared.wal), owner, "calvin_route") - }); - let routed = - route_write_to_calvin(shared, tenant_id, database_id, vshard_id, plan).await; - if let Some(superseded) = superseded { - superseded.finish().await; - } - let routed = routed?; - return Ok(PendingWrite::done(SubmitOutcome { - response: routed - .unwrap_or_else(|| bare_ok_response(crate::types::RequestId::new(0))), - })); + let target = AdmissionTarget { + tenant_id, + database_id, + vshard_id, + plan: &plan, + may_route: calvin_route_keeps(&plan, event_source, 0), + }; + let admitted = match admit_write(shared, target, ordering, deadline).await { + Ok(admitted) => admitted, + Err(error) => { + // No record of this write reaches a core. + if let Some(minted) = caller_minted { + minted.cancel(&shared.wal, owner, 0).await?; } - }; + return Err(error); + } + }; + let (admission, admission_guard, order_guard) = match admitted { + AdmissionOutcome::Proceed { + admission, + admission_guard, + order_guard, + } => (admission, admission_guard, order_guard), + AdmissionOutcome::RouteToCalvin => { + // The scheduler journals the write in its own sequenced record. + // The scheduler applies the write from its own records, so + // the caller's records never apply. + let superseded = caller_minted.map(|minted| { + minted.supersede(std::sync::Arc::clone(&shared.wal), owner, "calvin_route") + }); + let routed = route_write_to_calvin( + shared, + tenant_id, + database_id, + vshard_id, + plan, + event_source, + ) + .await; + if let Some(superseded) = superseded { + superseded.finish().await; + } + let routed = routed?; + return Ok(PendingWrite::done(SubmitOutcome { + response: routed + .unwrap_or_else(|| bare_ok_response(crate::types::RequestId::new(0))), + })); + } + }; // Order this write's row records (document rows and edges) against every // other row write of its vShard, from before the append below through its @@ -371,6 +393,11 @@ pub(crate) async fn enqueue_write( wal_lsn, resolved_now_ms, commit_hlc, + // A write that applies a data-group entry versions its rows by + // the entry's position, which every replica shares. + entry_version: wal_append_outcome.change_position.map(|position| { + nodedb_types::WriteVersion::logged(position.epoch, position.log_index) + }), admission, journal, }, diff --git a/nodedb/src/control/server/dispatch_utils/submit_write/funnel/response.rs b/nodedb/src/control/server/dispatch_utils/submit_write/funnel/response.rs index d4ce8d8df..a4e3df114 100644 --- a/nodedb/src/control/server/dispatch_utils/submit_write/funnel/response.rs +++ b/nodedb/src/control/server/dispatch_utils/submit_write/funnel/response.rs @@ -122,7 +122,7 @@ pub(super) async fn collect_classify_and_finish( OwnedResponse::Answered { response, closed } => { // A failed cancel holds the window and fails the write here. closed?; - response + *response } OwnedResponse::DeadlineExceeded => { observe(shared); diff --git a/nodedb/src/control/server/dispatch_utils/write_abort.rs b/nodedb/src/control/server/dispatch_utils/write_abort.rs index 0ed1cdba0..088202080 100644 --- a/nodedb/src/control/server/dispatch_utils/write_abort.rs +++ b/nodedb/src/control/server/dispatch_utils/write_abort.rs @@ -36,9 +36,8 @@ use crate::bridge::envelope::ErrorCode; /// a redelivery here can too. That covers admission and capacity verdicts, /// a task that expired before it started, concurrency retries, the staging /// byte budget, a sync hold, which depends on the core's own stream mark, -/// and `RetryableRefusal`, -/// which a committed-redo apply answers with after it rolled a failed -/// install back. +/// `RetryableRefusal`, which a committed-redo apply answers with after it +/// rolled a failed install back, and a fail-stopped core's refusal. pub(crate) fn refusal_is_final(code: &ErrorCode) -> bool { write_definitely_not_applied(code) && !is_transient_verdict(code) } @@ -48,6 +47,7 @@ pub(crate) fn refusal_is_final(code: &ErrorCode) -> bool { fn is_transient_verdict(code: &ErrorCode) -> bool { match code { ErrorCode::RetryableRefusal { .. } + | ErrorCode::CoreFailStopped { .. } | ErrorCode::SyncNotApplied { .. } | ErrorCode::RateExceeded { .. } | ErrorCode::CollectionDraining { .. } @@ -149,6 +149,8 @@ pub(crate) fn write_definitely_not_applied(code: &ErrorCode) -> bool { // Documented as applying nothing: the identical frame is expected to be // re-sent, and the retry carries its own record. | ErrorCode::RetryableRefusal { .. } + // A fail-stopped core refuses every request before it runs. + | ErrorCode::CoreFailStopped { .. } // Concurrency verdicts that abort the whole attempt before install. | ErrorCode::ConflictRetry | ErrorCode::OllpRetryRequired diff --git a/nodedb/src/control/server/exchange/all_cores/base_snapshot.rs b/nodedb/src/control/server/exchange/all_cores/base_snapshot.rs index 3c761b384..363eaa917 100644 --- a/nodedb/src/control/server/exchange/all_cores/base_snapshot.rs +++ b/nodedb/src/control/server/exchange/all_cores/base_snapshot.rs @@ -94,8 +94,8 @@ mod tests { payload: Payload::from_vec(payload.to_vec()), watermark_lsn: Lsn::ZERO, error_code: code.map(Box::new), - read_set_valid: None, - read_version_lsn: Lsn::ZERO, + stage_vote: None, + read_versions: crate::types::ReadVersions::new(), write_set: Vec::new(), } } diff --git a/nodedb/src/control/server/exchange/all_cores/bsp.rs b/nodedb/src/control/server/exchange/all_cores/bsp.rs index 7ea8ab90b..ebc1d96a2 100644 --- a/nodedb/src/control/server/exchange/all_cores/bsp.rs +++ b/nodedb/src/control/server/exchange/all_cores/bsp.rs @@ -60,7 +60,8 @@ pub(super) async fn fan_bsp_all_cores( Ok(NodeLevelResult { payload, watermark_lsn, - read_version_lsn: Lsn::ZERO, + read_versions: crate::types::ReadVersions::new(), + not_found: false, }) } diff --git a/nodedb/src/control/server/exchange/all_cores/dispatch.rs b/nodedb/src/control/server/exchange/all_cores/dispatch.rs index 5a720abef..10bb2d30c 100644 --- a/nodedb/src/control/server/exchange/all_cores/dispatch.rs +++ b/nodedb/src/control/server/exchange/all_cores/dispatch.rs @@ -18,10 +18,13 @@ use super::wcc::fan_wcc_all_cores; pub struct NodeLevelResult { pub payload: Vec, pub watermark_lsn: Lsn, - /// Max per-collection read-version LSN of the fanned read. `Lsn::ZERO` for - /// non-read results. Distinct from `watermark_lsn` — the comparand for + /// The read versions the fanned read observed, one per vShard. Empty for + /// non-read results. Distinct from `watermark_lsn`: the comparand for /// cross-shard OCC read validation. - pub read_version_lsn: Lsn, + pub read_versions: crate::types::ReadVersions, + /// The plan's one owning core refused it with `NotFound`: a read that + /// found no row. `read_versions` holds the versions it observed. + pub not_found: bool, } /// Fan `plan` across all local Data-Plane cores, merge per-core payloads, and @@ -77,7 +80,8 @@ pub(crate) async fn execute_plan_all_local_cores( Ok(NodeLevelResult { payload: envelope, watermark_lsn: outcome.watermark_lsn, - read_version_lsn: Lsn::ZERO, + read_versions: outcome.read_versions, + not_found: false, }) } @@ -195,8 +199,6 @@ pub(crate) async fn execute_plan_all_local_cores( | MetaOp::RebuildIndex { .. } | MetaOp::PutSynonymGroup { .. } | MetaOp::DeleteSynonymGroup { .. } - | MetaOp::RecordCalvinWriteVersions { .. } - | MetaOp::CalvinFlush { .. } | MetaOp::CalvinDrop { .. } => { generic_gather(state, tenant_id, database_id, plan, trace_id, txn_id).await } @@ -263,16 +265,18 @@ async fn generic_gather( ) .await?; return Ok(NodeLevelResult { + not_found: crate::control::local_dispatch::is_not_found(&resp), payload: resp.payload.to_vec(), watermark_lsn: resp.watermark_lsn, - read_version_lsn: resp.read_version_lsn, + read_versions: resp.read_versions, }); } let outcome = gather_all_cores(state, tenant_id, database_id, plan, trace_id, txn_id).await?; Ok(NodeLevelResult { payload: outcome.merged_array, watermark_lsn: outcome.watermark_lsn, - read_version_lsn: outcome.read_version_lsn, + read_versions: outcome.read_versions, + not_found: false, }) } @@ -302,22 +306,21 @@ async fn single_blob_gather( gather_every_core(state, tenant_id, database_id, plan, trace_id, "single-blob").await?; let mut watermark_lsn = Lsn::ZERO; - let mut read_version_lsn = Lsn::ZERO; + let mut read_versions = crate::types::ReadVersions::new(); let mut payloads: Vec> = Vec::with_capacity(responses.len()); for resp in responses { if resp.watermark_lsn > watermark_lsn { watermark_lsn = resp.watermark_lsn; } - if resp.read_version_lsn > read_version_lsn { - read_version_lsn = resp.read_version_lsn; - } + read_versions.merge(&resp.read_versions); payloads.push(resp.payload.as_ref().to_vec()); } Ok(NodeLevelResult { payload: pick_single_blob(payloads, op)?, watermark_lsn, - read_version_lsn, + read_versions, + not_found: false, }) } diff --git a/nodedb/src/control/server/exchange/all_cores/home_versions.rs b/nodedb/src/control/server/exchange/all_cores/home_versions.rs index 2b46f11f4..950b81069 100644 --- a/nodedb/src/control/server/exchange/all_cores/home_versions.rs +++ b/nodedb/src/control/server/exchange/all_cores/home_versions.rs @@ -4,9 +4,8 @@ //! vShards it owns, under this node's leader lease on each probe's group. //! //! A write to a vShard runs on the one core the dispatcher's router assigns -//! it, and raises that core's watermark and the collection's write floor -//! there. So a probe is answered by that core alone. A write that another core -//! took, to another vShard, never moves the answer. +//! it, and raises the vShard's versions there. So a probe is answered by that +//! core alone. A write to another vShard never moves the answer. //! //! A node that lost leadership of a group can miss writes a newer leader //! committed. So a probe is answered only while this node holds its group's @@ -104,7 +103,8 @@ pub(super) async fn fan_home_versions( Ok(NodeLevelResult { payload, watermark_lsn, - read_version_lsn: Lsn::ZERO, + read_versions: crate::types::ReadVersions::new(), + not_found: false, }) } diff --git a/nodedb/src/control/server/exchange/all_cores/read_cut.rs b/nodedb/src/control/server/exchange/all_cores/read_cut.rs index 54a7bd1d8..a66df9898 100644 --- a/nodedb/src/control/server/exchange/all_cores/read_cut.rs +++ b/nodedb/src/control/server/exchange/all_cores/read_cut.rs @@ -76,6 +76,7 @@ async fn resolve_read_cut(state: &SharedState, marker: u64) -> crate::Result) -> GatherOutcome { raw: payload, merged_array, watermark_lsn: Lsn::ZERO, - read_version_lsn: Lsn::ZERO, + read_versions: crate::types::ReadVersions::new(), shard_watermarks: Vec::new(), } } diff --git a/nodedb/src/control/server/exchange/core_outcome.rs b/nodedb/src/control/server/exchange/core_outcome.rs index 6b8746d42..a1d3c66c7 100644 --- a/nodedb/src/control/server/exchange/core_outcome.rs +++ b/nodedb/src/control/server/exchange/core_outcome.rs @@ -28,6 +28,19 @@ pub(crate) fn classify_core_response(collected: crate::Result) -> Core Ok(Some(resp)) } +/// The versions every core reported, a `NotFound` refusal's included. A +/// refusal observed its vShards with no matching row, and a read validates +/// that observation at the versions the refusal reported. +pub(crate) fn reported_versions<'a>( + collected: impl IntoIterator>, +) -> crate::types::ReadVersions { + let mut versions = crate::types::ReadVersions::new(); + for resp in collected.into_iter().flatten() { + versions.merge(&resp.read_versions); + } + versions +} + /// Require every core to answer, and return the answers in core order. /// /// The first error in core order fails the whole gather. It crosses as the @@ -59,8 +72,8 @@ mod tests { payload: Payload::from_vec(payload.to_vec()), watermark_lsn: Lsn::ZERO, error_code: code.map(Box::new), - read_set_valid: None, - read_version_lsn: Lsn::ZERO, + stage_vote: None, + read_versions: crate::types::ReadVersions::new(), write_set: Vec::new(), } } diff --git a/nodedb/src/control/server/exchange/gather.rs b/nodedb/src/control/server/exchange/gather.rs index 7a1ada834..3cf519820 100644 --- a/nodedb/src/control/server/exchange/gather.rs +++ b/nodedb/src/control/server/exchange/gather.rs @@ -20,7 +20,9 @@ use futures::future::join_all; use crate::bridge::envelope::{PhysicalPlan, Priority, Request, Response}; use crate::control::arrow_convert; use crate::control::gateway::core::QueryContext; -use crate::control::server::exchange::core_outcome::{classify_core_response, require_every_core}; +use crate::control::server::exchange::core_outcome::{ + classify_core_response, reported_versions, require_every_core, +}; use crate::control::server::exchange::read_scope::ReadScope; use crate::control::server::payload_merge::{encode_msgpack_array, extract_msgpack_elements}; use crate::control::server::result_stream::ResultStream; @@ -126,6 +128,7 @@ pub(crate) fn eager_dispatch_to_all_cores_until( wal_lsn: None, resolved_now_ms: None, commit_hlc: None, + entry_version: None, admission: crate::bridge::envelope::Admission::Exempt( crate::bridge::envelope::ExemptReason::Read, ), @@ -154,12 +157,11 @@ pub struct GatherOutcome { /// Maximum watermark LSN seen across all responding cores. Retained as the /// scalar fence value for Strong-consistency callers that need one LSN. pub watermark_lsn: Lsn, - /// Max per-collection read-version LSN across all responding cores. A read - /// homes to exactly ONE core, so the non-owning cores report `Lsn::ZERO` and - /// the owning core's value dominates the fold — yielding the scanned - /// collection's `coll_write_lsn` at read time (the sound comparand for - /// cross-shard OCC read validation, distinct from `watermark_lsn`). - pub read_version_lsn: Lsn, + /// The read versions every responding core reported, one per vShard. A + /// vShard lives on ONE core, so only that core reports it: the scanned + /// collection's version at read time (the comparand for cross-shard OCC + /// read validation, distinct from `watermark_lsn`). + pub read_versions: crate::types::ReadVersions, /// Per-shard watermark LSNs — one `(vshard, watermark_lsn)` per responding /// core, NOT collapsed to the max. The transaction read-set records one /// entry per participating shard from this, so a predicate read fanned over @@ -219,6 +221,9 @@ pub(crate) async fn gather_all_cores( }); let results: Vec<(usize, crate::Result)> = join_all(response_futures).await; + // The read versions of every core, one per vShard: a vShard lives on ONE + // core, so only that core reports it. + let read_versions = reported_versions(results.iter().map(|(_, result)| result)); let answered = require_every_core(results.into_iter().map(|(core_id, result)| { classify_core_response(result).map(|resp| resp.map(|resp| (core_id, resp))) }))?; @@ -226,11 +231,6 @@ pub(crate) async fn gather_all_cores( let mut raw = Vec::new(); let mut all_elements: Vec> = Vec::new(); let mut max_lsn = Lsn::ZERO; - // Max-fold of the per-collection read-version across cores: a collection - // homes to ONE core, so non-owning cores contribute `Lsn::ZERO` and the - // owning core's value dominates. Kept as a single scalar (not per-core) — - // the read plan targets one collection, so one non-zero value survives. - let mut max_read_version = Lsn::ZERO; let mut shard_watermarks: Vec<(VShardId, Lsn)> = Vec::new(); for (core_id, resp) in answered { @@ -242,9 +242,6 @@ pub(crate) async fn gather_all_cores( if resp.watermark_lsn > max_lsn { max_lsn = resp.watermark_lsn; } - if resp.read_version_lsn > max_read_version { - max_read_version = resp.read_version_lsn; - } if resp.payload.is_empty() { continue; @@ -261,7 +258,7 @@ pub(crate) async fn gather_all_cores( raw, merged_array, watermark_lsn: max_lsn, - read_version_lsn: max_read_version, + read_versions, shard_watermarks, }) } @@ -382,8 +379,10 @@ pub(crate) async fn gather_all_vshards( // The gateway already fails with a typed `crate::Error` — a shard's // `Error::DataPlane` code included. Re-wrapping it in `Dispatch` will // rewrite every such verdict as SQLSTATE XX000, so it passes through. - let (payloads, shard_watermarks, read_version_lsn): (Vec>, Vec<(VShardId, Lsn)>, Lsn) = - Box::pin(gateway.execute_internal_with_watermarks(&ctx, plan)).await?; + let (payloads, shard_watermarks, read_versions) = + Box::pin(gateway.execute_internal_outcome(&ctx, plan)) + .await? + .into_parts(); let mut all_elements: Vec> = Vec::new(); let mut raw = Vec::new(); @@ -407,7 +406,7 @@ pub(crate) async fn gather_all_vshards( raw, merged_array, watermark_lsn, - read_version_lsn, + read_versions, shard_watermarks, }) } diff --git a/nodedb/src/control/server/exchange/owning_core.rs b/nodedb/src/control/server/exchange/owning_core.rs index a90eaea45..3701141e9 100644 --- a/nodedb/src/control/server/exchange/owning_core.rs +++ b/nodedb/src/control/server/exchange/owning_core.rs @@ -27,7 +27,7 @@ use super::gather::GatherOutcome; /// collection's rows. /// /// The returned outcome carries that core's own `watermark_lsn` / -/// `read_version_lsn` and exactly one `shard_watermarks` entry keyed to the +/// `read_versions` and exactly one `shard_watermarks` entry keyed to the /// collection's vShard — matching the cluster `dispatch_local` path so an /// in-transaction read records the same OCC read-set entry the write-set uses /// (writes home to the same `CollectionKey` vShard). Aggregate @@ -60,7 +60,7 @@ pub async fn gather_single_owning_core( raw: payload_bytes.to_vec(), merged_array, watermark_lsn: resp.watermark_lsn, - read_version_lsn: resp.read_version_lsn, + read_versions: resp.read_versions.clone(), shard_watermarks: vec![(vshard_id, resp.watermark_lsn)], }) } diff --git a/nodedb/src/control/server/exchange/received.rs b/nodedb/src/control/server/exchange/received.rs index 322071d70..7fe890862 100644 --- a/nodedb/src/control/server/exchange/received.rs +++ b/nodedb/src/control/server/exchange/received.rs @@ -72,9 +72,10 @@ pub(crate) async fn execute_received_plan( ) .await?; Ok(NodeLevelResult { + not_found: crate::control::local_dispatch::is_not_found(&resp), payload: resp.payload.to_vec(), watermark_lsn: resp.watermark_lsn, - read_version_lsn: resp.read_version_lsn, + read_versions: resp.read_versions, }) } ReceivedRoute::AllCores => { diff --git a/nodedb/src/control/server/exchange/resolve/capture.rs b/nodedb/src/control/server/exchange/resolve/capture.rs index fa49e558f..a4c5fc886 100644 --- a/nodedb/src/control/server/exchange/resolve/capture.rs +++ b/nodedb/src/control/server/exchange/resolve/capture.rs @@ -4,7 +4,7 @@ use nodedb_physical::physical_plan::PhysicalPlan; -use crate::types::Lsn; +use crate::types::ReadVersions; /// One base collection's read observation from a distributed read whose result /// is materialized on the coordinator. @@ -13,16 +13,16 @@ use crate::types::Lsn; /// collection under a root `Exchange{Gather}`, including both sides of a /// gathered `HashJoin`) and the SHUFFLE path (`resolve::shuffle` / /// `resolve::shuffle_aggregate` — one per join side). Each carries that -/// collection's own bare full-collection scan plan and the REAL per-collection -/// read-version LSN the gather/producers observed. The record seam re-derives -/// the collection / engine / read key from `scan_plan` (a single-collection -/// scan, so it is NOT collapsed to just the left side the way a `HashJoin` plan -/// is) and stamps the entry at `read_version_lsn`, so the commit-time OCC +/// collection's own bare full-collection scan plan and the REAL read versions +/// the gather/producers observed. The record seam re-derives the collection / +/// engine / read key from `scan_plan` (a single-collection scan, so it is NOT +/// collapsed to just the left side the way a `HashJoin` plan is) and stamps +/// the entry at the collection's version in `read_versions`, so the commit-time OCC /// validator re-homes and revalidates each collection's vshard independently. /// This closes the serializability hole where a build-side (or non-left) /// collection never appeared in the read-set and a concurrent write to it went /// undetected. pub struct DistributedReadCapture { pub scan_plan: PhysicalPlan, - pub read_version_lsn: Lsn, + pub read_versions: ReadVersions, } diff --git a/nodedb/src/control/server/exchange/resolve/exchange/gather_arm.rs b/nodedb/src/control/server/exchange/resolve/exchange/gather_arm.rs index cd31c918b..315c71364 100644 --- a/nodedb/src/control/server/exchange/resolve/exchange/gather_arm.rs +++ b/nodedb/src/control/server/exchange/resolve/exchange/gather_arm.rs @@ -97,8 +97,8 @@ pub(super) async fn resolve_gather( let outcome: GatherOutcome = gather_all_vshards(state, child, ctx).await?; - // Record the probe/single-collection read at its OWN observed - // read-version (the gathered collection's `coll_write_lsn`), scoped to + // Record the probe/single-collection read at its OWN observed read + // versions (the gathered collection's version per vShard), scoped to // a bare single-collection scan so the commit-time OCC validator // re-homes and revalidates exactly that collection's vshard. A // `HashJoin` plan will otherwise collapse to the left collection @@ -114,7 +114,7 @@ pub(super) async fn resolve_gather( { captures.push(DistributedReadCapture { scan_plan, - read_version_lsn: outcome.read_version_lsn, + read_versions: outcome.read_versions.clone(), }); } @@ -124,7 +124,7 @@ pub(super) async fn resolve_gather( outcome.merged_array }; Ok(Resolved::Gathered( - outcome_to_response(payload, outcome.watermark_lsn, outcome.read_version_lsn), + outcome_to_response(payload, outcome.watermark_lsn, outcome.read_versions), outcome.shard_watermarks, std::mem::take(captures), )) @@ -143,7 +143,7 @@ pub(super) async fn resolve_broadcast( outcome_to_response( outcome.merged_array, outcome.watermark_lsn, - outcome.read_version_lsn, + outcome.read_versions, ), outcome.shard_watermarks, std::mem::take(captures), diff --git a/nodedb/src/control/server/exchange/resolve/exchange/post_process_arm.rs b/nodedb/src/control/server/exchange/resolve/exchange/post_process_arm.rs index 4a6027f2c..35ae1c939 100644 --- a/nodedb/src/control/server/exchange/resolve/exchange/post_process_arm.rs +++ b/nodedb/src/control/server/exchange/resolve/exchange/post_process_arm.rs @@ -225,7 +225,7 @@ pub(super) async fn materialize_child_rows( { captures.push(DistributedReadCapture { scan_plan, - read_version_lsn: outcome.read_version_lsn, + read_versions: outcome.read_versions.clone(), }); } diff --git a/nodedb/src/control/server/exchange/resolve/join_input.rs b/nodedb/src/control/server/exchange/resolve/join_input.rs index 515dea2f4..9f1130e6e 100644 --- a/nodedb/src/control/server/exchange/resolve/join_input.rs +++ b/nodedb/src/control/server/exchange/resolve/join_input.rs @@ -117,7 +117,7 @@ pub(super) async fn resolve_join_input( { captures.push(DistributedReadCapture { scan_plan, - read_version_lsn: outcome.read_version_lsn, + read_versions: outcome.read_versions.clone(), }); } let merged = if as_aggregate { @@ -209,7 +209,7 @@ pub(super) async fn gather_join_build_side( if let Some(scan_plan) = capture_plan { captures.push(DistributedReadCapture { scan_plan, - read_version_lsn: outcome.read_version_lsn, + read_versions: outcome.read_versions.clone(), }); } diff --git a/nodedb/src/control/server/exchange/resolve/peers.rs b/nodedb/src/control/server/exchange/resolve/peers.rs index 36a5f90c7..52f7333c2 100644 --- a/nodedb/src/control/server/exchange/resolve/peers.rs +++ b/nodedb/src/control/server/exchange/resolve/peers.rs @@ -93,23 +93,23 @@ pub(super) fn distinct_data_node_count(routing: &RoutingTable) -> usize { } /// Send one `ShuffleProduceRequest` and map the reply / RPC error to a typed -/// coordinator error, returning the producer's observed per-collection -/// read-version LSN on a clean produce. Fail-fast: a producer-reported terminal -/// error aborts. Shared by both the shuffle-JOIN and shuffle-AGGREGATE -/// resolvers, which each max-fold the returned LSN over their producers. +/// coordinator error, returning the producer's observed read versions on a +/// clean produce. Fail-fast: a producer-reported terminal error aborts. Shared +/// by both the shuffle-JOIN and shuffle-AGGREGATE resolvers, which each fold +/// the returned versions over their producers. pub(super) async fn send_produce( transport: &nodedb_cluster::NexarTransport, node: u64, req: ShuffleProduceRequest, -) -> crate::Result { +) -> crate::Result { match transport .send_rpc(node, RaftRpc::ShuffleProduceRequest(req)) .await { Ok(RaftRpc::ShuffleProduceResponse(ShuffleProduceResponse { error: None, - read_version_lsn, - })) => Ok(read_version_lsn), + read_versions, + })) => Ok(crate::types::ReadVersions::from_wire(&read_versions)), Ok(RaftRpc::ShuffleProduceResponse(ShuffleProduceResponse { error: Some(e), .. })) => { Err(crate::Error::Internal { detail: format!("shuffle produce failed on node {node}: {e:?}"), diff --git a/nodedb/src/control/server/exchange/resolve/shuffle.rs b/nodedb/src/control/server/exchange/resolve/shuffle.rs index f44818d68..af0f31fb1 100644 --- a/nodedb/src/control/server/exchange/resolve/shuffle.rs +++ b/nodedb/src/control/server/exchange/resolve/shuffle.rs @@ -294,23 +294,23 @@ pub async fn resolve_shuffle_join( probe_produce_futures.push(send_produce(transport, node, req)); } // Await ALL producers (both sides CONCURRENTLY); any error fails the whole - // shuffle (no partial join). Max-fold each side's producers' observed - // per-collection read-version LSN independently: build ↔ right_collection, - // probe ↔ left_collection. Each side's producers scan the SAME single - // collection, so its max is that collection's `coll_write_lsn` at read time — - // the sound OCC read-version comparand recorded for that side. + // shuffle (no partial join). Fold each side's producers' observed read + // versions independently: build ↔ right_collection, probe ↔ + // left_collection. Each side's producers scan the SAME single collection, + // so the fold holds that collection's version on each vShard at read time: + // the OCC read-version comparand recorded for that side. let (build_results, probe_results) = join( join_all(build_produce_futures), join_all(probe_produce_futures), ) .await; - let mut build_rv: u64 = 0; + let mut build_rv = crate::types::ReadVersions::new(); for result in build_results { - build_rv = build_rv.max(result?); + build_rv.merge(&result?); } - let mut probe_rv: u64 = 0; + let mut probe_rv = crate::types::ReadVersions::new(); for result in probe_results { - probe_rv = probe_rv.max(result?); + probe_rv.merge(&result?); } // 8. After ALL producers succeed, dispatch consumers CONCURRENTLY — one per @@ -364,7 +364,7 @@ pub async fn resolve_shuffle_join( // Two per-side read captures carry each side's REAL observed read version: // the probe capture (left_collection) at `probe_rv`, the build capture // (right_collection) at `build_rv` — folded from the producers' - // `ShuffleProduceResponse.read_version_lsn`. The record seam records one + // `ShuffleProduceResponse.read_versions`. The record seam records one // read-set entry per capture, re-homing and revalidating each side's vshard // independently, so a concurrent write to EITHER side between the in-txn read // and commit is detected (the build side is recorded too). The response's own scalar stays `ZERO`: the captures @@ -374,15 +374,15 @@ pub async fn resolve_shuffle_join( let captures = vec![ DistributedReadCapture { scan_plan: probe_scan, - read_version_lsn: Lsn::new(probe_rv), + read_versions: probe_rv, }, DistributedReadCapture { scan_plan: build_scan, - read_version_lsn: Lsn::new(build_rv), + read_versions: build_rv, }, ]; Ok(Resolved::Gathered( - outcome_to_response(merged, Lsn::ZERO, Lsn::ZERO), + outcome_to_response(merged, Lsn::ZERO, crate::types::ReadVersions::new()), Vec::new(), captures, )) diff --git a/nodedb/src/control/server/exchange/resolve/shuffle_aggregate.rs b/nodedb/src/control/server/exchange/resolve/shuffle_aggregate.rs index f6e94d43d..1ad4e28d6 100644 --- a/nodedb/src/control/server/exchange/resolve/shuffle_aggregate.rs +++ b/nodedb/src/control/server/exchange/resolve/shuffle_aggregate.rs @@ -257,13 +257,13 @@ pub async fn resolve_shuffle_aggregate( produce_futures.push(send_produce(transport, node, req)); } // Await ALL producers; any error fails the whole shuffle (no partial result). - // Max-fold every producer's observed per-collection read-version LSN: the - // producers all scan the SAME single source collection, so the max is that - // collection's `coll_write_lsn` at read time — the sound comparand the - // coordinator records for cross-shard OCC read validation of this aggregate. - let mut max_read_version_lsn: u64 = 0; + // Fold every producer's observed read versions: the producers all scan the + // SAME single source collection, so the fold holds that collection's + // version on each vShard at read time: the comparand the coordinator + // records for cross-shard OCC read validation of this aggregate. + let mut read_versions = crate::types::ReadVersions::new(); for result in join_all(produce_futures).await { - max_read_version_lsn = max_read_version_lsn.max(result?); + read_versions.merge(&result?); } // 7. After ALL producers succeed, dispatch consumers CONCURRENTLY — one per @@ -332,16 +332,16 @@ pub async fn resolve_shuffle_aggregate( } let merged = encode_msgpack_array(&elements); - // The producers report the source collection's `coll_write_lsn` at read time - // on their `ShuffleProduceResponse`; the max-fold above is the read version - // this aggregate observed. Pass it as the OCC read-version comparand so an + // The producers report the source collection's versions at read time on + // their `ShuffleProduceResponse`; the fold above is what this aggregate + // observed. Pass it as the OCC read-version comparand so an // in-transaction distributed aggregate records a sound read-set entry (the // aggregate is single-collection, so `record_read_set` attributes it to the // right collection). The core-global `watermark_lsn` is NOT threaded through // the shuffle transport and stays `ZERO`; using it as the read version will // skip required aborts (it advances on writes to ANY collection). Ok(Resolved::Gathered( - outcome_to_response(merged, Lsn::ZERO, Lsn::new(max_read_version_lsn)), + outcome_to_response(merged, Lsn::ZERO, read_versions), Vec::new(), Vec::new(), )) diff --git a/nodedb/src/control/server/exchange/response.rs b/nodedb/src/control/server/exchange/response.rs index 354357553..25503976f 100644 --- a/nodedb/src/control/server/exchange/response.rs +++ b/nodedb/src/control/server/exchange/response.rs @@ -4,7 +4,7 @@ use crate::bridge::envelope::{Response, Status}; use crate::control::server::result_stream::ResultStream; -use crate::types::{Lsn, RequestId}; +use crate::types::{Lsn, ReadVersions, RequestId}; /// Materialize a [`ResultStream`] into a synthetic successful Response. /// @@ -14,26 +14,30 @@ use crate::types::{Lsn, RequestId}; pub(crate) async fn stream_to_response(stream: ResultStream) -> crate::Result { let (merged, watermark_lsn) = crate::control::server::result_stream::materialize(stream).await?; - // A streamed result carries no per-collection read-version: the stream's - // frames carry only per-batch watermarks, and nothing consumes a read-version - // from this path — the streaming branch is gated on `txn_id.is_none()` - // (`resolve/exchange.rs`), so a stream never serves an in-transaction read and - // never reaches the read-set capture that would compare one. - Ok(outcome_to_response(merged, watermark_lsn, Lsn::ZERO)) + // A streamed result carries no read versions: the stream's frames carry + // only per-batch watermarks, and nothing consumes a read version from this + // path. The streaming branch is gated on `txn_id.is_none()` + // (`resolve/exchange.rs`), so a stream never serves an in-transaction read + // and never reaches the read-set capture. + Ok(outcome_to_response( + merged, + watermark_lsn, + ReadVersions::new(), + )) } /// Build a synthetic successful Response from a gathered merged-array payload. /// -/// `read_version_lsn` is the gathered collection's `coll_write_lsn` at read time -/// (`GatherOutcome::read_version_lsn`) — the comparand cross-shard OCC read -/// validation consumes. It is distinct from `watermark_lsn` (the core-global -/// fence) and must be passed through: dropping it to `Lsn::ZERO` silently strips -/// the read's version and leaves validation to compare against whatever floor the +/// `read_versions` are the gathered read's versions at read time +/// (`GatherOutcome::read_versions`): the comparand cross-shard OCC read +/// validation consumes. They are distinct from `watermark_lsn` (the core-global +/// fence) and must be passed through: dropping them silently strips the read's +/// versions and leaves validation to compare against whatever floor the /// session's own writes supply. pub(super) fn outcome_to_response( merged_array: Vec, watermark_lsn: Lsn, - read_version_lsn: Lsn, + read_versions: ReadVersions, ) -> Response { Response { request_id: RequestId::new(0), @@ -43,8 +47,8 @@ pub(super) fn outcome_to_response( payload: crate::bridge::envelope::Payload::from_vec(merged_array), watermark_lsn, error_code: None, - read_set_valid: None, - read_version_lsn, + stage_vote: None, + read_versions, write_set: Vec::new(), } } diff --git a/nodedb/src/control/server/graph_dispatch/bfs.rs b/nodedb/src/control/server/graph_dispatch/bfs.rs index 333c962c2..77104e945 100644 --- a/nodedb/src/control/server/graph_dispatch/bfs.rs +++ b/nodedb/src/control/server/graph_dispatch/bfs.rs @@ -90,7 +90,7 @@ pub async fn cross_core_bfs_with_options( &mut presence_reads, ) .await?; - presence_reads.publish(shared, tenant_id, database_id, None); + presence_reads.publish(tenant_id, database_id, None); frontier.retain(|node| present.contains(node) && visited.insert(node.clone())); all_discovered.extend(frontier.iter().cloned()); @@ -130,12 +130,7 @@ pub async fn cross_core_bfs_with_options( } // Every vShard the walk expanded joins the transaction read-set. - reads.publish( - shared, - tenant_id, - database_id, - collection.map(str::to_owned), - ); + reads.publish(tenant_id, database_id, collection.map(str::to_owned)); Ok(ok_response(encode_path(&all_discovered)?)) } diff --git a/nodedb/src/control/server/graph_dispatch/bsp_pagerank/coord.rs b/nodedb/src/control/server/graph_dispatch/bsp_pagerank/coord.rs index 807823fbf..6ad7529de 100644 --- a/nodedb/src/control/server/graph_dispatch/bsp_pagerank/coord.rs +++ b/nodedb/src/control/server/graph_dispatch/bsp_pagerank/coord.rs @@ -166,19 +166,14 @@ pub async fn run_bsp_pagerank( // The count phase reads every owner's partition first. A write after it // on any owned vShard changes what later supersteps read, so the count - // phase's watermarks are the ones the transaction read-set keeps. + // phase's versions are the ones the transaction read-set keeps. let mut reads = ShardReadLog::new(); for count in &counts { if let Some(target) = targets.iter().find(|t| t.node_id == count.node_id) { - reads.note( - target.owned_vshards.iter().copied(), - count.watermark_lsn, - target.node_id, - ); + reads.note(target.owned_vshards.iter().copied(), &count.read_versions); } } reads.publish( - state, tenant_id, database_id, Some(qualified(database_id, ¶ms.collection)), diff --git a/nodedb/src/control/server/graph_dispatch/bsp_pagerank/scatter.rs b/nodedb/src/control/server/graph_dispatch/bsp_pagerank/scatter.rs index 61ff23148..9ecb5432e 100644 --- a/nodedb/src/control/server/graph_dispatch/bsp_pagerank/scatter.rs +++ b/nodedb/src/control/server/graph_dispatch/bsp_pagerank/scatter.rs @@ -59,8 +59,8 @@ pub(super) struct ShardDispatch { pub(super) struct ShardResult { pub(super) node_id: u64, pub(super) result: BspSuperstepResult, - /// The highest watermark the node's cores served the superstep at. - pub(super) watermark_lsn: crate::types::Lsn, + /// The versions the node's cores reported for the superstep. + pub(super) read_versions: crate::types::ReadVersions, } /// Parameters for [`scatter_superstep`]. @@ -139,7 +139,7 @@ pub(super) async fn scatter_superstep( Ok::(ShardResult { node_id, result, - watermark_lsn: read.watermark_lsn, + read_versions: read.read_versions, }) }) }); diff --git a/nodedb/src/control/server/graph_dispatch/bsp_wcc/coord.rs b/nodedb/src/control/server/graph_dispatch/bsp_wcc/coord.rs index 4657099b5..3a1c80607 100644 --- a/nodedb/src/control/server/graph_dispatch/bsp_wcc/coord.rs +++ b/nodedb/src/control/server/graph_dispatch/bsp_wcc/coord.rs @@ -77,19 +77,14 @@ pub async fn run_bsp_wcc( .map(|sr| (sr.node_id, sr.result.system_as_of)), )?; - // Every owner's partition was read once, at the watermark it served. + // Every owner's partition was read once, at the versions it reported. let mut reads = ShardReadLog::new(); for sr in &results { if let Some(target) = targets.iter().find(|t| t.node_id == sr.node_id) { - reads.note( - target.owned_vshards.iter().copied(), - sr.watermark_lsn, - target.node_id, - ); + reads.note(target.owned_vshards.iter().copied(), &sr.read_versions); } } reads.publish( - state, tenant_id, database_id, Some(qualified(database_id, ¶ms.collection)), diff --git a/nodedb/src/control/server/graph_dispatch/bsp_wcc/scatter.rs b/nodedb/src/control/server/graph_dispatch/bsp_wcc/scatter.rs index 719dd50fa..0231c040e 100644 --- a/nodedb/src/control/server/graph_dispatch/bsp_wcc/scatter.rs +++ b/nodedb/src/control/server/graph_dispatch/bsp_wcc/scatter.rs @@ -28,8 +28,8 @@ use nodedb_physical::physical_plan::{GraphOp, WccSuperstepPlan, WccSuperstepResu pub(super) struct ShardWccResult { pub(super) node_id: u64, pub(super) result: WccSuperstepResult, - /// The highest watermark the node's cores served the round at. - pub(super) watermark_lsn: crate::types::Lsn, + /// The versions the node's cores reported for the round. + pub(super) read_versions: crate::types::ReadVersions, } /// The inputs of one WCC round. @@ -100,7 +100,7 @@ pub(super) async fn scatter_wcc_round( Ok::(ShardWccResult { node_id, result, - watermark_lsn: read.watermark_lsn, + read_versions: read.read_versions, }) }) }); diff --git a/nodedb/src/control/server/graph_dispatch/cluster_resolve.rs b/nodedb/src/control/server/graph_dispatch/cluster_resolve.rs index 5ce92b04c..b75c79d91 100644 --- a/nodedb/src/control/server/graph_dispatch/cluster_resolve.rs +++ b/nodedb/src/control/server/graph_dispatch/cluster_resolve.rs @@ -31,15 +31,15 @@ pub(in crate::control::server::graph_dispatch) struct DispatchSuperstepParams<'a pub(in crate::control::server::graph_dispatch) linearizable: bool, } -/// One owner node's answer to a graph superstep: its payload and the highest -/// watermark its cores served the plan at. +/// One owner node's answer to a graph superstep: its payload and the +/// versions its cores reported. pub(in crate::control::server::graph_dispatch) struct NodeRead { pub(in crate::control::server::graph_dispatch) payload: Payload, - pub(in crate::control::server::graph_dispatch) watermark_lsn: crate::types::Lsn, + pub(in crate::control::server::graph_dispatch) read_versions: crate::types::ReadVersions, } /// Dispatch a single already-built graph-superstep `plan` to one owner node and -/// return its node-level payload and served watermark. The LOCAL node fans the plan across all its +/// return its node-level payload and reported versions. The LOCAL node fans the plan across all its /// Data-Plane cores via `execute_plan_all_local_cores` (per-core results merged /// into one payload); a REMOTE node gets one `RouteDecision::Remote` dispatch via /// `dispatch_route`. An empty payload denotes a zero-vertex shard — the caller's @@ -81,7 +81,7 @@ pub(in crate::control::server::graph_dispatch) async fn dispatch_superstep_to_no .await?; Ok(NodeRead { payload: Payload::from_vec(node_result.payload), - watermark_lsn: node_result.watermark_lsn, + read_versions: node_result.read_versions, }) } else { // Remote node: one dispatch via the gateway. @@ -106,12 +106,7 @@ pub(in crate::control::server::graph_dispatch) async fn dispatch_superstep_to_no linearizable, }) .await?; - let watermark_lsn = outcome - .shard_watermarks - .iter() - .map(|(_, lsn)| *lsn) - .max() - .unwrap_or(crate::types::Lsn::ZERO); + let read_versions = outcome.read_versions; let payload = outcome .payloads .into_iter() @@ -122,7 +117,7 @@ pub(in crate::control::server::graph_dispatch) async fn dispatch_superstep_to_no })?; Ok(NodeRead { payload, - watermark_lsn, + read_versions, }) } } diff --git a/nodedb/src/control/server/graph_dispatch/helpers.rs b/nodedb/src/control/server/graph_dispatch/helpers.rs index 7aa3abbc5..37bff36be 100644 --- a/nodedb/src/control/server/graph_dispatch/helpers.rs +++ b/nodedb/src/control/server/graph_dispatch/helpers.rs @@ -22,8 +22,8 @@ pub(super) fn ok_response(payload: Vec) -> Response { payload: crate::bridge::envelope::Payload::from_vec(payload), watermark_lsn: Lsn::ZERO, error_code: None, - read_set_valid: None, - read_version_lsn: crate::types::Lsn::ZERO, + stage_vote: None, + read_versions: crate::types::ReadVersions::new(), write_set: Vec::new(), } } diff --git a/nodedb/src/control/server/graph_dispatch/hop.rs b/nodedb/src/control/server/graph_dispatch/hop.rs index ca7080fc0..19ad04d0a 100644 --- a/nodedb/src/control/server/graph_dispatch/hop.rs +++ b/nodedb/src/control/server/graph_dispatch/hop.rs @@ -80,8 +80,8 @@ pub(super) struct HopOutput { /// Deduplicated destination node IDs after merging local + remote /// expansion. Feeds the next frontier. pub merged_destinations: Vec, - /// The key vShard of every frontier node, at the watermark its expansion - /// served. + /// The key vShard of every frontier node, at the version its expansion + /// reported. pub reads: ShardReadLog, } @@ -156,12 +156,14 @@ pub(super) async fn execute_neighbor_hop( // Single-node mode: no routing table — every frontier node is local. if shared.cluster_routing.is_none() { - let (rows, _watermark) = expand_local(shared, scope, frontier).await?; + let (rows, versions) = expand_local(shared, scope, frontier).await?; let merged = dedup_destinations(&rows); + let mut reads = ShardReadLog::new(); + reads.note(key_vshards(frontier), &versions); return Ok(HopOutput { rows, merged_destinations: merged, - reads: ShardReadLog::new(), + reads, }); } @@ -174,8 +176,8 @@ pub(super) async fn execute_neighbor_hop( let mut all_rows: Vec = if local_nodes.is_empty() { Vec::new() } else { - let (rows, watermark) = expand_local(shared, scope, &local_nodes).await?; - reads.note(key_vshards(&local_nodes), watermark, shared.node_id); + let (rows, versions) = expand_local(shared, scope, &local_nodes).await?; + reads.note(key_vshards(&local_nodes), &versions); rows }; @@ -307,12 +309,12 @@ impl ExpandScope<'_> { } /// Expand a locally-owned subset on all local Data-Plane cores. Returns the -/// crossed edges and the highest watermark a core served them at. +/// crossed edges and the versions the cores reported. async fn expand_local( shared: &SharedState, scope: ExpandScope<'_>, node_ids: &[String], -) -> crate::Result<(Vec, crate::types::Lsn)> { +) -> crate::Result<(Vec, crate::types::ReadVersions)> { let plan = scope.plan(node_ids.to_vec()); // A linearizable hop confirms the groups its plan reads first. if scope.linearizable { @@ -327,13 +329,13 @@ async fn expand_local( scope.txn_id, ) .await?; - Ok((decode_neighbor_rows(&resp.payload)?, resp.watermark_lsn)) + Ok((decode_neighbor_rows(&resp.payload)?, resp.read_versions)) } /// Expand the remote-owned subsets concurrently: ship a typed /// `NeighborsMulti` plan to each owning node via [`dispatch_route`] and /// decode every returned payload with the shared `{src,label,node}` decoder. -/// Notes each owner's vShard in `reads` at the watermark it served. +/// Notes each owner's vShard in `reads` at the version it reported. async fn expand_remote( shared: &SharedState, scope: ExpandScope<'_>, @@ -389,7 +391,7 @@ async fn expand_remote( linearizable, }) .await - .map(|outcome| (read_vshard, node_id, outcome)) + .map(|outcome| (read_vshard, outcome)) }) }); @@ -399,8 +401,8 @@ async fn expand_remote( for result in results { // A remote dispatch error is fatal: a dropped owner means a partial // reachable set. - let (read_vshard, node_id, outcome) = result?; - reads.note_leg([read_vshard], &outcome.shard_watermarks, node_id); + let (read_vshard, outcome) = result?; + reads.note([read_vshard], &outcome.read_versions); for payload in outcome.payloads { rows.extend(decode_neighbor_rows(&payload)?); } diff --git a/nodedb/src/control/server/graph_dispatch/keyed_read.rs b/nodedb/src/control/server/graph_dispatch/keyed_read.rs index d75ba2a56..d54974b01 100644 --- a/nodedb/src/control/server/graph_dispatch/keyed_read.rs +++ b/nodedb/src/control/server/graph_dispatch/keyed_read.rs @@ -85,7 +85,7 @@ pub async fn read_on_vshard( txn_id, ) .await?; - reads.note([vshard_id], response.watermark_lsn, state.node_id); + reads.note([vshard_id], &response.read_versions); response.payload } RouteDecision::Remote { @@ -113,7 +113,7 @@ pub async fn read_on_vshard( linearizable, }) .await?; - reads.note_leg([vshard_id], &outcome.shard_watermarks, node_id); + reads.note([vshard_id], &outcome.read_versions); let payloads = outcome.payloads; Payload::from_vec(match payloads.len() { 1 => payloads.into_iter().next().unwrap_or_default(), @@ -135,7 +135,7 @@ pub async fn read_on_vshard( } }; // The node's key vShard joins the transaction read-set. - reads.publish(state, tenant_id, database_id, collection); + reads.publish(tenant_id, database_id, collection); Ok(payload) } diff --git a/nodedb/src/control/server/graph_dispatch/match_broadcast.rs b/nodedb/src/control/server/graph_dispatch/match_broadcast.rs index 08d7d9ce0..96de6264e 100644 --- a/nodedb/src/control/server/graph_dispatch/match_broadcast.rs +++ b/nodedb/src/control/server/graph_dispatch/match_broadcast.rs @@ -32,7 +32,9 @@ use futures::future::join_all; use crate::bridge::envelope::{Payload, PhysicalPlan, Response}; -use crate::control::server::exchange::core_outcome::{classify_core_response, require_every_core}; +use crate::control::server::exchange::core_outcome::{ + classify_core_response, reported_versions, require_every_core, +}; use crate::control::server::exchange::gather::eager_dispatch_to_all_cores; use crate::control::server::payload_merge::{encode_msgpack_array, extract_msgpack_elements}; use crate::data::executor::handlers::graph_match::{ @@ -73,6 +75,8 @@ pub struct MatchBroadcastOutcome { pub partial: bool, /// The highest watermark any core served the MATCH at. pub watermark_lsn: crate::types::Lsn, + /// The versions every core reported, one per vShard it holds. + pub read_versions: crate::types::ReadVersions, } /// Locate a top-level map value by key in a msgpack map payload. @@ -265,6 +269,7 @@ pub async fn broadcast_match_to_all_cores( }); let results: Vec> = join_all(response_futures).await; + let read_versions = reported_versions(&results); // `NotFound` is an empty CSR slice on that core. Any other core error fails // the MATCH: rows from the surviving cores are not the full answer. let answered = require_every_core(results.into_iter().map(classify_core_response))?; @@ -302,6 +307,7 @@ pub async fn broadcast_match_to_all_cores( resume, partial, watermark_lsn, + read_versions, }) } diff --git a/nodedb/src/control/server/graph_dispatch/match_scatter/coord.rs b/nodedb/src/control/server/graph_dispatch/match_scatter/coord.rs index 01765db8d..901c0798a 100644 --- a/nodedb/src/control/server/graph_dispatch/match_scatter/coord.rs +++ b/nodedb/src/control/server/graph_dispatch/match_scatter/coord.rs @@ -214,7 +214,6 @@ pub async fn scatter_match( let rows_payload = dedup_and_encode(&coordinator.completed)?; // Every vShard round 0 read joins the transaction read-set. reads.publish( - state, tenant_id, database_id, pattern_collection(database_id, &query_bytes), diff --git a/nodedb/src/control/server/graph_dispatch/match_scatter/round_zero.rs b/nodedb/src/control/server/graph_dispatch/match_scatter/round_zero.rs index 4b6aba2c8..e134bf6fe 100644 --- a/nodedb/src/control/server/graph_dispatch/match_scatter/round_zero.rs +++ b/nodedb/src/control/server/graph_dispatch/match_scatter/round_zero.rs @@ -44,7 +44,7 @@ struct RoundZeroOwners { /// Every leg walks every vShard its node leads, so round 0 reads every vShard /// of the graph. A MATCH result depends on every one of them: an edge written /// to any vShard can add a match. The returned log notes each vShard under the -/// leg it was dispatched to, at the watermark that leg served, from the same +/// leg it was dispatched to, at the versions that leg reported, from the same /// routing snapshot that picked the legs. Later rounds read vShards round 0 /// already noted. pub(super) async fn scatter_round_zero( @@ -120,7 +120,7 @@ pub(super) async fn scatter_round_zero( }) .await?; let mut log = ShardReadLog::new(); - log.note_leg(leg_vshards, &outcome.shard_watermarks, node_id); + log.note(leg_vshards, &outcome.read_versions); Ok::<_, crate::Error>((collect_remote_envelopes(node_id, outcome.payloads)?, log)) }) }); @@ -132,7 +132,7 @@ pub(super) async fn scatter_round_zero( let mut out: Vec = Vec::new(); let mut log = ShardReadLog::new(); let local_outcome = local_outcome?; - log.note(local_vshards, local_outcome.watermark_lsn, state.node_id); + log.note(local_vshards, &local_outcome.read_versions); out.push(TaggedShardResult { emitting_node: state.node_id, rows: decode_rows(&local_outcome.rows_payload)?, diff --git a/nodedb/src/control/server/graph_dispatch/rag_fusion/coord.rs b/nodedb/src/control/server/graph_dispatch/rag_fusion/coord.rs index 54ab49b42..4c0f32f54 100644 --- a/nodedb/src/control/server/graph_dispatch/rag_fusion/coord.rs +++ b/nodedb/src/control/server/graph_dispatch/rag_fusion/coord.rs @@ -129,7 +129,7 @@ async fn run(state: &SharedState, scope: RagScope, plan: &PhysicalPlan) -> crate ) .await? else { - reads.publish(state, tenant_id, database_id, Some(qualified)); + reads.publish(tenant_id, database_id, Some(qualified)); return Ok(not_found_response()); }; let legs = exported.legs; @@ -270,7 +270,7 @@ async fn run(state: &SharedState, scope: RagScope, plan: &PhysicalPlan) -> crate detail: format!("{op_name} encode: {e}"), })?; // Every vShard the fusion read joins the transaction read-set. - reads.publish(state, tenant_id, database_id, Some(qualified)); + reads.publish(tenant_id, database_id, Some(qualified)); Ok(ok_payload_response(Payload::from_vec(payload))) } diff --git a/nodedb/src/control/server/graph_dispatch/rag_fusion/stages.rs b/nodedb/src/control/server/graph_dispatch/rag_fusion/stages.rs index 30bc7018a..1c08c512d 100644 --- a/nodedb/src/control/server/graph_dispatch/rag_fusion/stages.rs +++ b/nodedb/src/control/server/graph_dispatch/rag_fusion/stages.rs @@ -62,10 +62,6 @@ pub(super) async fn export_legs( } = scope; let shared = gateway_shared(state)?; let decision = resolve_live_decision(state, vshard); - let served_by = match decision { - crate::control::gateway::RouteDecision::Remote { node_id, .. } => node_id, - _ => state.node_id, - }; let route = TaskRoute { plan: with_stage(plan, RagStage::ExportLegs), decision, @@ -83,7 +79,7 @@ pub(super) async fn export_legs( linearizable, }) .await?; - reads.note_leg([vshard], &outcome.shard_watermarks, served_by); + reads.note([vshard], &outcome.read_versions); let watermark_lsn = outcome .shard_watermarks .iter() diff --git a/nodedb/src/control/server/graph_dispatch/shard_reads.rs b/nodedb/src/control/server/graph_dispatch/shard_reads.rs index e8fd86ed6..1d3917a34 100644 --- a/nodedb/src/control/server/graph_dispatch/shard_reads.rs +++ b/nodedb/src/control/server/graph_dispatch/shard_reads.rs @@ -3,35 +3,28 @@ //! The vShards a cross-shard graph read observed, for the transaction //! read-set. //! -//! A coordinator notes each vShard its legs read, at the watermark the -//! serving node reported, then publishes the log once the read finishes -//! (`session::graph_reads`). The request's protocol records the published +//! A coordinator notes each vShard its legs read, at the version the serving +//! core reported, then publishes the log once the read finishes +//! (`session::pending_shard_reads`). The request's protocol records the published //! reads into the transaction read-set, so commit validation checks every -//! vShard the read depended on. A read that runs on one node only (no -//! cluster) publishes nothing: its cores' watermarks sit in this node's WAL, -//! which single-shard SI already compares against. +//! vShard the read depended on. A version is the data-group log position of +//! the write that set it, the same on every replica, so a leg any replica +//! served compares with the vShard leader's versions. use std::collections::BTreeMap; -use crate::control::server::shared::session::graph_reads::{ - GraphShardReads, ShardObservation, note, +use nodedb_types::WriteVersion; + +use crate::control::server::shared::session::pending_shard_reads::{ + ShardObservation, ShardReads, note, }; -use crate::control::state::SharedState; -use crate::types::{DatabaseId, Lsn, TenantId, VShardId}; +use crate::control::server::shared::session::read_set::EngineTag; +use crate::types::{DatabaseId, ReadVersions, TenantId, VShardId}; -/// Every vShard a graph read observed, each at its earliest served watermark -/// and with the node that served it. +/// Every vShard a graph read observed, each at its earliest observed version. #[derive(Debug, Default)] pub(crate) struct ShardReadLog { - served: BTreeMap, -} - -/// One vShard's observation: the earliest watermark, and the node whose WAL -/// numbers it (`0` once two nodes served the vShard). -#[derive(Debug, Clone, Copy, PartialEq, Eq)] -pub(crate) struct Served { - pub(crate) watermark: Lsn, - pub(crate) node: u64, + versions: BTreeMap, } impl ShardReadLog { @@ -39,85 +32,60 @@ impl ShardReadLog { Self::default() } - /// Note that `node` served `vshards` at `watermark`. A vShard read twice - /// keeps its earlier watermark: the first observation is the one the - /// result rests on. A vShard two nodes served has no one node whose - /// versions compare with the read, so its node becomes `0`. - pub(crate) fn note( - &mut self, - vshards: impl IntoIterator, - watermark: Lsn, - node: u64, - ) { + /// Note that a leg read `vshards` and reported `versions`. A vShard the + /// leg reported no version of had no write its serving core knew of, so + /// the leg observed it at `WriteVersion::ZERO`. A vShard read twice keeps + /// its earlier version: the first observation is the one the result rests + /// on. + pub(crate) fn note(&mut self, vshards: impl IntoIterator, versions: &ReadVersions) { for vshard in vshards { - self.served - .entry(vshard) - .and_modify(|seen| { - seen.watermark = seen.watermark.min(watermark); - if seen.node != node { - seen.node = 0; - } - }) - .or_insert(Served { watermark, node }); + let version = versions.of(VShardId::new(vshard)).unwrap_or_default(); + self.note_version(vshard, version); } } - /// Note one leg's response from `node`: `vshards` read at the highest - /// watermark the leg reported. - pub(crate) fn note_leg( - &mut self, - vshards: impl IntoIterator, - watermarks: &[(VShardId, Lsn)], - node: u64, - ) { - let watermark = watermarks - .iter() - .map(|(_, lsn)| *lsn) - .max() - .unwrap_or(Lsn::ZERO); - self.note(vshards, watermark, node); + fn note_version(&mut self, vshard: u32, version: WriteVersion) { + self.versions + .entry(vshard) + .and_modify(|seen| *seen = (*seen).min(version)) + .or_insert(version); } /// Fold `other` into this log. pub(crate) fn merge(&mut self, other: ShardReadLog) { - for (vshard, served) in other.served { - self.note([vshard], served.watermark, served.node); + for (vshard, version) in other.versions { + self.note_version(vshard, version); } } /// Hand the log to the running request for its transaction read-set. /// `collection` is the database-qualified collection the read scoped, or - /// `None` when it walked every collection. Without a cluster the log is - /// dropped (see the module doc). + /// `None` when it walked every collection. pub(crate) fn publish( self, - state: &SharedState, tenant_id: TenantId, database_id: DatabaseId, collection: Option, ) { - if state.cluster_routing.is_none() { - return; - } - note(GraphShardReads { + note(ShardReads { + engine: EngineTag::Graph, tenant_id, database_id, collection, shards: self - .served + .versions .into_iter() - .map(|(vshard, served)| ShardObservation { + .map(|(vshard, version)| ShardObservation { vshard: VShardId::new(vshard), - watermark: served.watermark, - node: served.node, + version, }) .collect(), }); } #[cfg(test)] - pub(crate) fn served(&self) -> &BTreeMap { - &self.served + pub(crate) fn versions(&self) -> &BTreeMap { + &self.versions } } @@ -140,52 +108,43 @@ pub(crate) fn key_vshards<'a>(nodes: impl IntoIterator) -> Ve mod tests { use super::*; - fn watermark(log: &ShardReadLog, vshard: u32) -> Option { - log.served().get(&vshard).map(|served| served.watermark) + fn at(index: u64) -> WriteVersion { + WriteVersion::logged(1, index) } - #[test] - fn a_vshard_read_twice_keeps_its_earliest_watermark() { - let mut log = ShardReadLog::new(); - log.note([3, 4], Lsn::new(20), 1); - log.note([3], Lsn::new(10), 1); - log.note([4], Lsn::new(30), 1); - assert_eq!(watermark(&log, 3), Some(Lsn::new(10))); - assert_eq!(watermark(&log, 4), Some(Lsn::new(20))); - assert_eq!(log.served().get(&3).map(|s| s.node), Some(1)); + fn reported(pairs: &[(u32, u64)]) -> ReadVersions { + let mut versions = ReadVersions::new(); + for (vshard, index) in pairs { + versions.note(VShardId::new(*vshard), at(*index)); + } + versions } #[test] - fn a_vshard_two_nodes_served_has_no_node() { + fn a_vshard_read_twice_keeps_its_earliest_version() { let mut log = ShardReadLog::new(); - log.note([3], Lsn::new(20), 1); - log.note([3], Lsn::new(30), 2); - assert_eq!(log.served().get(&3).map(|s| s.node), Some(0)); + log.note([3, 4], &reported(&[(3, 20), (4, 20)])); + log.note([3], &reported(&[(3, 10)])); + log.note([4], &reported(&[(4, 30)])); + assert_eq!(log.versions().get(&3), Some(&at(10))); + assert_eq!(log.versions().get(&4), Some(&at(20))); } #[test] - fn a_leg_is_noted_at_its_highest_reported_watermark() { + fn a_vshard_the_leg_reported_no_version_of_is_observed_at_zero() { let mut log = ShardReadLog::new(); - log.note_leg( - [7], - &[ - (VShardId::new(1), Lsn::new(5)), - (VShardId::new(2), Lsn::new(9)), - ], - 2, - ); - assert_eq!(watermark(&log, 7), Some(Lsn::new(9))); + log.note([7], &reported(&[(2, 9)])); + assert_eq!(log.versions().get(&7), Some(&WriteVersion::ZERO)); } #[test] - fn merging_keeps_the_earliest_watermark_per_vshard() { + fn merging_keeps_the_earliest_version_per_vshard() { let mut left = ShardReadLog::new(); - left.note([1], Lsn::new(8), 1); + left.note([1], &reported(&[(1, 8)])); let mut right = ShardReadLog::new(); - right.note([1], Lsn::new(4), 1); - right.note([2], Lsn::new(6), 1); + right.note([1, 2], &reported(&[(1, 4), (2, 6)])); left.merge(right); - assert_eq!(watermark(&left, 1), Some(Lsn::new(4))); - assert_eq!(watermark(&left, 2), Some(Lsn::new(6))); + assert_eq!(left.versions().get(&1), Some(&at(4))); + assert_eq!(left.versions().get(&2), Some(&at(6))); } } diff --git a/nodedb/src/control/server/graph_dispatch/shortest_path.rs b/nodedb/src/control/server/graph_dispatch/shortest_path.rs index 31019b246..16b89fe5a 100644 --- a/nodedb/src/control/server/graph_dispatch/shortest_path.rs +++ b/nodedb/src/control/server/graph_dispatch/shortest_path.rs @@ -109,7 +109,7 @@ pub async fn cross_core_shortest_path( &mut presence_reads, ) .await?; - presence_reads.publish(shared, tenant_id, database_id, None); + presence_reads.publish(tenant_id, database_id, None); if !present.contains(&src) || !present.contains(&dst) { return Ok(ok_response(encode_path::(&[])?)); } @@ -182,7 +182,7 @@ pub async fn cross_core_shortest_path( } // Every vShard the walk expanded joins the transaction read-set. - reads.publish(shared, tenant_id, database_id, collection); + reads.publish(tenant_id, database_id, collection); Ok(ok_response(encode_path(&path)?)) } diff --git a/nodedb/src/control/server/graph_dispatch/traverse_subgraph.rs b/nodedb/src/control/server/graph_dispatch/traverse_subgraph.rs index bc9037b5d..56ed05afe 100644 --- a/nodedb/src/control/server/graph_dispatch/traverse_subgraph.rs +++ b/nodedb/src/control/server/graph_dispatch/traverse_subgraph.rs @@ -210,7 +210,7 @@ pub(crate) async fn walk_subgraph( &mut presence_reads, ) .await?; - presence_reads.publish(shared, tenant_id, database_id, None); + presence_reads.publish(tenant_id, database_id, None); if !present.contains(&start) { return Ok(SubgraphWalk { node_order: Vec::new(), @@ -259,7 +259,7 @@ pub(crate) async fn walk_subgraph( } // Every vShard the walk expanded joins the transaction read-set. - reads.publish(shared, tenant_id, database_id, collection); + reads.publish(tenant_id, database_id, collection); Ok(SubgraphWalk { node_order, edges: edges.into_vec(), diff --git a/nodedb/src/control/server/graph_dispatch/whole_graph.rs b/nodedb/src/control/server/graph_dispatch/whole_graph.rs index 1cd21678b..75f68861f 100644 --- a/nodedb/src/control/server/graph_dispatch/whole_graph.rs +++ b/nodedb/src/control/server/graph_dispatch/whole_graph.rs @@ -67,6 +67,10 @@ pub async fn scatter_to_graph_owners( let node = execute_plan_all_local_cores(state, tenant_id, database_id, plan, TraceId::ZERO, None) .await?; + // This node holds every vShard, so the read observed all of them. + let mut reads = ShardReadLog::new(); + reads.note(0..VShardId::COUNT, &node.read_versions); + reads.publish(tenant_id, database_id, collection); return Ok(vec![Payload::from_vec(node.payload)]); } let targets = enumerate_shards(state)?.targets; @@ -93,17 +97,17 @@ pub async fn scatter_to_graph_owners( }, ) .await?; - Ok::<_, crate::Error>((target.owned_vshards, target.node_id, read)) + Ok::<_, crate::Error>((target.owned_vshards, read)) } }); let mut reads = ShardReadLog::new(); let mut payloads = Vec::new(); for leg in join_all(legs).await { - let (owned_vshards, node_id, read) = leg?; - reads.note(owned_vshards, read.watermark_lsn, node_id); + let (owned_vshards, read) = leg?; + reads.note(owned_vshards, &read.read_versions); payloads.push(read.payload); } - reads.publish(state, tenant_id, database_id, collection); + reads.publish(tenant_id, database_id, collection); Ok(payloads) } diff --git a/nodedb/src/control/server/http/routes/atomic_statement.rs b/nodedb/src/control/server/http/routes/atomic_statement.rs new file mode 100644 index 000000000..ca75cb211 --- /dev/null +++ b/nodedb/src/control/server/http/routes/atomic_statement.rs @@ -0,0 +1,100 @@ +// SPDX-License-Identifier: BUSL-1.1 + +//! The shared atomic routes for the HTTP query routes: `/v1/query`, +//! `/v1/query/stream` and WebSocket RPC. +//! +//! Each route calls [`route_http_statement`] after admission. A statement +//! the shared router takes runs there, through the same implicit +//! transaction, implicit-edge gate and Calvin commit native and pgwire use, +//! and answers as JSON rows. Every other statement comes back for the +//! route's own per-task loop. + +use crate::control::server::response_shape::cell::row_to_wire_json; +use crate::control::server::response_shape::types::FoldedTag; +use crate::control::server::shared::session::DetachedTxnScope; +use crate::control::server::shared::statement_exec::{ + AtomicRoute, AtomicStatement, StatementAnswer, StatementExec, calvin_answer, + route_atomic_statement, +}; + +/// What the shared router did with a statement. +pub(super) enum RoutedStatement { + /// An atomic route ran the statement. Its answer as JSON rows. + Answered(Vec), + /// No atomic route applies. The route runs its per-task loop. + PerTask(AtomicStatement), +} + +/// Route `statement` through the shared atomic routes. +/// +/// These routes carry no session: no transaction block is open, and the +/// cross-shard mode is the default, `Strict`. An implicit transaction runs +/// on a private session of its own. +pub(super) async fn route_http_statement( + exec: &StatementExec<'_>, + statement: AtomicStatement, +) -> crate::Result { + let detached = DetachedTxnScope::new(); + let session = detached.ctx(); + let answer = match route_atomic_statement(exec, statement, &session).await { + Ok(AtomicRoute::Implicit(answer)) => Ok(answer), + Ok(AtomicRoute::Calvin(applied)) => calvin_answer(exec, applied), + Ok(AtomicRoute::PerTask(statement)) => return Ok(RoutedStatement::PerTask(statement)), + Err(error) => Err(error), + }; + let answer = answer.map_err(|error| error.into_error(exec.tenant_id()))?; + Ok(RoutedStatement::Answered(answer_json_rows(answer))) +} + +/// A statement's answer as JSON rows, each keyed by its cell keys as +/// `shape_http_payload` keys a row. A write that answers no rows answers one +/// `{"affected": n}` row: the shape a single write's count takes on these +/// routes. A derived task (a balance move, an implicit edge) adds no count. +fn answer_json_rows(answer: StatementAnswer) -> Vec { + let mut rows: Vec = answer + .rows + .iter() + .flat_map(|shaped| { + shaped + .rows + .iter() + .map(|row| serde_json::Value::Object(row_to_wire_json(row))) + }) + .collect(); + if rows.is_empty() + && let Some(FoldedTag::Dml(outcome)) = answer.tag + { + rows.push(serde_json::json!({ "affected": outcome.affected })); + } + rows +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::control::server::response_shape::types::DmlOutcome; + + fn answer(tag: Option) -> StatementAnswer { + StatementAnswer { + rows: Vec::new(), + warnings: Vec::new(), + tag, + last_lsn: 0, + } + } + + #[test] + fn a_counted_write_answers_one_affected_row() { + let rows = answer_json_rows(answer(Some(FoldedTag::Dml(DmlOutcome { + verb: "INSERT", + affected: 2, + })))); + assert_eq!(rows, vec![serde_json::json!({ "affected": 2 })]); + } + + #[test] + fn an_opaque_statement_answers_no_rows() { + assert!(answer_json_rows(answer(Some(FoldedTag::Opaque))).is_empty()); + assert!(answer_json_rows(answer(None)).is_empty()); + } +} diff --git a/nodedb/src/control/server/http/routes/health.rs b/nodedb/src/control/server/http/routes/health.rs index 8fdfb9c35..85519e9e8 100644 --- a/nodedb/src/control/server/http/routes/health.rs +++ b/nodedb/src/control/server/http/routes/health.rs @@ -431,9 +431,9 @@ mod tests { vshard_id: 12, epoch: 40, position: 3, - reason: "flush_failed", - step: "flush", - error: "CalvinFlush returned Error".to_string(), + reason: "redo_apply_failed", + step: "redo_apply", + error: "the stamped redo install returned Error".to_string(), }); let (status, body) = healthz_body(state).await; @@ -444,8 +444,8 @@ mod tests { assert_eq!(body["vshard_id"], 12); assert_eq!(body["epoch"], 40); assert_eq!(body["position"], 3); - assert_eq!(body["halt_reason"], "flush_failed"); - assert_eq!(body["step"], "flush"); + assert_eq!(body["halt_reason"], "redo_apply_failed"); + assert_eq!(body["step"], "redo_apply"); } #[tokio::test] diff --git a/nodedb/src/control/server/http/routes/mod.rs b/nodedb/src/control/server/http/routes/mod.rs index 43ba25de3..f9e5ae0db 100644 --- a/nodedb/src/control/server/http/routes/mod.rs +++ b/nodedb/src/control/server/http/routes/mod.rs @@ -1,5 +1,6 @@ // SPDX-License-Identifier: BUSL-1.1 +mod atomic_statement; pub mod auth_key; pub mod auth_session; pub mod cdc; diff --git a/nodedb/src/control/server/http/routes/query/materialized/append.rs b/nodedb/src/control/server/http/routes/query/materialized/append.rs index a3e6b55a8..71e2b86cd 100644 --- a/nodedb/src/control/server/http/routes/query/materialized/append.rs +++ b/nodedb/src/control/server/http/routes/query/materialized/append.rs @@ -1,16 +1,9 @@ // SPDX-License-Identifier: BUSL-1.1 //! Per-task helpers the dispatch loop and the orchestrated plans share: -//! authorization without the clone-write check, metering, and shaping one -//! task's answer into the statement's JSON rows. - -use std::sync::Arc; - -use nodedb_physical::physical_task::PhysicalTask; +//! metering, and shaping one task's answer into the statement's JSON rows. use crate::bridge::envelope::Status; -use crate::control::security::audit::ArcAuditEmitter; -use crate::control::security::identity::AuthenticatedIdentity; use crate::control::security::request_scope::RequestAuthScope; use crate::control::server::response_shape::redaction::QueryRedaction; use crate::control::server::response_shape::request::MaterializedShapeRequest; @@ -25,31 +18,6 @@ use super::super::super::result_shape::{ }; use super::encode::response_error; -/// Authorize one task with no clone-write check. Used only for the -/// Control-Plane orchestrated plans (see `orchestrated`), which are never -/// clone-write shapes. -pub(super) fn authorize_materialized_task( - shared: &crate::control::state::SharedState, - identity: &AuthenticatedIdentity, - task: &PhysicalTask, -) -> crate::Result { - let emitter = ArcAuditEmitter(Arc::clone(&shared.audit)); - crate::control::server::shared::authorization::authorize_task_set( - identity, - std::slice::from_ref(task), - &shared.permissions, - &shared.roles, - &emitter, - ) - .map_err(crate::Error::from)? - .into_tasks() - .into_iter() - .next() - .ok_or_else(|| crate::Error::Internal { - detail: "authorization returned an empty capability set".into(), - }) -} - /// Meter one task's dispatch after its rows are appended to `result_rows` — /// the row count is the delta since `rows_before`. pub(super) fn meter_task_dispatch( diff --git a/nodedb/src/control/server/http/routes/query/materialized/mod.rs b/nodedb/src/control/server/http/routes/query/materialized/mod.rs index b16412010..1f5693dfa 100644 --- a/nodedb/src/control/server/http/routes/query/materialized/mod.rs +++ b/nodedb/src/control/server/http/routes/query/materialized/mod.rs @@ -2,12 +2,13 @@ //! `/v1/query`: materialized (buffer-then-respond) SQL execution. //! -//! - `request.rs`: the entry point through admission. +//! - `request.rs`: the entry point through admission and the shared atomic +//! routes. //! - `shape.rs`: the per-task dispatch loop. //! - `orchestrated.rs`: the plans that run on the Control Plane instead of //! the gateway route, cluster array ops among them. //! - `append.rs`: shaping each task's answer into JSON rows, and the -//! per-task authorization and metering helpers. +//! per-task metering helper. //! - `encode.rs`: maps errors onto the HTTP surface. mod append; diff --git a/nodedb/src/control/server/http/routes/query/materialized/orchestrated.rs b/nodedb/src/control/server/http/routes/query/materialized/orchestrated.rs index 3cb1ef0c8..50d6705ea 100644 --- a/nodedb/src/control/server/http/routes/query/materialized/orchestrated.rs +++ b/nodedb/src/control/server/http/routes/query/materialized/orchestrated.rs @@ -19,10 +19,10 @@ use nodedb_physical::physical_task::PhysicalTask; use crate::bridge::envelope::{PhysicalPlan, Response, Status}; use crate::control::security::identity::AuthenticatedIdentity; use crate::control::server::shared::cluster_array_dispatch::{is_cluster_array, run_cluster_array}; +use crate::control::server::shared::statement_exec::authorize_one_task; use crate::control::state::SharedState; use super::super::super::super::auth::ApiError; -use super::append::authorize_materialized_task; use super::encode::{gateway_error, response_error}; /// The payload an orchestrated plan answered with. @@ -39,7 +39,7 @@ pub(super) async fn run_orchestrated( identity: &AuthenticatedIdentity, task: &PhysicalTask, ) -> Result, ApiError> { - let authorize = || authorize_materialized_task(shared, identity, task).map_err(gateway_error); + let authorize = || authorize_one_task(shared, identity, task).map_err(gateway_error); if crate::control::array_catalog::ddl::is_array_ddl(&task.plan) { let response = diff --git a/nodedb/src/control/server/http/routes/query/materialized/request.rs b/nodedb/src/control/server/http/routes/query/materialized/request.rs index 99ff58e79..7c8671318 100644 --- a/nodedb/src/control/server/http/routes/query/materialized/request.rs +++ b/nodedb/src/control/server/http/routes/query/materialized/request.rs @@ -7,10 +7,12 @@ use axum::http::HeaderMap; use axum::response::IntoResponse; use crate::control::security::audit::ArcAuditEmitter; +use crate::control::sequence::SessionSequenceAccess; use crate::control::server::shared::authorization::authorize_database; use crate::control::server::shared::plan_admission::{ PlanAdmissionRequest, plan_authorize_and_admit, }; +use crate::control::server::shared::statement_exec::{AtomicStatement, StatementExec}; use super::super::super::super::auth::{ ApiError, AppState, build_request_scope, resolve_auth_parts, @@ -18,6 +20,7 @@ use super::super::super::super::auth::{ use super::super::super::super::peer::PeerAddr; use super::super::super::super::transport::ClientTransport; use super::super::super::super::types::{HttpQueryRequest, HttpQueryResponse}; +use super::super::super::atomic_statement::{RoutedStatement, route_http_statement}; use super::super::super::result_shape::ddl_results_to_json; use super::super::{DatabaseQueryParam, resolve_database_id}; use super::encode::ddl_error_to_api; @@ -136,6 +139,41 @@ pub async fn query( // A statement admitted under a lease this node then loses ends with a // retryable error: a read mid-flight, a write only before dispatch. lease_scope.check_not_revoked().map_err(ApiError::from)?; + + // The shared atomic routes run a statement whose tasks commit together: + // the implicit transaction, the implicit-edge gate, and a Calvin commit + // for a write that spans vShards. + // HTTP carries no session: `nextval` advances the registry, `currval` + // reports "not yet called in this session". + let sequences = SessionSequenceAccess::for_session(&state.shared, None, database_id, tenant_id); + let exec = StatementExec { + state: &state.shared, + identity: &identity, + scope: &scope, + output_schema: Some(&output_schema), + database_id, + sequences: &sequences, + client_session: None, + }; + let routed = route_http_statement( + &exec, + AtomicStatement { + tasks, + lease_scope, + sum_target_reads: admission.sum_target_reads, + }, + ) + .await + .map_err(ApiError::from)?; + let AtomicStatement { + tasks, lease_scope, .. + } = match routed { + RoutedStatement::Answered(rows) => { + return Ok((rate_limit_headers, axum::Json(HttpQueryResponse::ok(rows)))); + } + RoutedStatement::PerTask(statement) => statement, + }; + let read_only = tasks .iter() .all(|task| !crate::control::server::shared::write_admission::plan_is_write(&task.plan)); diff --git a/nodedb/src/control/server/http/routes/query/ndjson.rs b/nodedb/src/control/server/http/routes/query/ndjson.rs index 4480375ca..3f85e492b 100644 --- a/nodedb/src/control/server/http/routes/query/ndjson.rs +++ b/nodedb/src/control/server/http/routes/query/ndjson.rs @@ -9,6 +9,7 @@ use axum::response::IntoResponse; use crate::control::gateway::GatewayErrorMap; use crate::control::gateway::core::QueryContext; use crate::control::security::audit::ArcAuditEmitter; +use crate::control::sequence::SessionSequenceAccess; use crate::control::server::response_shape::redaction::QueryRedaction; use crate::control::server::response_shape::request::MaterializedShapeRequest; use crate::control::server::response_shape::types::describe_plan; @@ -21,10 +22,14 @@ use crate::control::server::shared::plan_admission::{ PlanAdmissionRequest, plan_authorize_and_admit, }; use crate::control::server::shared::quota_admission::admit_quota_for_dispatch; +use crate::control::server::shared::statement_exec::{ + AtomicStatement, StatementExec, authorize_one_task, +}; use super::super::super::auth::{ApiError, AppState, build_request_scope, resolve_auth_parts}; use super::super::super::peer::PeerAddr; use super::super::super::transport::ClientTransport; +use super::super::atomic_statement::{RoutedStatement, route_http_statement}; use super::super::query_stream::{NdjsonBody, ndjson_body_stream, try_open_stream}; use super::super::result_shape::{HttpShaped, passthrough_to_ndjson, shape_http_payload}; use super::{DatabaseQueryParam, resolve_database_id}; @@ -119,14 +124,56 @@ pub async fn query_ndjson( Ok(admission) => admission, Err(error) => return ApiError::from(error).into_response(), }; - let tasks = admission.tasks; let output_schema = admission.output_schema; - let mut lease_scope = Some(admission.lease_scope); let trace_id = crate::control::trace_context::generate_trace_id(); let _request = state.shared.tenant_request_guard(tenant_id); + // The shared atomic routes run a statement whose tasks commit together: + // the implicit transaction, the implicit-edge gate, and a Calvin commit + // for a write that spans vShards. HTTP carries no session: `nextval` + // advances the registry, `currval` reports "not yet called in this + // session". + let sequences = SessionSequenceAccess::for_session(&state.shared, None, database_id, tenant_id); + let exec = StatementExec { + state: &state.shared, + identity: &identity, + scope: &scope, + output_schema: Some(&output_schema), + database_id, + sequences: &sequences, + client_session: None, + }; + let routed = route_http_statement( + &exec, + AtomicStatement { + tasks: admission.tasks, + lease_scope: admission.lease_scope, + sum_target_reads: admission.sum_target_reads, + }, + ) + .await; + let (tasks, lease_scope) = match routed { + Ok(RoutedStatement::PerTask(statement)) => (statement.tasks, statement.lease_scope), + Ok(RoutedStatement::Answered(rows)) => { + let mut ndjson = String::new(); + for row in rows { + ndjson.push_str(&row.to_string()); + ndjson.push('\n'); + } + return ndjson_response(ndjson, rate_limit_headers); + } + // A statement that stopped answers one error line, as a task that + // stopped in the loop below does. + Err(error) => { + let (_status, message) = GatewayErrorMap::to_http(&error); + let line = format!("{}\n", serde_json::json!({ "error": message })); + return ndjson_response(line, rate_limit_headers); + } + }; + let mut lease_scope = Some(lease_scope); + // `Body::from_stream` polls the data-plane stream under normal HTTP backpressure // while its captured lease scope stays alive until body completion or disconnect. match try_open_stream( @@ -225,7 +272,7 @@ pub async fn query_ndjson( let dispatch = async { let result: crate::Result>> = if crate::control::array_catalog::ddl::is_array_ddl(&task.plan) { - match authorize_ndjson_task(&state.shared, &identity, &task) { + match authorize_one_task(&state.shared, &identity, &task) { Ok(authorized_task) => { crate::control::array_catalog::ddl::run_authorized_array_ddl( &state.shared, @@ -239,7 +286,7 @@ pub async fn query_ndjson( } else if is_cluster_array(&task.plan) { // This node's array coordinator routes the op to the // shards that own its cells. - match authorize_ndjson_task(&state.shared, &identity, &task) { + match authorize_one_task(&state.shared, &identity, &task) { Ok(authorized_task) => run_cluster_array(&state.shared, authorized_task) .await .map(|payload| vec![payload]), @@ -251,7 +298,7 @@ pub async fn query_ndjson( nodedb_physical::physical_plan::DocumentOp::InsertSelect { .. } ) ) { - match authorize_ndjson_task(&state.shared, &identity, &task) { + match authorize_one_task(&state.shared, &identity, &task) { Ok(authorized_task) => { crate::control::insert_select::run_authorized_insert_select( &state.shared, @@ -271,7 +318,7 @@ pub async fn query_ndjson( } ) ) { - match authorize_ndjson_task(&state.shared, &identity, &task) { + match authorize_one_task(&state.shared, &identity, &task) { Ok(authorized_task) => { crate::control::merge_orchestrator::run_authorized_merge( &state.shared, @@ -291,7 +338,7 @@ pub async fn query_ndjson( } ) ) { - match authorize_ndjson_task(&state.shared, &identity, &task) { + match authorize_one_task(&state.shared, &identity, &task) { Ok(authorized_task) => { crate::control::update_from_join_orchestrator::run_authorized_update_from_join( &state.shared, @@ -307,7 +354,7 @@ pub async fn query_ndjson( { // A governed columnar predicate UPDATE/DELETE resolves to a concrete row set // before proposing. - match authorize_ndjson_task(&state.shared, &identity, &task) { + match authorize_one_task(&state.shared, &identity, &task) { Ok(authorized_task) => { crate::control::write_resolve::run_authorized_write_resolve( &state.shared, @@ -425,36 +472,18 @@ pub async fn query_ndjson( } } - let mut response = Response::builder() + ndjson_response(ndjson, rate_limit_headers) +} + +/// A materialized NDJSON body with the request's rate-limit headers. +fn ndjson_response( + ndjson: String, + rate_limit_headers: axum::http::HeaderMap, +) -> axum::response::Response { + let mut response = axum::response::Response::builder() .header("Content-Type", "application/x-ndjson") .body(axum::body::Body::from(ndjson)) .unwrap_or_else(|_| (StatusCode::INTERNAL_SERVER_ERROR, "encoding error").into_response()); response.headers_mut().extend(rate_limit_headers); response } - -/// Authorize one task with no clone-write check — used only by the -/// Control-Plane orchestrator branches ahead of the general dispatch tail, -/// whose plan shapes (`InsertSelect`, `Merge`, `UpdateFromJoin`, a governed -/// predicate resolution) are never clone-write shapes. -fn authorize_ndjson_task( - shared: &crate::control::state::SharedState, - identity: &crate::control::security::identity::AuthenticatedIdentity, - task: &nodedb_physical::physical_task::PhysicalTask, -) -> crate::Result { - let emitter = ArcAuditEmitter(Arc::clone(&shared.audit)); - crate::control::server::shared::authorization::authorize_task_set( - identity, - std::slice::from_ref(task), - &shared.permissions, - &shared.roles, - &emitter, - ) - .map_err(crate::Error::from)? - .into_tasks() - .into_iter() - .next() - .ok_or_else(|| crate::Error::Internal { - detail: "authorization returned an empty capability set".into(), - }) -} diff --git a/nodedb/src/control/server/http/routes/query_stream.rs b/nodedb/src/control/server/http/routes/query_stream.rs index 27cd25374..d1fff32f5 100644 --- a/nodedb/src/control/server/http/routes/query_stream.rs +++ b/nodedb/src/control/server/http/routes/query_stream.rs @@ -269,7 +269,7 @@ mod tests { Ok(RowBatch { payload: json_object_batch(start, n), watermark_lsn: Lsn::ZERO, - read_version_lsn: Lsn::ZERO, + read_versions: crate::types::ReadVersions::new(), }) } diff --git a/nodedb/src/control/server/http/routes/ws_rpc/execute_sql.rs b/nodedb/src/control/server/http/routes/ws_rpc/execute_sql.rs index 57611f31c..874abe777 100644 --- a/nodedb/src/control/server/http/routes/ws_rpc/execute_sql.rs +++ b/nodedb/src/control/server/http/routes/ws_rpc/execute_sql.rs @@ -8,6 +8,7 @@ use crate::control::gateway::core::QueryContext; use crate::control::security::audit::ArcAuditEmitter; use crate::control::security::identity::AuthenticatedIdentity; use crate::control::security::request_scope::RequestAuthScope; +use crate::control::sequence::SessionSequenceAccess; use crate::control::server::response_shape::redaction::{QueryRedaction, redact_decoded_value}; use crate::control::server::shared::authorization::authorize_database; use crate::control::server::shared::cluster_array_dispatch::{is_cluster_array, run_cluster_array}; @@ -16,11 +17,17 @@ use crate::control::server::shared::plan_admission::{ PlanAdmissionRequest, plan_authorize_and_admit, }; use crate::control::server::shared::quota_admission::admit_quota_for_dispatch; +use crate::control::server::shared::statement_exec::{ + AtomicStatement, StatementExec, authorize_one_task, +}; use crate::control::state::SharedState; use crate::types::{DatabaseId, TraceId}; +use super::super::atomic_statement::{RoutedStatement, route_http_statement}; + /// Execute SQL and return result as JSON. /// +/// A statement whose tasks commit together takes the shared atomic routes. /// A Control-Plane orchestrated task (array DDL, a cluster array op, /// `INSERT ... SELECT`) runs here. Every other task routes through the /// gateway. @@ -64,12 +71,46 @@ pub async fn execute_sql( trace_id, }) .await?; - let tasks = admission.tasks; - let lease_scope = admission.lease_scope; + let output_schema = admission.output_schema; // A statement admitted under a lease this node then loses ends with a // retryable error: a read mid-flight, a write only before dispatch. - lease_scope.check_not_revoked()?; + admission.lease_scope.check_not_revoked()?; + + // The shared atomic routes run a statement whose tasks commit together: + // the implicit transaction, the implicit-edge gate, and a Calvin commit + // for a write that spans vShards. WebSocket RPC carries no session: + // `nextval` advances the registry, `currval` reports "not yet called in + // this session". + let sequences = SessionSequenceAccess::for_session(shared, None, database_id, tenant_id); + let exec = StatementExec { + state: shared, + identity, + scope: &scope, + output_schema: Some(&output_schema), + database_id, + sequences: &sequences, + client_session: None, + }; + let routed = { + let _request = shared.tenant_request_guard(tenant_id); + route_http_statement( + &exec, + AtomicStatement { + tasks: admission.tasks, + lease_scope: admission.lease_scope, + sum_target_reads: admission.sum_target_reads, + }, + ) + .await? + }; + let AtomicStatement { + tasks, lease_scope, .. + } = match routed { + RoutedStatement::Answered(rows) => return Ok(rows_result(rows)), + RoutedStatement::PerTask(statement) => statement, + }; + let read_only = tasks .iter() .all(|task| !crate::control::server::shared::write_admission::plan_is_write(&task.plan)); @@ -99,7 +140,7 @@ pub async fn execute_sql( // Array DDL proposes a replicated catalog entry, never a core task. if crate::control::array_catalog::ddl::is_array_ddl(&task.plan) { - let authorized_task = authorize_ws_rpc_task(shared, identity, &task)?; + let authorized_task = authorize_one_task(shared, identity, &task)?; let resp = crate::control::array_catalog::ddl::run_authorized_array_ddl( shared, authorized_task, @@ -117,7 +158,7 @@ pub async fn execute_sql( // A cluster array op runs through this node's array coordinator, // which routes it to the shards that own its cells. if is_cluster_array(&task.plan) { - let authorized_task = authorize_ws_rpc_task(shared, identity, &task)?; + let authorized_task = authorize_one_task(shared, identity, &task)?; let payload = run_cluster_array(shared, authorized_task).await?; if !payload.is_empty() { let json = @@ -140,7 +181,7 @@ pub async fn execute_sql( nodedb_physical::physical_plan::DocumentOp::InsertSelect { .. }, ) = &task.plan { - let authorized_task = authorize_ws_rpc_task(shared, identity, &task)?; + let authorized_task = authorize_one_task(shared, identity, &task)?; match crate::control::insert_select::run_authorized_insert_select( shared, authorized_task, @@ -194,7 +235,7 @@ pub async fn execute_sql( }, ) = &task.plan { - let authorized_task = authorize_ws_rpc_task(shared, identity, &task)?; + let authorized_task = authorize_one_task(shared, identity, &task)?; match crate::control::merge_orchestrator::run_authorized_merge( shared, authorized_task, @@ -247,7 +288,7 @@ pub async fn execute_sql( }, ) = &task.plan { - let authorized_task = authorize_ws_rpc_task(shared, identity, &task)?; + let authorized_task = authorize_one_task(shared, identity, &task)?; match crate::control::update_from_join_orchestrator::run_authorized_update_from_join( shared, authorized_task, @@ -277,7 +318,7 @@ pub async fn execute_sql( // A governed columnar predicate UPDATE/DELETE resolves to a concrete row set // before proposing. if let Some(resolver) = crate::control::write_resolve::resolver_for_plan(&task.plan) { - let authorized_task = authorize_ws_rpc_task(shared, identity, &task)?; + let authorized_task = authorize_one_task(shared, identity, &task)?; match crate::control::write_resolve::run_authorized_write_resolve( shared, authorized_task, @@ -399,15 +440,7 @@ pub async fn execute_sql( meter_task(shared, &scope, &plan_metering_info, rows_before, &results); } - let outcome: crate::Result = match results.len() { - 0 => Ok(serde_json::Value::Null), - 1 => Ok(results - .into_iter() - .next() - .unwrap_or(serde_json::Value::Null)), - _ => Ok(serde_json::Value::Array(results)), - }; - outcome + Ok(rows_result(results)) }; if read_only { lease_scope.guard(body).await? @@ -416,30 +449,14 @@ pub async fn execute_sql( } } -/// Authorize one task with no clone-write check — used only by the -/// Control-Plane orchestrator branches ahead of the general dispatch tail, -/// whose plan shapes (`InsertSelect`, `Merge`, `UpdateFromJoin`, a governed -/// predicate resolution) are never clone-write shapes. -fn authorize_ws_rpc_task( - shared: &SharedState, - identity: &AuthenticatedIdentity, - task: &nodedb_physical::physical_task::PhysicalTask, -) -> crate::Result { - let emitter = ArcAuditEmitter(Arc::clone(&shared.audit)); - crate::control::server::shared::authorization::authorize_task_set( - identity, - std::slice::from_ref(task), - &shared.permissions, - &shared.roles, - &emitter, - ) - .map_err(crate::Error::from)? - .into_tasks() - .into_iter() - .next() - .ok_or_else(|| crate::Error::Internal { - detail: "authorization returned an empty capability set".into(), - }) +/// The RPC result for a statement's JSON rows: `null` for none, the row +/// itself for one, an array for more. +fn rows_result(mut rows: Vec) -> serde_json::Value { + match rows.len() { + 0 => serde_json::Value::Null, + 1 => rows.pop().unwrap_or(serde_json::Value::Null), + _ => serde_json::Value::Array(rows), + } } /// Meter one task's dispatch after its rows are pushed onto `results` — diff --git a/nodedb/src/control/server/ilp_batch/dispatch.rs b/nodedb/src/control/server/ilp_batch/dispatch.rs index 877d1fba4..b79363828 100644 --- a/nodedb/src/control/server/ilp_batch/dispatch.rs +++ b/nodedb/src/control/server/ilp_batch/dispatch.rs @@ -96,7 +96,7 @@ pub(crate) async fn flush_authenticated_ilp_batch( /// At most one schema-projection merge is ever in flight, process-wide. /// /// The merge is a replicated catalog DDL, and every catalog DDL already -/// serializes on `SharedState::metadata_ddl_lock`. A second concurrent merge +/// serializes on `MetadataDdlState::lock`. A second concurrent merge /// can therefore only park a second task on a lock that admits one holder, /// so one permit is both the useful and the safe bound — ingest can never /// queue merges no matter how many ILP connections or OTLP requests are live. diff --git a/nodedb/src/control/server/native/dispatch/cluster_array.rs b/nodedb/src/control/server/native/dispatch/cluster_array.rs index 7a66b1bc8..1495fa7de 100644 --- a/nodedb/src/control/server/native/dispatch/cluster_array.rs +++ b/nodedb/src/control/server/native/dispatch/cluster_array.rs @@ -2,9 +2,9 @@ //! `ClusterArray` plan dispatch for the native protocol. //! -//! `sql_loop.rs` and `single_task.rs` intercept a `PhysicalPlan::ClusterArray` -//! task right after their own in-transaction routing gate and delegate to the -//! shared, protocol-neutral core +//! `single_task.rs` intercepts a `PhysicalPlan::ClusterArray` task right +//! after its own in-transaction routing gate and delegates to the shared, +//! protocol-neutral core //! (`shared::cluster_array_dispatch::execute_cluster_array`), then convert //! the outcome into native wire columns/rows or a count-bearing outcome. Metering //! is not applied on this path, matching pgwire's `ClusterArray` diff --git a/nodedb/src/control/server/native/dispatch/direct_ops.rs b/nodedb/src/control/server/native/dispatch/direct_ops.rs index 8a2c0d3f9..5a1d0878b 100644 --- a/nodedb/src/control/server/native/dispatch/direct_ops.rs +++ b/nodedb/src/control/server/native/dispatch/direct_ops.rs @@ -218,6 +218,7 @@ pub(crate) async fn handle_direct_op( &mut tasks, tenant_id, ctx.database_id(), + txn_id, TraceId::ZERO, ) .await @@ -243,6 +244,7 @@ pub(crate) async fn handle_direct_op( &mut tasks, tenant_id, ctx.database_id(), + txn_id, TraceId::ZERO, ) .await @@ -263,8 +265,9 @@ pub(crate) async fn handle_direct_op( Err(error) => return error_to_native(seq, &crate::Error::from(error)), }; - // A write that fires a BEFORE, INSTEAD OF or SYNC AFTER body, or a - // MERGE into an edge-bearing collection, runs in one transaction, as + // A write that fires a BEFORE, INSTEAD OF or SYNC AFTER body, a MERGE + // into an edge-bearing collection, or a join-expanding write into the + // source of a cross-shard materialized sum, runs in one transaction, as // the SQL path's does. The loop there clone-checks, authorizes and // meters each task itself. if statement_needs_implicit_txn(ctx.state, &tasks) { diff --git a/nodedb/src/control/server/native/dispatch/direct_orchestrated.rs b/nodedb/src/control/server/native/dispatch/direct_orchestrated.rs index 377d2e361..ef055c4c9 100644 --- a/nodedb/src/control/server/native/dispatch/direct_orchestrated.rs +++ b/nodedb/src/control/server/native/dispatch/direct_orchestrated.rs @@ -44,18 +44,22 @@ pub(super) async fn dispatch_orchestrated( PhysicalPlan::Document(nodedb_physical::physical_plan::DocumentOp::InsertSelect { .. }) ) { let task = bare_task(ctx, plan, vshard_id); - let authorized = match super::sql_gateway::authorize_native_task(ctx, &task) { - Ok(authorized) => authorized, - Err(error) => return Some(error_to_native(seq, &error)), - }; - let _request = ctx.state.tenant_request_guard(tenant_id); - let result = - crate::control::insert_select::run_authorized_insert_select(ctx.state, authorized) - .await; - return Some(match result { - Ok(resp) => data_plane_response_to_native(ctx, seq, plan, &resp), - Err(e) => error_to_native(seq, &e), - }); + // A copy into the source of a cross-shard materialized sum runs in the + // implicit transaction, which ships each balance to its target. + if !statement_needs_implicit_txn(ctx.state, std::slice::from_ref(&task)) { + let authorized = match super::sql_gateway::authorize_native_task(ctx, &task) { + Ok(authorized) => authorized, + Err(error) => return Some(error_to_native(seq, &error)), + }; + let _request = ctx.state.tenant_request_guard(tenant_id); + let result = + crate::control::insert_select::run_authorized_insert_select(ctx.state, authorized) + .await; + return Some(match result { + Ok(resp) => data_plane_response_to_native(ctx, seq, plan, &resp), + Err(e) => error_to_native(seq, &e), + }); + } } // Autocommit `MERGE` orchestrates on the Control Plane; never reaches the @@ -68,8 +72,10 @@ pub(super) async fn dispatch_orchestrated( }) ) { let task = bare_task(ctx, plan, vshard_id); - // A MERGE into an edge-bearing collection runs in the implicit - // transaction, which stages each removed row's edge tasks. + // A MERGE into an edge-bearing collection, or into the source of a + // cross-shard materialized sum, runs in the implicit transaction. It + // stages each removed row's edge tasks and ships each balance to its + // target. if !statement_needs_implicit_txn(ctx.state, std::slice::from_ref(&task)) { let authorized = match super::sql_gateway::authorize_native_task(ctx, &task) { Ok(authorized) => authorized, @@ -96,20 +102,24 @@ pub(super) async fn dispatch_orchestrated( }) ) { let task = bare_task(ctx, plan, vshard_id); - let authorized = match super::sql_gateway::authorize_native_task(ctx, &task) { - Ok(authorized) => authorized, - Err(error) => return Some(error_to_native(seq, &error)), - }; - let _request = ctx.state.tenant_request_guard(tenant_id); - let result = - crate::control::update_from_join_orchestrator::run_authorized_update_from_join( - ctx.state, authorized, - ) - .await; - return Some(match result { - Ok(resp) => data_plane_response_to_native(ctx, seq, plan, &resp), - Err(e) => error_to_native(seq, &e), - }); + // An update of the source of a cross-shard materialized sum runs in the + // implicit transaction, which ships each balance to its target. + if !statement_needs_implicit_txn(ctx.state, std::slice::from_ref(&task)) { + let authorized = match super::sql_gateway::authorize_native_task(ctx, &task) { + Ok(authorized) => authorized, + Err(error) => return Some(error_to_native(seq, &error)), + }; + let _request = ctx.state.tenant_request_guard(tenant_id); + let result = + crate::control::update_from_join_orchestrator::run_authorized_update_from_join( + ctx.state, authorized, + ) + .await; + return Some(match result { + Ok(resp) => data_plane_response_to_native(ctx, seq, plan, &resp), + Err(e) => error_to_native(seq, &e), + }); + } } // A governed predicate resolves to a concrete row set before proposing — see diff --git a/nodedb/src/control/server/native/dispatch/edge_recon_gate.rs b/nodedb/src/control/server/native/dispatch/edge_recon_gate.rs index e82f61c0a..a042734b6 100644 --- a/nodedb/src/control/server/native/dispatch/edge_recon_gate.rs +++ b/nodedb/src/control/server/native/dispatch/edge_recon_gate.rs @@ -1,101 +1,49 @@ // SPDX-License-Identifier: BUSL-1.1 -//! Implicit-edge OLLP/Calvin routing gate for the native protocol. +//! The implicit-edge OLLP/Calvin gate for native direct ops. //! -//! A dependent predicate (`BulkDelete`/`BulkUpdate`) on an edge-bearing -//! schemaless collection MUST route through the OLLP/Calvin coordinator so the -//! implicit edge tasks are derived from a pre-exec reconnaissance scan and -//! committed ATOMICALLY with the document write. Without this gate the native -//! path classifies such a delete as `SingleShard` and dispatches directly to -//! the Data Plane — leaving mirrored CSR edges dangling. -//! -//! This module exposes a single entry point: [`try_edge_recon_dispatch`], which -//! implements the same two-guard check as the pgwire `execute.rs` gate and -//! returns the native protocol outcome when the gate fires. +//! The gate itself is shared (`shared::statement_exec::try_edge_recon`). +//! This adapter reads the session's transaction state and renders the +//! applied batch as a native frame. use nodedb_types::protocol::NativeResponse; -use crate::control::planner::calvin::{ - dispatch_authorized_dependent_edge_recon, plan_needs_implicit_edge_recon, -}; use crate::control::server::shared::authorization::AuthorizedTaskSet; use crate::control::server::shared::session::TransactionState; +use crate::control::server::shared::statement_exec::{EdgeRecon, try_edge_recon}; use nodedb_physical::physical_task::PhysicalTask; use super::{DispatchCtx, SqlOutcome, error_to_native}; -/// Attempt to route `tasks` through the implicit-edge OLLP/Calvin path. -/// -/// Returns `Some(outcome)` when the gate fires (the task set contains a -/// `BulkDelete`/`BulkUpdate` on an edge-bearing collection that is not inside -/// an explicit transaction block). The -/// caller MUST return this outcome immediately — the tasks have been consumed. -/// -/// Returns `None` when the gate does not fire; the caller proceeds with the -/// normal classify/dispatch path. -/// -/// A genuine catalog I/O error propagates as `Some(Err-shaped SqlOutcome)` so -/// the caller surfaces it correctly — misrouting on a real I/O fault will -/// silently skip edge cleanup (dangling edges). +/// Route `tasks` through the implicit-edge OLLP/Calvin path when the shared +/// gate fires for them. The caller returns a fired outcome at once: the +/// tasks are consumed. pub(super) async fn try_edge_recon_dispatch( ctx: &DispatchCtx<'_>, seq: u64, tasks: Vec, authorized: AuthorizedTaskSet, ) -> EdgeReconResult { - // Guard 1: not inside an explicit transaction block. The native - // `handle_begin`/`handle_commit`/`handle_rollback` helpers drive the SAME - // `SessionStore` transaction-state machine that pgwire uses, so this guard - // is identical across both protocol paths. Edge-bearing predicate writes - // inside an explicit native transaction block are NOT recon-routed (same - // limitation as pgwire — buffering a multi-step OLLP inside an explicit txn - // will require full two-phase commit across the outer txn boundary). - if ctx.sessions.transaction_state(ctx.peer_addr) == TransactionState::InBlock { - return EdgeReconResult::NotFired(tasks, authorized); - } - - // Guard 2: at least one BulkDelete/BulkUpdate targets an edge-bearing - // collection. A genuine catalog I/O error propagates rather than falling - // through — misrouting on a real fault will skip edge cleanup. - let (_coll, database_id) = - match plan_needs_implicit_edge_recon(ctx.state, &tasks, ctx.tenant_id()) { - Err(e) => return EdgeReconResult::Outcome(resp(error_to_native(seq, &e))), - Ok(None) => return EdgeReconResult::NotFired(tasks, authorized), - Ok(Some(pair)) => pair, - }; - - // Capture the batch's plans before `tasks` is moved into the recon call, so - // the response can shape a RETURNING dependent delete/update's rows and read - // a plain write's affected count from the applied Response the coordinator - // drains. - let plans: Vec<_> = tasks.iter().map(|t| t.plan.clone()).collect(); - - // Both guards passed — run the OLLP/Calvin coordinator. - let outcome = dispatch_authorized_dependent_edge_recon( - ctx.state, - authorized, - ctx.identity, - ctx.tenant_id(), - database_id, - ) - .await; - - EdgeReconResult::Outcome(match outcome { - Ok(recon) => { - // A RETURNING dependent write surfaces its deleted/updated rows; a - // plain write reports the affected count its own mutation returned. - resp(super::conversion::calvin_native_response( + // The native `handle_begin` / `handle_commit` / `handle_rollback` drive + // the same `SessionStore` state machine pgwire uses. + let in_txn_block = ctx.sessions.transaction_state(ctx.peer_addr) == TransactionState::InBlock; + match try_edge_recon(ctx.state, ctx.identity, in_txn_block, tasks, authorized).await { + Ok(EdgeRecon::NotFired(tasks, authorized)) => EdgeReconResult::NotFired(tasks, authorized), + // A RETURNING dependent write surfaces its deleted or updated rows. A + // plain write reports the count its own mutation returned. + Ok(EdgeRecon::Applied(applied)) => { + EdgeReconResult::Outcome(resp(super::conversion::calvin_native_response( seq, - recon.apply_result, - &plans, + applied.apply_result, + &applied.plans, ctx.state, - database_id, + applied.database_id, ctx.tenant_id(), ctx.auth_context(), - )) + ))) } - Err(e) => resp(error_to_native(seq, &e)), - }) + Err(error) => EdgeReconResult::Outcome(resp(error_to_native(seq, &error))), + } } /// Result returned by [`try_edge_recon_dispatch`]. diff --git a/nodedb/src/control/server/native/dispatch/graph_match.rs b/nodedb/src/control/server/native/dispatch/graph_match.rs index fb70e0cea..b1ba9fbfd 100644 --- a/nodedb/src/control/server/native/dispatch/graph_match.rs +++ b/nodedb/src/control/server/native/dispatch/graph_match.rs @@ -110,9 +110,8 @@ pub(crate) async fn handle_graph_match( Err(error) => return error_to_native(seq, &error), }; - // A MATCH issued inside a native transaction records a collection-scoped - // predicate read at the shard's watermark, identical to every other read - // seam. Single-shard direct op means one watermark and one entry. + // A MATCH issued inside a native transaction records the versions its + // core reported, identical to every other read seam. if (response.status == Status::Ok || response.error_code.as_deref() == Some(&crate::bridge::envelope::ErrorCode::NotFound)) && ctx.sessions.transaction_state(ctx.peer_addr) @@ -125,11 +124,9 @@ pub(crate) async fn handle_graph_match( ctx.tenant_id(), crate::control::server::shared::session::ResponseReads { plan: &plan_for_response, - watermarks: &[(vshard_id, response.watermark_lsn)], - read_version_lsn: response.read_version_lsn, + read_versions: &response.read_versions, found: response.status == Status::Ok, distributed_reads: &[], - read_lsn_vshard: vshard_id, }, ) .await; @@ -167,7 +164,7 @@ pub(crate) async fn handle_graph_match( /// local MATCH yields once its envelope is unwrapped. A result the scatter /// cannot finish is refused with `54001`, never returned partial. The /// scatter notes every vShard it read, and the session loop records them into -/// the transaction read-set (`session::graph_reads`). +/// the transaction read-set (`session::pending_shard_reads`). async fn match_across_shards( ctx: &DispatchCtx<'_>, seq: u64, diff --git a/nodedb/src/control/server/native/dispatch/mod.rs b/nodedb/src/control/server/native/dispatch/mod.rs index 64f4bcd72..c691fa0ef 100644 --- a/nodedb/src/control/server/native/dispatch/mod.rs +++ b/nodedb/src/control/server/native/dispatch/mod.rs @@ -23,7 +23,6 @@ mod single_task; mod sorted_read_op; mod sql; mod sql_admin; -mod sql_dispatch_task; mod sql_fold; mod sql_gateway; mod sql_loop; diff --git a/nodedb/src/control/server/native/dispatch/raw_dispatch.rs b/nodedb/src/control/server/native/dispatch/raw_dispatch.rs index 298667222..93a95aab2 100644 --- a/nodedb/src/control/server/native/dispatch/raw_dispatch.rs +++ b/nodedb/src/control/server/native/dispatch/raw_dispatch.rs @@ -178,8 +178,8 @@ async fn dispatch_external_crdt_apply( payload: Payload::from_vec(outcome.payload), watermark_lsn: Lsn::ZERO, error_code: None, - read_set_valid: None, - read_version_lsn: outcome.write_version, + stage_vote: None, + read_versions: outcome.write_versions, write_set: Vec::new(), }) } diff --git a/nodedb/src/control/server/native/dispatch/single_task.rs b/nodedb/src/control/server/native/dispatch/single_task.rs index d141355ee..ba28d0ed5 100644 --- a/nodedb/src/control/server/native/dispatch/single_task.rs +++ b/nodedb/src/control/server/native/dispatch/single_task.rs @@ -30,8 +30,8 @@ use super::{ /// the single-shard edge loop share one code path. /// /// Routes through the same protocol-neutral in-transaction staging gate -/// (`route_in_tx_write`) the SQL-planned dispatch loops (`sql_loop.rs`, -/// pgwire's `execute_dml_hooks.rs`) already use. Outside a transaction block +/// (`route_in_tx_write`) the SQL-planned dispatch loops (the shared +/// `statement_exec` loop, pgwire's `execute_dml_hooks.rs`) already use. Outside a transaction block /// the task comes back unchanged (`InTxnRoute::Read`, or `Autocommit` for a /// write), and the gateway gives a write its durable route. Inside a /// transaction block, a stageable write (e.g. `KvBatchPut`) @@ -63,7 +63,7 @@ pub(super) async fn dispatch_single_task( // Covers the `Read` / `Autocommit` dispatch below (reads, and writes that // apply now). `Staged` meters itself inside // `staging_gate::stage_write` — the single choke-point every `Staged` - // route (this file, `sql_loop.rs`, the expander's per-op staging, and + // route (this file, the shared `statement_exec` loop, the expander's per-op staging, and // pgwire's `execute_dml_hooks.rs`) dispatches through, so it is metered // there once rather than duplicated in every caller's closure. `Buffered` // performs no dispatch at all here — it is metered at COMMIT replay @@ -122,8 +122,8 @@ pub(super) async fn dispatch_single_task( payload: Payload::from_vec(outcome.payload), watermark_lsn: Lsn::new(0), error_code: None, - read_set_valid: None, - read_version_lsn: crate::types::Lsn::ZERO, + stage_vote: None, + read_versions: crate::types::ReadVersions::new(), write_set: Vec::new(), }; return data_plane_response_to_native(ctx, seq, &plan_for_staged_response, &synthetic); @@ -178,7 +178,7 @@ pub(super) async fn dispatch_single_task( } let plan_for_response = task.plan.clone(); - let task_vshard = task.vshard_id; + let task_database_id = task.database_id; match dispatch_authorized_single_task( ctx, task.tenant_id, @@ -189,6 +189,15 @@ pub(super) async fn dispatch_single_task( .await { Ok(resp) => { + // A committed write floors the session's later reads at the + // versions it stamped, as on every other protocol. + ctx.sessions.note_own_write_response( + ctx.peer_addr, + task_database_id, + ctx.tenant_id(), + &plan_for_response, + &resp, + ); // Track direct-op reads, including NotFound phantom observations, // identically to native SQL and pgwire conflict detection. let records_read = resp.status == Status::Ok @@ -205,11 +214,9 @@ pub(super) async fn dispatch_single_task( ctx.tenant_id(), crate::control::server::shared::session::ResponseReads { plan: &plan_for_response, - watermarks: &[(task_vshard, resp.watermark_lsn)], - read_version_lsn: resp.read_version_lsn, + read_versions: &resp.read_versions, found: resp.status == Status::Ok, distributed_reads: &[], - read_lsn_vshard: task_vshard, }, ) .await; diff --git a/nodedb/src/control/server/native/dispatch/sql_dispatch_task.rs b/nodedb/src/control/server/native/dispatch/sql_dispatch_task.rs deleted file mode 100644 index 78a07f960..000000000 --- a/nodedb/src/control/server/native/dispatch/sql_dispatch_task.rs +++ /dev/null @@ -1,142 +0,0 @@ -// SPDX-License-Identifier: BUSL-1.1 - -//! The single-task dispatch primitive the native SQL loop calls. -//! -//! Decides HOW one already-planned task reaches an engine — Control-Plane -//! orchestrators, Exchange resolution, then the gateway — while `sql_loop.rs` -//! decides what to do with the answers. - -use nodedb_types::TraceId; - -use crate::bridge::envelope::Response; -use crate::control::server::exchange::DistributedReadCapture; -use crate::control::server::exchange::resolve::{Resolved, resolve_and_materialize}; -use crate::types::{Lsn, VShardId}; -use nodedb_physical::physical_task::PhysicalTask; - -use super::DispatchCtx; -use super::sql_gateway::dispatch_task_via_gateway; - -/// Dispatch a single `PhysicalTask`, returning the response plus per-shard -/// watermark LSNs a single-node fan gather observed. -/// -/// Multi-arm writes run via Control-Plane orchestrators; everything else -/// routes through `dispatch_task_via_gateway`. -pub(super) async fn dispatch_task( - ctx: &DispatchCtx<'_>, - mut task: PhysicalTask, -) -> crate::Result<(Response, Vec<(VShardId, Lsn)>, Vec)> { - if let crate::bridge::envelope::PhysicalPlan::Document( - nodedb_physical::physical_plan::DocumentOp::InsertSelect { .. }, - ) = &task.plan - { - let authorized = super::sql_gateway::authorize_native_task(ctx, &task)?; - let resp = - crate::control::insert_select::run_authorized_insert_select(ctx.state, authorized) - .await?; - return Ok((resp, Vec::new(), Vec::new())); - } - - // Autocommit `MERGE` orchestrates on the Control Plane (`control::merge_orchestrator`). - if let crate::bridge::envelope::PhysicalPlan::Document( - nodedb_physical::physical_plan::DocumentOp::Merge { - target_collection: _, - source_collection: _, - source_alias: _, - target_join_col: _, - source_join_col: _, - clauses: _, - returning: _, - resolved_inserts: None, - resolved_insert_identities: _, - source_rows: _, - rls_filters: _, - rls_write_check: _, - resolved_sum_targets: _, - declared_primary_key: _, - }, - ) = &task.plan - { - let authorized = super::sql_gateway::authorize_native_task(ctx, &task)?; - let resp = - crate::control::merge_orchestrator::run_authorized_merge(ctx.state, authorized).await?; - return Ok((resp, Vec::new(), Vec::new())); - } - - // Autocommit `UPDATE ... FROM ` scans the source on its own core and - // ships it into the plan, since the source's vShard can live on a different core. - if let crate::bridge::envelope::PhysicalPlan::Document( - nodedb_physical::physical_plan::DocumentOp::UpdateFromJoin { - target_collection: _, - source_collection: _, - source_alias: _, - target_join_col: _, - source_join_col: _, - updates: _, - target_filters: _, - returning: _, - source_rows: None, - rls_filters: _, - rls_write_check: _, - resolved_sum_targets: _, - declared_primary_key: _, - }, - ) = &task.plan - { - let authorized = super::sql_gateway::authorize_native_task(ctx, &task)?; - let resp = crate::control::update_from_join_orchestrator::run_authorized_update_from_join( - ctx.state, authorized, - ) - .await?; - return Ok((resp, Vec::new(), Vec::new())); - } - - // A governed predicate resolves to a concrete row set before proposing - // (`control::write_resolve`). - if let Some(resolver) = crate::control::write_resolve::resolver_for_plan(&task.plan) { - let authorized = super::sql_gateway::authorize_native_task(ctx, &task)?; - let resp = crate::control::write_resolve::run_authorized_write_resolve( - ctx.state, authorized, resolver, - ) - .await?; - return Ok((resp, Vec::new(), Vec::new())); - } - - // Array DDL proposes a replicated catalog entry, as on pgwire. - if crate::control::array_catalog::ddl::is_array_ddl(&task.plan) { - let authorized = super::sql_gateway::authorize_native_task(ctx, &task)?; - let resp = - crate::control::array_catalog::ddl::run_authorized_array_ddl(ctx.state, authorized) - .await?; - return Ok((resp, Vec::new(), Vec::new())); - } - - // Materialize catalog providers and resolve Exchange nodes before dispatch. - // The native protocol has no weaker read consistency: every read is - // strong, so every leg confirms its group where it is served. - let scope = crate::control::server::exchange::ReadScope { - database_id: task.database_id, - tenant_id: task.tenant_id, - trace_id: TraceId::ZERO, - txn_id: task.txn_id, - linearizable: true, - }; - match resolve_and_materialize(ctx.state, ctx.identity, task.plan, scope).await? { - Resolved::Gathered(resp, shard_watermarks, dist_reads) => { - return Ok((resp, shard_watermarks, dist_reads)); - } - Resolved::Plan(resolved_plan) => { - let resolved_plan = *resolved_plan; - task.plan = resolved_plan; - } - // Native path materializes the stream into a Response, preserving gather-then-return. - Resolved::Stream(s) => { - let resp = crate::control::server::exchange::gather::stream_to_response(s).await?; - return Ok((resp, Vec::new(), Vec::new())); - } - } - - // Everything else routes through the gateway. - let resp = dispatch_task_via_gateway(ctx, task).await?; - Ok((resp, Vec::new(), Vec::new())) -} diff --git a/nodedb/src/control/server/native/dispatch/sql_fold.rs b/nodedb/src/control/server/native/dispatch/sql_fold.rs index 9852f8b61..3228fcfa8 100644 --- a/nodedb/src/control/server/native/dispatch/sql_fold.rs +++ b/nodedb/src/control/server/native/dispatch/sql_fold.rs @@ -1,149 +1,67 @@ // SPDX-License-Identifier: BUSL-1.1 -//! Folding each task's answer into one native SQL statement's response: its -//! columns and rows, its warnings, and its one command tag. +//! Rendering one native SQL statement's answer: its columns and rows, its +//! warnings, and its one command tag, or the error frame it stopped with. use nodedb_types::protocol::NativeResponse; -use nodedb_types::value::Value; -use crate::bridge::envelope::PhysicalPlan; -use crate::control::sequence::SessionSequenceAccess; -use crate::control::server::response_shape::compose::{ShapeOutcome, shape_response_materialized}; -use crate::control::server::response_shape::redaction::QueryRedaction; -use crate::control::server::response_shape::request::MaterializedShapeRequest; -use crate::control::server::response_shape::schema::OutputSchema; -use crate::control::server::response_shape::types::{ - DmlOutcome, FoldedTag, PlanKind, StatementTag, describe_plan, staged_dml_outcome, -}; -use crate::control::server::shared::session::staging_gate::{StagedTagKind, StagedWriteOutcome}; -use crate::types::DatabaseId; +use crate::control::server::native::sqlstate_code::sqlstate_error; +use crate::control::server::response_shape::types::FoldedTag; +use crate::control::server::shared::statement_exec::{StatementAnswer, StatementError}; use super::{ - DispatchCtx, apply_dml_outcome, dml_fold_error_to_native, shape_error_to_native, - to_native_columns_rows, + apply_dml_outcome, dml_fold_error_to_native, error_code_to_native, error_response_to_native, + error_to_native, error_to_native_with_sqlstate, shape_error_to_native, to_native_columns_rows, }; -/// What shaping a task's answer reads. -pub(super) struct FoldShape<'a> { - pub seq: u64, - pub output_schema: Option<&'a OutputSchema>, - pub database_id: DatabaseId, - /// The concrete session access, never `&dyn SequenceAccess`: the trait - /// object is not `Sync`, and the shape lives across the dispatch awaits - /// of a `Send` connection future. - pub sequences: &'a SessionSequenceAccess<'a>, -} - -/// One statement's response, accumulated across its tasks. -#[derive(Default)] -pub(super) struct StatementFold { - pub columns: Option>, - pub rows: Vec>, - pub warnings: Vec, - pub tag: StatementTag, - pub last_lsn: u64, -} - -impl StatementFold { - /// Fold one task's count-bearing outcome into the statement's tag. - pub(super) fn fold_dml( - &mut self, - seq: u64, - outcome: DmlOutcome, - ) -> Result<(), Box> { - self.tag - .fold(outcome) - .map_err(|e| Box::new(dml_fold_error_to_native(seq, &e))) +/// The native frame for a statement's answer or error. +pub(super) fn statement_result_to_native( + seq: u64, + result: Result, +) -> NativeResponse { + match result { + Ok(answer) => answer_to_native(seq, answer), + Err(error) => statement_error_to_native(seq, error), } +} - /// Shape `payload`, the answer `plan` gave as `plan_kind`, and fold its - /// rows. Returns the row count, `None` when the payload carries no rows. - pub(super) fn fold_rows( - &mut self, - ctx: &DispatchCtx<'_>, - shape: &FoldShape<'_>, - plan: &PhysicalPlan, - plan_kind: PlanKind, - payload: &[u8], - ) -> Result, Box> { - let redaction = QueryRedaction::for_plan(ctx.tenant_id(), ctx.auth_context(), plan); - match shape_response_materialized(MaterializedShapeRequest { - payload, - plan, - plan_kind, - projection: shape.output_schema, - state: ctx.state, - database_id: shape.database_id, - tenant_id: ctx.tenant_id(), - redaction: Some(redaction.ctx(&ctx.state.redaction)), - sequences: Some(shape.sequences), - }) { - Ok(ShapeOutcome::Rows(mut shaped)) => { - if let Some(notice) = shaped.notice.take() { - self.warnings.push(notice); - } - let (cols, rows) = to_native_columns_rows(&shaped); - if !cols.is_empty() && self.columns.is_none() { - self.columns = Some(cols); - } - let count = rows.len() as u64; - self.rows.extend(rows); - Ok(Some(count)) - } - Ok(ShapeOutcome::Passthrough) => Ok(None), - Err(e) => Err(Box::new(shape_error_to_native(shape.seq, &e))), +/// The native frame for a statement's answer. The columns are the first +/// non-empty set a task's rows named. +pub(super) fn answer_to_native(seq: u64, answer: StatementAnswer) -> NativeResponse { + let mut r = NativeResponse::ok(seq); + r.watermark_lsn = answer.last_lsn; + r.warnings = answer.warnings; + let mut columns = None; + let mut rows = Vec::new(); + for shaped in &answer.rows { + let (task_columns, task_rows) = to_native_columns_rows(shaped); + if !task_columns.is_empty() && columns.is_none() { + columns = Some(task_columns); } + rows.extend(task_rows); } - - /// Fold a write its transaction staged. A write that answers `RETURNING` - /// folds its rows in place of a count, as an autocommit one does. A - /// computed-value payload (`RawPayload`) is shaped instead of counted. - pub(super) fn fold_staged( - &mut self, - ctx: &DispatchCtx<'_>, - shape: &FoldShape<'_>, - plan: &PhysicalPlan, - outcome: StagedWriteOutcome, - ) -> Result<(), Box> { - if !outcome.returning_rows.is_empty() { - for rows in &outcome.returning_rows { - self.fold_rows(ctx, shape, plan, PlanKind::ReturningRows, rows)?; - } - return Ok(()); - } - if !matches!(outcome.kind, StagedTagKind::RawPayload) { - return self.fold_dml( - shape.seq, - staged_dml_outcome(outcome.kind, outcome.affected), - ); - } - if outcome.payload.is_empty() { - self.tag.fold_opaque(); - return Ok(()); - } - // The value rides in the payload, never as a count. - if self - .fold_rows(ctx, shape, plan, describe_plan(plan), &outcome.payload)? - .is_none() - { - self.tag.fold_opaque(); - } - Ok(()) + if !rows.is_empty() { + r.columns = columns; + r.rows = Some(rows); } + match answer.tag { + Some(FoldedTag::Dml(outcome)) => apply_dml_outcome(&mut r, outcome), + Some(FoldedTag::Opaque) | None => {} + } + r +} - /// The statement's one response. - pub(super) fn finish(self, seq: u64) -> NativeResponse { - let mut r = NativeResponse::ok(seq); - r.watermark_lsn = self.last_lsn; - r.warnings = self.warnings; - if !self.rows.is_empty() { - r.columns = self.columns; - r.rows = Some(self.rows); - } - match self.tag.finish() { - Some(FoldedTag::Dml(outcome)) => apply_dml_outcome(&mut r, outcome), - Some(FoldedTag::Opaque) | None => {} +/// The native error frame for why a statement stopped. +pub(super) fn statement_error_to_native(seq: u64, error: StatementError) -> NativeResponse { + match error { + StatementError::Error(error) => error_to_native(seq, &error), + StatementError::Quota(error) => error_to_native_with_sqlstate(seq, "53400", &error), + StatementError::TenantIsolation => { + sqlstate_error(seq, "42501", "tenant isolation violation") } - r + StatementError::Rejected(code) => error_code_to_native(seq, code.as_ref()), + StatementError::Response(response) => error_response_to_native(seq, &response), + StatementError::DmlFold(error) => dml_fold_error_to_native(seq, &error), + StatementError::Shape(error) => shape_error_to_native(seq, &error), } } diff --git a/nodedb/src/control/server/native/dispatch/sql_gateway.rs b/nodedb/src/control/server/native/dispatch/sql_gateway.rs index 061a31aeb..6ae96a7c1 100644 --- a/nodedb/src/control/server/native/dispatch/sql_gateway.rs +++ b/nodedb/src/control/server/native/dispatch/sql_gateway.rs @@ -1,102 +1,24 @@ // SPDX-License-Identifier: BUSL-1.1 -//! Gateway-based SQL task dispatch for the native protocol. -//! -//! Tasks route through `Gateway::execute_response`, which handles -//! cluster-aware routing, typed `NotLeader` retry, and plan caching. A -//! transaction meta-op runs on its own vShard's core instead. This is native's -//! SQL-TEXT opcode path — distinct from `raw_dispatch.rs`, which serves only -//! native's direct-op opcodes. +//! Task authorization for the native protocol's Control-Plane orchestrated +//! plans. The SQL path's single-task dispatch lives in +//! `shared::statement_exec::dispatch`. -use crate::bridge::envelope::Response; -use std::sync::Arc; - -use crate::control::gateway::core::QueryContext as GatewayQueryContext; -use crate::control::gateway::router::is_task_vshard_scoped; -use crate::control::server::shared::clone_write::CloneCheckedOutcome; -use crate::types::TraceId; use nodedb_physical::physical_task::PhysicalTask; use super::DispatchCtx; -/// Authorize one task with no clone-write check — used only by the -/// Control-Plane orchestrator branches ahead of this file's gateway dispatch, -/// whose plan shapes (`InsertSelect`, `Merge`, `UpdateFromJoin`, a governed -/// predicate resolution, array DDL) are never clone-write shapes. +/// Authorize one task with no clone-write check. Used only by the +/// Control-Plane orchestrator branches, whose plan shapes (`InsertSelect`, +/// `Merge`, `UpdateFromJoin`, a governed predicate resolution, array DDL, a +/// cluster array op, a graph owner op) are never clone-write shapes. pub(super) fn authorize_native_task( ctx: &DispatchCtx<'_>, task: &PhysicalTask, ) -> crate::Result { - let emitter = crate::control::security::audit::ArcAuditEmitter(Arc::clone(&ctx.state.audit)); - crate::control::server::shared::authorization::authorize_task_set( + crate::control::server::shared::statement_exec::authorize_one_task( + ctx.state, ctx.identity, - std::slice::from_ref(task), - &ctx.state.permissions, - &ctx.state.roles, - &emitter, - ) - .map_err(crate::Error::from)? - .into_tasks() - .into_iter() - .next() - .ok_or_else(|| crate::Error::Internal { - detail: "authorization returned an empty capability set".into(), - }) -} - -/// Dispatch a single `PhysicalTask` through the gateway. -/// -/// Returns the Data-Plane `Response` shape, with a `NotFound` verdict as an -/// error status. -pub(super) async fn dispatch_task_via_gateway( - ctx: &DispatchCtx<'_>, - task: PhysicalTask, -) -> crate::Result { - let tenant_id = task.tenant_id; - // Clone CoW write-path interception, then authorization, run once per - // task before dispatch — same protocol-neutral gate raw_dispatch runs. - let emitter = crate::control::security::audit::ArcAuditEmitter(Arc::clone(&ctx.state.audit)); - let checked = match crate::control::server::shared::clone_write::intercept_and_authorize( - crate::control::server::shared::clone_write::InterceptAndAuthorizeParams { - state: ctx.state, - task, - identity: ctx.identity, - tenant_id, - permissions: &ctx.state.permissions, - roles: &ctx.state.roles, - emitter: &emitter, - }, + task, ) - .await? - { - CloneCheckedOutcome::Handled(resp) => return Ok(resp), - CloneCheckedOutcome::Proceed(checked) => checked, - }; - let database_id = checked.database_id(); - let txn_id = checked.txn_id(); - - let gateway = ctx.state.installed_gateway()?; - // A staged write and the other transaction meta-ops run on the core of - // the task's own vShard. The gateway will route them to vShard 0. A - // write takes the durable route, a read the read route. - if is_task_vshard_scoped(checked.plan()) { - return crate::control::server::dispatch_utils::dispatch_authorized_task_by_class( - ctx.state, - checked, - TraceId::generate(), - ) - .await; - } - let gw_ctx = GatewayQueryContext { - tenant_id, - trace_id: TraceId::generate(), - database_id, - // Propagate the in-block transaction id so gateway local dispatch - // resolves the per-txn staging overlay. - txn_id, - linearizable: true, - }; - // The typed error passes through unchanged. The native frame renders its - // SQLSTATE and numeric code from it. - gateway.execute_response(&gw_ctx, checked).await } diff --git a/nodedb/src/control/server/native/dispatch/sql_loop.rs b/nodedb/src/control/server/native/dispatch/sql_loop.rs index 742697c10..58ca09b2a 100644 --- a/nodedb/src/control/server/native/dispatch/sql_loop.rs +++ b/nodedb/src/control/server/native/dispatch/sql_loop.rs @@ -1,54 +1,26 @@ // SPDX-License-Identifier: BUSL-1.1 -//! Per-task dispatch loop for the DataFusion-planned SQL path. +//! The native SQL path's per-task loop: the shared statement loop +//! (`shared::statement_exec`), rendered as one native frame. //! //! A statement in a transaction (the client's block, or the implicit //! transaction a statement whose writes fire a BEFORE, INSTEAD OF or SYNC //! AFTER body runs in) routes each task through the shared `txn_route`, the -//! route pgwire takes too: the write's trigger bodies join the transaction, -//! a shadowed clone takes its copy-on-write steps, and the write stages. The -//! single-task dispatch primitive lives in `sql_dispatch_task.rs`, and the -//! folding of each task's answer in `sql_fold.rs`. +//! route pgwire takes too. The frame renders in `sql_fold.rs`. -use std::ops::ControlFlow; use std::sync::Arc; -use nodedb_types::protocol::NativeResponse; - -use crate::bridge::envelope::Status; -use crate::control::planner::calvin::write_class::{ - plan_counts_toward_statement_tag, plans_have_user_write, -}; use crate::control::sequence::SessionSequenceAccess; use crate::control::server::response_shape::schema::OutputSchema; -use crate::control::server::response_shape::types::{ - PlanKind, describe_plan, payload_to_dml_outcome, replaced_write_outcome, -}; -use crate::control::server::shared::metering::{PlanMeteringInfo, meter_dispatch}; -use crate::control::server::shared::quota_admission::admit_quota_for_dispatch; use crate::control::server::shared::session::DmlTxnCtx; use crate::control::server::shared::session::read_set::ReadSetEntry; -use crate::control::server::shared::session::staging_gate::StagingGateError; -use crate::control::server::shared::txn_route::{ - StatementEvents, TxnTaskContext, TxnTaskOutcome, route_txn_task, -}; +use crate::control::server::shared::statement_exec::{self, StatementExec}; use crate::types::DatabaseId; use nodedb_physical::physical_task::PhysicalTask; -use super::sql_dispatch_task::dispatch_task; -use super::sql_fold::{FoldShape, StatementFold}; +use super::DispatchCtx; +use super::sql_fold::statement_result_to_native; use super::streaming::SqlOutcome; -use super::{ - DispatchCtx, error_code_to_native, error_response_to_native, error_to_native, - error_to_native_with_sqlstate, -}; -use crate::control::server::native::sqlstate_code::sqlstate_error; - -/// Wrap a materialized response as a non-streaming [`SqlOutcome`]. -#[inline] -fn resp(r: NativeResponse) -> SqlOutcome { - SqlOutcome::Response(Box::new(r)) -} /// A planned statement the loop dispatches. pub(super) struct PlannedStatement<'a> { @@ -61,18 +33,23 @@ pub(super) struct PlannedStatement<'a> { pub sum_target_reads: Vec, } -/// Run the per-task dispatch loop for a planned, non-streamed task set, -/// materializing all rows/columns/affected-count into a single -/// [`SqlOutcome::Response`]. `txn` is the statement's transaction, `None` +/// The session's sequence access for a statement in `database_id`: the +/// registry plus this connection's `currval` map. +pub(super) fn session_sequences<'a>( + ctx: &DispatchCtx<'a>, + database_id: DatabaseId, +) -> SessionSequenceAccess<'a> { + SessionSequenceAccess::for_session( + ctx.state, + ctx.sessions.sequence_values(ctx.peer_addr), + database_id, + ctx.tenant_id(), + ) +} + +/// Run the per-task loop for a planned, non-streamed task set, rendered as +/// one [`SqlOutcome::Response`]. `txn` is the statement's transaction, `None` /// for an autocommit statement. -/// -/// The statement's `rows_affected` and `command` come from one -/// [`crate::control::server::response_shape::types::StatementTag`] folded -/// over every task, the same fold pgwire renders as its command tag: a -/// count-bearing task contributes its reported count and verb, an opaque task -/// (buffered write, index or graph maintenance, computed-value payload) -/// contributes nothing. Neither field is ever synthesised from the task -/// count. pub(super) async fn run_dispatch_loop( ctx: &DispatchCtx<'_>, seq: u64, @@ -86,258 +63,30 @@ pub(super) async fn run_dispatch_loop( plan_lease_scope, sum_target_reads, } = statement; - if let Some(txn) = txn - && !sum_target_reads.is_empty() - { - txn.sessions - .record_read_entries(txn.session_id, sum_target_reads); - } - let mut fold = StatementFold::default(); - // Checked once rather than per task — metering is disabled by default, so - // this keeps the per-task extraction below (which clones the collection - // name) a true no-op on the hot path for every deployment that hasn't - // turned it on. - let metering_enabled = ctx.state.metering_config.enabled; - // A derived implicit-edge write beside the user's own never answers the - // statement, exactly as Calvin's deposit rule has it. - let has_user_write = plans_have_user_write(tasks.iter().map(|t| &t.plan)); - // Session-scoped sequence access for the statement's Control-Plane - // computed columns: the registry plus this connection's `currval` map. - let session_sequences = ctx.sessions.sequence_values(ctx.peer_addr); - let sequences = SessionSequenceAccess::for_session( - ctx.state, - session_sequences, - database_id, - ctx.tenant_id(), - ); - let shape = FoldShape { - seq, + let sequences = session_sequences(ctx, database_id); + let exec = StatementExec { + state: ctx.state, + identity: ctx.identity, + scope: &ctx.scope, output_schema, database_id, sequences: &sequences, + client_session: Some(DmlTxnCtx { + sessions: ctx.sessions, + session_id: ctx.peer_addr.into(), + }), }; - let route_ctx = txn.map(|txn| TxnTaskContext { - state: ctx.state, - identity: ctx.identity, - auth: ctx.auth_context(), + let result = statement_exec::run_statement_loop( + &exec, + statement_exec::PlannedStatement { + tasks, + plan_lease_scope, + sum_target_reads, + }, txn, - lease_scope: &plan_lease_scope, - fire_triggers: true, - }); - let mut statement_events = StatementEvents::default(); - - for task in tasks { - if task.tenant_id != ctx.tenant_id() { - return resp(sqlstate_error(seq, "42501", "tenant isolation violation")); - } - - // Cloned before routing consumes `task`: a staged write's rows and - // computed-value payload shape against it. - let plan = task.plan.clone(); - // Metering needs the collection/engine shape after this task's - // dispatch succeeds. It covers the direct dispatch below (reads, and - // writes that apply now). A staged write meters itself inside the - // staging gate, and a buffered write meters at COMMIT. - let plan_metering_info = metering_enabled.then(|| PlanMeteringInfo::extract(&plan)); - - // A spent hard quota refuses the task before it runs; the charging - // call at the end of this loop is on the success path and so can - // never refuse anything itself. - if let Some(info) = &plan_metering_info - && let Err(e) = admit_quota_for_dispatch(ctx.state, &ctx.scope, info) - { - return resp(error_to_native_with_sqlstate(seq, "53400", &e)); - } - - // The task to dispatch below: the input task, or the one routing - // hands back. Every other routing outcome answers the task here. - let task = if let Some(route) = route_ctx.as_ref() { - let routed = route_txn_task( - route, - task, - &mut statement_events, - |stage_task| async move { - dispatch_task(ctx, stage_task) - .await - .map(|(resp, _, _)| resp) - }, - ) - .await; - let answered = match routed { - Ok(TxnTaskOutcome::Dispatch(routed_task)) => ControlFlow::Continue(*routed_task), - Ok(TxnTaskOutcome::Buffered) => { - // Applied at COMMIT: no count and no verb yet. - fold.tag.fold_opaque(); - ControlFlow::Break(Ok(())) - } - Ok(TxnTaskOutcome::Staged(outcome)) => { - ControlFlow::Break(fold.fold_staged(ctx, &shape, &plan, outcome)) - } - Ok(TxnTaskOutcome::CloneHandled(clone_resp)) => ControlFlow::Break( - fold_answer( - ctx, - &shape, - &mut fold, - &plan, - has_user_write, - clone_resp.payload.as_ref(), - ) - .map(drop), - ), - Ok(TxnTaskOutcome::InsteadOf) => { - ControlFlow::Break(match replaced_write_outcome(describe_plan(&plan)) { - Some(outcome) => fold.fold_dml(seq, outcome), - None => { - fold.tag.fold_opaque(); - Ok(()) - } - }) - } - Err(StagingGateError::Dispatch(e)) => return resp(error_to_native(seq, &e)), - Err(StagingGateError::Rejected { code }) => { - return resp(error_code_to_native(seq, code.as_ref())); - } - }; - match answered { - ControlFlow::Continue(routed_task) => routed_task, - ControlFlow::Break(Ok(())) => continue, - ControlFlow::Break(Err(e)) => return SqlOutcome::Response(e), - } - } else { - task - }; - - // `ClusterArray` plans are handled entirely on the Control Plane by - // the `ArrayCoordinator` — they must never reach the SPSC bridge or - // the trigger/DML machinery. A write in a transaction reshapes into - // per-shard `ArrayOp` tasks inside the staging gate above and never - // reaches here as `ClusterArray`; only reads (`Slice`/`Agg`) and - // autocommit `Put`/`Delete` arrive as this plan. Intercepted here, - // before `dispatch_task` will otherwise route it through the gateway - // toward the Data Plane. Metering is not applied here, matching - // pgwire's `ClusterArray` short-circuit, which also does not meter - // this path. - if matches!( - task.plan, - crate::bridge::envelope::PhysicalPlan::ClusterArray(_) - ) { - let authorized = match super::sql_gateway::authorize_native_task(ctx, &task) { - Ok(a) => a, - Err(e) => return resp(error_to_native(seq, &e)), - }; - match super::cluster_array::dispatch_cluster_array_task(ctx, authorized, output_schema) - .await - { - Ok(super::cluster_array::ClusterArrayOutcome::Rows { - columns, - rows, - notice, - }) => { - if let Some(n) = notice { - fold.warnings.push(n); - } - if !columns.is_empty() && fold.columns.is_none() { - fold.columns = Some(columns); - } - fold.rows.extend(rows); - } - Ok(super::cluster_array::ClusterArrayOutcome::Affected(outcome)) => { - if let Err(e) = fold.fold_dml(seq, outcome) { - return SqlOutcome::Response(e); - } - } - Err(e) => return resp(error_to_native(seq, &e)), - } - continue; - } - - let task_vshard = task.vshard_id; - let task_database_id = task.database_id; - let (task_resp, shard_watermarks, dist_reads) = match dispatch_task(ctx, task).await { - Ok(r) => r, - Err(e) => return resp(error_to_native(seq, &e)), - }; - - // Track reads for snapshot-isolation / cross-shard conflict detection - // at the protocol-neutral layer, into the statement's transaction. - // Recorded BEFORE the error short-circuit so an absent-key point read - // (a `NotFound` from the Data Plane) is captured too; a "not found" is - // a validatable phantom observation. A multi-core fan read records one - // entry per participating shard from the gather's per-shard - // watermarks; a single read falls back to its one watermark. - let records_read = task_resp.status == Status::Ok - || task_resp.error_code.as_deref() - == Some(&crate::bridge::envelope::ErrorCode::NotFound); - if records_read && let Some(txn) = txn { - let watermarks = if shard_watermarks.is_empty() { - vec![(task_vshard, task_resp.watermark_lsn)] - } else { - shard_watermarks - }; - crate::control::server::shared::session::record_reads_for_response( - ctx.state, - txn.sessions, - txn.session_id, - ctx.tenant_id(), - crate::control::server::shared::session::ResponseReads { - plan: &plan, - watermarks: &watermarks, - read_version_lsn: task_resp.read_version_lsn, - found: task_resp.status == Status::Ok, - distributed_reads: &dist_reads, - read_lsn_vshard: task_vshard, - }, - ) - .await; - } - - if task_resp.status == Status::Error { - return resp(error_response_to_native(seq, &task_resp)); - } - - // --- TRUNCATE RESTART IDENTITY --- - // Autocommit only: a buffered truncate restarts its sequences at - // COMMIT. Same rule as the pgwire dispatch loop. - if let Some((collection, true)) = plan.truncate_target() { - ctx.state - .sequence_registry - .restart_sequences_for_collection( - task_database_id.as_u64(), - ctx.tenant_id().as_u64(), - collection.as_str(), - ); - } - - fold.last_lsn = task_resp.watermark_lsn.as_u64(); - - let task_rows = match fold_answer( - ctx, - &shape, - &mut fold, - &plan, - has_user_write, - task_resp.payload.as_ref(), - ) { - Ok(rows) => rows, - Err(e) => return SqlOutcome::Response(e), - }; - - // Metered here, once per successfully dispatched task (see the scope - // note on `plan_metering_info` above for the tasks this does not - // cover) — `task_resp.status == Status::Error` returned above, so - // every path reaching here is the success path. - if let Some(info) = &plan_metering_info { - meter_dispatch(ctx.state, &ctx.scope, info, task_rows); - } - } - - if let Some(route) = route_ctx.as_ref() - && let Err(e) = statement_events.fire(route).await - { - return resp(error_to_native(seq, &e)); - } - - resp(fold.finish(seq)) + ) + .await; + SqlOutcome::Response(Box::new(statement_result_to_native(seq, result))) } /// Run a statement in an implicit transaction: it commits when every task @@ -347,75 +96,39 @@ pub(super) async fn run_implicit_statement( seq: u64, statement: PlannedStatement<'_>, ) -> SqlOutcome { + let PlannedStatement { + tasks, + output_schema, + database_id, + plan_lease_scope, + sum_target_reads, + } = statement; + let sequences = session_sequences(ctx, database_id); + let exec = StatementExec { + state: ctx.state, + identity: ctx.identity, + scope: &ctx.scope, + output_schema, + database_id, + sequences: &sequences, + client_session: Some(DmlTxnCtx { + sessions: ctx.sessions, + session_id: ctx.peer_addr.into(), + }), + }; let client = DmlTxnCtx { sessions: ctx.sessions, session_id: ctx.peer_addr.into(), }; - let outcome = crate::control::trigger::statement_txn::with_statement_txn_lifted( - ctx.state, - ctx.identity, - &client, - true, - |error| resp(error_to_native(seq, &error)), - async |txn: &DmlTxnCtx<'_>| { - let outcome = run_dispatch_loop(ctx, seq, statement, Some(txn)).await; - match &outcome { - SqlOutcome::Response(r) - if r.status == nodedb_types::protocol::ResponseStatus::Error => - { - Err(outcome) - } - _ => Ok(outcome), - } + let result = statement_exec::run_implicit_statement( + &exec, + statement_exec::PlannedStatement { + tasks, + plan_lease_scope, + sum_target_reads, }, + &client, ) .await; - match outcome { - Ok(outcome) | Err(outcome) => outcome, - } -} - -/// Fold a dispatched task's answer `payload`: its rows, or its count. Returns -/// the task's own row count for metering, `None` when it carries none. -fn fold_answer( - ctx: &DispatchCtx<'_>, - shape: &FoldShape<'_>, - fold: &mut StatementFold, - plan: &crate::bridge::envelope::PhysicalPlan, - has_user_write: bool, - payload: &[u8], -) -> Result, Box> { - let plan_kind = describe_plan(plan); - let counts_toward_tag = plan_counts_toward_statement_tag(plan, has_user_write); - let count_bearing = matches!(plan_kind, PlanKind::DmlResult(_) | PlanKind::DmlResultByOp); - if payload.is_empty() && !count_bearing { - // Not a count-bearing plan (graph / vector / index write): no count - // and no verb to report. A count-bearing plan with no payload falls - // through so the count reader refuses it. - fold.tag.fold_opaque(); - return Ok(None); - } - if let Some(rows) = fold.fold_rows(ctx, shape, plan, plan_kind, payload)? { - return Ok(Some(rows)); - } - // A count-bearing write reports the rows it touched and its verb; an - // opaque execution reports neither. Counting one per dispatched task - // instead will report a row for a delete that removed nothing and for an - // `ON CONFLICT DO NOTHING` insert that skipped. - if !counts_toward_tag { - fold.tag.fold_opaque(); - return Ok(None); - } - match payload_to_dml_outcome(payload, plan_kind) { - Ok(Some(outcome)) => { - let affected = outcome.affected; - fold.fold_dml(shape.seq, outcome)?; - Ok(Some(affected)) - } - Ok(None) => { - fold.tag.fold_opaque(); - Ok(None) - } - Err(e) => Err(Box::new(error_to_native(shape.seq, &e))), - } + SqlOutcome::Response(Box::new(statement_result_to_native(seq, result))) } diff --git a/nodedb/src/control/server/native/dispatch/sql_planned.rs b/nodedb/src/control/server/native/dispatch/sql_planned.rs index fa2d3eadf..d6ef1ed56 100644 --- a/nodedb/src/control/server/native/dispatch/sql_planned.rs +++ b/nodedb/src/control/server/native/dispatch/sql_planned.rs @@ -8,26 +8,22 @@ use nodedb_types::protocol::NativeResponse; use std::sync::Arc; -use crate::control::planner::calvin::{ - CrossShardTxnMode, DispatchClass, TxnDispatchPosition, classify_dispatch, - dispatch_authorized_tasks_to_calvin, -}; -use crate::control::security::audit::ArcAuditEmitter; use crate::control::server::native::sqlstate_code::sqlstate_error; use crate::control::server::shared::check_constraint::{ enforce_statement_checks, enforce_statement_enum_labels, }; use crate::control::server::shared::plan_admission::{ - PlanAdmissionRequest, plan_authorize_and_admit, + PlanAdmissionRequest, plan_authorize_and_admit_in_txn, }; use crate::control::server::shared::session::{DmlTxnCtx, TransactionState}; -use crate::control::server::shared::txn_route::{ - statement_needs_implicit_txn, unbufferable_joined_statement, +use crate::control::server::shared::statement_exec::{ + AtomicRoute, AtomicStatement, StatementExec, route_atomic_statement, }; -use crate::control::server::shared::write_admission::{all_writes_bufferable, plan_is_write}; +use crate::control::server::shared::write_admission::plan_is_write; use super::sql::resp; -use super::sql_loop::{PlannedStatement, run_dispatch_loop, run_implicit_statement}; +use super::sql_fold::{answer_to_native, statement_error_to_native}; +use super::sql_loop::{PlannedStatement, run_dispatch_loop, session_sequences}; use super::streaming::{SqlOutcome, try_open_sql_stream}; use super::{DispatchCtx, ddl_result_to_native, error_to_native}; @@ -100,157 +96,90 @@ pub(super) async fn execute_planned( // still follows authorization inside the unit, so denied requests consume // no descriptor lease. The scope stays alive through all dispatch and // response shaping below. - let admission = match plan_authorize_and_admit(PlanAdmissionRequest { - state: ctx.state, - query_ctx: ctx.query_ctx, - scope: &scope, - sql: &clean_sql, - trace_id: TraceId::ZERO, - }) + // The derived writes read their source rows through the open + // transaction's staging overlay, so they see this transaction's earlier + // statements. + let admission = match plan_authorize_and_admit_in_txn( + PlanAdmissionRequest { + state: ctx.state, + query_ctx: ctx.query_ctx, + scope: &scope, + sql: &clean_sql, + trace_id: TraceId::ZERO, + }, + ctx.sessions.tx_id(ctx.peer_addr), + ) .await { Ok(admission) => admission, Err(error) => return resp(error_to_native(seq, &error)), }; - let mut tasks = admission.tasks; let output_schema = admission.output_schema; - // Re-derived here rather than carried from `admission`: Calvin dispatch - // and implicit-edge reconciliation are trusted internal mechanisms that - // never reach the clone-checked dispatch boundary (they don't call - // `dispatch_authorized_to_data_plane` / `Gateway::execute`), so a plain - // batch authorize is correct for them. `run_dispatch_loop` below clone-checks - // and authorizes each task itself, immediately before its own dispatch. - let mut authorized_tasks = - match crate::control::server::shared::authorization::authorize_task_set( - ctx.identity, - &tasks, - &ctx.state.permissions, - &ctx.state.roles, - &ArcAuditEmitter(Arc::clone(&ctx.state.audit)), - ) { - Ok(authorized) => authorized, - Err(error) => return resp(error_to_native(seq, &crate::Error::from(error))), - }; - let mut lease_scope = Some(admission.lease_scope); - // Covers the images every cross-shard materialized-sum balance in `tasks` - // was settled from, so Calvin's OCC check aborts rather than committing a - // total folded from an image that has since moved. - let sum_target_reads = admission.sum_target_reads; - - if tasks.is_empty() { + if admission.tasks.is_empty() { return resp(NativeResponse::status_row(seq, "OK")); } - // A statement outside a transaction block whose writes fire a BEFORE, - // INSTEAD OF or SYNC AFTER body, or a MERGE into an edge-bearing - // collection, runs in an implicit transaction ahead of every autocommit - // route: its writes stage on their vShards' leaders and commit together - // at its end. + // The shared atomic routes, the ones every protocol takes: the implicit + // transaction, the implicit-edge OLLP/Calvin gate, and a Calvin commit + // for a write that spans vShards. The native session's + // `cross_shard_txn` parameter picks the cross-shard mode. An unset value + // is the documented default, `CrossShardTxnMode::Strict`. let in_txn_block = ctx.sessions.transaction_state(ctx.peer_addr) == TransactionState::InBlock; - if !in_txn_block && statement_needs_implicit_txn(ctx.state, &tasks) { - if !all_writes_bufferable(&tasks) { - return resp(error_to_native(seq, &unbufferable_joined_statement())); - } - let Some(lease_scope) = lease_scope.take() else { - return resp(sqlstate_error( + let session = DmlTxnCtx { + sessions: ctx.sessions, + session_id: ctx.peer_addr.into(), + }; + let sequences = session_sequences(ctx, database_id); + let exec = StatementExec { + state: ctx.state, + identity: ctx.identity, + scope: &ctx.scope, + output_schema: Some(&output_schema), + database_id, + sequences: &sequences, + client_session: Some(DmlTxnCtx { + sessions: ctx.sessions, + session_id: ctx.peer_addr.into(), + }), + }; + let routed = route_atomic_statement( + &exec, + AtomicStatement { + tasks: admission.tasks, + lease_scope: admission.lease_scope, + // Covers the images every cross-shard materialized-sum balance in + // the tasks was settled from, so Calvin's OCC check aborts rather + // than committing a total folded from an image that has since + // moved. + sum_target_reads: admission.sum_target_reads, + }, + &session, + ) + .await; + let AtomicStatement { + tasks, + lease_scope, + sum_target_reads, + } = match routed { + Ok(AtomicRoute::Implicit(answer)) => return resp(answer_to_native(seq, answer)), + // A RETURNING write surfaces its rows from the applied response. A + // plain write reports the count its own mutation returned. + Ok(AtomicRoute::Calvin(applied)) => { + return resp(super::conversion::calvin_native_response( seq, - nodedb_types::error::sqlstate::INTERNAL_ERROR, - "internal error: query lease scope missing before SQL dispatch", + applied.apply_result, + &applied.plans, + ctx.state, + applied.database_id, + ctx.tenant_id(), + ctx.auth_context(), )); - }; - return run_implicit_statement( - ctx, - seq, - PlannedStatement { - tasks, - output_schema: Some(&output_schema), - database_id, - plan_lease_scope: Arc::new(lease_scope), - sum_target_reads, - }, - ) - .await; - } - - // Implicit-edge DELETE/UPDATE routing gate (native-protocol parity with - // pgwire). See `edge_recon_gate` for the full invariant and guard - // documentation. Returns early when the gate fires, consuming `tasks`. - { - use super::edge_recon_gate::{EdgeReconResult, try_edge_recon_dispatch}; - match try_edge_recon_dispatch(ctx, seq, tasks, authorized_tasks).await { - EdgeReconResult::Outcome(outcome) => return outcome, - EdgeReconResult::NotFired(returned_tasks, returned_authorized) => { - tasks = returned_tasks; - authorized_tasks = returned_authorized; - } } - } - - // Cross-shard write parity with pgwire: classify the planned task set and, - // for a strict multi-shard write, route the whole batch through the Calvin - // sequencer so it commits atomically. Single-shard (and best-effort) keep - // the existing per-task gateway/SPSC dispatch loop below unchanged. - // Autocommit single-statement dispatch: no session read-set to widen with. - let sum_read_vshards = match crate::control::planner::calvin::read_vshards_of(&sum_target_reads) - { - Ok(vshards) => vshards, - Err(error) => return resp(error_to_native(seq, &error)), + Ok(AtomicRoute::PerTask(statement)) => statement, + Err(error) => return resp(statement_error_to_native(seq, error)), }; - match classify_dispatch(&tasks, &sum_read_vshards) { - DispatchClass::SingleShard { .. } => {} - DispatchClass::MultiShard { .. } => { - // Dispatching to Calvin here applies the statement durably at - // statement time, escaping the transaction buffer. Inside a block, - // fall through to the per-task staging gate when the gate can - // buffer every write — COMMIT flushes the whole buffer through - // Calvin. Anything else is refused, matching pgwire. - if in_txn_block && !all_writes_bufferable(&tasks) { - return resp(error_to_native( - seq, - &crate::Error::CrossShardInExplicitTransaction, - )); - } - - // Native has no per-session `cross_shard_txn` parameter wired, so it - // reads the same `SessionStore` accessor pgwire uses; an unset value - // defaults to `CrossShardTxnMode::Strict` (the documented default), - // so native multi-shard writes route through Calvin by default. - let cross_shard_mode = ctx.sessions.cross_shard_txn_mode(ctx.peer_addr); - if !in_txn_block && cross_shard_mode == CrossShardTxnMode::Strict { - return match dispatch_authorized_tasks_to_calvin( - ctx.state, - authorized_tasks, - ctx.tenant_id(), - cross_shard_mode, - TxnDispatchPosition::Autocommit, - &sum_target_reads, - None, - ) - .await - { - // Calvin committed. A RETURNING write surfaces its rows from - // the applied Response; a plain write reports the affected - // count its own mutation returned. - Ok(apply_resp) => { - let plans: Vec<_> = tasks.iter().map(|t| t.plan.clone()).collect(); - resp(super::conversion::calvin_native_response( - seq, - apply_resp, - &plans, - ctx.state, - database_id, - ctx.tenant_id(), - ctx.auth_context(), - )) - } - Err(e) => resp(error_to_native(seq, &e)), - }; - } - // An in-block statement and `BestEffortNonAtomic` both fall - // through to the per-task loop below. - } - } + let mut lease_scope = Some(lease_scope); // A native lazy stream outlives this handler, so transfer its descriptor // leases to the session-owned `SqlStream` before returning it. The session @@ -294,10 +223,6 @@ pub(super) async fn execute_planned( } let read_only = tasks.iter().all(|task| !plan_is_write(&task.plan)); let guard_scope = Arc::clone(&lease_scope); - let client_txn = DmlTxnCtx { - sessions: ctx.sessions, - session_id: ctx.peer_addr.into(), - }; let run = run_dispatch_loop( ctx, seq, @@ -308,7 +233,7 @@ pub(super) async fn execute_planned( plan_lease_scope: lease_scope, sum_target_reads, }, - in_txn_block.then_some(&client_txn), + in_txn_block.then_some(&session), ); if read_only { match guard_scope.guard(run).await { diff --git a/nodedb/src/control/server/native/session/run.rs b/nodedb/src/control/server/native/session/run.rs index 34edd61b6..8bb0f6758 100644 --- a/nodedb/src/control/server/native/session/run.rs +++ b/nodedb/src/control/server/native/session/run.rs @@ -299,10 +299,13 @@ impl NativeSession { }; // Crash-injection coverage verifies that a panic after a request // mutates transaction state still runs detached connection cleanup. - crate::fail_point!("native_session::after_request"); - // Cross-shard graph reads the request made join the transaction's + crate::fail_point!( + crate::fail_point::FailScope::Node(self.state.node_id), + "native_session::after_request" + ); + // Cross-shard reads the request made join the transaction's // read-set. - crate::control::server::shared::session::graph_reads::record_pending( + crate::control::server::shared::session::pending_shard_reads::record_pending( &self.sessions, self.peer_addr.into(), ); diff --git a/nodedb/src/control/server/native/session/session_stream.rs b/nodedb/src/control/server/native/session/session_stream.rs index e3157af60..d3844f4ea 100644 --- a/nodedb/src/control/server/native/session/session_stream.rs +++ b/nodedb/src/control/server/native/session/session_stream.rs @@ -218,7 +218,7 @@ mod tests { Ok(RowBatch { payload: json_batch(start, n), watermark_lsn: Lsn::ZERO, - read_version_lsn: Lsn::ZERO, + read_versions: crate::types::ReadVersions::new(), }) } @@ -352,7 +352,7 @@ mod tests { vec![Ok(RowBatch { payload: msgpack_empty_batch(3), watermark_lsn: Lsn::ZERO, - read_version_lsn: Lsn::ZERO, + read_versions: crate::types::ReadVersions::new(), })], usize::MAX, ) diff --git a/nodedb/src/control/server/pgwire/catalog/dropped_collections.rs b/nodedb/src/control/server/pgwire/catalog/dropped_collections.rs index 2dfaf47fb..a54f73297 100644 --- a/nodedb/src/control/server/pgwire/catalog/dropped_collections.rs +++ b/nodedb/src/control/server/pgwire/catalog/dropped_collections.rs @@ -120,6 +120,7 @@ async fn query_collection_size( wal_lsn: None, resolved_now_ms: None, commit_hlc: None, + entry_version: None, admission: crate::bridge::envelope::Admission::Exempt( crate::bridge::envelope::ExemptReason::AlreadyOrdered, ), diff --git a/nodedb/src/control/server/pgwire/handler/core.rs b/nodedb/src/control/server/pgwire/handler/core.rs index 1568f7ad1..3e2f962a2 100644 --- a/nodedb/src/control/server/pgwire/handler/core.rs +++ b/nodedb/src/control/server/pgwire/handler/core.rs @@ -199,9 +199,9 @@ impl ExtendedQueryHandler for NodeDbPgHandler { .send(PgWireBackendMessage::NoticeResponse(notice)) .await; } - // Cross-shard graph reads the statement made join the transaction's + // Cross-shard reads the statement made join the transaction's // read-set. - crate::control::server::shared::session::graph_reads::record_pending( + crate::control::server::shared::session::pending_shard_reads::record_pending( &self.sessions, session_id, ); diff --git a/nodedb/src/control/server/pgwire/handler/core/simple_query.rs b/nodedb/src/control/server/pgwire/handler/core/simple_query.rs index 7fafc80f8..25645ecf9 100644 --- a/nodedb/src/control/server/pgwire/handler/core/simple_query.rs +++ b/nodedb/src/control/server/pgwire/handler/core/simple_query.rs @@ -125,9 +125,9 @@ impl SimpleQueryHandler for NodeDbPgHandler { .send(PgWireBackendMessage::NoticeResponse(notice)) .await; } - // Cross-shard graph reads the query made join the transaction's + // Cross-shard reads the query made join the transaction's // read-set. - crate::control::server::shared::session::graph_reads::record_pending( + crate::control::server::shared::session::pending_shard_reads::record_pending( &self.sessions, session_id, ); diff --git a/nodedb/src/control/server/pgwire/handler/dispatch/entry.rs b/nodedb/src/control/server/pgwire/handler/dispatch/entry.rs index cd6f60c8c..77e1b3536 100644 --- a/nodedb/src/control/server/pgwire/handler/dispatch/entry.rs +++ b/nodedb/src/control/server/pgwire/handler/dispatch/entry.rs @@ -11,7 +11,6 @@ use std::sync::Arc; use crate::bridge::envelope::Response; use crate::control::security::identity::AuthenticatedIdentity; use crate::control::server::exchange::resolve::DistributedReadCapture; -use crate::types::{Lsn, VShardId}; use nodedb_physical::physical_task::PhysicalTask; use super::super::core::NodeDbPgHandler; @@ -30,30 +29,27 @@ impl NodeDbPgHandler { identity: &AuthenticatedIdentity, linearizable: bool, ) -> crate::Result { - let mut shard_watermarks = Vec::new(); let mut distributed_reads = Vec::new(); self.dispatch_task_inner( task, user_id, identity, linearizable, - &mut shard_watermarks, &mut distributed_reads, ) .await } - /// Dispatch a task and return the response, per-shard watermark LSNs a fan - /// gather observed, and per-side read captures a shuffle JOIN produced. - /// Used by the transactional read-recording seam. - pub(in crate::control::server::pgwire::handler) async fn dispatch_authorized_task_with_watermarks( + /// Dispatch a task and return the response and the per-side read captures + /// a distributed read produced. Used by the transactional read-recording + /// seam. + pub(in crate::control::server::pgwire::handler) async fn dispatch_authorized_task_with_captures( &self, task: PhysicalTask, user_id: Option>, identity: &AuthenticatedIdentity, linearizable: bool, - ) -> crate::Result<(Response, Vec<(VShardId, Lsn)>, Vec)> { - let mut shard_watermarks = Vec::new(); + ) -> crate::Result<(Response, Vec)> { let mut distributed_reads = Vec::new(); let resp = self .dispatch_task_inner( @@ -61,10 +57,9 @@ impl NodeDbPgHandler { user_id, identity, linearizable, - &mut shard_watermarks, &mut distributed_reads, ) .await?; - Ok((resp, shard_watermarks, distributed_reads)) + Ok((resp, distributed_reads)) } } diff --git a/nodedb/src/control/server/pgwire/handler/dispatch/replicated.rs b/nodedb/src/control/server/pgwire/handler/dispatch/replicated.rs index 72ebdd4d0..2c9237cdd 100644 --- a/nodedb/src/control/server/pgwire/handler/dispatch/replicated.rs +++ b/nodedb/src/control/server/pgwire/handler/dispatch/replicated.rs @@ -28,11 +28,15 @@ impl NodeDbPgHandler { let ReplicatedWrite { entry, proposer } = args; let request_id = self.next_request_id(); - // `write_version` is the post-write `coll_write_lsn`, surfaced so the session - // can floor a later read-set at it (read-your-writes for cross-shard OCC). - let (payload, write_version) = - crate::control::wal_replication::propose_replicated_entry(&self.state, proposer, entry) - .await?; + // `write_versions` are the versions the write stamped. The session + // floors its later reads of the written vShards at them. + let (payload, write_versions) = crate::control::wal_replication::propose_replicated_entry( + &self.state, + proposer, + entry, + crate::control::wal_replication::statement_propose_deadline(&self.state), + ) + .await?; let response = Response { request_id, @@ -40,11 +44,11 @@ impl NodeDbPgHandler { attempt: 1, partial: false, payload: payload.into(), - // Authoritative participant WAL LSN — CDC ordering must use it, not zero. - watermark_lsn: write_version, + // A write carries no read watermark. + watermark_lsn: crate::types::Lsn::ZERO, error_code: None, - read_set_valid: None, - read_version_lsn: write_version, + stage_vote: None, + read_versions: write_versions, write_set: Vec::new(), }; diff --git a/nodedb/src/control/server/pgwire/handler/dispatch/routing.rs b/nodedb/src/control/server/pgwire/handler/dispatch/routing.rs index 0cb929392..5cd4c48da 100644 --- a/nodedb/src/control/server/pgwire/handler/dispatch/routing.rs +++ b/nodedb/src/control/server/pgwire/handler/dispatch/routing.rs @@ -15,7 +15,7 @@ use crate::control::server::exchange::resolve::{ DistributedReadCapture, Resolved, resolve_and_materialize, }; use crate::control::server::shared::write_admission::plan_is_write; -use crate::types::{Lsn, ReadConsistency, TraceId, VShardId}; +use crate::types::{ReadConsistency, TraceId}; use nodedb_physical::physical_task::PhysicalTask; use super::super::core::NodeDbPgHandler; @@ -29,7 +29,6 @@ impl NodeDbPgHandler { user_id: Option>, identity: &AuthenticatedIdentity, linearizable: bool, - shard_watermarks: &mut Vec<(VShardId, Lsn)>, distributed_reads: &mut Vec, ) -> crate::Result { use crate::control::security::identity::{Permission, required_permission}; @@ -167,8 +166,7 @@ impl NodeDbPgHandler { linearizable, }; match resolve_and_materialize(&self.state, identity, task.plan, scope).await? { - Resolved::Gathered(resp, wms, caps) => { - *shard_watermarks = wms; + Resolved::Gathered(resp, _watermarks, caps) => { *distributed_reads = caps; return Ok(resp); } @@ -243,11 +241,7 @@ impl NodeDbPgHandler { txn_id: checked.txn_id(), linearizable, }; - let outcome = gateway.execute_with_watermarks(&ctx, checked).await; - if let Ok((_, wms, _)) = &outcome { - *shard_watermarks = wms.clone(); - } - return owner_response(outcome); + return owner_response(gateway.execute_outcome(&ctx, checked).await); } } } diff --git a/nodedb/src/control/server/pgwire/handler/routing/dispatch_loop/run.rs b/nodedb/src/control/server/pgwire/handler/routing/dispatch_loop/run.rs index 9dcb4878d..36748de24 100644 --- a/nodedb/src/control/server/pgwire/handler/routing/dispatch_loop/run.rs +++ b/nodedb/src/control/server/pgwire/handler/routing/dispatch_loop/run.rs @@ -15,13 +15,10 @@ use std::sync::Arc; use pgwire::api::results::Response; use pgwire::error::{ErrorInfo, PgWireError, PgWireResult}; -use crate::control::planner::calvin::write_class::{ - plan_counts_toward_statement_tag, plans_have_user_write, -}; use crate::control::security::identity::AuthenticatedIdentity; use crate::control::security::request_scope::RequestAuthScope; use crate::control::server::response_shape::redaction::QueryRedaction; -use crate::control::server::response_shape::types::{ShapedRows, StatementTag}; +use crate::control::server::response_shape::types::{ShapedRows, StatementTag, TaskTagRole}; use crate::control::server::shared::ddl::neutral::maintenance::auto_analyze; use crate::control::server::shared::metering::{PlanMeteringInfo, meter_dispatch}; use crate::control::server::shared::quota_admission::admit_quota_for_dispatch; @@ -104,8 +101,11 @@ impl NodeDbPgHandler { let mut returning_rows: Option = None; // The statement's ONE command tag, folded over its write tasks the // same way. `execute_sql` calls this loop once per `;`-separated - // statement, so the fold never crosses a statement boundary. - let mut statement_tag = StatementTag::default(); + // statement, so the fold never crosses a statement boundary. Built + // over every task, so a derived write beside the user's own (an + // implicit edge, a cross-shard balance move) adds no verb and no + // count, in a transaction or out of one. + let mut statement_tag = StatementTag::for_plans(tasks.iter().map(|t| &t.plan)); let mut responses = Vec::with_capacity(tasks.len()); // Session-scoped sequence access for the statement's Control-Plane // computed columns, resolved once: every task of one statement @@ -117,9 +117,6 @@ impl NodeDbPgHandler { // the collection name) a true no-op on the hot path for every // deployment that hasn't turned it on. let metering_enabled = self.state.metering_config.enabled; - // A derived implicit-edge write beside the user's own never answers - // the statement, exactly as Calvin's deposit rule has it. - let has_user_write = plans_have_user_write(tasks.iter().map(|t| &t.plan)); // A strong session reads linearizably, in and out of a transaction. let strong_reads = self.sessions.read_consistency(session_id).requires_leader(); @@ -156,6 +153,7 @@ impl NodeDbPgHandler { // rows into the statement's one result set. if let Some(route) = route_ctx.as_ref() { let plan = task.plan.clone(); + let tag_role = statement_tag.role_of(&plan); let task_database_id = task.database_id; let hooks = execute_dml_hooks::PreDispatchContext { identity, @@ -172,7 +170,7 @@ impl NodeDbPgHandler { { StatementTxnOutcome::Dispatch(routed_task) => task = *routed_task, StatementTxnOutcome::Write(handled) => { - handled.fold_into(&mut statement_tag)?; + handled.fold_into(&mut statement_tag, tag_role)?; continue; } StatementTxnOutcome::Returning(returning) => { @@ -182,7 +180,7 @@ impl NodeDbPgHandler { response: &staged_rows_response(rows), plan: &plan, plan_kind: PlanKind::ReturningRows, - counts_toward_tag: false, + tag_role: TaskTagRole::Opaque, projection, result_formats, session_id, @@ -202,7 +200,7 @@ impl NodeDbPgHandler { match self.clone_write_answer(hooks, &plan, &resp)? { PreDispatchHandled::Rows(response) => responses.push(response), PreDispatchHandled::Write(handled) => { - handled.fold_into(&mut statement_tag)?; + handled.fold_into(&mut statement_tag, tag_role)?; } } continue; @@ -245,17 +243,16 @@ impl NodeDbPgHandler { { ClusterArrayResult::Rows(response) => responses.push(response), ClusterArrayResult::Dml(outcome) => statement_tag - .fold(outcome) + .fold(statement_tag.role_of(&task.plan), outcome) .map_err(|e| dml_fold_error_to_pg(&e))?, } continue; } let plan_kind = describe_plan(&task.plan); - let counts_toward_tag = plan_counts_toward_statement_tag(&task.plan, has_user_write); + let tag_role = statement_tag.role_of(&task.plan); let resp_post_set_op = task.post_set_op; let task_database_id = task.database_id; - let task_vshard = task.vshard_id; let plan_for_response = task.plan.clone(); // Extracted from the clone above, before dispatch — metering // needs the collection/engine shape after this task's dispatch @@ -332,7 +329,7 @@ impl NodeDbPgHandler { execute_dml_hooks::PreDispatchOutcome::Handled(PreDispatchHandled::Write( handled, )) => { - handled.fold_into(&mut statement_tag)?; + handled.fold_into(&mut statement_tag, tag_role)?; continue; } execute_dml_hooks::PreDispatchOutcome::Proceed(proceed) => { @@ -350,8 +347,8 @@ impl NodeDbPgHandler { let user_id: Option> = Some(std::sync::Arc::from(identity.username.as_str())); let linearizable = strong_reads && !plan_is_write(&task.plan); - let (resp, shard_watermarks, distributed_reads) = self - .dispatch_authorized_task_with_watermarks(task, user_id, identity, linearizable) + let (resp, distributed_reads) = self + .dispatch_authorized_task_with_captures(task, user_id, identity, linearizable) .await .map_err(|e| { let (severity, code, message) = error_to_sqlstate(&e); @@ -368,11 +365,9 @@ impl NodeDbPgHandler { client_session: session_id, txn, plan: &plan_for_response, - vshard: task_vshard, database_id: task_database_id, }, &resp, - shard_watermarks, &distributed_reads, ) .await; @@ -425,7 +420,7 @@ impl NodeDbPgHandler { response: &resp, plan: &plan_for_response, plan_kind, - counts_toward_tag, + tag_role, projection, result_formats, session_id, @@ -489,8 +484,8 @@ fn staged_rows_response(rows: Vec) -> crate::bridge::envelope::Response { payload: crate::bridge::envelope::Payload::from_vec(rows), watermark_lsn: crate::types::Lsn::ZERO, error_code: None, - read_set_valid: None, - read_version_lsn: crate::types::Lsn::ZERO, + stage_vote: None, + read_versions: crate::types::ReadVersions::new(), write_set: Vec::new(), } } diff --git a/nodedb/src/control/server/pgwire/handler/routing/dispatch_loop/task.rs b/nodedb/src/control/server/pgwire/handler/routing/dispatch_loop/task.rs index 8b9c925c7..b3189e979 100644 --- a/nodedb/src/control/server/pgwire/handler/routing/dispatch_loop/task.rs +++ b/nodedb/src/control/server/pgwire/handler/routing/dispatch_loop/task.rs @@ -15,7 +15,7 @@ use crate::control::server::response_shape::redaction::QueryRedaction; use crate::control::server::response_shape::request::MaterializedShapeRequest; use crate::control::server::response_shape::schema::OutputSchema; use crate::control::server::response_shape::types::{ - ShapedRows, StatementTag, payload_to_dml_outcome, + ShapedRows, StatementTag, TaskTagRole, payload_to_dml_outcome, }; use crate::control::server::shared::session::SessionId; use crate::types::{DatabaseId, TenantId}; @@ -30,9 +30,9 @@ pub(super) struct ShapeTaskParams<'a> { pub(super) response: &'a crate::bridge::envelope::Response, pub(super) plan: &'a PhysicalPlan, pub(super) plan_kind: PlanKind, - /// Whether this task's count answers the statement. False for a derived - /// implicit-edge write beside the user's own, which folds as opaque. - pub(super) counts_toward_tag: bool, + /// Whether this task's count answers the statement. `Opaque` for a + /// derived write beside the user's own, which folds as opaque. + pub(super) tag_role: TaskTagRole, pub(super) projection: Option<&'a OutputSchema>, pub(super) result_formats: &'a Format, pub(super) session_id: SessionId, @@ -64,7 +64,7 @@ impl NodeDbPgHandler { response, plan, plan_kind, - counts_toward_tag, + tag_role, projection, result_formats, session_id, @@ -113,7 +113,7 @@ impl NodeDbPgHandler { Ok(Some(task_rows)) } ShapeOutcome::Passthrough => { - if !counts_toward_tag { + if tag_role == TaskTagRole::Opaque { statement_tag.fold_opaque(); return Ok(None); } @@ -121,7 +121,7 @@ impl NodeDbPgHandler { .map_err(|e| error_to_pg(&e))? { Some(outcome) => statement_tag - .fold(outcome) + .fold(tag_role, outcome) .map_err(|e| dml_fold_error_to_pg(&e))?, None => statement_tag.fold_opaque(), } diff --git a/nodedb/src/control/server/pgwire/handler/routing/dispatch_loop/tracking.rs b/nodedb/src/control/server/pgwire/handler/routing/dispatch_loop/tracking.rs index 3a63315aa..e79dae379 100644 --- a/nodedb/src/control/server/pgwire/handler/routing/dispatch_loop/tracking.rs +++ b/nodedb/src/control/server/pgwire/handler/routing/dispatch_loop/tracking.rs @@ -4,13 +4,12 @@ //! validates at COMMIT, and the session's own committed write version. use crate::bridge::envelope::{ErrorCode, PhysicalPlan, Response, Status}; -use crate::control::security::identity::{AuthenticatedIdentity, Permission, required_permission}; +use crate::control::security::identity::AuthenticatedIdentity; use crate::control::server::exchange::resolve::DistributedReadCapture; -use crate::control::server::shared::plan_util::extract_collection; use crate::control::server::shared::session::{ DmlTxnCtx, ResponseReads, SessionId, record_reads_for_response, }; -use crate::types::{DatabaseId, Lsn, VShardId}; +use crate::types::DatabaseId; use super::super::super::core::NodeDbPgHandler; @@ -23,29 +22,25 @@ pub(super) struct DispatchedTask<'a> { /// COMMIT. `None` for an autocommit statement. pub txn: Option<&'a DmlTxnCtx<'a>>, pub plan: &'a PhysicalPlan, - pub vshard: VShardId, pub database_id: DatabaseId, } impl NodeDbPgHandler { /// Record what one dispatched task observed and wrote. /// - /// A read in a transaction joins the transaction's read set, for - /// snapshot-isolation and cross-shard conflict checks. An absent-key - /// point read (a `NotFound` from the Data Plane) joins it too: a "not - /// found" is a validatable phantom observation. A genuine dispatch - /// failure records nothing. + /// A read in a transaction joins the transaction's read set, for commit + /// conflict checks. An absent-key point read (a `NotFound` from the Data + /// Plane) joins it too: a "not found" is a validatable phantom + /// observation. A genuine dispatch failure records nothing. /// - /// A successful write records its committed per-collection version on - /// the client's session, so a later transaction's read-set capture is - /// floored at it (the read-your-writes floor for cross-shard OCC). An - /// autocommit write floors a later transaction's read too, so this - /// records outside a transaction as well. + /// A successful write records the versions it stamped on the client's + /// session, so a later transaction's read-set capture is floored at them + /// (the read-your-writes floor). An autocommit write floors a later + /// transaction's read too, so this records outside a transaction as well. pub(super) async fn track_dispatched_task( &self, task: DispatchedTask<'_>, resp: &Response, - shard_watermarks: Vec<(VShardId, Lsn)>, distributed_reads: &[DistributedReadCapture], ) { let DispatchedTask { @@ -53,17 +48,11 @@ impl NodeDbPgHandler { client_session, txn, plan, - vshard, database_id, } = task; let records_read = resp.status == Status::Ok || resp.error_code.as_deref() == Some(&ErrorCode::NotFound); if records_read && let Some(txn) = txn { - let watermarks = if shard_watermarks.is_empty() { - vec![(vshard, resp.watermark_lsn)] - } else { - shard_watermarks - }; record_reads_for_response( &self.state, txn.sessions, @@ -71,28 +60,20 @@ impl NodeDbPgHandler { identity.tenant_id, ResponseReads { plan, - watermarks: &watermarks, - read_version_lsn: resp.read_version_lsn, + read_versions: &resp.read_versions, found: resp.status == Status::Ok, distributed_reads, - read_lsn_vshard: vshard, }, ) .await; } - if resp.status == Status::Ok - && resp.read_version_lsn > Lsn::ZERO - && matches!(required_permission(plan), Permission::Write) - && let Some(collection) = extract_collection(plan) - { - self.sessions.note_own_write( - client_session, - database_id, - identity.tenant_id, - collection, - resp.read_version_lsn, - ); - } + self.sessions.note_own_write_response( + client_session, + database_id, + identity.tenant_id, + plan, + resp, + ); } } diff --git a/nodedb/src/control/server/pgwire/handler/routing/execute.rs b/nodedb/src/control/server/pgwire/handler/routing/execute.rs index 35bdf6000..3c7c9a272 100644 --- a/nodedb/src/control/server/pgwire/handler/routing/execute.rs +++ b/nodedb/src/control/server/pgwire/handler/routing/execute.rs @@ -80,12 +80,16 @@ impl NodeDbPgHandler { // this pass settled was folded from; they travel on the // dispatch read-set so Calvin's OCC check aborts rather than // committing a total folded from an image that has moved. + // The source rows are read through the open transaction's + // staging overlay, so they include its earlier statements. + let read_txn = self.sessions.tx_id(session_id); let sum_target_reads = crate::control::planner::materialized_sum::resolve_materialized_sum_targets( &self.state, &mut tasks, tenant_id, edge_database_id, + read_txn, crate::types::TraceId::ZERO, ) .await @@ -111,6 +115,7 @@ impl NodeDbPgHandler { &mut tasks, tenant_id, edge_database_id, + read_txn, crate::types::TraceId::ZERO, ) .await @@ -173,8 +178,10 @@ impl NodeDbPgHandler { let effective_schema = Some(&effective_schema_owned); // A statement outside a transaction block whose writes fire a - // BEFORE, INSTEAD OF or SYNC AFTER body, or a MERGE into an - // edge-bearing collection, runs in an implicit transaction, + // BEFORE, INSTEAD OF or SYNC AFTER body, a MERGE into an + // edge-bearing collection, or an INSERT ... SELECT, UPDATE ... FROM + // or MERGE into the source of a cross-shard materialized sum, + // runs in an implicit transaction, // ahead of every autocommit route: // its writes stage on their vShards' leaders and commit together // at its end. diff --git a/nodedb/src/control/server/pgwire/handler/routing/execute_dml_hooks.rs b/nodedb/src/control/server/pgwire/handler/routing/execute_dml_hooks.rs index 740afc556..8055e9256 100644 --- a/nodedb/src/control/server/pgwire/handler/routing/execute_dml_hooks.rs +++ b/nodedb/src/control/server/pgwire/handler/routing/execute_dml_hooks.rs @@ -14,7 +14,7 @@ use crate::control::security::auth_context::AuthContext; use crate::control::security::identity::AuthenticatedIdentity; use crate::control::server::response_shape::schema::OutputSchema; use crate::control::server::response_shape::types::{ - DmlOutcome, StatementTag, payload_to_dml_outcome, + DmlOutcome, StatementTag, TaskTagRole, payload_to_dml_outcome, }; use crate::control::server::shared::session::SessionId; use crate::control::trigger::dml_hook::DmlWriteInfo; @@ -39,11 +39,16 @@ pub(super) enum HandledWrite { } impl HandledWrite { - /// Fold this contribution into the statement's tag. - pub(super) fn fold_into(self, statement_tag: &mut StatementTag) -> PgWireResult<()> { + /// Fold this contribution, made by a task whose role is `role`, into the + /// statement's tag. + pub(super) fn fold_into( + self, + statement_tag: &mut StatementTag, + role: TaskTagRole, + ) -> PgWireResult<()> { match self { HandledWrite::Dml(outcome) => statement_tag - .fold(outcome) + .fold(role, outcome) .map_err(|e| dml_fold_error_to_pg(&e)), HandledWrite::Opaque => { statement_tag.fold_opaque(); diff --git a/nodedb/src/control/server/pgwire/handler/routing/gateway_dispatch.rs b/nodedb/src/control/server/pgwire/handler/routing/gateway_dispatch.rs index a5e3c6e91..de326a381 100644 --- a/nodedb/src/control/server/pgwire/handler/routing/gateway_dispatch.rs +++ b/nodedb/src/control/server/pgwire/handler/routing/gateway_dispatch.rs @@ -13,9 +13,6 @@ use pgwire::api::results::Response; use pgwire::error::{ErrorInfo, PgWireError, PgWireResult}; use crate::control::gateway::GatewayErrorMap; -use crate::control::planner::calvin::write_class::{ - plan_counts_toward_statement_tag, plans_have_user_write, -}; use crate::control::security::identity::AuthenticatedIdentity; use crate::control::security::request_scope::RequestAuthScope; use crate::control::server::response_shape::redaction::QueryRedaction; @@ -128,16 +125,15 @@ impl NodeDbPgHandler { linearizable: self.sessions.read_consistency(session_id).requires_leader(), }; - // A derived implicit-edge write beside the user's own never answers - // the statement, exactly as Calvin's deposit rule has it. - let has_user_write = plans_have_user_write(tasks.iter().map(|t| &t.plan)); - let mut fold = GatewayFold::with_capacity(tasks.len()); + // A derived write beside the user's own never answers the statement, + // exactly as Calvin's deposit rule has it. + let mut fold = GatewayFold::for_tasks(&tasks); for task in tasks { let plan_kind = describe_plan(&task.plan); // The task moves into authorization below; a row-producing task // keeps its plan for shaping the rows it answers with. let shape_plan = plan_produces_rows(plan_kind).then(|| task.plan.clone()); - let counts_toward_tag = plan_counts_toward_statement_tag(&task.plan, has_user_write); + let tag_role = fold.role_of(&task.plan); let metering_info = PlanMeteringInfo::extract(&task.plan); let emitter = crate::control::security::audit::ArcAuditEmitter(std::sync::Arc::clone( &self.state.audit, @@ -175,7 +171,7 @@ impl NodeDbPgHandler { resp.payload.as_ref(), plan_kind, shape_plan.as_ref(), - counts_toward_tag, + tag_role, &shaping, )?; meter_gateway_task( @@ -213,7 +209,7 @@ impl NodeDbPgHandler { &[], plan_kind, shape_plan.as_ref(), - counts_toward_tag, + tag_role, &shaping, )?; } @@ -223,7 +219,7 @@ impl NodeDbPgHandler { payload, plan_kind, shape_plan.as_ref(), - counts_toward_tag, + tag_role, &shaping, )? { task_rows = Some(task_rows.unwrap_or(0) + rows); diff --git a/nodedb/src/control/server/pgwire/handler/routing/gateway_fold.rs b/nodedb/src/control/server/pgwire/handler/routing/gateway_fold.rs index 86619e320..94edbe118 100644 --- a/nodedb/src/control/server/pgwire/handler/routing/gateway_fold.rs +++ b/nodedb/src/control/server/pgwire/handler/routing/gateway_fold.rs @@ -15,9 +15,10 @@ use crate::control::server::response_shape::redaction::QueryRedaction; use crate::control::server::response_shape::request::MaterializedShapeRequest; use crate::control::server::response_shape::schema::OutputSchema; use crate::control::server::response_shape::types::{ - ShapedRows, StatementTag, payload_to_dml_outcome, + ShapedRows, StatementTag, TaskTagRole, payload_to_dml_outcome, }; use crate::control::server::shared::session::SessionId; +use nodedb_physical::physical_task::PhysicalTask; use super::super::super::command_tag::push_folded_tag; use super::super::super::types::{dml_fold_error_to_pg, error_to_pg, shape_error_to_pg}; @@ -47,13 +48,19 @@ pub(super) struct GatewayFold { } impl GatewayFold { - pub(super) fn with_capacity(tasks: usize) -> Self { + /// An empty fold for the statement that planned `tasks`. + pub(super) fn for_tasks(tasks: &[PhysicalTask]) -> Self { Self { - responses: Vec::with_capacity(tasks), + responses: Vec::with_capacity(tasks.len()), returning_rows: None, - statement_tag: StatementTag::default(), + statement_tag: StatementTag::for_plans(tasks.iter().map(|t| &t.plan)), } } + + /// The role of the task that runs `plan` in this statement. + pub(super) fn role_of(&self, plan: &PhysicalPlan) -> TaskTagRole { + self.statement_tag.role_of(plan) + } } impl NodeDbPgHandler { @@ -87,8 +94,8 @@ impl NodeDbPgHandler { /// the fold, `RETURNING` rows accumulate into one result set, a /// passthrough payload folds its count and verb into the statement's /// tag, and an opaque execution folds as such. A task whose count does - /// not answer the statement (`counts_toward_tag` false: a derived - /// implicit-edge write beside the user's own) folds as opaque. + /// not answer the statement (`tag_role` is `Opaque`: a derived write + /// beside the user's own) folds as opaque. /// /// A row-producing payload shapes exactly as a locally dispatched one /// does, through `shape_response_materialized` with the task's plan. A @@ -107,7 +114,7 @@ impl NodeDbPgHandler { payload: &[u8], plan_kind: PlanKind, shape_plan: Option<&PhysicalPlan>, - counts_toward_tag: bool, + tag_role: TaskTagRole, shaping: &GatewayShaping<'_>, ) -> PgWireResult> { let outcome = match (plan_produces_rows(plan_kind), shape_plan) { @@ -154,7 +161,7 @@ impl NodeDbPgHandler { Ok(Some(rows)) } ShapeOutcome::Passthrough => { - if !counts_toward_tag { + if tag_role == TaskTagRole::Opaque { fold.statement_tag.fold_opaque(); return Ok(None); } @@ -163,7 +170,7 @@ impl NodeDbPgHandler { match payload_to_dml_outcome(payload, plan_kind).map_err(|e| error_to_pg(&e))? { Some(outcome) => fold .statement_tag - .fold(outcome) + .fold(tag_role, outcome) .map_err(|e| dml_fold_error_to_pg(&e))?, None => fold.statement_tag.fold_opaque(), } diff --git a/nodedb/src/control/server/pgwire/handler/transaction_cmds/errors.rs b/nodedb/src/control/server/pgwire/handler/transaction_cmds/errors.rs index 54f4d9a4d..f21bba9b3 100644 --- a/nodedb/src/control/server/pgwire/handler/transaction_cmds/errors.rs +++ b/nodedb/src/control/server/pgwire/handler/transaction_cmds/errors.rs @@ -56,24 +56,4 @@ mod tests { other => panic!("expected PgWireError::UserError, got {other:?}"), } } - - #[test] - fn ollp_mismatch_is_not_57014() { - // Verify the OLLP mismatch arm (a genuine invariant violation) stays - // XX000 — a distinct code path from coordinator deadline cancellation. - let err = PgWireError::UserError(Box::new(ErrorInfo::new( - "ERROR".to_owned(), - "XX000".to_owned(), - "OLLP mismatch outcome on non-dependent Calvin path".to_owned(), - ))); - match err { - PgWireError::UserError(info) => { - assert_eq!( - info.code, "XX000", - "OLLP mismatch must stay XX000 (internal_error)" - ); - } - other => panic!("unexpected error variant: {other:?}"), - } - } } diff --git a/nodedb/src/control/server/response_shape/calvin_fold.rs b/nodedb/src/control/server/response_shape/calvin_fold.rs index 380d34b50..53947df71 100644 --- a/nodedb/src/control/server/response_shape/calvin_fold.rs +++ b/nodedb/src/control/server/response_shape/calvin_fold.rs @@ -15,13 +15,10 @@ use super::redaction::QueryRedaction; use super::request::MaterializedShapeRequest; use super::schema::OutputSchema; use super::types::{ - DmlFoldError, DmlOutcome, FoldedTag, PlanKind, ShapedRows, StatementTag, describe_plan, - dml_outcome_by_op, dml_outcome_from_payload, + DmlFoldError, DmlOutcome, FoldedTag, PlanKind, ShapedRows, StatementTag, TaskTagRole, + describe_plan, dml_outcome_by_op, dml_outcome_from_payload, }; use crate::bridge::envelope::Response; -use crate::control::planner::calvin::write_class::{ - plan_counts_toward_statement_tag, plans_have_user_write, -}; use crate::control::security::auth_context::AuthContext; use crate::control::state::SharedState; use crate::types::{DatabaseId, TenantId}; @@ -78,11 +75,11 @@ pub fn fold_calvin_batch( apply_resp: Option<&Response>, ctx: &CalvinFoldCtx<'_>, ) -> Result { - let has_user_write = plans_have_user_write(plans.iter().copied()); let mut rows: Option = None; - let mut tag = StatementTag::default(); + let mut tag = StatementTag::for_plans(plans.iter().copied()); for plan in plans { - if !plan_counts_toward_statement_tag(plan, has_user_write) { + let role = tag.role_of(plan); + if role == TaskTagRole::Opaque { tag.fold_opaque(); continue; } @@ -90,7 +87,9 @@ pub fn fold_calvin_batch( CalvinTaskOutcome::Rows(shaped) => { rows.get_or_insert(shaped); } - CalvinTaskOutcome::Dml(outcome) => tag.fold(outcome).map_err(CalvinFoldError::Verb)?, + CalvinTaskOutcome::Dml(outcome) => { + tag.fold(role, outcome).map_err(CalvinFoldError::Verb)? + } CalvinTaskOutcome::Opaque => tag.fold_opaque(), } } diff --git a/nodedb/src/control/server/response_shape/types/dml_outcome.rs b/nodedb/src/control/server/response_shape/types/dml_outcome.rs index fbf32f7fb..6efa44556 100644 --- a/nodedb/src/control/server/response_shape/types/dml_outcome.rs +++ b/nodedb/src/control/server/response_shape/types/dml_outcome.rs @@ -7,6 +7,11 @@ //! The payload-to-outcome readers here are the one place a Data Plane //! count payload becomes a [`DmlOutcome`]; pgwire and native both call them. +use nodedb_physical::physical_plan::PhysicalPlan; + +use crate::control::planner::calvin::write_class::{ + plan_counts_toward_statement_tag, plans_have_user_write, +}; use crate::control::server::shared::sql::staging_predicates::{ StagedTagKind, extract_ingest_rejections, extract_kv_conflict_op, rejected_lines_notice, require_affected_count, @@ -57,10 +62,32 @@ pub enum DmlFoldError { /// - Any other verb mix: [`DmlFoldError::VerbMismatch`]. Never a silent pick. /// - An opaque task (`PlanKind::Execution`) adds nothing when a DML outcome /// is folded before or after it. Only-opaque folds render as `OK`. -#[derive(Debug, Default)] +/// - A task whose [`TaskTagRole`] is `Opaque` folds as an opaque task, whatever +/// outcome it reports. A derived write beside the user's own has that role: +/// a materialized-sum balance move or an implicit graph edge write. Its +/// count describes a row the statement never named. +/// +/// The tag is built over the statement's full plan set +/// ([`Self::for_plans`]), because one task alone cannot tell a derived write +/// beside the user's own from a statement that is only that write. +#[derive(Debug)] pub struct StatementTag { dml: Option, opaque: bool, + /// Whether the statement carries the user's own write. When it does, a + /// derived write folds as opaque. + has_user_write: bool, +} + +/// Whether one task's outcome answers the statement. Read from +/// [`StatementTag::role_of`] for the task's plan. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum TaskTagRole { + /// The statement's own write: its verb and count fold into the tag. + Counts, + /// A task whose outcome adds no verb and no count: a derived write beside + /// the user's own, or a task whose rows answer in place of a count. + Opaque, } /// What a statement's fold produced. @@ -72,8 +99,32 @@ pub enum FoldedTag { } impl StatementTag { - /// Fold one task's count-bearing outcome into the statement's tag. - pub fn fold(&mut self, outcome: DmlOutcome) -> Result<(), DmlFoldError> { + /// An empty tag for the statement that planned `plans`: every task the + /// statement dispatches, stages or buffers. + pub fn for_plans<'a>(plans: impl IntoIterator) -> Self { + Self { + dml: None, + opaque: false, + has_user_write: plans_have_user_write(plans), + } + } + + /// The role of the task that runs `plan` in this statement. + pub fn role_of(&self, plan: &PhysicalPlan) -> TaskTagRole { + if plan_counts_toward_statement_tag(plan, self.has_user_write) { + TaskTagRole::Counts + } else { + TaskTagRole::Opaque + } + } + + /// Fold one task's count-bearing outcome into the statement's tag. A task + /// whose role is [`TaskTagRole::Opaque`] folds as opaque. + pub fn fold(&mut self, role: TaskTagRole, outcome: DmlOutcome) -> Result<(), DmlFoldError> { + if role == TaskTagRole::Opaque { + self.fold_opaque(); + return Ok(()); + } let Some(current) = self.dml else { self.dml = Some(outcome); return Ok(()); @@ -257,43 +308,116 @@ fn merged_verb(first: &'static str, second: &'static str) -> Option<&'static str mod tests { use super::*; + use nodedb_physical::physical_plan::{DocumentOp, GraphOp}; + use nodedb_types::{DatabaseId, QualifiedCollection, Surrogate}; + fn outcome(verb: &'static str, affected: u64) -> DmlOutcome { DmlOutcome { verb, affected } } + /// A tag for a statement with no plans: every task counts. + fn tag() -> StatementTag { + StatementTag::for_plans(std::iter::empty()) + } + + const COUNTS: TaskTagRole = TaskTagRole::Counts; + + fn entries() -> QualifiedCollection { + QualifiedCollection::new(DatabaseId::DEFAULT, "entries") + } + + /// The sum source's own insert. + fn source_insert() -> PhysicalPlan { + PhysicalPlan::Document(DocumentOp::PointInsert { + collection: entries(), + document_id: "e3".to_owned(), + value: Vec::new(), + if_absent: false, + surrogate: Surrogate::new(11), + returning: None, + rls_filters: Vec::new(), + resolved_sum_targets: Vec::new(), + deferred_sum_targets: vec!["accts".to_owned()], + }) + } + + /// The sum source's own delete. + fn source_delete() -> PhysicalPlan { + PhysicalPlan::Document(DocumentOp::PointDelete { + collection: entries(), + document_id: "e0".into(), + surrogate: None, + pk_bytes: Vec::new(), + returning: None, + rls_filters: Vec::new(), + rls_write_check: nodedb_types::RlsWriteCheck::pending_injection(), + resolved_sum_targets: Vec::new(), + }) + } + + /// The cross-shard balance move the planner appends beside a sum-source + /// write. + fn balance_delta(delta: &str) -> PhysicalPlan { + PhysicalPlan::Document(DocumentOp::ApplyBalanceDelta { + collection: QualifiedCollection::new(DatabaseId::DEFAULT, "accts"), + document_id: "acc-0".to_owned(), + surrogate: Surrogate::new(7), + column: "balance".to_owned(), + delta: delta.to_owned(), + join_column: "account_id".to_owned(), + join_value: "acc-0".to_owned(), + declared_primary_key: None, + }) + } + + /// Fold each task's staged outcome in plan order, as the in-transaction + /// routes of both protocols do. + fn fold_staged(plans: &[(PhysicalPlan, StagedTagKind)]) -> Option { + let mut tag = StatementTag::for_plans(plans.iter().map(|(plan, _)| plan)); + for (plan, kind) in plans { + tag.fold(tag.role_of(plan), staged_dml_outcome(*kind, 1)) + .expect("a derived write never mixes its verb into the tag"); + } + tag.finish() + } + #[test] fn empty_fold_finishes_to_none() { - assert_eq!(StatementTag::default().finish(), None); + assert_eq!(tag().finish(), None); } #[test] fn same_verb_sums_affected() { - let mut tag = StatementTag::default(); - tag.fold(outcome("INSERT", 1)).expect("first fold"); - tag.fold(outcome("INSERT", 0)).expect("second fold"); - tag.fold(outcome("INSERT", 1)).expect("third fold"); + let mut tag = tag(); + tag.fold(COUNTS, outcome("INSERT", 1)).expect("first fold"); + tag.fold(COUNTS, outcome("INSERT", 0)).expect("second fold"); + tag.fold(COUNTS, outcome("INSERT", 1)).expect("third fold"); assert_eq!(tag.finish(), Some(FoldedTag::Dml(outcome("INSERT", 2)))); } #[test] fn insert_and_update_fold_to_insert_in_either_order() { - let mut tag = StatementTag::default(); - tag.fold(outcome("INSERT", 1)).expect("insert"); - tag.fold(outcome("UPDATE", 2)).expect("update after insert"); - assert_eq!(tag.finish(), Some(FoldedTag::Dml(outcome("INSERT", 3)))); - - let mut tag = StatementTag::default(); - tag.fold(outcome("UPDATE", 2)).expect("update"); - tag.fold(outcome("INSERT", 1)).expect("insert after update"); - assert_eq!(tag.finish(), Some(FoldedTag::Dml(outcome("INSERT", 3)))); + let mut first = tag(); + first.fold(COUNTS, outcome("INSERT", 1)).expect("insert"); + first + .fold(COUNTS, outcome("UPDATE", 2)) + .expect("update after insert"); + assert_eq!(first.finish(), Some(FoldedTag::Dml(outcome("INSERT", 3)))); + + let mut second = tag(); + second.fold(COUNTS, outcome("UPDATE", 2)).expect("update"); + second + .fold(COUNTS, outcome("INSERT", 1)) + .expect("insert after update"); + assert_eq!(second.finish(), Some(FoldedTag::Dml(outcome("INSERT", 3)))); } #[test] fn other_verb_mix_is_an_error() { - let mut tag = StatementTag::default(); - tag.fold(outcome("INSERT", 1)).expect("insert"); + let mut tag = tag(); + tag.fold(COUNTS, outcome("INSERT", 1)).expect("insert"); assert_eq!( - tag.fold(outcome("DELETE", 1)), + tag.fold(COUNTS, outcome("DELETE", 1)), Err(DmlFoldError::VerbMismatch { first: "INSERT", second: "DELETE", @@ -303,21 +427,85 @@ mod tests { #[test] fn opaque_never_changes_a_dml_tag() { - let mut tag = StatementTag::default(); + let mut tag = tag(); tag.fold_opaque(); - tag.fold(outcome("DELETE", 4)).expect("delete"); + tag.fold(COUNTS, outcome("DELETE", 4)).expect("delete"); tag.fold_opaque(); assert_eq!(tag.finish(), Some(FoldedTag::Dml(outcome("DELETE", 4)))); } #[test] fn only_opaque_finishes_to_opaque() { - let mut tag = StatementTag::default(); + let mut tag = tag(); tag.fold_opaque(); tag.fold_opaque(); assert_eq!(tag.finish(), Some(FoldedTag::Opaque)); } + /// An opaque-role task adds no verb and no count, whatever it reports. + #[test] + fn an_opaque_role_folds_as_opaque() { + let mut tag = tag(); + tag.fold(COUNTS, outcome("DELETE", 1)).expect("delete"); + tag.fold(TaskTagRole::Opaque, outcome("UPDATE", 1)) + .expect("an opaque role never mixes its verb"); + assert_eq!(tag.finish(), Some(FoldedTag::Dml(outcome("DELETE", 1)))); + } + + /// A DELETE of a sum-source row in a transaction block stages the delete + /// and the cross-shard balance move. The balance move is a derived + /// write, so the statement tags `DELETE 1`, never a verb mismatch. + #[test] + fn a_staged_sum_source_delete_tags_delete_1() { + let folded = fold_staged(&[ + (source_delete(), StagedTagKind::Delete), + (balance_delta("-4"), StagedTagKind::Update), + ]); + assert_eq!(folded, Some(FoldedTag::Dml(outcome("DELETE", 1)))); + } + + /// An INSERT of a sum-source row in a transaction block tags `INSERT 0 1`. + /// The balance move's row is not counted as an inserted row. + #[test] + fn a_staged_sum_source_insert_tags_insert_0_1() { + let folded = fold_staged(&[ + (source_insert(), StagedTagKind::Insert), + (balance_delta("4"), StagedTagKind::Update), + ]); + assert_eq!(folded, Some(FoldedTag::Dml(outcome("INSERT", 1)))); + } + + /// The task order does not matter: a balance move folded first still + /// adds nothing. + #[test] + fn a_derived_write_folded_first_adds_nothing() { + let folded = fold_staged(&[ + (balance_delta("-4"), StagedTagKind::Update), + (source_delete(), StagedTagKind::Delete), + ]); + assert_eq!(folded, Some(FoldedTag::Dml(outcome("DELETE", 1)))); + } + + /// An implicit graph edge write beside the user's own delete is derived + /// too. Its count never adds to the deleted rows. + #[test] + fn an_implicit_edge_delete_adds_no_count() { + let edge_cleanup = PhysicalPlan::Graph(GraphOp::EdgeDeleteBatch { edges: Vec::new() }); + let folded = fold_staged(&[ + (source_delete(), StagedTagKind::Delete), + (edge_cleanup, StagedTagKind::Delete), + ]); + assert_eq!(folded, Some(FoldedTag::Dml(outcome("DELETE", 1)))); + } + + /// A statement that is only a derived-shaped write is the user's own + /// write, so it counts. + #[test] + fn a_lone_derived_shaped_write_counts() { + let folded = fold_staged(&[(balance_delta("4"), StagedTagKind::Update)]); + assert_eq!(folded, Some(FoldedTag::Dml(outcome("UPDATE", 1)))); + } + #[test] fn truncate_is_the_only_count_less_verb() { assert!(!outcome("TRUNCATE", 0).carries_count()); diff --git a/nodedb/src/control/server/response_shape/types/mod.rs b/nodedb/src/control/server/response_shape/types/mod.rs index 96176689b..33c0bfc5e 100644 --- a/nodedb/src/control/server/response_shape/types/mod.rs +++ b/nodedb/src/control/server/response_shape/types/mod.rs @@ -6,7 +6,7 @@ pub mod dml_outcome; pub mod plan_kind; pub mod shaped; -pub use dml_outcome::{DmlFoldError, DmlOutcome, FoldedTag, StatementTag}; +pub use dml_outcome::{DmlFoldError, DmlOutcome, FoldedTag, StatementTag, TaskTagRole}; pub(crate) use dml_outcome::{ dml_outcome_by_op, dml_outcome_from_payload, payload_to_dml_outcome, replaced_write_outcome, staged_dml_outcome, diff --git a/nodedb/src/control/server/result_stream.rs b/nodedb/src/control/server/result_stream.rs index 1f94856b9..95d111e65 100644 --- a/nodedb/src/control/server/result_stream.rs +++ b/nodedb/src/control/server/result_stream.rs @@ -24,16 +24,16 @@ use crate::types::Lsn; /// /// `payload` is a standalone msgpack array (the exact bytes produced by a single /// Data-Plane scan chunk); `watermark_lsn` is that frame's read watermark and -/// `read_version_lsn` its per-collection read version. +/// `read_versions` its read versions. pub struct RowBatch { /// Standalone msgpack array of row elements for this frame. pub payload: Vec, /// Watermark LSN reported by the Data Plane for this frame. pub watermark_lsn: Lsn, - /// Per-collection read-version LSN for this frame (the scanned collection's - /// `coll_write_lsn` at read time) — the sound comparand for cross-shard OCC - /// read validation, distinct from the core-global `watermark_lsn`. - pub read_version_lsn: Lsn, + /// The read versions this frame observed, one per vShard: the comparand + /// for cross-shard OCC read validation, distinct from the core-global + /// `watermark_lsn`. + pub read_versions: crate::types::ReadVersions, } /// A pinned, boxed stream of [`RowBatch`]es, fallible per item. @@ -103,7 +103,7 @@ pub(crate) fn stream_response_channel( yield RowBatch { payload: resp.payload.to_vec(), watermark_lsn: resp.watermark_lsn, - read_version_lsn: resp.read_version_lsn, + read_versions: resp.read_versions.clone(), }; if is_terminal { return; @@ -156,8 +156,8 @@ mod tests { payload: Payload::from_vec(array_payload(n)), watermark_lsn: Lsn::ZERO, error_code: None, - read_set_valid: None, - read_version_lsn: crate::types::Lsn::ZERO, + stage_vote: None, + read_versions: crate::types::ReadVersions::new(), write_set: Vec::new(), } } @@ -171,8 +171,8 @@ mod tests { payload: Payload::from_vec(array_payload(n)), watermark_lsn: Lsn::ZERO, error_code: None, - read_set_valid: None, - read_version_lsn: crate::types::Lsn::ZERO, + stage_vote: None, + read_versions: crate::types::ReadVersions::new(), write_set: Vec::new(), } } @@ -186,8 +186,8 @@ mod tests { payload: Payload::from_vec(vec![0u8; bytes]), watermark_lsn: Lsn::ZERO, error_code: None, - read_set_valid: None, - read_version_lsn: crate::types::Lsn::ZERO, + stage_vote: None, + read_versions: crate::types::ReadVersions::new(), write_set: Vec::new(), } } @@ -201,8 +201,8 @@ mod tests { payload: Payload::empty(), watermark_lsn: Lsn::ZERO, error_code: Some(Box::new(code)), - read_set_valid: None, - read_version_lsn: crate::types::Lsn::ZERO, + stage_vote: None, + read_versions: crate::types::ReadVersions::new(), write_set: Vec::new(), } } diff --git a/nodedb/src/control/server/shared/authorization/requirements/collect.rs b/nodedb/src/control/server/shared/authorization/requirements/collect.rs index b2a72725a..5ca6a3bf4 100644 --- a/nodedb/src/control/server/shared/authorization/requirements/collect.rs +++ b/nodedb/src/control/server/shared/authorization/requirements/collect.rs @@ -108,7 +108,6 @@ fn collect_requirements(plan: &PhysicalPlan, out: &mut Vec add_collection_requirement(name, required_permission(plan), out), PhysicalPlan::Meta(MetaOp::TransactionBatch { plans, .. }) | PhysicalPlan::Meta(MetaOp::ResolveTxn { plans, .. }) - | PhysicalPlan::Meta(MetaOp::RecordCalvinWriteVersions { plans, .. }) | PhysicalPlan::Meta(MetaOp::CalvinExecuteStatic { plans, .. }) | PhysicalPlan::Meta(MetaOp::CalvinExecuteActive { plans, .. }) => { for nested in plans { @@ -252,7 +251,6 @@ fn requires_tenant_fallback(plan: &PhysicalPlan) -> bool { PhysicalPlan::Meta( MetaOp::TransactionBatch { plans, .. } | MetaOp::ResolveTxn { plans, .. } - | MetaOp::RecordCalvinWriteVersions { plans, .. } | MetaOp::CalvinExecuteStatic { plans, .. } | MetaOp::CalvinExecuteActive { plans, .. }, ) => plans.is_empty(), diff --git a/nodedb/src/control/server/shared/clone_read/dispatch.rs b/nodedb/src/control/server/shared/clone_read/dispatch.rs index 0ffc86b98..61b445050 100644 --- a/nodedb/src/control/server/shared/clone_read/dispatch.rs +++ b/nodedb/src/control/server/shared/clone_read/dispatch.rs @@ -208,8 +208,8 @@ pub(super) fn empty_response(state: &SharedState) -> Response { payload: Payload::empty(), watermark_lsn: crate::types::Lsn::ZERO, error_code: None, - read_set_valid: None, - read_version_lsn: crate::types::Lsn::ZERO, + stage_vote: None, + read_versions: crate::types::ReadVersions::new(), write_set: Vec::new(), } } diff --git a/nodedb/src/control/server/shared/clone_write/probes.rs b/nodedb/src/control/server/shared/clone_write/probes.rs index de59fdc6a..341083029 100644 --- a/nodedb/src/control/server/shared/clone_write/probes.rs +++ b/nodedb/src/control/server/shared/clone_write/probes.rs @@ -290,6 +290,7 @@ pub(super) async fn dispatch_data_plane_raw( wal_lsn: None, resolved_now_ms: None, commit_hlc: None, + entry_version: None, admission: crate::bridge::envelope::Admission::Exempt( crate::bridge::envelope::ExemptReason::Read, ), diff --git a/nodedb/src/control/server/shared/clone_write/util.rs b/nodedb/src/control/server/shared/clone_write/util.rs index d55ee1f0c..c00b961a9 100644 --- a/nodedb/src/control/server/shared/clone_write/util.rs +++ b/nodedb/src/control/server/shared/clone_write/util.rs @@ -33,8 +33,8 @@ pub(super) fn synthetic_affected_response( payload: payload.into(), watermark_lsn, error_code: None, - read_set_valid: None, - read_version_lsn: crate::types::Lsn::ZERO, + stage_vote: None, + read_versions: crate::types::ReadVersions::new(), write_set: Vec::new(), } } diff --git a/nodedb/src/control/server/shared/cluster_array_dispatch.rs b/nodedb/src/control/server/shared/cluster_array_dispatch.rs index 6e065ed54..a816b659d 100644 --- a/nodedb/src/control/server/shared/cluster_array_dispatch.rs +++ b/nodedb/src/control/server/shared/cluster_array_dispatch.rs @@ -7,9 +7,10 @@ //! trigger/DML machinery. Each transport's dispatch loop intercepts a //! `PhysicalPlan::ClusterArray` task after authorization: //! -//! - pgwire and native call [`execute_cluster_array`] and render the -//! [`ClusterArrayShaped`] outcome in their own wire format. The adapters -//! live in `pgwire::handler::routing::cluster_array` and +//! - pgwire, native and the shared statement loop +//! (`shared::statement_exec`) call [`execute_cluster_array`] and render the +//! [`ClusterArrayShaped`] outcome in their own shape. The protocol +//! adapters live in `pgwire::handler::routing::cluster_array` and //! `native::dispatch::cluster_array`. //! - HTTP and WebSocket RPC call [`run_cluster_array`] and shape the raw //! payload the way they shape a gateway payload. diff --git a/nodedb/src/control/server/shared/ddl/neutral/collection/dml/parse/dispatch.rs b/nodedb/src/control/server/shared/ddl/neutral/collection/dml/parse/dispatch.rs index 73730c139..25d020762 100644 --- a/nodedb/src/control/server/shared/ddl/neutral/collection/dml/parse/dispatch.rs +++ b/nodedb/src/control/server/shared/ddl/neutral/collection/dml/parse/dispatch.rs @@ -128,13 +128,16 @@ pub(in crate::control::server::shared::ddl::neutral::collection) async fn plan_a // The entries cover the row images every cross-shard balance this pass // settled was folded from. They travel on the dispatch read-set so the // Calvin OCC check aborts, before any row moves, if those images have been - // written since. + // written since. The source rows are read through the open transaction's + // staging overlay, so they include this transaction's earlier statements. + let read_txn = txn_ctx.sessions.tx_id(txn_ctx.session_id); let sum_target_reads = crate::control::planner::materialized_sum::resolve_materialized_sum_targets( state, &mut tasks, tenant_id, database_id, + read_txn, TraceId::ZERO, ) .await @@ -159,6 +162,7 @@ pub(in crate::control::server::shared::ddl::neutral::collection) async fn plan_a &mut tasks, tenant_id, database_id, + read_txn, TraceId::ZERO, ) .await diff --git a/nodedb/src/control/server/shared/ddl/neutral/collection/purge/dispatch.rs b/nodedb/src/control/server/shared/ddl/neutral/collection/purge/dispatch.rs index bd8222ede..e07ee1859 100644 --- a/nodedb/src/control/server/shared/ddl/neutral/collection/purge/dispatch.rs +++ b/nodedb/src/control/server/shared/ddl/neutral/collection/purge/dispatch.rs @@ -85,6 +85,7 @@ pub async fn dispatch_unregister_collection( wal_lsn: None, resolved_now_ms: None, commit_hlc: None, + entry_version: None, admission: crate::bridge::envelope::Admission::Exempt( crate::bridge::envelope::ExemptReason::AlreadyOrdered, ), diff --git a/nodedb/src/control/server/shared/ddl/neutral/kv_atomic/dispatch.rs b/nodedb/src/control/server/shared/ddl/neutral/kv_atomic/dispatch.rs index 4863e4a11..24cbc8084 100644 --- a/nodedb/src/control/server/shared/ddl/neutral/kv_atomic/dispatch.rs +++ b/nodedb/src/control/server/shared/ddl/neutral/kv_atomic/dispatch.rs @@ -190,7 +190,7 @@ async fn dispatch_autocommit( /// Map a dispatch error to the client-facing error. A Data-Plane verdict /// arrives here as `Error::DataPlane` on the replicated route, and it renders /// the same way the local route's error status does. -fn error_to_ddl(error: &crate::Error) -> DdlError { +pub(crate) fn error_to_ddl(error: &crate::Error) -> DdlError { match error { crate::Error::DataPlane(code) => data_plane_error(Some(code.clone())), other => { diff --git a/nodedb/src/control/server/shared/ddl/neutral/kv_atomic/mod.rs b/nodedb/src/control/server/shared/ddl/neutral/kv_atomic/mod.rs index 2bb312865..972b75cc8 100644 --- a/nodedb/src/control/server/shared/ddl/neutral/kv_atomic/mod.rs +++ b/nodedb/src/control/server/shared/ddl/neutral/kv_atomic/mod.rs @@ -19,6 +19,6 @@ pub mod dispatch; pub mod handlers; pub(crate) use dispatch::{ - dispatch_and_respond, parse_function_args, single_text_col, split_args, unquote, + dispatch_and_respond, error_to_ddl, parse_function_args, single_text_col, split_args, unquote, }; pub use handlers::{kv_cas, kv_getset, kv_incr, kv_incr_float}; diff --git a/nodedb/src/control/server/shared/ddl/neutral/kv_sorted_index/dispatch.rs b/nodedb/src/control/server/shared/ddl/neutral/kv_sorted_index/dispatch.rs index b33617619..d0f10a046 100644 --- a/nodedb/src/control/server/shared/ddl/neutral/kv_sorted_index/dispatch.rs +++ b/nodedb/src/control/server/shared/ddl/neutral/kv_sorted_index/dispatch.rs @@ -155,8 +155,8 @@ fn verdict_response(code: ErrorCode) -> Response { payload: crate::bridge::envelope::Payload::empty(), watermark_lsn: crate::types::Lsn::ZERO, error_code: Some(Box::new(code)), - read_set_valid: None, - read_version_lsn: crate::types::Lsn::ZERO, + stage_vote: None, + read_versions: crate::types::ReadVersions::new(), write_set: Vec::new(), } } diff --git a/nodedb/src/control/server/shared/ddl/neutral/maintenance/distributed.rs b/nodedb/src/control/server/shared/ddl/neutral/maintenance/distributed.rs index 20f4d0084..7cf174703 100644 --- a/nodedb/src/control/server/shared/ddl/neutral/maintenance/distributed.rs +++ b/nodedb/src/control/server/shared/ddl/neutral/maintenance/distributed.rs @@ -49,6 +49,7 @@ pub fn dispatch_maintenance_to_all_cores( wal_lsn: None, resolved_now_ms: None, commit_hlc: None, + entry_version: None, admission: crate::bridge::envelope::Admission::Exempt( crate::bridge::envelope::ExemptReason::AlreadyOrdered, ), diff --git a/nodedb/src/control/server/shared/ddl/neutral/mod.rs b/nodedb/src/control/server/shared/ddl/neutral/mod.rs index a7094a850..171977a0d 100644 --- a/nodedb/src/control/server/shared/ddl/neutral/mod.rs +++ b/nodedb/src/control/server/shared/ddl/neutral/mod.rs @@ -83,6 +83,7 @@ pub mod timeseries; pub mod topic; pub mod topic_subscribe; pub mod transfer; +mod transfer_cross_shard; pub mod tree_ops; pub mod trigger; pub mod typeguard; diff --git a/nodedb/src/control/server/shared/ddl/neutral/rate_gate.rs b/nodedb/src/control/server/shared/ddl/neutral/rate_gate.rs index 7574d56e6..abbabf120 100644 --- a/nodedb/src/control/server/shared/ddl/neutral/rate_gate.rs +++ b/nodedb/src/control/server/shared/ddl/neutral/rate_gate.rs @@ -431,8 +431,8 @@ mod tests { payload: Payload::empty(), watermark_lsn: Lsn::ZERO, error_code: code.map(Box::new), - read_set_valid: None, - read_version_lsn: Lsn::ZERO, + stage_vote: None, + read_versions: crate::types::ReadVersions::new(), write_set: Vec::new(), } } diff --git a/nodedb/src/control/server/shared/ddl/neutral/router/string_engine_ops.rs b/nodedb/src/control/server/shared/ddl/neutral/router/string_engine_ops.rs index 52653aa95..9d0c904fe 100644 --- a/nodedb/src/control/server/shared/ddl/neutral/router/string_engine_ops.rs +++ b/nodedb/src/control/server/shared/ddl/neutral/router/string_engine_ops.rs @@ -368,9 +368,7 @@ mod tests { }; assert!(!crdt_apply_forbidden_in_transaction(&ctx)); - sessions - .begin(addr, crate::types::Lsn::new(1), 0) - .expect("begin"); + sessions.begin(addr, 0).expect("begin"); assert!(crdt_apply_forbidden_in_transaction(&ctx)); sessions.fail_transaction(addr); assert!(crdt_apply_forbidden_in_transaction(&ctx)); diff --git a/nodedb/src/control/server/shared/ddl/neutral/router/string_versioning.rs b/nodedb/src/control/server/shared/ddl/neutral/router/string_versioning.rs index 1d5334675..bd764f2e4 100644 --- a/nodedb/src/control/server/shared/ddl/neutral/router/string_versioning.rs +++ b/nodedb/src/control/server/shared/ddl/neutral/router/string_versioning.rs @@ -286,9 +286,7 @@ mod tests { session_id: SessionId::from(&addr), }; assert!(!restore_forbidden_in_transaction(&ctx)); - sessions - .begin(addr, crate::types::Lsn::new(1), 0) - .expect("begin"); + sessions.begin(addr, 0).expect("begin"); assert!(restore_forbidden_in_transaction(&ctx)); sessions.fail_transaction(addr); assert!(restore_forbidden_in_transaction(&ctx)); diff --git a/nodedb/src/control/server/shared/ddl/neutral/tenant/move_tenant/cutover.rs b/nodedb/src/control/server/shared/ddl/neutral/tenant/move_tenant/cutover.rs index 892202da6..71fadc467 100644 --- a/nodedb/src/control/server/shared/ddl/neutral/tenant/move_tenant/cutover.rs +++ b/nodedb/src/control/server/shared/ddl/neutral/tenant/move_tenant/cutover.rs @@ -100,9 +100,11 @@ pub async fn run( } // Fails the cutover after the re-issue and before the proposal. - crate::fail_point_err!("move_tenant::cutover::before_proposal", |detail: String| { - failed(format!("fail point: {detail}")) - }); + crate::fail_point_err!( + crate::fail_point::FailScope::Node(state.node_id), + "move_tenant::cutover::before_proposal", + |detail: String| failed(format!("fail point: {detail}")) + ); // The arrays are read under the DDL preparation lease, so the commit // rekeys exactly the arrays the catalog holds when it is proposed. Its apply rekeyed the arrays and diff --git a/nodedb/src/control/server/shared/ddl/neutral/transfer.rs b/nodedb/src/control/server/shared/ddl/neutral/transfer.rs index f09f1555f..90c5cbf40 100644 --- a/nodedb/src/control/server/shared/ddl/neutral/transfer.rs +++ b/nodedb/src/control/server/shared/ddl/neutral/transfer.rs @@ -17,6 +17,8 @@ //! //! Both dispatch to the Data Plane as dedicated KvOp variants. The entire //! read-validate-write executes in a single TPC core pass — no TOCTOU race. +//! A `TRANSFER_ITEM` between collections on two vShards runs as one +//! read-dependent Calvin transaction instead (see `transfer_cross_shard`). use crate::control::security::identity::AuthenticatedIdentity; use crate::control::server::shared::session::DmlTxnCtx; @@ -134,22 +136,13 @@ pub async fn transfer_item( let source_owner = unquote(&args[3]); let dest_owner = unquote(&args[4]); - // Cross-collection transfers must be on the same vshard. - // Validate this upfront to prevent silent failures. + // A move between collections on two vShards runs as one read-dependent + // Calvin transaction. A move within one vShard runs on its core. let vshard_src = nodedb_types::CollectionKey::from_bare(DatabaseId::DEFAULT, &source_collection).vshard(); let vshard_dst = nodedb_types::CollectionKey::from_bare(DatabaseId::DEFAULT, &dest_collection).vshard(); - if source_collection != dest_collection && vshard_src != vshard_dst { - return Err(ddl_err( - "0A000", - format!( - "TRANSFER_ITEM: cross-shard transfer not supported \ - (source '{}' and dest '{}' map to different vShards)", - source_collection, dest_collection - ), - )); - } + let cross_shard = vshard_src != vshard_dst; let item_key = format!("{source_owner}:{item_id}"); let dest_key = format!("{dest_owner}:{item_id}"); @@ -187,6 +180,18 @@ pub async fn transfer_item( dest_rls_write_check: nodedb_types::RlsWriteCheck::pending_injection(), }); + if cross_shard { + return super::transfer_cross_shard::transfer_item_across_shards( + state, + identity, + txn_ctx, + plan, + [source_collection.as_str(), dest_collection.as_str()], + vshard_src, + ) + .await; + } + dispatch_and_respond( state, identity, diff --git a/nodedb/src/control/server/shared/ddl/neutral/transfer_cross_shard.rs b/nodedb/src/control/server/shared/ddl/neutral/transfer_cross_shard.rs new file mode 100644 index 000000000..713b07f3b --- /dev/null +++ b/nodedb/src/control/server/shared/ddl/neutral/transfer_cross_shard.rs @@ -0,0 +1,130 @@ +// SPDX-License-Identifier: BUSL-1.1 + +//! `SELECT TRANSFER_ITEM(...)` between collections on two vShards. +//! +//! The move runs as one read-dependent Calvin transaction (see +//! `control::planner::calvin::read_dependent`): the destination write takes +//! the bytes the source vShard holds under the transaction's locks, and both +//! halves commit or neither does. The handler applies the gates a +//! same-vShard move passes: both collections' grants, the row-level +//! security injection, the clone-write check, and the descriptor write lease +//! held until the move finishes. +//! +//! Inside a transaction block the move is refused: its two halves cannot +//! stage into one vShard's overlay. + +use nodedb_physical::physical_plan::{KvOp, PhysicalPlan}; +use nodedb_physical::physical_task::{PhysicalTask, PostSetOp}; + +use crate::control::planner::calvin::read_dependent::{ + CrossShardItemMove, move_item_across_shards, +}; +use crate::control::security::audit::ArcAuditEmitter; +use crate::control::security::identity::{AuthenticatedIdentity, Permission}; +use crate::control::server::shared::clone_write::{ + CloneCheckedOutcome, InterceptAndAuthorizeParams, intercept_and_authorize, +}; +use crate::control::server::shared::session::{DmlTxnCtx, TransactionState}; +use crate::control::state::SharedState; +use crate::types::{DatabaseId, VShardId}; + +use super::super::result::{DdlError, DdlResult}; +use super::kv_atomic::{error_to_ddl, single_text_col}; +use super::read_gate::CollectionReadGate; + +/// Run `plan`, a `KvOp::TransferItem` between `collections` (source, then +/// destination) whose vShards differ, as one Calvin transaction. +/// `source_vshard` is the source collection's vShard. +pub(super) async fn transfer_item_across_shards( + state: &SharedState, + identity: &AuthenticatedIdentity, + txn_ctx: &DmlTxnCtx<'_>, + mut plan: PhysicalPlan, + collections: [&str; 2], + source_vshard: VShardId, +) -> Result, DdlError> { + if txn_ctx.sessions.transaction_state(txn_ctx.session_id) != TransactionState::Idle { + return Err(error_to_ddl(&crate::Error::CrossShardInExplicitTransaction)); + } + let database_id = DatabaseId::DEFAULT; + let gate = CollectionReadGate::for_request(state, identity, database_id); + for collection in collections { + gate.authorize(collection)?; + gate.authorize_permission(collection, Permission::Write)?; + } + gate.inject_rls(&mut plan)?; + + let emitter = ArcAuditEmitter(std::sync::Arc::clone(&state.audit)); + let checked = intercept_and_authorize(InterceptAndAuthorizeParams { + state, + task: PhysicalTask { + tenant_id: identity.tenant_id, + vshard_id: source_vshard, + database_id, + plan, + post_set_op: PostSetOp::None, + txn_id: None, + }, + identity, + tenant_id: identity.tenant_id, + permissions: &state.permissions, + roles: &state.roles, + emitter: &emitter, + }) + .await + .map_err(|e| error_to_ddl(&e))?; + let checked = match checked { + CloneCheckedOutcome::Handled(response) => { + let text = crate::data::executor::response_codec::decode_payload_to_json( + response.payload.as_ref(), + ); + return Ok(vec![single_text_col("transfer_item", text)]); + } + CloneCheckedOutcome::Proceed(checked) => checked, + }; + // The lease holds every collection's descriptor until the move finished, + // so a drain of either waits for it. + let (authorized, _lease) = checked.into_parts(); + let PhysicalPlan::Kv(KvOp::TransferItem { + source_collection, + dest_collection, + item_key, + dest_key, + surrogate, + source_rls_write_check, + dest_rls_write_check: _, + }) = authorized.into_physical_task().plan + else { + return Err(DdlError::internal( + "TRANSFER_ITEM: the authorized task is not the item move", + )); + }; + let reply = serde_json::json!({ + "item_key": String::from_utf8_lossy(&item_key), + "dest_key": String::from_utf8_lossy(&dest_key), + "source_collection": source_collection.as_str(), + "dest_collection": dest_collection.as_str(), + }); + + move_item_across_shards( + state, + identity, + CrossShardItemMove { + tenant_id: identity.tenant_id, + database_id, + source_collection, + dest_collection, + item_key, + dest_key, + surrogate, + source_rls_write_check, + }, + ) + .await + .map_err(|e| error_to_ddl(&e))?; + + let payload = nodedb_types::json_to_msgpack(&reply) + .map_err(|e| DdlError::internal(format!("TRANSFER_ITEM reply: {e}")))?; + let text = crate::data::executor::response_codec::decode_payload_to_json(&payload); + Ok(vec![single_text_col("transfer_item", text)]) +} diff --git a/nodedb/src/control/server/shared/ddl/neutral/version_history/restore.rs b/nodedb/src/control/server/shared/ddl/neutral/version_history/restore.rs index 7a3a39111..ae7ea0dba 100644 --- a/nodedb/src/control/server/shared/ddl/neutral/version_history/restore.rs +++ b/nodedb/src/control/server/shared/ddl/neutral/version_history/restore.rs @@ -146,7 +146,7 @@ struct RestoreDeltaParams<'a> { async fn persist_restore_delta( state: &SharedState, params: RestoreDeltaParams<'_>, -) -> crate::Result> { +) -> crate::Result { let RestoreDeltaParams { tenant_id, database_id, @@ -182,7 +182,7 @@ async fn persist_restore_delta( }, ) .await?; - Ok((outcome.write_version != crate::types::Lsn::ZERO).then_some(outcome.write_version)) + Ok(outcome.write_versions) } /// Parse: RESTORE collection SET VERSION = 'checkpoint' WHERE id = 'doc-id' @@ -275,7 +275,7 @@ mod tests { /// A Data Plane core for a one-node cluster that answers the restore /// admission: the generate returns `delta`, the preview returns a result - /// with `digest`, and the fenced apply echoes its WAL LSN. It answers + /// with `digest`, and the fenced apply echoes its write version. It answers /// every other request, the cluster's Raft applies included, with `Ok`. /// It records each restore step it sees, marking a step whose delta or /// digest differs from the expected one. @@ -295,18 +295,32 @@ mod tests { while let Ok(request) = side.request_rx.try_pop() { let request = request.inner; let request_id = request.request_id; - let wal_lsn = request.wal_lsn.unwrap_or(Lsn::ZERO); - let (step, payload, read_version_lsn) = match request.plan { - PhysicalPlan::Crdt(CrdtOp::RestoreToVersion { .. }) => { - (Some("generate"), core.delta.clone(), Lsn::ZERO) - } + let applied_versions = crate::types::ReadVersions::single( + request.vshard_id, + request + .entry_version + .unwrap_or(nodedb_types::WriteVersion::local_after( + nodedb_types::WriteVersion::ZERO, + request.wal_lsn.unwrap_or(Lsn::ZERO).as_u64(), + )), + ); + let (step, payload, read_versions) = match request.plan { + PhysicalPlan::Crdt(CrdtOp::RestoreToVersion { .. }) => ( + Some("generate"), + core.delta.clone(), + crate::types::ReadVersions::new(), + ), PhysicalPlan::Crdt(CrdtOp::PreviewApply { delta, .. }) => { let step = if delta == core.delta { "preview" } else { "preview: wrong delta" }; - (Some(step), preview.clone(), Lsn::ZERO) + ( + Some(step), + preview.clone(), + crate::types::ReadVersions::new(), + ) } PhysicalPlan::Crdt(CrdtOp::Apply { delta, @@ -320,9 +334,9 @@ mod tests { } else { "apply: wrong delta or digest" }; - (Some(step), Vec::new(), wal_lsn) + (Some(step), Vec::new(), applied_versions) } - _ => (None, Vec::new(), Lsn::ZERO), + _ => (None, Vec::new(), crate::types::ReadVersions::new()), }; if let Some(step) = step { core.seen @@ -340,8 +354,8 @@ mod tests { payload: payload.into(), watermark_lsn: Lsn::ZERO, error_code: None, - read_set_valid: None, - read_version_lsn, + stage_vote: None, + read_versions, write_set: Vec::new(), }, }) @@ -424,15 +438,15 @@ mod tests { ) .await .expect("persist restore delta"); - let lsn = outcome.map(|outcome| outcome.write_version); + let versions = outcome.map(|outcome| outcome.write_versions); assert_eq!( core.seen(), ["generate", "preview", "apply"], "restore admission must generate, preview, then apply" ); assert!( - lsn.is_some(), - "a restore must allocate and return a durable WAL LSN" + versions.is_some_and(|versions| !versions.is_empty()), + "a restore must return the version its apply stamped" ); state.wal.sync().expect("sync wal"); diff --git a/nodedb/src/control/server/shared/ddl/sqlstate.rs b/nodedb/src/control/server/shared/ddl/sqlstate.rs index 414c22ad9..e18b39213 100644 --- a/nodedb/src/control/server/shared/ddl/sqlstate.rs +++ b/nodedb/src/control/server/shared/ddl/sqlstate.rs @@ -69,6 +69,13 @@ pub fn error_code_to_sqlstate(code: &ErrorCode) -> (&'static str, &'static str, sqlstate::SERIALIZATION_FAILURE, format!("write refused without applying, retry: {reason}"), ), + // The fail-stopped core applied nothing, and another replica or a + // restart serves the statement: the class drivers retry on. + ErrorCode::CoreFailStopped { core_id, detail } => ( + "ERROR", + sqlstate::SERIALIZATION_FAILURE, + format!("core {core_id} is fail-stopped and applied nothing, retry: {detail}"), + ), ErrorCode::NotFound => ("ERROR", sqlstate::NO_DATA, "not found".into()), // `resource` is what makes the denial actionable: it says whether a // row-level-security policy refused the row or a grant is missing, and diff --git a/nodedb/src/control/server/shared/ddl/sync_dispatch/dispatch.rs b/nodedb/src/control/server/shared/ddl/sync_dispatch/dispatch.rs index f1998526b..09cf73d83 100644 --- a/nodedb/src/control/server/shared/ddl/sync_dispatch/dispatch.rs +++ b/nodedb/src/control/server/shared/ddl/sync_dispatch/dispatch.rs @@ -200,6 +200,7 @@ async fn dispatch_plan(state: &SharedState, dispatch: PlanDispatch) -> crate::Re wal_lsn: None, resolved_now_ms: None, commit_hlc: None, + entry_version: None, admission: crate::bridge::envelope::Admission::Exempt( crate::bridge::envelope::ExemptReason::AlreadyOrdered, ), diff --git a/nodedb/src/control/server/shared/ddl/user_dispatch.rs b/nodedb/src/control/server/shared/ddl/user_dispatch.rs index c2a817eb2..a507c446d 100644 --- a/nodedb/src/control/server/shared/ddl/user_dispatch.rs +++ b/nodedb/src/control/server/shared/ddl/user_dispatch.rs @@ -492,8 +492,8 @@ mod tests { payload: Payload::empty(), watermark_lsn: Lsn::ZERO, error_code: None, - read_set_valid: None, - read_version_lsn: Lsn::ZERO, + stage_vote: None, + read_versions: crate::types::ReadVersions::new(), write_set: Vec::new(), }, }) diff --git a/nodedb/src/control/server/shared/metering.rs b/nodedb/src/control/server/shared/metering.rs index 21750b6d8..c1a151d5e 100644 --- a/nodedb/src/control/server/shared/metering.rs +++ b/nodedb/src/control/server/shared/metering.rs @@ -962,8 +962,8 @@ mod tests { payload: crate::bridge::envelope::Payload::empty(), watermark_lsn: crate::types::Lsn::new(0), error_code: None, - read_set_valid: None, - read_version_lsn: crate::types::Lsn::ZERO, + stage_vote: None, + read_versions: crate::types::ReadVersions::new(), write_set: Vec::new(), } } diff --git a/nodedb/src/control/server/shared/mod.rs b/nodedb/src/control/server/shared/mod.rs index 6d1f001ad..f4c1a7df8 100644 --- a/nodedb/src/control/server/shared/mod.rs +++ b/nodedb/src/control/server/shared/mod.rs @@ -17,6 +17,7 @@ pub mod retry; pub mod returning; pub mod session; pub mod sql; +pub mod statement_exec; pub mod txn_control; pub mod txn_route; pub mod write_admission; diff --git a/nodedb/src/control/server/shared/plan_admission.rs b/nodedb/src/control/server/shared/plan_admission.rs index 83a578681..d31666655 100644 --- a/nodedb/src/control/server/shared/plan_admission.rs +++ b/nodedb/src/control/server/shared/plan_admission.rs @@ -74,12 +74,30 @@ pub struct PlanAdmissionRequest<'a> { /// Plan `sql`, authorize it, expand implicit graph edges, authorize the expanded /// set, and acquire the descriptor leases — retrying the whole unit while a /// descriptor drain is in flight. +/// +/// For a transport with no transaction block: every derived read sees +/// committed base only. A session transport uses +/// [`plan_authorize_and_admit_in_txn`]. pub async fn plan_authorize_and_admit( request: PlanAdmissionRequest<'_>, +) -> crate::Result { + plan_authorize_and_admit_in_txn(request, None).await +} + +/// [`plan_authorize_and_admit`] for a statement that runs in the open +/// transaction `read_txn`, `None` outside a transaction block. +/// +/// The derived writes read their source rows through that transaction's +/// staging overlay. A materialized-sum balance or a period lock then derives +/// from the rows as this transaction sees them, including the rows its earlier +/// statements staged. +pub async fn plan_authorize_and_admit_in_txn( + request: PlanAdmissionRequest<'_>, + read_txn: Option, ) -> crate::Result { let request = &request; retry_on_schema_change(&request.state.lease_drain, move || { - plan_authorize_and_admit_once(request) + plan_authorize_and_admit_once(request, read_txn) }) .await } @@ -88,6 +106,7 @@ pub async fn plan_authorize_and_admit( /// re-invocation with no partial state carried between attempts. async fn plan_authorize_and_admit_once( request: &PlanAdmissionRequest<'_>, + read_txn: Option, ) -> crate::Result { let state = request.state; let query_ctx = request.query_ctx; @@ -137,8 +156,15 @@ async fn plan_authorize_and_admit_once( let _preauthorized_tasks = authorize_task_set(identity, &tasks, &state.permissions, &state.roles, &emitter)?; - let sum_target_reads = - append_derived_tasks(state, &mut tasks, tenant_id, database_id, trace_id).await?; + let sum_target_reads = append_derived_tasks( + state, + &mut tasks, + tenant_id, + database_id, + read_txn, + trace_id, + ) + .await?; // Deliberate gate: proves the final task set is authorizable before a // descriptor lease is acquired. The caller re-derives the capability per @@ -166,11 +192,16 @@ async fn plan_authorize_and_admit_once( /// /// Returns the read-set entries covering the images every cross-shard /// balance was settled from. The caller unions them into its read set. +/// +/// `read_txn` is the open transaction the statement runs in, `None` outside a +/// transaction block. Each derived write reads its source rows through that +/// transaction's staging overlay. pub async fn append_derived_tasks( state: &SharedState, tasks: &mut Vec, tenant_id: crate::types::TenantId, database_id: crate::types::DatabaseId, + read_txn: Option, trace_id: TraceId, ) -> crate::Result> { crate::control::planner::implicit_edges::append_implicit_edge_tasks( @@ -181,13 +212,39 @@ pub async fn append_derived_tasks( trace_id, ) .await?; + append_sum_and_period_targets(state, tasks, tenant_id, database_id, read_txn, trace_id).await +} +/// Resolve every write's materialized-sum targets, append one +/// `ApplyBalanceDelta` task per cross-shard balance, and resolve every +/// write's period-lock reference row. +/// +/// A plain statement runs this through [`append_derived_tasks`]. The +/// in-transaction expanders run it on the point writes they emit. So every +/// write that moves a sum source ships its cross-shard balance on a task +/// homed on the target's vShard. +/// +/// Returns the read-set entries covering the images every cross-shard +/// balance was settled from. The caller unions them into its read set. +/// +/// `read_txn` is the open transaction the statement runs in, `None` outside a +/// transaction block. Each source read goes through that transaction's +/// staging overlay. +pub async fn append_sum_and_period_targets( + state: &SharedState, + tasks: &mut Vec, + tenant_id: crate::types::TenantId, + database_id: crate::types::DatabaseId, + read_txn: Option, + trace_id: TraceId, +) -> crate::Result> { let sum_target_reads = crate::control::planner::materialized_sum::resolve_materialized_sum_targets( state, tasks, tenant_id, database_id, + read_txn, trace_id, ) .await?; @@ -209,6 +266,7 @@ pub async fn append_derived_tasks( tasks, tenant_id, database_id, + read_txn, trace_id, ) .await?; diff --git a/nodedb/src/control/server/shared/response_payload.rs b/nodedb/src/control/server/shared/response_payload.rs index 1c1f04a4b..905ba3f42 100644 --- a/nodedb/src/control/server/shared/response_payload.rs +++ b/nodedb/src/control/server/shared/response_payload.rs @@ -44,8 +44,8 @@ mod tests { payload: Payload::from_vec(payload.to_vec()), watermark_lsn: Lsn::ZERO, error_code: code.map(Box::new), - read_set_valid: None, - read_version_lsn: Lsn::ZERO, + stage_vote: None, + read_versions: crate::types::ReadVersions::new(), write_set: Vec::new(), } } diff --git a/nodedb/src/control/server/shared/returning/inject.rs b/nodedb/src/control/server/shared/returning/inject.rs index 6b3ea3fba..2ce734d75 100644 --- a/nodedb/src/control/server/shared/returning/inject.rs +++ b/nodedb/src/control/server/shared/returning/inject.rs @@ -413,8 +413,6 @@ pub fn inject_returning_spec(plan: &mut PhysicalPlan, spec: ReturningSpec) { | MetaOp::DropTxnOverlay { .. } | MetaOp::MarkSavepoint { .. } | MetaOp::RollbackToSavepoint { .. } - | MetaOp::RecordCalvinWriteVersions { .. } - | MetaOp::CalvinFlush { .. } | MetaOp::CalvinDrop { .. } | MetaOp::ResolveTxn { .. } | MetaOp::CalvinResolve { .. } diff --git a/nodedb/src/control/server/shared/session/commit/conflict.rs b/nodedb/src/control/server/shared/session/commit/conflict.rs deleted file mode 100644 index 8521a5ffc..000000000 --- a/nodedb/src/control/server/shared/session/commit/conflict.rs +++ /dev/null @@ -1,56 +0,0 @@ -// SPDX-License-Identifier: BUSL-1.1 - -//! Snapshot-isolation write-conflict validation for a single-shard COMMIT. - -use crate::control::state::SharedState; - -use super::super::connection::SessionId; -use super::super::outcome::{AbortReason, CommitOutcome}; -use super::super::read_set::ReadSetEntry; -use super::super::store::SessionStore; - -/// Snapshot-isolation write-conflict check for a single-shard interactive -/// COMMIT. If any read key's collection advanced past both the read LSN and the -/// transaction snapshot LSN — and the transaction did not write that collection -/// itself (read-your-own-write is excluded) — the WAL moved under the reader: -/// records the read-set hot-key aborts and returns a serialization abort. The -/// caller owns the session rollback (via `release_and_rollback`) so it can -/// first release the transaction's read reservations while the reservation owner -/// is still set — the rollback clears it. Returns `None` when there is no -/// conflict (or no snapshot, i.e. not in a transaction). -/// -/// This is a single-shard validation: it compares against the global WAL -/// `next_lsn`, so it is only sound for a transaction whose participants are one -/// shard, and is run exclusively on the `SingleShard` / read-only paths. -pub(super) fn si_conflict_abort( - sessions: &SessionStore, - session_id: SessionId, - state: &SharedState, - read_set: &[ReadSetEntry], - written_collections: &std::collections::HashSet, -) -> Option { - let snapshot_lsn = sessions.snapshot_lsn(session_id)?; - let current_lsn = state.wal.next_lsn(); - let current = crate::types::Lsn::new(current_lsn.as_u64().saturating_sub(1)); - for entry in read_set { - // A homed read's watermark can come from another node's WAL, so this - // node's WAL cannot judge it. `homed_reads` validates it on its home. - if entry.home.is_some() { - continue; - } - let collection = &entry.collection; - let read_lsn = entry.read_lsn; - if written_collections.contains(collection) { - continue; - } - if current > read_lsn && current > snapshot_lsn { - // WAL advanced past what we read — concurrent write detected. The - // caller releases reservations and rolls the session back. - super::super::hot_key::record_read_set_aborts(state, read_set); - return Some(CommitOutcome::Aborted { - reason: AbortReason::Serialization, - }); - } - } - None -} diff --git a/nodedb/src/control/server/shared/session/commit/mod.rs b/nodedb/src/control/server/shared/session/commit/mod.rs index 13b97e7cc..26eea7e14 100644 --- a/nodedb/src/control/server/shared/session/commit/mod.rs +++ b/nodedb/src/control/server/shared/session/commit/mod.rs @@ -2,9 +2,9 @@ //! Protocol-neutral COMMIT orchestration shared by pgwire and native sessions. -pub mod conflict; -mod homed_reads; pub mod metering; +mod read_split; +mod read_validation; pub mod restart_identity; pub mod run; pub mod single_shard; diff --git a/nodedb/src/control/server/shared/session/commit/read_split.rs b/nodedb/src/control/server/shared/session/commit/read_split.rs new file mode 100644 index 000000000..69d2db4f5 --- /dev/null +++ b/nodedb/src/control/server/shared/session/commit/read_split.rs @@ -0,0 +1,121 @@ +// SPDX-License-Identifier: BUSL-1.1 + +//! Which reads of a COMMIT enlist Calvin participants, and which this node +//! checks itself before dispatch. +//! +//! A homed read is one vShard of a cross-shard graph or array read. A +//! whole-graph read homes on every vShard. On a node with no cluster routing, +//! every vShard is local, so this node checks such reads against its own +//! cores and they enlist no participant. A single-node transaction that makes +//! a cross-shard read and writes one vShard then keeps its single-vShard +//! commit. In a cluster, every read enlists its vShard, so a participant +//! checks it inside the Calvin barrier. + +use crate::control::state::SharedState; + +use super::super::connection::SessionId; +use super::super::outcome::CommitOutcome; +use super::super::read_set::ReadSetEntry; +use super::super::store::SessionStore; +use super::read_validation::stale_read_abort; + +/// A COMMIT's read set, split by who validates each read. +pub(super) struct SplitReads<'a> { + all: &'a [ReadSetEntry], + /// `(enlisted, checked_here)` when this node checks some reads itself. + local: Option<(Vec, Vec)>, +} + +impl<'a> SplitReads<'a> { + pub(super) fn of(state: &SharedState, all: &'a [ReadSetEntry]) -> Self { + let local = (state.cluster_routing.is_none() && all.iter().any(|read| read.home.is_some())) + .then(|| { + let (here, enlisted): (Vec<_>, Vec<_>) = + all.iter().cloned().partition(|read| read.home.is_some()); + (enlisted, here) + }); + Self { all, local } + } + + /// The reads that enlist their vShard as a participant. + pub(super) fn enlisted(&self) -> &[ReadSetEntry] { + match &self.local { + Some((enlisted, _)) => enlisted, + None => self.all, + } + } + + /// The reads this node checks before dispatch. + fn checked_here(&self) -> &[ReadSetEntry] { + match &self.local { + Some((_, here)) => here, + None => &[], + } + } + + /// Abort the transaction when a read this node checks itself is no + /// longer current. Releases the reservations and rolls the session back + /// before returning the outcome. `None` when every such read holds, or + /// there is none. + pub(super) async fn stale_here( + &self, + state: &SharedState, + sessions: &SessionStore, + session_id: SessionId, + written_collections: &std::collections::HashSet, + ) -> Option { + let here = self.checked_here(); + if here.is_empty() { + return None; + } + stale_read_abort(state, sessions, session_id, here, written_collections).await + } +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::control::server::shared::session::read_set::{EngineTag, ReadKey, ReadOrigin}; + use crate::types::{DatabaseId, TenantId, VShardId}; + + fn read(collection: &str, home: Option) -> ReadSetEntry { + ReadSetEntry { + engine: EngineTag::Graph, + database_id: DatabaseId::DEFAULT, + tenant_id: TenantId::new(1), + collection: collection.to_owned(), + key: ReadKey::Predicate, + read_version: nodedb_types::WriteVersion::ZERO, + origin: ReadOrigin::Session, + home: home.map(VShardId::new), + } + } + + fn single_node() -> (std::sync::Arc, tempfile::TempDir) { + let dir = tempfile::tempdir().expect("tempdir"); + let wal = std::sync::Arc::new( + crate::wal::WalManager::open_for_testing(&dir.path().join("test.wal")).expect("wal"), + ); + let (dispatcher, _sides) = crate::bridge::dispatch::Dispatcher::new(1, 64); + let state = SharedState::new(dispatcher, wal).expect("shared state"); + (state, dir) + } + + #[test] + fn a_single_node_checks_homed_reads_itself() { + let (state, _dir) = single_node(); + let reads = [read("docs", None), read("", Some(3)), read("", Some(9))]; + let split = SplitReads::of(&state, &reads); + assert_eq!(split.enlisted(), &reads[..1]); + assert_eq!(split.checked_here(), &reads[1..]); + } + + #[test] + fn a_read_set_with_no_homed_read_enlists_every_read() { + let (state, _dir) = single_node(); + let reads = [read("docs", None)]; + let split = SplitReads::of(&state, &reads); + assert_eq!(split.enlisted(), &reads[..]); + assert!(split.checked_here().is_empty()); + } +} diff --git a/nodedb/src/control/server/shared/session/commit/homed_reads.rs b/nodedb/src/control/server/shared/session/commit/read_validation.rs similarity index 71% rename from nodedb/src/control/server/shared/session/commit/homed_reads.rs rename to nodedb/src/control/server/shared/session/commit/read_validation.rs index 44aa92d87..6397554f7 100644 --- a/nodedb/src/control/server/shared/session/commit/homed_reads.rs +++ b/nodedb/src/control/server/shared/session/commit/read_validation.rs @@ -1,29 +1,34 @@ // SPDX-License-Identifier: BUSL-1.1 -//! Commit-time validation of a read-only transaction's homed reads. +//! Commit-time validation of a transaction's reads on their vShards' leaders. //! -//! A homed read is one vShard a cross-shard graph read observed (see -//! `session::graph_reads`). A read-write COMMIT goes through Calvin, whose -//! participants check it. A read-only COMMIT dispatches nothing, so this -//! module checks it: it asks each vShard's leader for the home's current -//! version and compares it to the recorded read version. +//! A read-write COMMIT that goes through Calvin is validated by its +//! participants. Every other COMMIT is validated here: a read-only one, and a +//! single-shard one this node applies itself. Each read entry validates on one +//! vShard: its home, else its collection's vShard. This module asks each +//! vShard's leader for its current version and compares it with the version +//! the read recorded. +//! +//! A version is the data-group log position of the write that set it. Every +//! replica numbers a vShard's writes alike, so a read served by a follower, +//! or by a leader that has since lost leadership, compares with the current +//! leader's versions directly. //! //! The check sends one `MetaOp::HomeVersions` per leader node, carrying every -//! home that node leads. The leader hands each probe to the one core that owns -//! the probe's vShard (`exchange::all_cores::home_versions`), and that core -//! answers: +//! vShard that node leads. The leader hands each probe to the one core that +//! owns the probe's vShard (`exchange::all_cores::home_versions`), and that +//! core answers: //! -//! - A read of one collection gets the collection's write floor on that core. -//! - A read of every collection gets the core watermark. +//! - A read of one collection gets the collection's version on the vShard. +//! - A read of every collection gets the vShard's latest version. //! -//! The leader answers a home only under its leader lease on the home's group, -//! applied through the lease read index. A node that lost leadership refuses -//! the home with the leader and term it knows, and the check asks that leader. +//! The leader answers a probe only under its leader lease on the vShard's +//! group, applied through the lease read index. A node that lost leadership +//! refuses the probe with the leader and term it knows, and the check asks +//! that leader. //! -//! A version above the recorded read version means a write landed on that -//! core after the read. Single-shard SI (`conflict::si_conflict_abort`) -//! compares against this node's WAL, so it never judges a homed read: the -//! read's watermark can come from another node's WAL. +//! A version above the recorded read version means a write landed on the +//! vShard after the read. use std::collections::BTreeMap; @@ -39,7 +44,8 @@ use crate::control::gateway::{RouteDecision, TaskRoute}; use crate::control::server::exchange::all_cores::execute_plan_all_local_cores; use crate::control::server::graph_dispatch::cluster_resolve::gateway_shared; use crate::control::state::SharedState; -use crate::types::{DatabaseId, Lsn, TenantId, TraceId, VShardId}; +use crate::types::{DatabaseId, TenantId, TraceId, VShardId}; +use nodedb_types::WriteVersion; use super::super::connection::SessionId; use super::super::outcome::{AbortReason, CommitOutcome}; @@ -60,14 +66,13 @@ const MAX_CHECK_ROUNDS: u32 = 4; /// The wait before a later round, times the round number. const RETRY_BACKOFF: std::time::Duration = std::time::Duration::from_millis(50); -/// One home read: its tenant scope, the vShard and collection it read, and -/// the node that served it. +/// One validated read: its tenant scope, and the vShard and collection it +/// read. #[derive(Debug, Clone, PartialEq, Eq, PartialOrd, Ord, Hash)] struct Home { database_id: u64, tenant_id: u64, probe: HomeVersionProbe, - served_by: u64, } /// One check request: every home one leader leads, in one tenant scope. @@ -78,17 +83,23 @@ struct CheckTarget { tenant_id: u64, } -/// Abort the transaction when a homed read in `read_set` is no longer -/// current, or when a home cannot be checked. Releases the read -/// reservations and rolls the session back before returning the outcome. -/// `None` means every homed read still holds. -pub(super) async fn homed_read_abort( +/// Abort the transaction when a read in `read_set` is no longer current, or +/// when a read cannot be checked. Releases the read reservations and rolls +/// the session back before returning the outcome. `None` means every read +/// still holds. +/// +/// A session read of a collection this transaction writes is skipped. It can +/// have read the transaction's own staged write, which moves no committed +/// version, and the own-write exclusion of the Calvin builders drops it the +/// same way. A plan-derivation read is always checked. +pub(super) async fn stale_read_abort( state: &SharedState, sessions: &SessionStore, session_id: SessionId, read_set: &[ReadSetEntry], + written_collections: &std::collections::HashSet, ) -> Option { - let reason = match homed_reads_changed(state, read_set).await { + let reason = match reads_changed(state, read_set, written_collections).await { Ok(false) => return None, Ok(true) => { super::super::hot_key::record_read_set_aborts(state, read_set); @@ -100,33 +111,32 @@ pub(super) async fn homed_read_abort( Some(CommitOutcome::Aborted { reason }) } -/// Whether any homed read in `read_set` is no longer current. -/// -/// Entries without a home are skipped: SI validates them. +/// Whether any checked read in `read_set` is no longer current. /// /// A read of every collection (a graph read with no collection scope) is -/// checked against the core watermark. That is the conservative rule: any -/// write the core took raises it, to any collection and to any vShard the core -/// owns. Such a check can abort a transaction whose read still holds. It never -/// misses a write that changed the read. +/// checked against the vShard's latest version. Any write to the vShard +/// raises it, so such a check can abort a transaction whose read still holds. +/// It never misses a write that changed the read. /// -/// A leader answers only under its leader lease. A home it refuses is asked +/// A leader answers only under its leader lease. A probe it refuses is asked /// again at the leader the refusal names, or at the routing table's leader -/// when it names none, for at most [`MAX_CHECK_ROUNDS`] rounds. A home still +/// when it names none, for at most [`MAX_CHECK_ROUNDS`] rounds. A probe still /// unanswered then aborts the commit: no answer from a node that lost /// leadership ever lets it through. -/// -/// A version is a position in the answering node's WAL. A home answered by a -/// node other than the one that served the read counts as changed: its -/// versions do not compare with the read's. -async fn homed_reads_changed( +async fn reads_changed( state: &SharedState, read_set: &[ReadSetEntry], + written_collections: &std::collections::HashSet, ) -> crate::Result { - // The earliest read version per home is the one that must still hold. - let mut homes: BTreeMap = BTreeMap::new(); + // The earliest read version per probe is the one that must still hold. + let mut homes: BTreeMap = BTreeMap::new(); for entry in read_set { - let Some(vshard) = entry.home else { + if !entry.origin.survives_own_write_exclusion() + && written_collections.contains(&entry.collection) + { + continue; + } + let Some(vshard) = entry.validation_vshard() else { continue; }; let home = Home { @@ -136,10 +146,9 @@ async fn homed_reads_changed( vshard: vshard.as_u32(), collection: (!entry.collection.is_empty()).then(|| entry.collection.clone()), }, - served_by: entry.home_node, }; - let version = homes.entry(home).or_insert(entry.read_version_lsn); - *version = (*version).min(entry.read_version_lsn); + let version = homes.entry(home).or_insert(entry.read_version); + *version = (*version).min(entry.read_version); } if homes.is_empty() { return Ok(false); @@ -183,10 +192,6 @@ async fn homed_reads_changed( } }); for (target, asked, answers) in futures::future::try_join_all(checks).await? { - let answered_by = match target.leader { - Leader::Local => state.node_id, - Leader::Remote(node_id) => node_id, - }; let by_probe: BTreeMap = answers .into_iter() .map(|answer| (answer.probe, answer.answer)) @@ -195,19 +200,13 @@ async fn homed_reads_changed( let read_version = *homes .get(&home) .ok_or_else(|| unasked_home(target, &home))?; - let answer = *by_probe + let answer = by_probe .get(&home.probe) + .copied() .ok_or_else(|| unasked_home(target, &home))?; match answer { - // Versions are the answering node's WAL positions. Only the - // node that served the read numbers them in the read's own - // domain, so an answer from any other node counts as a - // change. - HomeAnswer::Version(_) if answered_by != home.served_by => { - return Ok(true); - } HomeAnswer::Version(version) => { - if Lsn::new(version) > read_version { + if version > read_version { return Ok(true); } } @@ -236,7 +235,7 @@ async fn homed_reads_changed( return Ok(false); } Err(last_refusal.unwrap_or_else(|| crate::Error::Internal { - detail: "homed read check: a home stayed unanswered; retry the commit".into(), + detail: "read check: a vShard stayed unanswered; retry the commit".into(), })) } @@ -253,7 +252,7 @@ fn leader_of(state: &SharedState, vshard: VShardId) -> crate::Result { }), RouteDecision::Broadcast { .. } => Err(crate::Error::Internal { detail: format!( - "homed read check: vShard {} resolved to a broadcast route", + "read check: vShard {} resolved to a broadcast route", vshard.as_u32() ), }), @@ -310,14 +309,14 @@ async fn current_versions( for payload in payloads { let decoded: Vec = zerompk::from_msgpack(&payload).map_err(|e| crate::Error::Internal { - detail: format!("homed read check: a leader's answer does not decode: {e}"), + detail: format!("read check: a leader's answer does not decode: {e}"), })?; answers.extend(decoded); } if answers.len() != asked { return Err(crate::Error::Internal { detail: format!( - "homed read check: {:?} answered {} of {asked} homes; retry the commit", + "read check: {:?} answered {} of {asked} vShards; retry the commit", target.leader, answers.len() ), @@ -329,7 +328,7 @@ async fn current_versions( fn unasked_home(target: CheckTarget, home: &Home) -> crate::Error { crate::Error::Internal { detail: format!( - "homed read check: {:?} gave no answer for vShard {} ({:?}); retry the commit", + "read check: {:?} gave no answer for vShard {} ({:?}); retry the commit", target.leader, home.probe.vshard, home.probe.collection ), } diff --git a/nodedb/src/control/server/shared/session/commit/run.rs b/nodedb/src/control/server/shared/session/commit/run.rs index 33b4b84f5..b0b861dc2 100644 --- a/nodedb/src/control/server/shared/session/commit/run.rs +++ b/nodedb/src/control/server/shared/session/commit/run.rs @@ -21,9 +21,9 @@ use super::super::outcome::{AbortReason, CommitOutcome, TxnDataPlane}; use super::super::overlay_drop::drop_txn_overlay; use super::super::reservation_release; use super::super::store::SessionStore; -use super::conflict::si_conflict_abort; -use super::homed_reads::homed_read_abort; use super::metering::meter_committed_buffered_writes; +use super::read_split::SplitReads; +use super::read_validation::stale_read_abort; use super::restart_identity::restart_truncated_identities; use super::single_shard::{commit_publishes, dispatch_single_shard}; @@ -83,9 +83,10 @@ pub async fn run_commit( state: &SharedState, dp: &impl TxnDataPlane, ) -> CommitOutcome { - // A graph read made earlier in the same request as this COMMIT is still - // pending in the connection scope. It joins the read-set before validation. - super::super::graph_reads::record_pending(sessions, session_id); + // A cross-shard read made earlier in the same request as this COMMIT is + // still pending in the connection scope. It joins the read-set before + // validation. + super::super::pending_shard_reads::record_pending(sessions, session_id); let read_set = sessions.take_read_set(session_id); // The engine side effects this transaction's index DDL deferred. They run // once its catalog entries landed, below. An abort before then drops them @@ -116,7 +117,10 @@ pub async fn run_commit( // The interactive-COMMIT read-set widens dispatch classification: a txn that // writes shard X but read shard Y participates in {X, Y} and must route // through Calvin with Y as a participant. Autocommit has no session read-set. - let read_vshards = match read_vshards_of(&read_set) { + // A single node checks its homed cross-shard reads itself: they enlist no + // Calvin participant (`read_split`). + let split_reads = SplitReads::of(state, &read_set); + let read_vshards = match read_vshards_of(split_reads.enlisted()) { Ok(vshards) => vshards, Err(error) => { reservation_release::release_and_rollback(state, sessions, session_id).await; @@ -140,19 +144,12 @@ pub async fn run_commit( // anything afterward that can fail in a way compensation will undo. if buffered.is_empty() { // Read-only interactive transaction: no writes to classify, but it can - // still serialization-conflict against concurrent writers. Run the - // single-shard SI validation only — classifying an empty buffer will + // still serialization-conflict against concurrent writers. Each read + // is checked on its vShard's leader. Classifying an empty buffer will // misread a lone cross-shard READ as `MultiShard` and wrongly reject it. if let Some(outcome) = - si_conflict_abort(sessions, session_id, state, &read_set, &written_collections) + stale_read_abort(state, sessions, session_id, &read_set, &written_collections).await { - // Release read reservations (owner still set), then roll back. - reservation_release::release_and_rollback(state, sessions, session_id).await; - return outcome; - } - // SI skips homed reads (the vShards a cross-shard graph read observed). - // Each is checked on its home vShard instead. - if let Some(outcome) = homed_read_abort(state, sessions, session_id, &read_set).await { return outcome; } // No buffered DML for this DDL to race, so finalize it here, @@ -207,11 +204,17 @@ pub async fn run_commit( match classify_dispatch(&buffered, &read_vshards) { DispatchClass::MultiShard { .. } => { // Flush the buffered cross-shard batch through Calvin's durable - // Vote/Verdict barrier (`run_commit_calvin`), leader-routed. SI is - // a single-shard validation and is intentionally NOT run here — - // Calvin performs its own cross-shard OCC over `versioned_reads` - // and returns a serialization abort (SQLSTATE 40001) on an ABORT + // Vote/Verdict barrier (`run_commit_calvin`), leader-routed. + // Calvin's participants validate `versioned_reads` themselves + // and return a serialization abort (SQLSTATE 40001) on an ABORT // verdict. + if let Some(outcome) = split_reads + .stale_here(state, sessions, session_id, &written_collections) + .await + { + compensate_finalized_ddl(state, &ddl_compensation).await; + return outcome; + } if let Some(reason) = commit_calvin::run_commit_calvin( sessions, session_id, @@ -219,7 +222,7 @@ pub async fn run_commit( commit_calvin::CalvinCommit { buffered: &buffered, tenant_id, - reads: &read_set, + reads: split_reads.enlisted(), event_source: dp.event_source(), publishes: &publishes, applied_key: dp.applied_key().zip(dp.applied_key_vshard()), @@ -250,6 +253,13 @@ pub async fn run_commit( // A commit that writes an edge takes it too: every edge // version takes a Calvin ordinal, so it orders against a // TRUNCATE's cut on every replica. + if let Some(outcome) = split_reads + .stale_here(state, sessions, session_id, &written_collections) + .await + { + compensate_finalized_ddl(state, &ddl_compensation).await; + return outcome; + } if let Some(reason) = commit_calvin::run_commit_calvin( sessions, session_id, @@ -257,7 +267,7 @@ pub async fn run_commit( commit_calvin::CalvinCommit { buffered: &buffered, tenant_id, - reads: &read_set, + reads: split_reads.enlisted(), event_source: dp.event_source(), publishes: &publishes, applied_key: dp.applied_key().zip(dp.applied_key_vshard()), @@ -272,20 +282,14 @@ pub async fn run_commit( return CommitOutcome::Aborted { reason }; } } else { - if let Some(outcome) = si_conflict_abort( + if let Some(outcome) = stale_read_abort( + state, sessions, session_id, - state, &read_set, &written_collections, - ) { - compensate_finalized_ddl(state, &ddl_compensation).await; - reservation_release::release_and_rollback(state, sessions, session_id) - .await; - return outcome; - } - if let Some(outcome) = - homed_read_abort(state, sessions, session_id, &read_set).await + ) + .await { compensate_finalized_ddl(state, &ddl_compensation).await; return outcome; diff --git a/nodedb/src/control/server/shared/session/commit/single_shard.rs b/nodedb/src/control/server/shared/session/commit/single_shard.rs index 2ea89a2a4..e61fe5d36 100644 --- a/nodedb/src/control/server/shared/session/commit/single_shard.rs +++ b/nodedb/src/control/server/shared/session/commit/single_shard.rs @@ -14,9 +14,9 @@ use nodedb_physical::physical_task::{PhysicalTask, PostSetOp}; use crate::bridge::envelope::{PhysicalPlan, Status}; use crate::control::gateway::RouteDecision; use crate::control::state::SharedState; -use crate::control::wal_replication::encode::transaction_redo_entry; -use crate::control::wal_replication::propose_replicated_entry; -use crate::control::wal_replication::transaction_redo::{RedoTarget, TransactionRedoPayload}; +use crate::control::wal_replication::transaction_redo::{ + RedoTarget, TransactionRedoPayload, propose_transaction_redo, +}; use crate::wal::RedoPublish; use super::super::outcome::{AbortReason, TxnDataPlane}; @@ -201,19 +201,14 @@ async fn commit_redo( target: RedoTarget, payload: &TransactionRedoPayload, ) -> Result { - if let Err(e) = inject_commit_failure() { + if let Err(e) = inject_commit_failure(state.node_id) { return Err(AbortReason::Dispatch(e)); } - let proposer = state.async_raft_proposer().map_err(AbortReason::Dispatch)?; - // The proposer forwards to the group leader and returns once the entry is - // committed and applied on this node by the apply loop. - let entry = transaction_redo_entry( - target.tenant_id, - target.database_id, - target.vshard_id, - payload, - ); - match propose_replicated_entry(state, proposer, entry).await { + // The proposal forwards to the group leader and returns once the redo is + // committed and applied on this node by the apply loop. A redo past the + // entry limit travels as chunks, then its final entry. + let deadline = crate::control::wal_replication::statement_propose_deadline(state); + match propose_transaction_redo(state, target, payload, deadline).await { Ok((applied, _)) => Ok(super::ts_rejections::applied_rejected_by_collection( &applied, )), @@ -225,11 +220,13 @@ async fn commit_redo( } } -/// The `commit::single_shard_redo_commit` fail point. Compiles to `Ok(())` -/// outside the `failpoints` feature. -fn inject_commit_failure() -> crate::Result<()> { - crate::fail_point_err!("commit::single_shard_redo_commit", |detail| { - crate::Error::Internal { detail } - }); +/// The `commit::single_shard_redo_commit` fail point of node `node_id`. +/// Compiles to `Ok(())` outside the `failpoints` feature. +fn inject_commit_failure(node_id: u64) -> crate::Result<()> { + crate::fail_point_err!( + crate::fail_point::FailScope::Node(node_id), + "commit::single_shard_redo_commit", + |detail| crate::Error::Internal { detail } + ); Ok(()) } diff --git a/nodedb/src/control/server/shared/session/commit_calvin.rs b/nodedb/src/control/server/shared/session/commit_calvin.rs index bbeec076f..592e41b5b 100644 --- a/nodedb/src/control/server/shared/session/commit_calvin.rs +++ b/nodedb/src/control/server/shared/session/commit_calvin.rs @@ -160,9 +160,10 @@ async fn submit_commit_calvin( // independent single-vShard Calvin transaction, sequenced through the // SAME deterministic funnel the contended point-write path uses // (`build_single_vshard_tx_class` + `submit_calvin_routed`): the - // scheduler resolves it into a `TransactionRedo`, WAL-appends it, and - // the Calvin flush sets `epoch_system_ms` so every engine's bitemporal - // stamp is epoch-anchored and byte-identical on replay. + // scheduler resolves it into a `TransactionRedo` and proposes it as a + // stamped data-group entry. The install sets `epoch_system_ms`, so + // every engine's bitemporal stamp is epoch-anchored and + // byte-identical on replay. // // Non-atomic ACROSS vShards is preserved by construction: each group // is a separate submit-and-await with its own single-participant @@ -196,10 +197,9 @@ async fn submit_commit_calvin( } else { event_source }; - // Empty read-set: best-effort performs no cross-shard OCC (the - // multi-shard COMMIT path never ran `si_conflict_abort`), so each - // group carries no versioned reads — matching the single-vShard - // submit `route_write_to_calvin` uses. + // Empty read-set: best-effort performs no cross-shard OCC, so + // each group carries no versioned reads — matching the + // single-vShard submit `route_write_to_calvin` uses. // A timeseries ingest resolves to its rows here, before it is // sequenced: every replica resolves a sequenced group on its own. let tasks = diff --git a/nodedb/src/control/server/shared/session/conn_scope.rs b/nodedb/src/control/server/shared/session/conn_scope.rs index cd4994f79..35de2f8d2 100644 --- a/nodedb/src/control/server/shared/session/conn_scope.rs +++ b/nodedb/src/control/server/shared/session/conn_scope.rs @@ -15,7 +15,7 @@ use std::time::Instant; use super::audit_context::AuditCtx; use super::ddl_buffer::DdlBuffer; use super::ephemeral_sequence::EphemeralSequences; -use super::graph_reads::GraphShardReads; +use super::pending_shard_reads::ShardReads; /// Per-connection session slots. Each field is owned by its own module, which /// exposes the accessors; nothing outside reaches through this struct. @@ -33,12 +33,9 @@ pub(super) struct ConnScope { /// Client-facing notices raised while the current statement ran, below /// the response shaper (see [`super::statement_notice`]). pub(super) statement_notices: RefCell>, - /// Cross-shard graph reads the current request made, not yet recorded - /// into the transaction read-set (see [`super::graph_reads`]). - pub(super) graph_reads: RefCell>, - /// The node that served each vShard the current statement read (see - /// [`super::served_reads`]). - pub(super) served_reads: RefCell>, + /// Cross-shard reads the current request made, not yet recorded into the + /// transaction read-set (see [`super::pending_shard_reads`]). + pub(super) pending_shard_reads: RefCell>, } impl ConnScope { @@ -49,8 +46,7 @@ impl ConnScope { ephemeral_sequences: RefCell::new(EphemeralSequences::new()), statement_deadline: Cell::new(None), statement_notices: RefCell::new(Vec::new()), - graph_reads: RefCell::new(Vec::new()), - served_reads: super::served_reads::empty(), + pending_shard_reads: RefCell::new(Vec::new()), } } } diff --git a/nodedb/src/control/server/shared/session/ddl_compensate.rs b/nodedb/src/control/server/shared/session/ddl_compensate.rs index 651de26d5..e698bc197 100644 --- a/nodedb/src/control/server/shared/session/ddl_compensate.rs +++ b/nodedb/src/control/server/shared/session/ddl_compensate.rs @@ -73,7 +73,7 @@ async fn propose_reversals( entry: Box::new(MetadataEntry::Batch { entries }), }; let log_index = propose_and_await(state, handle, &prepared).await?; - if state.metadata_ddl_applied_token.load(Ordering::Acquire) != token { + if state.metadata_ddl.applied_token.load(Ordering::Acquire) != token { return Err(crate::Error::Config { detail: "commit compensation: DDL preparation ownership was superseded before apply" .into(), diff --git a/nodedb/src/control/server/shared/session/ddl_flush.rs b/nodedb/src/control/server/shared/session/ddl_flush.rs index 3499474d7..296a862cf 100644 --- a/nodedb/src/control/server/shared/session/ddl_flush.rs +++ b/nodedb/src/control/server/shared/session/ddl_flush.rs @@ -373,7 +373,8 @@ async fn finalize_reserved( // cancels the record first, so a finalize that found nothing applied // nothing. if state - .metadata_ddl_applied_token + .metadata_ddl + .applied_token .load(std::sync::atomic::Ordering::Acquire) != token { @@ -713,7 +714,8 @@ mod tests { ); assert!( state - .metadata_ddl_owner + .metadata_ddl + .owner .lock() .expect("owner lock") .is_none(), diff --git a/nodedb/src/control/server/shared/session/deadline.rs b/nodedb/src/control/server/shared/session/deadline.rs index 6831efd36..cbe6447db 100644 --- a/nodedb/src/control/server/shared/session/deadline.rs +++ b/nodedb/src/control/server/shared/session/deadline.rs @@ -57,8 +57,7 @@ impl Drop for StatementScope { } } -/// Pin the deadline for the statement about to run, and start the -/// statement's served-read notes empty. +/// Pin the deadline for the statement about to run. /// /// The instant is fixed ONCE here, so every request the statement fans out into /// shares it and the statement is bounded end to end rather than per hop. @@ -67,8 +66,6 @@ impl Drop for StatementScope { /// store is a no-op and every envelope site falls back to the node default, so /// installing the guard is always safe. pub fn enter(statement_timeout: Option, default_deadline_secs: u64) -> StatementScope { - // A statement starts with no reads served (`served_reads`). - super::served_reads::clear(); let previous = current(); store(Some( Instant::now() + statement_budget(statement_timeout, default_deadline_secs), diff --git a/nodedb/src/control/server/shared/session/expander_stage.rs b/nodedb/src/control/server/shared/session/expander_stage.rs index a03666cdc..7927ae991 100644 --- a/nodedb/src/control/server/shared/session/expander_stage.rs +++ b/nodedb/src/control/server/shared/session/expander_stage.rs @@ -17,6 +17,10 @@ use super::store::SessionStore; use crate::bridge::envelope::{PhysicalPlan, Response}; use crate::control::insert_select::resolve_and_emit_insert_select_ops; use crate::control::merge_orchestrator::resolve_and_emit_merge_ops; +use crate::control::planner::calvin::write_class::{ + plan_counts_toward_statement_tag, plans_have_user_write, +}; +use crate::control::server::shared::plan_admission::append_sum_and_period_targets; use crate::control::state::SharedState; use crate::control::update_from_join_orchestrator::resolve_and_emit_update_from_join_ops; use nodedb_physical::physical_plan::DocumentOp; @@ -61,7 +65,8 @@ where /// every other plan, and outside a transaction block. /// /// The ops are `PointInsert` for an inserted row, `PointPut` for an updated -/// row's post-image, and `PointDelete` for a removed row. +/// row's post-image, `PointDelete` for a removed row, and an +/// `ApplyBalanceDelta` for each cross-shard materialized-sum balance they move. pub(crate) async fn expand_in_tx( state: &SharedState, sessions: &SessionStore, @@ -72,7 +77,7 @@ pub(crate) async fn expand_in_tx( if sessions.transaction_state(session_id) != TransactionState::InBlock { return Ok(None); } - let expanded = match &task.plan { + let (mut ops, kind) = match &task.plan { PhysicalPlan::Document(DocumentOp::Merge { resolved_inserts: None, .. @@ -105,13 +110,35 @@ pub(crate) async fn expand_in_tx( } // A `BatchInsert` page is autocommit-shaped; only point ops give an overlay // post-image, per-row undo, and row-level redo, so expand back to points. - PhysicalPlan::Document(DocumentOp::BatchInsert { .. }) => ( - expand_batch_insert(task).map_err(StagingGateError::Dispatch)?, - StagedTagKind::Insert, - ), + // The page carries the statement's resolution and deferrals already. + PhysicalPlan::Document(DocumentOp::BatchInsert { .. }) => { + return Ok(Some(( + expand_batch_insert(task).map_err(StagingGateError::Dispatch)?, + StagedTagKind::Insert, + ))); + } _ => return Ok(None), }; - Ok(Some(expanded)) + // The expanded point writes never passed statement admission. They take + // its sum and period-lock pass here. A cross-shard balance ships on an + // `ApplyBalanceDelta` task homed on the target's vShard, and the source + // write does not fold it. The pre-image each balance settles from is read + // through the transaction's staging overlay, the same view the expansion + // resolved its rows against. + let reads = append_sum_and_period_targets( + state, + &mut ops, + task.tenant_id, + task.database_id, + sessions.tx_id(session_id), + crate::types::TraceId::ZERO, + ) + .await + .map_err(StagingGateError::Dispatch)?; + // The images each shipped balance was settled from join the read set, so + // COMMIT aborts when a concurrent write moved them. + sessions.record_read_entries(session_id, reads); + Ok(Some((ops, kind))) } /// Expand a `BatchInsert` page into one `PointInsert` op per row. Nothing is @@ -186,8 +213,12 @@ where { // Each `stage_write` dispatches into the overlay AND buffers the concrete op // for COMMIT replay — the raw Merge/UpdateFromJoin/InsertSelect is never buffered. + // A balance task the expansion appended stages too, but its count describes + // a row the statement never named. + let has_user_write = plans_have_user_write(ops.iter().map(|op| &op.plan)); let mut affected = 0usize; for op in ops { + let counts = plan_counts_toward_statement_tag(&op.plan, has_user_write); // A removed row of an edge-bearing collection carries its node's edge // tasks, read before the removal stages. let node_delete_tasks = @@ -199,7 +230,9 @@ where .await .map_err(StagingGateError::Dispatch)?; let outcome = stage_write(state, sessions, session_id, op, &dispatch).await?; - affected += outcome.affected; + if counts { + affected += outcome.affected; + } super::node_delete_stage::stage_node_delete_tasks( state, sessions, diff --git a/nodedb/src/control/server/shared/session/graph_reads.rs b/nodedb/src/control/server/shared/session/graph_reads.rs deleted file mode 100644 index 545cd9291..000000000 --- a/nodedb/src/control/server/shared/session/graph_reads.rs +++ /dev/null @@ -1,197 +0,0 @@ -// SPDX-License-Identifier: BUSL-1.1 - -//! Cross-shard graph reads recorded into the transaction read-set. -//! -//! A cluster graph read (a MATCH, a walk, a gathered algorithm) reads edges on -//! many key vShards through a coordinator that holds no session. The -//! coordinator notes every vShard it read, each at the watermark that shard -//! served, in the connection scope ([`note`]). Each protocol records the noted -//! reads when the request ends ([`record_pending`]). -//! -//! Each vShard becomes one [`ReadSetEntry`] homed on that vShard. Commit -//! validation then checks the entry on the vShard that holds the edges: -//! -//! - A Calvin participant checks the collection's write floor on that vShard. -//! - A read-only commit fetches that floor from the vShard's leader -//! (`commit::homed_reads`). -//! -//! The read version is the served watermark. A collection's write floor on a -//! core never exceeds the core watermark, and a later write to the collection -//! on that core carries a higher LSN. So a floor above the recorded watermark -//! means the collection changed on that vShard after the read. - -use crate::types::{DatabaseId, Lsn, TenantId, VShardId}; - -use super::conn_scope::with_scope; -use super::connection::SessionId; -use super::read_set::{EngineTag, ReadKey, ReadOrigin, ReadSetEntry}; -use super::store::SessionStore; - -/// One vShard a graph read observed, at the watermark its core served, and -/// the node that served it (`0` when more than one did). -#[derive(Debug, Clone, Copy, PartialEq, Eq)] -pub struct ShardObservation { - pub vshard: VShardId, - pub watermark: Lsn, - pub node: u64, -} - -/// The vShards one cross-shard graph read observed. -#[derive(Debug, Clone, PartialEq, Eq)] -pub struct GraphShardReads { - pub tenant_id: TenantId, - pub database_id: DatabaseId, - /// The database-qualified collection the read scoped, or `None` when it - /// walked every collection. - pub collection: Option, - pub shards: Vec, -} - -/// Note a cross-shard graph read for the request now running. Outside a -/// connection scope there is no session to record it for, so it is dropped. -pub fn note(reads: GraphShardReads) { - if reads.shards.is_empty() { - return; - } - let mut pending = Some(reads); - with_scope((), |scope| { - if let Some(reads) = pending.take() { - scope.graph_reads.borrow_mut().push(reads); - } - }); -} - -/// Take every graph read the request noted, oldest first. -pub fn take() -> Vec { - with_scope(Vec::new(), |scope| { - std::mem::take(&mut *scope.graph_reads.borrow_mut()) - }) -} - -/// Record the graph reads the request noted into `session_id`'s transaction -/// read-set. Outside a transaction block the reads are dropped. -pub fn record_pending(sessions: &SessionStore, session_id: SessionId) { - let pending = take(); - if pending.is_empty() || !sessions.is_in_transaction_block(session_id) { - return; - } - for reads in pending { - let entries = homed_entries(sessions, session_id, reads); - sessions.record_read_entries(session_id, entries); - } -} - -/// One predicate entry per observed vShard, homed there. -fn homed_entries( - sessions: &SessionStore, - session_id: SessionId, - reads: GraphShardReads, -) -> Vec { - let GraphShardReads { - tenant_id, - database_id, - collection, - shards, - } = reads; - let collection = collection.unwrap_or_default(); - // The session's own committed writes to the collection raise the read - // version, as `record_read_set` does, so a read never aborts on the - // session's own earlier write. - let own_write_version = if collection.is_empty() { - Lsn::ZERO - } else { - sessions.own_write_version(session_id, database_id, tenant_id, &collection) - }; - shards - .into_iter() - .map(|shard| ReadSetEntry { - engine: EngineTag::Graph, - database_id, - tenant_id, - collection: collection.clone(), - key: ReadKey::Predicate, - read_lsn: shard.watermark, - read_version_lsn: shard.watermark.max(own_write_version), - origin: ReadOrigin::Session, - home: Some(shard.vshard), - home_node: shard.node, - }) - .collect() -} - -#[cfg(test)] -mod tests { - use super::super::conn_scope; - use super::*; - - fn session_id() -> SessionId { - SessionId::from( - "127.0.0.1:5611" - .parse::() - .expect("test address"), - ) - } - - fn reads(collection: Option<&str>) -> GraphShardReads { - GraphShardReads { - tenant_id: TenantId::new(1), - database_id: DatabaseId::DEFAULT, - collection: collection.map(str::to_owned), - shards: vec![ - ShardObservation { - vshard: VShardId::new(3), - watermark: Lsn::new(10), - node: 1, - }, - ShardObservation { - vshard: VShardId::new(9), - watermark: Lsn::new(20), - node: 2, - }, - ], - } - } - - fn store_in_block() -> (SessionStore, SessionId) { - let sessions = SessionStore::new(); - let session_id = session_id(); - sessions.ensure_session(match session_id { - SessionId::LegacySocket(addr) => addr, - SessionId::Connection(_) => unreachable!("legacy test identity"), - }); - sessions.begin(session_id, Lsn::new(5), 0).expect("begin"); - (sessions, session_id) - } - - #[tokio::test] - async fn each_observed_vshard_becomes_one_homed_entry() { - let (sessions, session_id) = store_in_block(); - conn_scope::scoped(async { - note(reads(Some("db1.edges"))); - record_pending(&sessions, session_id); - }) - .await; - let entries = sessions.take_read_set(session_id); - assert_eq!(entries.len(), 2); - assert_eq!(entries[0].home, Some(VShardId::new(3))); - assert_eq!(entries[0].read_version_lsn, Lsn::new(10)); - assert_eq!(entries[1].home, Some(VShardId::new(9))); - assert_eq!(entries[1].read_version_lsn, Lsn::new(20)); - assert_eq!(entries[1].home_node, 2); - assert!(entries.iter().all(|e| e.key == ReadKey::Predicate)); - assert!(entries.iter().all(|e| e.collection == "db1.edges")); - } - - #[tokio::test] - async fn a_read_outside_a_transaction_block_records_nothing() { - let sessions = SessionStore::new(); - let session_id = session_id(); - conn_scope::scoped(async { - note(reads(None)); - record_pending(&sessions, session_id); - assert!(take().is_empty(), "the pending reads are drained"); - }) - .await; - assert!(sessions.take_read_set(session_id).is_empty()); - } -} diff --git a/nodedb/src/control/server/shared/session/leader_forward.rs b/nodedb/src/control/server/shared/session/leader_forward.rs index 497ef245f..843a3c180 100644 --- a/nodedb/src/control/server/shared/session/leader_forward.rs +++ b/nodedb/src/control/server/shared/session/leader_forward.rs @@ -120,7 +120,7 @@ pub(crate) async fn forward_to_leader( let payload = outcome.payloads.into_iter().next().unwrap_or_default(); let mut resp = bare_ok_response(RequestId::new(0)); resp.payload = Payload::from_vec(payload); - resp.read_version_lsn = outcome.read_version_lsn; + resp.read_versions = outcome.read_versions; Ok(resp) } diff --git a/nodedb/src/control/server/shared/session/lifecycle.rs b/nodedb/src/control/server/shared/session/lifecycle.rs index 2fddfda86..620b81fb9 100644 --- a/nodedb/src/control/server/shared/session/lifecycle.rs +++ b/nodedb/src/control/server/shared/session/lifecycle.rs @@ -24,10 +24,6 @@ pub fn run_begin( session_id: SessionId, state: &SharedState, ) -> Result<(), crate::Error> { - let snapshot_lsn = { - let next = state.wal.next_lsn(); - crate::types::Lsn::new(next.as_u64().saturating_sub(1)) - }; // Last globally-applied Calvin epoch as the cross-shard snapshot anchor. // 0 in single-node / no-Calvin deployments (the atomic is never advanced). let snapshot_epoch = state @@ -36,7 +32,7 @@ pub fn run_begin( .load(std::sync::atomic::Ordering::Acquire); ddl_buffer::activate(); sessions - .begin(session_id, snapshot_lsn, snapshot_epoch) + .begin(session_id, snapshot_epoch) .map_err(|msg| crate::Error::BadRequest { detail: msg.to_owned(), }) @@ -198,8 +194,8 @@ mod tests { payload: Payload::empty(), watermark_lsn: Lsn::ZERO, error_code: None, - read_set_valid: None, - read_version_lsn: crate::types::Lsn::ZERO, + stage_vote: None, + read_versions: crate::types::ReadVersions::new(), write_set: Vec::new(), }) }) diff --git a/nodedb/src/control/server/shared/session/live.rs b/nodedb/src/control/server/shared/session/live.rs index 87da175b9..66c1c2c18 100644 --- a/nodedb/src/control/server/shared/session/live.rs +++ b/nodedb/src/control/server/shared/session/live.rs @@ -472,7 +472,7 @@ mod tests { let mut live = LiveSubscription::new("live_orders".into(), subscription); let event = |index, floor| { SequencedChangeEvent::new( - ChangePartition::Group(2), + ChangePartition(2), CdcOffset::data_event(0, index, 1), floor, DatabaseId::DEFAULT, diff --git a/nodedb/src/control/server/shared/session/mod.rs b/nodedb/src/control/server/shared/session/mod.rs index 508e095f4..45b8465a4 100644 --- a/nodedb/src/control/server/shared/session/mod.rs +++ b/nodedb/src/control/server/shared/session/mod.rs @@ -19,7 +19,6 @@ pub mod ddl_rollback; pub mod deadline; pub mod ephemeral_sequence; pub mod expander_stage; -pub mod graph_reads; mod hot_key; mod leader_forward; pub mod lifecycle; @@ -31,13 +30,13 @@ pub mod outcome; pub mod overlay_drop; mod own_writes; mod params; +pub mod pending_shard_reads; pub mod read_consistency; pub mod read_set; pub mod record_reads; mod reservation_release; pub mod savepoint_ops; mod sequence_values; -pub mod served_reads; pub mod set_validation; pub mod staging_gate; mod state; diff --git a/nodedb/src/control/server/shared/session/own_writes.rs b/nodedb/src/control/server/shared/session/own_writes.rs index a88ce822a..01564fe6e 100644 --- a/nodedb/src/control/server/shared/session/own_writes.rs +++ b/nodedb/src/control/server/shared/session/own_writes.rs @@ -2,73 +2,111 @@ //! Per-session read-your-writes floor tracking on `SessionStore`. //! -//! Records the highest committed write-version this session has observed for -//! each `(database, tenant, collection)` it has written, so a later -//! transaction's read-set capture can be floored at the session's OWN prior -//! committed writes. This extends the read-your-own-write exclusion already -//! applied to a transaction's buffered writes (see the Calvin static builder) -//! to the session's PRIOR committed autocommit writes: without it, a -//! cross-shard OCC validation would see the session's own committed -//! `coll_write_lsn` exceed a read that captured a stale floor and false-abort -//! with a serialization failure. +//! Records the highest committed write version this session holds for each +//! `(database, tenant, collection, vShard)` it has written. A later +//! transaction's read-set capture is floored at it. This extends the +//! read-your-own-write exclusion the Calvin static builder applies to buffered +//! writes to the session's prior committed writes. Without it, OCC validation +//! sees the session's own committed write above a read that captured an older +//! version, and aborts the transaction on its own write. //! -//! Soundness: the floor is only ever RAISED by this session's own committed -//! writes to that exact `(database, tenant, collection)`. A concurrent -//! OTHER-session write yields a higher `coll_write_lsn` that still exceeds the -//! floor, so a genuine conflict still aborts — this only removes the self-abort, -//! never a real one. +//! Only this session's own committed writes raise the floor. A concurrent +//! write by another session has a higher version that still exceeds the +//! floor, so a real conflict still aborts. + +use nodedb_types::WriteVersion; use super::connection::SessionId; -use crate::types::{DatabaseId, Lsn, TenantId}; +use crate::bridge::envelope::{PhysicalPlan, Response, Status}; +use crate::control::security::identity::{Permission, required_permission}; +use crate::control::server::shared::plan_util::extract_collection; +use crate::types::{DatabaseId, ReadVersions, TenantId, VShardId}; use super::store::SessionStore; +/// The scope of one own-write floor: database, tenant, collection, and the +/// vShard whose versions the floor is a position in. +pub type OwnWriteKey = (DatabaseId, TenantId, String, VShardId); + impl SessionStore { - /// Record a committed write-version for `(database, tenant, collection)` on - /// this session, keeping the maximum seen. `version` is the write's - /// committed per-collection version (`coll_write_lsn`), sourced from the - /// replicated-write response. A `Lsn::ZERO` version carries no floor and is - /// ignored. Persists for the life of the session. + /// Record the versions a dispatched write of `plan` stamped, taken from + /// its `response`. Every protocol calls this once per dispatched task. + /// A read, a failed write, or a write of no single collection records + /// nothing. + pub fn note_own_write_response( + &self, + addr: impl Into, + database_id: DatabaseId, + tenant_id: TenantId, + plan: &PhysicalPlan, + response: &Response, + ) { + if response.status != Status::Ok + || response.read_versions.is_empty() + || !matches!(required_permission(plan), Permission::Write) + { + return; + } + let Some(collection) = extract_collection(plan) else { + return; + }; + self.note_own_write( + addr, + database_id, + tenant_id, + collection, + &response.read_versions, + ); + } + + /// Record the versions a committed write to `(database, tenant, + /// collection)` stamped, keeping the maximum per vShard. `versions` come + /// from the write's response. A write that stamped none sets no floor. + /// The floors persist for the life of the session. pub fn note_own_write( &self, addr: impl Into, database_id: DatabaseId, tenant_id: TenantId, collection: &str, - version: Lsn, + versions: &ReadVersions, ) { - if version == Lsn::ZERO { + if versions.is_empty() { return; } self.write_session(addr, |session| { - let slot = session - .own_write_versions - .entry((database_id, tenant_id, collection.to_string())) - .or_insert(Lsn::ZERO); - if version > *slot { - *slot = version; + for shard in versions.iter() { + let key = ( + database_id, + tenant_id, + collection.to_string(), + VShardId::new(shard.vshard), + ); + let slot = session.own_write_versions.entry(key).or_default(); + *slot = (*slot).max(shard.version); } }); } - /// Return the session's own highest committed write-version for - /// `(database, tenant, collection)`, or `Lsn::ZERO` if the session never - /// wrote that collection (no floor to apply). + /// The session's own highest committed write version for `(database, + /// tenant, collection)` on `vshard`, or `WriteVersion::ZERO` when the + /// session never wrote it there. pub fn own_write_version( &self, addr: impl Into, database_id: DatabaseId, tenant_id: TenantId, collection: &str, - ) -> Lsn { + vshard: VShardId, + ) -> WriteVersion { self.read_session(addr, |session| { session .own_write_versions - .get(&(database_id, tenant_id, collection.to_string())) + .get(&(database_id, tenant_id, collection.to_string(), vshard)) .copied() - .unwrap_or(Lsn::ZERO) + .unwrap_or_default() }) - .unwrap_or(Lsn::ZERO) + .unwrap_or_default() } } @@ -78,6 +116,8 @@ mod tests { use super::*; + const HOME: VShardId = VShardId::new(4); + fn addr() -> SocketAddr { "127.0.0.1:5601".parse().expect("test addr") } @@ -89,83 +129,141 @@ mod tests { (sessions, a) } + fn at(index: u64) -> ReadVersions { + ReadVersions::single(HOME, WriteVersion::logged(1, index)) + } + + fn floor( + sessions: &SessionStore, + a: SocketAddr, + tenant: u64, + collection: &str, + ) -> WriteVersion { + sessions.own_write_version( + a, + DatabaseId::DEFAULT, + TenantId::new(tenant), + collection, + HOME, + ) + } + #[test] fn absent_collection_returns_zero() { let (sessions, a) = store_with_session(); - assert_eq!( - sessions.own_write_version(a, DatabaseId::DEFAULT, TenantId::new(1), "bread"), - Lsn::ZERO - ); + assert_eq!(floor(&sessions, a, 1, "bread"), WriteVersion::ZERO); } #[test] fn records_and_returns_own_write_version() { let (sessions, a) = store_with_session(); - sessions.note_own_write( - a, - DatabaseId::DEFAULT, - TenantId::new(1), - "bread", - Lsn::new(2), - ); - assert_eq!( - sessions.own_write_version(a, DatabaseId::DEFAULT, TenantId::new(1), "bread"), - Lsn::new(2) - ); + sessions.note_own_write(a, DatabaseId::DEFAULT, TenantId::new(1), "bread", &at(2)); + assert_eq!(floor(&sessions, a, 1, "bread"), WriteVersion::logged(1, 2)); } #[test] fn keeps_the_maximum_version() { let (sessions, a) = store_with_session(); - sessions.note_own_write( - a, - DatabaseId::DEFAULT, - TenantId::new(1), - "bread", - Lsn::new(5), - ); + sessions.note_own_write(a, DatabaseId::DEFAULT, TenantId::new(1), "bread", &at(5)); // A lower version never lowers the floor. + sessions.note_own_write(a, DatabaseId::DEFAULT, TenantId::new(1), "bread", &at(3)); + assert_eq!(floor(&sessions, a, 1, "bread"), WriteVersion::logged(1, 5)); + } + + #[test] + fn a_write_with_no_versions_sets_no_floor() { + let (sessions, a) = store_with_session(); sessions.note_own_write( a, DatabaseId::DEFAULT, TenantId::new(1), "bread", - Lsn::new(3), + &ReadVersions::new(), ); - assert_eq!( - sessions.own_write_version(a, DatabaseId::DEFAULT, TenantId::new(1), "bread"), - Lsn::new(5) + assert_eq!(floor(&sessions, a, 1, "bread"), WriteVersion::ZERO); + } + + fn kv_plan(collection: &str, write: bool) -> PhysicalPlan { + let collection = nodedb_types::QualifiedCollection::new(DatabaseId::DEFAULT, collection); + PhysicalPlan::Kv(if write { + nodedb_physical::physical_plan::KvOp::Put { + collection, + key: b"k".to_vec(), + value: b"v".to_vec(), + ttl_ms: 0, + surrogate: nodedb_types::Surrogate::new(1), + returning: None, + rls_filters: Vec::new(), + provenance: None, + } + } else { + nodedb_physical::physical_plan::KvOp::Get { + collection, + key: b"k".to_vec(), + rls_filters: Vec::new(), + surrogate_ceiling: None, + } + }) + } + + fn response(status: Status, versions: ReadVersions) -> Response { + let mut response = crate::control::server::shared::write_admission::bare_ok_response( + crate::types::RequestId::new(1), ); + response.status = status; + response.read_versions = versions; + response } #[test] - fn zero_version_is_ignored() { + fn a_dispatched_write_response_sets_the_floor() { let (sessions, a) = store_with_session(); - sessions.note_own_write(a, DatabaseId::DEFAULT, TenantId::new(1), "bread", Lsn::ZERO); - assert_eq!( - sessions.own_write_version(a, DatabaseId::DEFAULT, TenantId::new(1), "bread"), - Lsn::ZERO + sessions.note_own_write_response( + a, + DatabaseId::DEFAULT, + TenantId::new(1), + &kv_plan("bread", true), + &response(Status::Ok, at(9)), ); + assert_eq!(floor(&sessions, a, 1, "bread"), WriteVersion::logged(1, 9)); } #[test] - fn scoped_by_database_tenant_and_collection() { + fn a_read_or_a_failed_write_sets_no_floor() { let (sessions, a) = store_with_session(); - sessions.note_own_write( + sessions.note_own_write_response( a, DatabaseId::DEFAULT, TenantId::new(1), - "bread", - Lsn::new(7), + &kv_plan("bread", false), + &response(Status::Ok, at(9)), ); - // A different collection, tenant, or database sees no floor. - assert_eq!( - sessions.own_write_version(a, DatabaseId::DEFAULT, TenantId::new(1), "milk"), - Lsn::ZERO + sessions.note_own_write_response( + a, + DatabaseId::DEFAULT, + TenantId::new(1), + &kv_plan("bread", true), + &response(Status::Error, at(9)), ); + assert_eq!(floor(&sessions, a, 1, "bread"), WriteVersion::ZERO); + } + + #[test] + fn scoped_by_database_tenant_collection_and_vshard() { + let (sessions, a) = store_with_session(); + sessions.note_own_write(a, DatabaseId::DEFAULT, TenantId::new(1), "bread", &at(7)); + // A different collection, tenant, or vShard sees no floor. + assert_eq!(floor(&sessions, a, 1, "milk"), WriteVersion::ZERO); + assert_eq!(floor(&sessions, a, 2, "bread"), WriteVersion::ZERO); assert_eq!( - sessions.own_write_version(a, DatabaseId::DEFAULT, TenantId::new(2), "bread"), - Lsn::ZERO + sessions.own_write_version( + a, + DatabaseId::DEFAULT, + TenantId::new(1), + "bread", + VShardId::new(5), + ), + WriteVersion::ZERO ); } } diff --git a/nodedb/src/control/server/shared/session/pending_shard_reads.rs b/nodedb/src/control/server/shared/session/pending_shard_reads.rs new file mode 100644 index 000000000..10a29c55f --- /dev/null +++ b/nodedb/src/control/server/shared/session/pending_shard_reads.rs @@ -0,0 +1,233 @@ +// SPDX-License-Identifier: BUSL-1.1 + +//! Cross-shard reads recorded into the transaction read-set. +//! +//! Some reads span many vShards through a coordinator that holds no session: +//! +//! - A cluster graph read (a MATCH, a walk, a gathered algorithm) reads edges +//! on many key vShards. +//! - A cluster array read (a slice or an aggregate) reads cells on many tile +//! vShards. +//! +//! The coordinator notes every vShard it read, each at the version the serving +//! core reported, in the connection scope ([`note`]). Each protocol records +//! the noted reads when the request ends ([`record_pending`]). +//! +//! Each vShard becomes one [`ReadSetEntry`] homed on that vShard. Commit +//! validation then checks the entry on the vShard that holds the data: +//! +//! - A Calvin participant checks the collection's version on that vShard. +//! - Any other commit fetches that version from the vShard's leader +//! (`commit::read_validation`). +//! +//! A version is a data-group log position, the same on every replica. So a +//! version above the recorded one means the collection changed on that vShard +//! after the read, whichever replica served it. + +use nodedb_types::WriteVersion; + +use crate::types::{DatabaseId, TenantId, VShardId}; + +use super::conn_scope::with_scope; +use super::connection::SessionId; +use super::read_set::{EngineTag, ReadKey, ReadOrigin, ReadSetEntry}; +use super::store::SessionStore; + +/// One vShard a cross-shard read observed, at the version its serving core +/// reported (`WriteVersion::ZERO` when the core reported none). +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub struct ShardObservation { + pub vshard: VShardId, + pub version: WriteVersion, +} + +/// The vShards one cross-shard read observed. +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct ShardReads { + /// The engine that served the read: `Graph` or `ClusterArray`. + pub engine: EngineTag, + pub tenant_id: TenantId, + pub database_id: DatabaseId, + /// The database-qualified collection the read scoped, or `None` when it + /// read every collection. + pub collection: Option, + pub shards: Vec, +} + +/// Note a cross-shard read for the request now running. Outside a +/// connection scope there is no session to record it for, so it is dropped. +pub fn note(reads: ShardReads) { + if reads.shards.is_empty() { + return; + } + let mut pending = Some(reads); + with_scope((), |scope| { + if let Some(reads) = pending.take() { + scope.pending_shard_reads.borrow_mut().push(reads); + } + }); +} + +/// Take every cross-shard read the request noted, oldest first. +pub fn take() -> Vec { + with_scope(Vec::new(), |scope| { + std::mem::take(&mut *scope.pending_shard_reads.borrow_mut()) + }) +} + +/// Record the cross-shard reads the request noted into `session_id`'s +/// transaction read-set. Outside a transaction block the reads are dropped. +pub fn record_pending(sessions: &SessionStore, session_id: SessionId) { + let pending = take(); + if pending.is_empty() || !sessions.is_in_transaction_block(session_id) { + return; + } + for reads in pending { + let entries = homed_entries(sessions, session_id, reads); + sessions.record_read_entries(session_id, entries); + } +} + +/// One predicate entry per observed vShard, homed there. +fn homed_entries( + sessions: &SessionStore, + session_id: SessionId, + reads: ShardReads, +) -> Vec { + let ShardReads { + engine, + tenant_id, + database_id, + collection, + shards, + } = reads; + let collection = collection.unwrap_or_default(); + shards + .into_iter() + .map(|shard| { + // The session's own committed writes to the collection raise the + // read version, as `record_read_set` does, so a read never aborts + // on the session's own earlier write. + let own = if collection.is_empty() { + WriteVersion::ZERO + } else { + sessions.own_write_version( + session_id, + database_id, + tenant_id, + &collection, + shard.vshard, + ) + }; + ReadSetEntry { + engine, + database_id, + tenant_id, + collection: collection.clone(), + key: ReadKey::Predicate, + read_version: shard.version.max(own), + origin: ReadOrigin::Session, + home: Some(shard.vshard), + } + }) + .collect() +} + +#[cfg(test)] +mod tests { + use super::super::conn_scope; + use super::*; + + fn session_id() -> SessionId { + SessionId::from( + "127.0.0.1:5611" + .parse::() + .expect("test address"), + ) + } + + fn reads(engine: EngineTag, collection: Option<&str>) -> ShardReads { + ShardReads { + engine, + tenant_id: TenantId::new(1), + database_id: DatabaseId::DEFAULT, + collection: collection.map(str::to_owned), + shards: vec![ + ShardObservation { + vshard: VShardId::new(3), + version: WriteVersion::logged(1, 10), + }, + ShardObservation { + vshard: VShardId::new(9), + version: WriteVersion::logged(2, 20), + }, + ], + } + } + + fn store_in_block() -> (SessionStore, SessionId) { + let sessions = SessionStore::new(); + let session_id = session_id(); + sessions.ensure_session(match session_id { + SessionId::LegacySocket(addr) => addr, + SessionId::Connection(_) => unreachable!("legacy test identity"), + }); + sessions.begin(session_id, 0).expect("begin"); + (sessions, session_id) + } + + #[tokio::test] + async fn each_observed_vshard_becomes_one_homed_entry() { + let (sessions, session_id) = store_in_block(); + conn_scope::scoped(async { + note(reads(EngineTag::Graph, Some("db1.edges"))); + record_pending(&sessions, session_id); + }) + .await; + let entries = sessions.take_read_set(session_id); + assert_eq!(entries.len(), 2); + assert_eq!(entries[0].home, Some(VShardId::new(3))); + assert_eq!(entries[0].read_version, WriteVersion::logged(1, 10)); + assert_eq!(entries[1].home, Some(VShardId::new(9))); + assert_eq!(entries[1].read_version, WriteVersion::logged(2, 20)); + assert!(entries.iter().all(|e| e.key == ReadKey::Predicate)); + assert!(entries.iter().all(|e| e.collection == "db1.edges")); + assert!(entries.iter().all(|e| e.engine == EngineTag::Graph)); + } + + #[tokio::test] + async fn entries_carry_the_engine_that_served_the_read() { + let (sessions, session_id) = store_in_block(); + conn_scope::scoped(async { + note(reads(EngineTag::Graph, Some("db1.edges"))); + note(reads(EngineTag::ClusterArray, Some("db1.grid"))); + record_pending(&sessions, session_id); + }) + .await; + let entries = sessions.take_read_set(session_id); + assert_eq!(entries.len(), 4); + assert!( + entries[..2] + .iter() + .all(|e| e.engine == EngineTag::Graph && e.collection == "db1.edges") + ); + assert!( + entries[2..] + .iter() + .all(|e| e.engine == EngineTag::ClusterArray && e.collection == "db1.grid") + ); + } + + #[tokio::test] + async fn a_read_outside_a_transaction_block_records_nothing() { + let sessions = SessionStore::new(); + let session_id = session_id(); + conn_scope::scoped(async { + note(reads(EngineTag::ClusterArray, None)); + record_pending(&sessions, session_id); + assert!(take().is_empty(), "the pending reads are drained"); + }) + .await; + assert!(sessions.take_read_set(session_id).is_empty()); + } +} diff --git a/nodedb/src/control/server/shared/session/read_set.rs b/nodedb/src/control/server/shared/session/read_set.rs index 0158b414f..69d06f88c 100644 --- a/nodedb/src/control/server/shared/session/read_set.rs +++ b/nodedb/src/control/server/shared/session/read_set.rs @@ -1,6 +1,6 @@ // SPDX-License-Identifier: BUSL-1.1 -//! Protocol-neutral, LSN-versioned transaction read-set capture. +//! Protocol-neutral, versioned transaction read-set capture. //! //! Every read a transaction performs is recorded here as one or more //! [`ReadSetEntry`]s, keyed by the same `(database_id, tenant_id, collection, @@ -15,11 +15,15 @@ //! one no point key will name), while an absent KV point read keeps its precise `Point` //! key (the byte key any future write reuses). A //! scan / search / aggregate records [`ReadKey::Predicate`] (collection scope -//! — the day-one phantom-safe floor). A multi-shard read records one entry per -//! participating shard, each stamped with that shard's own watermark LSN. +//! — the day-one phantom-safe floor). //! Absent-key / empty-result reads are recorded too: a "not found" is a //! validatable phantom observation, not a no-op. //! +//! Each entry carries the write version its read observed on its validation +//! vShard. A version is the data-group log position of the write that set it, +//! so every replica of the vShard reports the same version for the same state. +//! A read served by a follower validates against the leader's versions. +//! //! No validation happens here — the entries are captured for the commit-time //! optimistic-concurrency check to consume. @@ -32,7 +36,8 @@ use crate::control::cluster::calvin::scheduler::lock::LockKey; use crate::control::planner::calvin::reservation::submit_reserve_read; use crate::control::server::shared::plan_util::{extract_collection, plan_engine, read_key_of}; use crate::control::state::SharedState; -use crate::types::{DatabaseId, KeyRepr, Lsn, TenantId, VShardId}; +use crate::types::{DatabaseId, KeyRepr, ReadVersions, TenantId, VShardId}; +use nodedb_types::WriteVersion; use super::connection::SessionId; use super::store::SessionStore; @@ -89,16 +94,19 @@ pub enum ReadOrigin { /// began. Its observation can legitimately be superseded by the /// transaction's own writes, so the own-write exclusion applies. Session, - /// A read the Control Plane performed BEFORE the transaction existed, whose - /// observed value a value this transaction writes was computed from — a - /// materialized-sum settlement's pre-image is the case that exists today. + /// A read the Control Plane performed at plan time, whose observed value a + /// value this transaction writes was computed from — a materialized-sum + /// settlement's pre-image is the case that exists today. /// - /// This is NOT a read-your-own-write. It observed COMMITTED base state at a - /// point in time, and the derived value the transaction ships is only - /// correct if that observation still holds at apply time. Dropping it - /// because the transaction happens to write the same collection will - /// discard the one check that catches a concurrent writer moving the base - /// row out from under the derivation, so it survives the exclusion. + /// This is NOT a read-your-own-write. Its version is the COMMITTED base + /// state at a point in time, and the derived value the transaction ships is + /// only correct if that observation still holds at apply time. Inside a + /// transaction the read also folds the transaction's staging overlay, but + /// a staged write moves no committed version, so the check still compares + /// committed base only. Dropping it because the transaction happens to + /// write the same collection will discard the one check that catches a + /// concurrent writer moving the base row out from under the derivation, so + /// it survives the exclusion. PlanDerivation, } @@ -113,7 +121,7 @@ impl ReadOrigin { } } -/// One LSN-versioned, predicate-aware read-set entry. Scoped by +/// One versioned, predicate-aware read-set entry. Scoped by /// `(database_id, tenant_id)` exactly like the write path so two tenants (or /// databases) never alias. #[derive(Debug, Clone, PartialEq, Eq)] @@ -123,78 +131,66 @@ pub struct ReadSetEntry { pub tenant_id: TenantId, pub collection: String, pub key: ReadKey, - pub read_lsn: Lsn, - /// Per-collection read-version LSN (the read collection's `coll_write_lsn` - /// at read time, a WAL LSN) — the SOUND comparand for cross-shard OCC - /// validation, which compares it against the same collection's recorded - /// write versions in that one domain. `read_lsn` above stays the core-global - /// watermark used by single-shard SI (`si_conflict_abort`). - pub read_version_lsn: Lsn, + /// The version the read observed on its validation vShard. Commit + /// validation compares it with that vShard's current version of the + /// collection. Every replica numbers versions alike, so the read can have + /// been served by any replica. + pub read_version: WriteVersion, /// Whether this observation is the transaction's own session read or a /// plan-time derivation read. Required at construction — the own-write /// exclusion in the `TxClass` builders reads it, and an entry that guessed /// will be silently dropped from validation. pub origin: ReadOrigin, /// The vShard whose write versions validate this entry, or `None` for the - /// collection's own vShard. A cross-shard graph read records one entry per - /// key vShard it read, each homed there. A homed entry with an empty + /// collection's own vShard. A cross-shard graph or array read records one + /// entry per vShard it read, each homed there. A homed entry with an empty /// `collection` observed every collection on its vShard. pub home: Option, - /// The node that served the read: of `home` for a homed read, of the - /// collection's vShard otherwise. Its WAL numbers the read's versions, so - /// only that node's current versions compare with them. `0` when no one - /// node is known to have served it, which every commit check treats as a - /// change. - pub home_node: u64, } -/// The node that served the running statement's read of `vshard`. -/// -/// The read dispatch notes it (`served_reads`). A read no dispatch noted ran -/// on this node's cores: it is this node when there is no cluster or this node -/// leads the vShard now, and unknown (`0`) otherwise. -pub(crate) fn serving_node(state: &SharedState, vshard: u32) -> u64 { - if let Some(node) = super::served_reads::served_by(vshard) { - return node; - } - if state.cluster_routing.is_none() { - return state.node_id; - } - match crate::control::gateway::live_leaders::resolve_live_decision(state, vshard) { - crate::control::gateway::RouteDecision::Local => state.node_id, - _ => 0, +impl ReadSetEntry { + /// The vShard whose versions validate this entry: its home, else its + /// collection's vShard. `None` for an unhomed entry whose collection + /// names no vShard. + pub fn validation_vshard(&self) -> Option { + match self.home { + Some(home) => Some(home), + None => { + nodedb_types::CollectionKey::from_qualified_str(self.database_id, &self.collection) + .ok() + .map(|key| key.vshard()) + } + } } } /// The observed read passed to [`record_read_set`]: the executed plan, the -/// responding shards' write-LSN watermarks, the read-version floor, and whether -/// a point read hit. +/// versions its responding cores reported, and whether a point read hit. pub struct ReadCapture<'a> { pub plan: &'a PhysicalPlan, - pub watermarks: &'a [(VShardId, Lsn)], - pub read_version_lsn: Lsn, + pub read_versions: &'a ReadVersions, pub found: bool, } /// Record a completed read into the session's transaction read-set. /// /// Transport-agnostic: every read post-dispatch seam calls this with the plan -/// that ran and the per-shard watermark(s) it observed. Records one -/// [`ReadSetEntry`] per `(vshard, watermark)` pair — a predicate read fanned -/// over N shards yields N entries, each carrying that shard's own watermark -/// LSN. A point read observes a single shard and yields one entry. +/// that ran and the versions it observed. +/// +/// A read of one collection records one entry that validates on the +/// collection's vShard at the version the read observed there. A vShard the +/// read reported no version of had no write the serving core knew of, so the +/// read observed it at `WriteVersion::ZERO`. A read that reported other +/// vShards too records one more entry per other reported vShard, homed +/// there. +/// +/// A read that names no collection observed every collection on each vShard +/// it reported. It records one entry per reported vShard, homed there. /// /// Guarded on the connection being inside a transaction block (the session /// write path drops the entries otherwise), so autocommit reads never touch -/// the read-set. Absent-key / empty-result reads MUST reach this with a -/// non-empty `watermarks` slice — a "not found" is a validatable observation. -/// -/// `read_version_lsn` is the read collection's per-collection write floor at -/// read time (a WAL LSN) — one scalar, since a read op -/// resolves to a single collection (joins collapse to one via -/// `extract_collection`). It stamps every entry produced here and is the SOUND -/// comparand cross-shard OCC validation consumes; the per-shard `watermarks` -/// still source the core-global `read_lsn` used by single-shard SI. +/// the read-set. Absent-key / empty-result reads are recorded too — a "not +/// found" is a validatable observation. /// /// `found` reports whether a point read observed a present row (`true` on a /// hit, `false` on a miss). It only affects document point reads — an absent @@ -208,13 +204,9 @@ pub async fn record_read_set( ) { let ReadCapture { plan, - watermarks, - read_version_lsn, + read_versions, found, } = capture; - if watermarks.is_empty() { - return; - } let engine = plan_engine(plan); let key = read_key_of(plan, found); @@ -228,45 +220,60 @@ pub async fn record_read_set( .get_current_database(session_id) .unwrap_or(DatabaseId::DEFAULT); - // Read-your-writes floor: raise the captured read-version to the session's - // OWN highest committed write-version for this collection. Without it, a - // read that observed a stale collection floor (0) before the session's own - // prior committed write was reflected on the serving core will, at - // cross-shard OCC validation, see that write's `coll_write_lsn` exceed the - // read-version and false-abort with a serialization failure on the session's - // OWN write. The floor is only ever raised by this session's own committed - // writes to this exact `(database, tenant, collection)`, so a concurrent - // OTHER-session write (higher `coll_write_lsn`) still exceeds the floor and - // still aborts — this removes only the self-abort. `read_lsn` (the per-shard - // watermark used by single-shard SI) is deliberately left untouched. - let own_write_version = - sessions.own_write_version(session_id, database_id, tenant_id, &collection); - let effective_read_version = read_version_lsn.max(own_write_version); - // The read validates on its collection's vShard, against the versions of - // the node that served it there. - let validation_vshard = - nodedb_types::CollectionKey::from_qualified_str(database_id, &collection) - .map(|key| key.vshard().as_u32()) - .unwrap_or_else(|_| watermarks[0].0.as_u32()); - let served_by = serving_node(state, validation_vshard); - - let entries: Vec = watermarks - .iter() - .map(|(_vshard, read_lsn)| ReadSetEntry { + let entry = |home: Option, observed: WriteVersion, validation: VShardId| { + // Read-your-writes floor: raise the observed version to the + // session's own highest committed write of this collection on the + // validation vShard. A read that ran before the serving replica + // applied the session's own prior write will otherwise abort on that + // write. Only this session's own writes raise the floor, so a + // concurrent write by another session still exceeds it and aborts. + let own = if collection.is_empty() { + WriteVersion::ZERO + } else { + sessions.own_write_version(session_id, database_id, tenant_id, &collection, validation) + }; + ReadSetEntry { engine, database_id, tenant_id, collection: collection.clone(), key: key.clone(), - read_lsn: *read_lsn, - read_version_lsn: effective_read_version, + read_version: observed.max(own), // Every entry captured here is a read the session issued inside its // own transaction, so the own-write exclusion applies to it. origin: ReadOrigin::Session, - home: None, - home_node: served_by, - }) - .collect(); + home, + } + }; + let entries: Vec = + match nodedb_types::CollectionKey::from_qualified_str(database_id, &collection) { + Ok(collection_key) if !collection.is_empty() => { + let validation = collection_key.vshard(); + let observed = read_versions.of(validation).unwrap_or_default(); + let mut entries = vec![entry(None, observed, validation)]; + entries.extend( + read_versions + .iter() + .map(|shard| VShardId::new(shard.vshard)) + .filter(|vshard| *vshard != validation) + .map(|vshard| { + let observed = read_versions.of(vshard).unwrap_or_default(); + entry(Some(vshard), observed, vshard) + }), + ); + entries + } + _ => read_versions + .iter() + .map(|shard| { + let vshard = VShardId::new(shard.vshard); + entry(Some(vshard), shard.version, vshard) + }) + .collect(), + }; + if entries.is_empty() { + return; + } sessions.record_read_entries(session_id, entries); @@ -376,6 +383,18 @@ fn lock_key_to_wire(key: &LockKey) -> LockKeyWire { src: *src, dst: *dst, }, + LockKey::Collection { collection } => LockKeyWire::Collection { + collection: collection.to_string(), + }, + LockKey::Unique { + collection, + index, + value, + } => LockKeyWire::Unique { + collection: collection.to_string(), + index: index.to_string(), + value: value.to_vec(), + }, } } @@ -417,7 +436,7 @@ mod tests { SessionId::LegacySocket(addr) => addr, SessionId::Connection(_) => unreachable!("legacy test identity"), }); - sessions.begin(session_id, Lsn::new(5), 0).expect("begin"); + sessions.begin(session_id, 0).expect("begin"); (sessions, session_id) } @@ -439,6 +458,18 @@ mod tests { (state, dir) } + /// The vShard a read of `collection` validates on. + fn home_of(collection: &str) -> VShardId { + nodedb_types::CollectionKey::from_qualified_str(DatabaseId::DEFAULT, collection) + .expect("collection key") + .vshard() + } + + /// `collection`'s vShard observed at log index `index`. + fn observed(collection: &str, index: u64) -> ReadVersions { + ReadVersions::single(home_of(collection), WriteVersion::logged(1, index)) + } + #[tokio::test] async fn point_read_records_point_key() { let (state, _dir) = test_state(); @@ -450,8 +481,7 @@ mod tests { TenantId::new(1), ReadCapture { plan: &kv_get("c", b"k1"), - watermarks: &[(VShardId::new(0), Lsn::new(7))], - read_version_lsn: Lsn::ZERO, + read_versions: &observed("c", 7), found: true, }, ) @@ -460,7 +490,9 @@ mod tests { assert_eq!(rs.len(), 1); assert_eq!(rs[0].engine, EngineTag::Kv); assert_eq!(rs[0].collection, "c"); - assert_eq!(rs[0].read_lsn, Lsn::new(7)); + assert_eq!(rs[0].read_version, WriteVersion::logged(1, 7)); + assert_eq!(rs[0].home, None); + assert_eq!(rs[0].validation_vshard(), Some(home_of("c"))); assert_eq!( rs[0].key, ReadKey::Point { @@ -482,8 +514,7 @@ mod tests { TenantId::new(1), ReadCapture { plan: &kv_batch_get("c"), - watermarks: &[(VShardId::new(0), Lsn::new(9))], - read_version_lsn: Lsn::ZERO, + read_versions: &observed("c", 9), found: true, }, ) @@ -494,11 +525,14 @@ mod tests { } #[tokio::test] - async fn multi_shard_read_records_one_entry_per_watermark() { + async fn a_read_records_every_vshard_it_reported() { let (state, _dir) = test_state(); let (sessions, a) = begun_session(); - // A predicate fanned over three cores records one entry per shard, each - // stamped with that shard's own watermark — NOT a single collapsed max. + // The collection's own vShard validates the unhomed entry. Another + // reported vShard validates an entry homed there. + let other = VShardId::new(home_of("c").as_u32() + 1); + let mut versions = observed("c", 11); + versions.note(other, WriteVersion::logged(1, 90)); record_read_set( &state, &sessions, @@ -506,21 +540,68 @@ mod tests { TenantId::new(1), ReadCapture { plan: &kv_batch_get("c"), - watermarks: &[ - (VShardId::new(0), Lsn::new(3)), - (VShardId::new(1), Lsn::new(11)), - (VShardId::new(2), Lsn::new(7)), - ], - read_version_lsn: Lsn::ZERO, + read_versions: &versions, + found: true, + }, + ) + .await; + let rs = sessions.take_read_set(a); + assert_eq!(rs.len(), 2); + assert_eq!(rs[0].home, None); + assert_eq!(rs[0].read_version, WriteVersion::logged(1, 11)); + assert_eq!(rs[1].home, Some(other)); + assert_eq!(rs[1].read_version, WriteVersion::logged(1, 90)); + assert_eq!(rs[1].collection, "c"); + } + + #[tokio::test] + async fn a_read_with_no_reported_version_observed_zero() { + let (state, _dir) = test_state(); + let (sessions, a) = begun_session(); + // The serving core knew no write of the vShard, so any write since + // the read moves it above the recorded version. + record_read_set( + &state, + &sessions, + a, + TenantId::new(1), + ReadCapture { + plan: &kv_get("c", b"k1"), + read_versions: &ReadVersions::new(), + found: false, + }, + ) + .await; + let rs = sessions.take_read_set(a); + assert_eq!(rs.len(), 1); + assert_eq!(rs[0].read_version, WriteVersion::ZERO); + } + + #[tokio::test] + async fn the_sessions_own_write_floors_the_read_version() { + let (state, _dir) = test_state(); + let (sessions, a) = begun_session(); + sessions.note_own_write( + a, + DatabaseId::DEFAULT, + TenantId::new(1), + "c", + &observed("c", 20), + ); + record_read_set( + &state, + &sessions, + a, + TenantId::new(1), + ReadCapture { + plan: &kv_get("c", b"k1"), + read_versions: &observed("c", 12), found: true, }, ) .await; let rs = sessions.take_read_set(a); - assert_eq!(rs.len(), 3); - let mut lsns: Vec = rs.iter().map(|e| e.read_lsn.as_u64()).collect(); - lsns.sort_unstable(); - assert_eq!(lsns, vec![3, 7, 11]); + assert_eq!(rs[0].read_version, WriteVersion::logged(1, 20)); } #[tokio::test] @@ -528,7 +609,7 @@ mod tests { let (state, _dir) = test_state(); let (sessions, a) = begun_session(); // A "not found" is a validatable phantom observation: the KV point entry - // is recorded (as `found = false`) at the current watermark. KV keeps the + // is recorded (as `found = false`) at the observed version. KV keeps the // precise byte key — the identity any future insert of that key reuses. record_read_set( &state, @@ -537,8 +618,7 @@ mod tests { TenantId::new(1), ReadCapture { plan: &kv_get("c", b"missing"), - watermarks: &[(VShardId::new(0), Lsn::new(5))], - read_version_lsn: Lsn::ZERO, + read_versions: &observed("c", 5), found: false, }, ) @@ -578,8 +658,7 @@ mod tests { TenantId::new(1), ReadCapture { plan: &doc_point_get("docs", 42), - watermarks: &[(VShardId::new(0), Lsn::new(7))], - read_version_lsn: Lsn::ZERO, + read_versions: &observed("docs", 7), found: true, }, ) @@ -608,8 +687,7 @@ mod tests { TenantId::new(1), ReadCapture { plan: &doc_point_get("docs", 999), - watermarks: &[(VShardId::new(0), Lsn::new(5))], - read_version_lsn: Lsn::ZERO, + read_versions: &observed("docs", 5), found: false, }, ) @@ -633,8 +711,7 @@ mod tests { TenantId::new(1), ReadCapture { plan: &kv_get("c", b"k1"), - watermarks: &[(VShardId::new(0), Lsn::new(7))], - read_version_lsn: Lsn::ZERO, + read_versions: &observed("c", 7), found: true, }, ) @@ -643,23 +720,35 @@ mod tests { } #[tokio::test] - async fn empty_watermarks_records_nothing() { + async fn a_read_of_no_collection_records_one_homed_entry_per_vshard() { let (state, _dir) = test_state(); let (sessions, a) = begun_session(); + // A read that names no collection observed every collection on each + // vShard it reported. + let plan = PhysicalPlan::Meta(nodedb_physical::physical_plan::MetaOp::HomeVersions { + probes: Vec::new(), + }); + let mut versions = ReadVersions::single(VShardId::new(3), WriteVersion::logged(1, 4)); + versions.note(VShardId::new(8), WriteVersion::logged(2, 6)); record_read_set( &state, &sessions, a, TenantId::new(1), ReadCapture { - plan: &kv_get("c", b"k1"), - watermarks: &[], - read_version_lsn: Lsn::ZERO, + plan: &plan, + read_versions: &versions, found: true, }, ) .await; - assert!(sessions.take_read_set(a).is_empty()); + let rs = sessions.take_read_set(a); + assert_eq!(rs.len(), 2); + assert_eq!(rs[0].home, Some(VShardId::new(3))); + assert_eq!(rs[0].read_version, WriteVersion::logged(1, 4)); + assert_eq!(rs[1].home, Some(VShardId::new(8))); + assert_eq!(rs[1].read_version, WriteVersion::logged(2, 6)); + assert!(rs.iter().all(|entry| entry.collection.is_empty())); } #[test] diff --git a/nodedb/src/control/server/shared/session/record_reads.rs b/nodedb/src/control/server/shared/session/record_reads.rs index f8340a545..bf7f6a01e 100644 --- a/nodedb/src/control/server/shared/session/record_reads.rs +++ b/nodedb/src/control/server/shared/session/record_reads.rs @@ -9,8 +9,8 @@ //! from its own single-collection scan plan and REAL observed read-version, so //! the commit-time OCC validator re-homes and revalidates each collection's //! vshard independently — while every other read records a single -//! collection-scoped entry from the executed plan and the responding shards' -//! watermarks. When captures are present the default single-collection entry is +//! collection-scoped entry from the executed plan and the versions its cores +//! reported. When captures are present the default single-collection entry is //! SKIPPED, because a `HashJoin` plan collapses to the left collection via //! `extract_collection` and would miss the build side entirely. This module //! hosts that decision so all transports funnel through one implementation @@ -19,7 +19,7 @@ use crate::bridge::envelope::PhysicalPlan; use crate::control::server::exchange::DistributedReadCapture; use crate::control::state::SharedState; -use crate::types::{Lsn, TenantId, VShardId}; +use crate::types::{ReadVersions, TenantId}; use super::connection::SessionId; use super::read_set::{ReadCapture, record_read_set}; @@ -27,22 +27,15 @@ use super::store::SessionStore; /// The observed reads produced by one dispatched response. /// -/// `plan` / `watermarks` / `read_version_lsn` / `found` describe the plain -/// single-collection observation (used when `distributed_reads` is empty). +/// `plan` / `read_versions` / `found` describe the plain single-collection +/// observation (used when `distributed_reads` is empty). /// `distributed_reads`, when non-empty, carries the per-collection captures of a /// distributed gather/shuffle read and takes precedence over the plain fields. -/// -/// `read_lsn_vshard` is the vshard stamped into each per-capture entry's -/// single-shard SI `read_lsn` slot (paired with [`Lsn::ZERO`], since the sound -/// cross-shard comparand is the capture's own `read_version_lsn`). It is -/// consulted only on the captures branch. pub struct ResponseReads<'a> { pub plan: &'a PhysicalPlan, - pub watermarks: &'a [(VShardId, Lsn)], - pub read_version_lsn: Lsn, + pub read_versions: &'a ReadVersions, pub found: bool, pub distributed_reads: &'a [DistributedReadCapture], - pub read_lsn_vshard: VShardId, } /// Record a dispatched response's reads into the session transaction read-set. @@ -68,8 +61,7 @@ pub async fn record_reads_for_response( tenant_id, ReadCapture { plan: &cap.scan_plan, - watermarks: &[(reads.read_lsn_vshard, Lsn::ZERO)], - read_version_lsn: cap.read_version_lsn, + read_versions: &cap.read_versions, found: false, }, ) @@ -83,8 +75,7 @@ pub async fn record_reads_for_response( tenant_id, ReadCapture { plan: reads.plan, - watermarks: reads.watermarks, - read_version_lsn: reads.read_version_lsn, + read_versions: reads.read_versions, found: reads.found, }, ) diff --git a/nodedb/src/control/server/shared/session/savepoint_ops.rs b/nodedb/src/control/server/shared/session/savepoint_ops.rs index ad25f5f44..44c0deb43 100644 --- a/nodedb/src/control/server/shared/session/savepoint_ops.rs +++ b/nodedb/src/control/server/shared/session/savepoint_ops.rs @@ -307,8 +307,8 @@ mod tests { payload: Payload::empty(), watermark_lsn: Lsn::ZERO, error_code: None, - read_set_valid: None, - read_version_lsn: crate::types::Lsn::ZERO, + stage_vote: None, + read_versions: crate::types::ReadVersions::new(), write_set: Vec::new(), }) }) @@ -344,7 +344,7 @@ mod tests { let store = SessionStore::new(); let addr: std::net::SocketAddr = "127.0.0.1:5203".parse().unwrap(); store.ensure_session(addr); - store.begin(addr, Lsn::new(1), 0).unwrap(); + store.begin(addr, 0).unwrap(); assert!(store.buffer_write(addr, staged_task(3))); let result = run_savepoint( @@ -386,7 +386,7 @@ mod tests { let store = SessionStore::new(); let addr: std::net::SocketAddr = "127.0.0.1:5201".parse().unwrap(); store.ensure_session(addr); - store.begin(addr, Lsn::new(1), 0).unwrap(); + store.begin(addr, 0).unwrap(); let tenant = TenantId::new(1); let dp = RecordingDp::default(); @@ -435,7 +435,7 @@ mod tests { let store = SessionStore::new(); let addr: std::net::SocketAddr = "127.0.0.1:5204".parse().unwrap(); store.ensure_session(addr); - store.begin(addr, Lsn::new(1), 0).unwrap(); + store.begin(addr, 0).unwrap(); let tenant = TenantId::new(1); let dp = RecordingDp::default(); assert!(store.buffer_write(addr, staged_task(2))); @@ -467,7 +467,7 @@ mod tests { let store = SessionStore::new(); let addr: std::net::SocketAddr = "127.0.0.1:5202".parse().unwrap(); store.ensure_session(addr); - store.begin(addr, Lsn::new(1), 0).unwrap(); + store.begin(addr, 0).unwrap(); let session = SessionId::from(&addr); let tenant = TenantId::new(1); let dp = RecordingDp::default(); diff --git a/nodedb/src/control/server/shared/session/served_reads.rs b/nodedb/src/control/server/shared/session/served_reads.rs deleted file mode 100644 index 625d59f55..000000000 --- a/nodedb/src/control/server/shared/session/served_reads.rs +++ /dev/null @@ -1,73 +0,0 @@ -// SPDX-License-Identifier: BUSL-1.1 - -//! The node that served each vShard the running statement read. -//! -//! A read's versions are positions in the WAL of the node that served it, so -//! only that node's versions compare with them at commit. Every read dispatch -//! notes the vShard it read and the node that answered. The read-set capture -//! stamps each entry with that node (`ReadSetEntry::home_node`). A vShard two -//! nodes served in one statement has no one node, noted as `0`, and a commit -//! then treats its read as changed. -//! -//! The notes live in the connection scope and start empty with each -//! statement (`deadline::enter`). Outside a connection scope nothing is noted. - -use std::collections::HashMap; - -use super::conn_scope::with_scope; - -/// Note that `node` served a read of `vshard` in the running statement. -pub fn note(vshard: u32, node: u64) { - with_scope((), |scope| { - scope - .served_reads - .borrow_mut() - .entry(vshard) - .and_modify(|seen| { - if *seen != node { - *seen = 0; - } - }) - .or_insert(node); - }); -} - -/// The node that served the running statement's read of `vshard`: `Some(0)` -/// when two nodes did, `None` when no read of it was noted. -pub fn served_by(vshard: u32) -> Option { - with_scope(None, |scope| { - scope.served_reads.borrow().get(&vshard).copied() - }) -} - -/// Forget every note: a statement starts with no reads served. -pub(super) fn clear() { - with_scope((), |scope| scope.served_reads.borrow_mut().clear()); -} - -/// An empty note map for a fresh connection scope. -pub(super) fn empty() -> std::cell::RefCell> { - std::cell::RefCell::new(HashMap::new()) -} - -#[cfg(test)] -mod tests { - use super::super::conn_scope; - use super::*; - - #[tokio::test] - async fn a_vshard_two_nodes_served_has_no_node() { - conn_scope::scoped(async { - note(3, 1); - note(3, 1); - note(4, 1); - note(4, 2); - assert_eq!(served_by(3), Some(1)); - assert_eq!(served_by(4), Some(0)); - assert_eq!(served_by(5), None); - clear(); - assert_eq!(served_by(3), None); - }) - .await; - } -} diff --git a/nodedb/src/control/server/shared/session/state.rs b/nodedb/src/control/server/shared/session/state.rs index 1833f54f4..221d40da9 100644 --- a/nodedb/src/control/server/shared/session/state.rs +++ b/nodedb/src/control/server/shared/session/state.rs @@ -18,7 +18,9 @@ pub(crate) fn now_unix_ms() -> u64 { use crate::control::lease::QueryLeaseScope; use crate::event::cdc::CdcOffset; -use crate::types::{DatabaseId, Lsn, TenantId, TxnId, VShardId}; +use crate::types::{DatabaseId, TenantId, TxnId, VShardId}; + +use super::own_writes::OwnWriteKey; use nodedb_physical::physical_task::PhysicalTask; /// One entry on the transaction's savepoint stack. @@ -146,10 +148,6 @@ pub struct ConnSession { /// rejected a line has an entry. COMMIT compares these counts with its /// authoritative resolve. pub tx_ts_preview_rejected: BTreeMap, - /// Snapshot LSN captured at BEGIN for snapshot isolation. - /// All reads within the transaction see data as of this LSN. - /// Concurrent writes after this point are invisible to the transaction. - pub tx_snapshot_lsn: Option, /// Snapshot epoch captured at BEGIN: the last globally-applied Calvin epoch, /// read from `CalvinLocalState::last_applied_epoch`. The cross-shard-valid /// version anchor for the transaction (0 in single-node / no-Calvin). `None` @@ -166,11 +164,11 @@ pub struct ConnSession { /// mark/rewind must fan over ALL of them. Ordered (BTree) for deterministic /// teardown. Empty until the first staged write. pub tx_vshards: BTreeSet, - /// Read-set: LSN-versioned, predicate-aware entries for write conflict + /// Read-set: versioned, predicate-aware entries for write conflict /// detection, captured on the shared read seam by every transport. At /// COMMIT, each entry is checked — if the entry's collection has a current - /// write-LSN past `read_lsn`, a concurrent write occurred and the - /// transaction is rejected with SERIALIZATION_FAILURE. + /// version past `read_version` on its vShard, a concurrent write occurred + /// and the transaction is rejected with SERIALIZATION_FAILURE. pub tx_read_set: Vec, /// Distinct vShards this transaction took a SHARED read reservation on. A /// hot-key read reserves under the transaction's single `tx_reservation_owner` @@ -241,17 +239,14 @@ pub struct ConnSession { /// is idle-eligible only when this is zero — a legitimately long-running /// statement (in flight) must never be idle-killed. pub in_flight: AtomicU32, - /// Highest committed write-version this session has observed for each - /// `(database, tenant, collection)` it has written, keyed identically to - /// the read/write namespace. Used to floor a later transaction's captured - /// `read_version_lsn` at the session's OWN prior committed writes — a - /// read-your-writes floor that removes cross-shard OCC self-aborts on a - /// collection the session itself last wrote, without ever masking a - /// concurrent OTHER-session write (whose higher `coll_write_lsn` still - /// exceeds the floor). Persists for the life of the session — a prior - /// autocommit write must still floor a later transaction's read — and is - /// therefore NOT cleared at transaction boundaries. - pub own_write_versions: HashMap<(DatabaseId, TenantId, String), Lsn>, + /// Highest committed write version this session holds for each + /// `(database, tenant, collection, vShard)` it has written. It floors a + /// later transaction's captured read version at the session's own prior + /// committed writes, so a read never aborts on the session's own write. A + /// concurrent write by another session has a higher version and still + /// aborts. It persists for the life of the session, so a prior autocommit + /// write floors a later transaction's read too. + pub own_write_versions: HashMap, /// Database- and tenant-scoped connection slots this session holds. /// /// pgwire takes them during the startup handshake, once the bound database @@ -321,7 +316,6 @@ impl ConnSession { tx_lease_scopes: Vec::new(), tx_body_tasks: BTreeSet::new(), tx_ts_preview_rejected: BTreeMap::new(), - tx_snapshot_lsn: None, tx_snapshot_epoch: None, tx_id: None, tx_vshards: BTreeSet::new(), diff --git a/nodedb/src/control/server/shared/session/store.rs b/nodedb/src/control/server/shared/session/store.rs index f0e49f772..e1f3a8d91 100644 --- a/nodedb/src/control/server/shared/session/store.rs +++ b/nodedb/src/control/server/shared/session/store.rs @@ -303,7 +303,6 @@ impl SessionStore { session.tx_lease_scopes.clear(); session.tx_body_tasks.clear(); session.tx_ts_preview_rejected.clear(); - session.tx_snapshot_lsn = None; session.tx_snapshot_epoch = None; session.tx_id = None; session.tx_vshards.clear(); @@ -412,13 +411,12 @@ mod tests { use std::sync::Arc; use crate::control::lease::QueryLeaseScope; - use crate::types::Lsn; let store = SessionStore::new(); let addr: std::net::SocketAddr = "127.0.0.1:6011".parse().expect("address"); store.ensure_session(addr); let scope = Arc::new(QueryLeaseScope::empty()); - store.begin(addr, Lsn::new(1), 0).expect("begin"); + store.begin(addr, 0).expect("begin"); assert!(store.buffer_write(addr, task())); assert!(store.attach_tx_lease_scope_since(addr, 0, Arc::clone(&scope))); store.rollback(addr).expect("rollback"); @@ -428,7 +426,7 @@ mod tests { assert_eq!(session.tx_buffer.len(), session.tx_lease_scopes.len()); }); - store.begin(addr, Lsn::new(2), 0).expect("begin"); + store.begin(addr, 0).expect("begin"); assert!(store.buffer_write(addr, task())); assert!(store.attach_tx_lease_scope_since(addr, 0, scope)); store.reset_for_database_switch(addr, DatabaseId::new(2)); diff --git a/nodedb/src/control/server/shared/session/transaction/buffer.rs b/nodedb/src/control/server/shared/session/transaction/buffer.rs index 6d41d559a..cda637a0b 100644 --- a/nodedb/src/control/server/shared/session/transaction/buffer.rs +++ b/nodedb/src/control/server/shared/session/transaction/buffer.rs @@ -223,7 +223,7 @@ mod tests { use nodedb_physical::physical_task::{PhysicalTask, PostSetOp}; use super::super::super::store::SessionStore; - use crate::types::{DatabaseId, Lsn, TenantId, VShardId}; + use crate::types::{DatabaseId, TenantId, VShardId}; fn ingest_task() -> PhysicalTask { PhysicalTask { @@ -254,7 +254,7 @@ mod tests { let store = SessionStore::new(); let addr: std::net::SocketAddr = "127.0.0.1:5311".parse().expect("addr"); store.ensure_session(addr); - store.begin(addr, Lsn::new(1), 0).expect("begin"); + store.begin(addr, 0).expect("begin"); assert!(store.buffer_write(addr, ingest_task())); store.note_ts_preview_rejected(addr, 0); diff --git a/nodedb/src/control/server/shared/session/transaction/lifecycle.rs b/nodedb/src/control/server/shared/session/transaction/lifecycle.rs index ef9a3216e..d8f51c17a 100644 --- a/nodedb/src/control/server/shared/session/transaction/lifecycle.rs +++ b/nodedb/src/control/server/shared/session/transaction/lifecycle.rs @@ -8,7 +8,7 @@ use std::sync::atomic::{AtomicU64, Ordering}; use nodedb_physical::physical_task::PhysicalTask; use crate::control::lease::QueryLeaseScope; -use crate::types::{Lsn, TxnId, VShardId}; +use crate::types::{TxnId, VShardId}; use super::super::connection::SessionId; use super::super::state::TransactionState; @@ -31,22 +31,19 @@ impl SessionStore { .unwrap_or(TransactionState::Idle) } - /// BEGIN — enter transaction block with snapshot isolation. + /// BEGIN — enter a transaction block. /// - /// Captures the current WAL LSN as the local snapshot point (single-shard - /// fast path) and the last globally-applied Calvin `snapshot_epoch` as the - /// cross-shard-valid version anchor. All reads within this transaction see - /// data as of this LSN. + /// Captures the last globally-applied Calvin `snapshot_epoch` as the + /// cross-shard version anchor. Commit validates each read by the write + /// version it observed, so BEGIN captures no WAL position. pub fn begin( &self, addr: impl Into, - current_lsn: Lsn, snapshot_epoch: u64, ) -> Result<(), &'static str> { self.write_session(addr, |session| match session.tx_state { TransactionState::Idle => { session.tx_state = TransactionState::InBlock; - session.tx_snapshot_lsn = Some(current_lsn); session.tx_snapshot_epoch = Some(snapshot_epoch); session.tx_read_set.clear(); session.tx_reservation_vshards.clear(); @@ -75,11 +72,6 @@ impl SessionStore { .unwrap_or(false) } - /// Get the snapshot LSN for the current transaction. - pub fn snapshot_lsn(&self, addr: impl Into) -> Option { - self.read_session(addr, |s| s.tx_snapshot_lsn)? - } - /// Get the cross-shard snapshot epoch for the current transaction. pub fn snapshot_epoch(&self, addr: impl Into) -> Option { self.read_session(addr, |s| s.tx_snapshot_epoch)? @@ -116,7 +108,6 @@ impl SessionStore { session.tx_body_tasks.clear(); session.tx_ts_preview_rejected.clear(); session.tx_state = TransactionState::Idle; - session.tx_snapshot_lsn = None; session.tx_snapshot_epoch = None; session.tx_id = None; session.tx_vshards.clear(); @@ -147,7 +138,6 @@ impl SessionStore { session.tx_ts_preview_rejected.clear(); session.pending_publishes.clear(); session.tx_state = TransactionState::Idle; - session.tx_snapshot_lsn = None; session.tx_snapshot_epoch = None; session.tx_id = None; session.tx_vshards.clear(); @@ -203,13 +193,13 @@ mod tests { assert_eq!(store.transaction_state(addr), TransactionState::Idle); - store.begin(addr, Lsn::new(1), 0).unwrap(); + store.begin(addr, 0).unwrap(); assert_eq!(store.transaction_state(addr), TransactionState::InBlock); store.commit(addr).unwrap(); assert_eq!(store.transaction_state(addr), TransactionState::Idle); - store.begin(addr, Lsn::new(1), 0).unwrap(); + store.begin(addr, 0).unwrap(); store.fail_transaction(addr); assert_eq!(store.transaction_state(addr), TransactionState::Failed); @@ -222,7 +212,7 @@ mod tests { let store = SessionStore::new(); let addr: std::net::SocketAddr = "127.0.0.1:6012".parse().expect("address"); store.ensure_session(addr); - store.begin(addr, Lsn::new(1), 0).expect("begin"); + store.begin(addr, 0).expect("begin"); let scope = Arc::new(QueryLeaseScope::empty()); assert!(store.buffer_write(addr, task())); diff --git a/nodedb/src/control/server/shared/session/transaction/savepoints.rs b/nodedb/src/control/server/shared/session/transaction/savepoints.rs index a4b352d7f..e07957fc0 100644 --- a/nodedb/src/control/server/shared/session/transaction/savepoints.rs +++ b/nodedb/src/control/server/shared/session/transaction/savepoints.rs @@ -140,7 +140,7 @@ mod tests { use crate::control::lease::QueryLeaseScope; use crate::control::server::shared::session::state::PendingOffsetCommit; - use crate::types::{DatabaseId, Lsn, TenantId, VShardId}; + use crate::types::{DatabaseId, TenantId, VShardId}; fn task() -> PhysicalTask { PhysicalTask { @@ -160,7 +160,7 @@ mod tests { let store = SessionStore::new(); let addr: std::net::SocketAddr = "127.0.0.1:6010".parse().expect("address"); store.ensure_session(addr); - store.begin(addr, Lsn::new(1), 0).expect("begin"); + store.begin(addr, 0).expect("begin"); let scope = Arc::new(QueryLeaseScope::empty()); assert!(store.buffer_write(addr, task())); @@ -184,7 +184,7 @@ mod tests { let store = SessionStore::new(); let addr: std::net::SocketAddr = "127.0.0.1:6013".parse().expect("address"); store.ensure_session(addr); - store.begin(addr, Lsn::new(1), 0).expect("begin"); + store.begin(addr, 0).expect("begin"); let before = PendingOffsetCommit { database_id: DatabaseId::DEFAULT, @@ -221,7 +221,7 @@ mod tests { let store = SessionStore::new(); let addr: std::net::SocketAddr = "127.0.0.1:6014".parse().expect("address"); store.ensure_session(addr); - store.begin(addr, Lsn::new(1), 0).expect("begin"); + store.begin(addr, 0).expect("begin"); store.create_savepoint(addr, "sp".into(), 1, 0); store.fail_transaction(addr); assert_eq!(store.transaction_state(addr), TransactionState::Failed); diff --git a/nodedb/src/control/server/shared/statement_exec/answer.rs b/nodedb/src/control/server/shared/statement_exec/answer.rs new file mode 100644 index 000000000..e7389f02d --- /dev/null +++ b/nodedb/src/control/server/shared/statement_exec/answer.rs @@ -0,0 +1,190 @@ +// SPDX-License-Identifier: BUSL-1.1 + +//! One statement's answer, folded across its tasks: its row sets, its +//! warnings and its one command tag. +//! +//! The fold is protocol-neutral. pgwire renders the same tag fold, native +//! renders a [`StatementAnswer`] as one response frame, and HTTP renders it as +//! JSON rows. + +use crate::bridge::envelope::PhysicalPlan; +use crate::control::server::response_shape::compose::{ShapeOutcome, shape_response_materialized}; +use crate::control::server::response_shape::redaction::QueryRedaction; +use crate::control::server::response_shape::request::MaterializedShapeRequest; +use crate::control::server::response_shape::types::{ + DmlOutcome, FoldedTag, PlanKind, ShapedRows, StatementTag, TaskTagRole, describe_plan, + payload_to_dml_outcome, staged_dml_outcome, +}; +use crate::control::server::shared::session::staging_gate::{StagedTagKind, StagedWriteOutcome}; +use nodedb_physical::physical_task::PhysicalTask; + +use super::error::StatementError; +use super::task_loop::StatementExec; + +/// A statement's finished answer. +pub(crate) struct StatementAnswer { + /// Each task's shaped rows, in dispatch order. A row set's notice is + /// already in `warnings`. + pub(crate) rows: Vec, + pub(crate) warnings: Vec, + /// The statement's one tag. `None` when no task folded into it. + pub(crate) tag: Option, + /// The watermark of the last task the statement dispatched. + pub(crate) last_lsn: u64, +} + +/// One statement's answer, accumulated across its tasks. +pub(crate) struct StatementFold { + rows: Vec, + warnings: Vec, + pub(crate) tag: StatementTag, + pub(crate) last_lsn: u64, +} + +impl StatementFold { + /// An empty fold for the statement that planned `tasks`. The tag is built + /// over every task, so a derived write beside the user's own adds no + /// verb and no count. + pub(crate) fn for_tasks(tasks: &[PhysicalTask]) -> Self { + Self { + rows: Vec::new(), + warnings: Vec::new(), + tag: StatementTag::for_plans(tasks.iter().map(|t| &t.plan)), + last_lsn: 0, + } + } + + /// Fold the count-bearing outcome of the task that ran `plan` into the + /// statement's tag. + pub(crate) fn fold_dml( + &mut self, + plan: &PhysicalPlan, + outcome: DmlOutcome, + ) -> Result<(), StatementError> { + self.tag + .fold(self.tag.role_of(plan), outcome) + .map_err(StatementError::DmlFold) + } + + /// Fold one shaped row set. Returns its row count. + pub(crate) fn fold_shaped(&mut self, mut shaped: ShapedRows) -> u64 { + if let Some(notice) = shaped.notice.take() { + self.warnings.push(notice); + } + let count = shaped.rows.len() as u64; + self.rows.push(shaped); + count + } + + /// Shape `payload`, the answer `plan` gave as `plan_kind`, and fold its + /// rows. Returns the row count, `None` when the payload carries no rows. + pub(crate) fn fold_rows( + &mut self, + exec: &StatementExec<'_>, + plan: &PhysicalPlan, + plan_kind: PlanKind, + payload: &[u8], + ) -> Result, StatementError> { + let redaction = QueryRedaction::for_plan(exec.tenant_id(), exec.scope.auth(), plan); + match shape_response_materialized(MaterializedShapeRequest { + payload, + plan, + plan_kind, + projection: exec.output_schema, + state: exec.state, + database_id: exec.database_id, + tenant_id: exec.tenant_id(), + redaction: Some(redaction.ctx(&exec.state.redaction)), + sequences: Some(exec.sequences), + }) { + Ok(ShapeOutcome::Rows(shaped)) => Ok(Some(self.fold_shaped(shaped))), + Ok(ShapeOutcome::Passthrough) => Ok(None), + Err(error) => Err(StatementError::Shape(error)), + } + } + + /// Fold a write its transaction staged. A write that answers `RETURNING` + /// folds its rows in place of a count, as an autocommit one does. A + /// computed-value payload (`RawPayload`) is shaped instead of counted. + pub(crate) fn fold_staged( + &mut self, + exec: &StatementExec<'_>, + plan: &PhysicalPlan, + outcome: StagedWriteOutcome, + ) -> Result<(), StatementError> { + if !outcome.returning_rows.is_empty() { + for rows in &outcome.returning_rows { + self.fold_rows(exec, plan, PlanKind::ReturningRows, rows)?; + } + return Ok(()); + } + if !matches!(outcome.kind, StagedTagKind::RawPayload) { + return self.fold_dml(plan, staged_dml_outcome(outcome.kind, outcome.affected)); + } + if outcome.payload.is_empty() { + self.tag.fold_opaque(); + return Ok(()); + } + // The value rides in the payload, never as a count. + if self + .fold_rows(exec, plan, describe_plan(plan), &outcome.payload)? + .is_none() + { + self.tag.fold_opaque(); + } + Ok(()) + } + + /// Fold a dispatched task's answer `payload`: its rows, or its count. + /// Returns the task's own row count for metering, `None` when it carries + /// none. + pub(crate) fn fold_answer( + &mut self, + exec: &StatementExec<'_>, + plan: &PhysicalPlan, + payload: &[u8], + ) -> Result, StatementError> { + let plan_kind = describe_plan(plan); + let count_bearing = matches!(plan_kind, PlanKind::DmlResult(_) | PlanKind::DmlResultByOp); + if payload.is_empty() && !count_bearing { + // Not a count-bearing plan (graph / vector / index write): no + // count and no verb to report. A count-bearing plan with no + // payload falls through so the count reader refuses it. + self.tag.fold_opaque(); + return Ok(None); + } + if let Some(rows) = self.fold_rows(exec, plan, plan_kind, payload)? { + return Ok(Some(rows)); + } + // A count-bearing write reports the rows it touched and its verb; an + // opaque execution reports neither. A count of one per dispatched + // task reports a row for a delete that removed nothing and for an + // `ON CONFLICT DO NOTHING` insert that skipped. + if self.tag.role_of(plan) == TaskTagRole::Opaque { + self.tag.fold_opaque(); + return Ok(None); + } + match payload_to_dml_outcome(payload, plan_kind) { + Ok(Some(outcome)) => { + let affected = outcome.affected; + self.fold_dml(plan, outcome)?; + Ok(Some(affected)) + } + Ok(None) => { + self.tag.fold_opaque(); + Ok(None) + } + Err(error) => Err(StatementError::Error(error)), + } + } + + /// The statement's answer. + pub(crate) fn finish(self) -> StatementAnswer { + StatementAnswer { + rows: self.rows, + warnings: self.warnings, + tag: self.tag.finish(), + last_lsn: self.last_lsn, + } + } +} diff --git a/nodedb/src/control/server/shared/statement_exec/atomic.rs b/nodedb/src/control/server/shared/statement_exec/atomic.rs new file mode 100644 index 000000000..1528899d7 --- /dev/null +++ b/nodedb/src/control/server/shared/statement_exec/atomic.rs @@ -0,0 +1,233 @@ +// SPDX-License-Identifier: BUSL-1.1 + +//! The routes that commit a planned statement's tasks together, taken ahead +//! of its per-task loop. Native, HTTP, WebSocket RPC and NDJSON call +//! [`route_atomic_statement`] for a planned SQL statement, so the same +//! statement takes the same route over each of them, as it does over pgwire. +//! +//! In order: +//! +//! 1. A statement outside a transaction block that +//! [`statement_needs_implicit_txn`] flags runs in an implicit +//! transaction: its writes stage on their vShards' leaders and commit +//! together at its end, through Calvin when they span vShards. +//! 2. A dependent predicate on an edge-bearing collection runs through the +//! implicit-edge OLLP/Calvin coordinator. +//! 3. A write that spans vShards commits through the Calvin sequencer, so a +//! derived task (a cross-shard balance move, an implicit edge) commits +//! with the user's write. + +use std::sync::Arc; + +use nodedb_physical::physical_task::PhysicalTask; + +use crate::control::lease::QueryLeaseScope; +use crate::control::planner::calvin::{ + CrossShardTxnMode, DispatchClass, TxnDispatchPosition, classify_dispatch, + dispatch_authorized_tasks_to_calvin, read_vshards_of, +}; +use crate::control::security::audit::ArcAuditEmitter; +use crate::control::server::response_shape::calvin_fold::{ + CalvinFoldCtx, CalvinFoldError, fold_calvin_batch, +}; +use crate::control::server::shared::authorization::authorize_task_set; +use crate::control::server::shared::metering::{PlanMeteringInfo, meter_dispatch}; +use crate::control::server::shared::session::DmlTxnCtx; +use crate::control::server::shared::session::read_set::ReadSetEntry; +use crate::control::server::shared::txn_route::{ + statement_needs_implicit_txn, unbufferable_joined_statement, +}; +use crate::control::server::shared::write_admission::all_writes_bufferable; +use crate::control::trigger::statement_txn::in_block; + +use super::answer::StatementAnswer; +use super::edge_recon::{CalvinApplied, EdgeRecon, try_edge_recon}; +use super::error::StatementError; +use super::task_loop::{PlannedStatement, StatementExec, run_implicit_statement}; + +/// A planned statement before it takes a route. +pub(crate) struct AtomicStatement { + pub(crate) tasks: Vec, + /// The statement's descriptor leases. They stay alive through dispatch. + pub(crate) lease_scope: QueryLeaseScope, + /// The row images the statement's cross-shard balances were settled + /// from. They widen Calvin's read set, so its OCC check aborts a balance + /// folded from an image that has since moved. + pub(crate) sum_target_reads: Vec, +} + +/// The route a statement took. +pub(crate) enum AtomicRoute { + /// The statement ran in its implicit transaction. + Implicit(StatementAnswer), + /// Calvin applied the statement's tasks as one transaction. + Calvin(CalvinApplied), + /// No atomic route applies. The caller runs its per-task loop. + PerTask(AtomicStatement), +} + +/// Route `statement`, planned on `session`, through the first atomic route +/// that applies. A transport with no session passes a detached one, which is +/// never in a transaction block and takes the default cross-shard mode. +pub(crate) async fn route_atomic_statement( + exec: &StatementExec<'_>, + statement: AtomicStatement, + session: &DmlTxnCtx<'_>, +) -> Result { + let AtomicStatement { + tasks, + lease_scope, + sum_target_reads, + } = statement; + let state = exec.state; + + // Calvin dispatch and edge reconciliation are trusted internal mechanisms + // that never reach the clone-checked dispatch boundary, so one batch + // authorize is correct for them. The per-task loop authorizes each task + // itself, immediately before its own dispatch. + let authorized = authorize_task_set( + exec.identity, + &tasks, + &state.permissions, + &state.roles, + &ArcAuditEmitter(Arc::clone(&state.audit)), + ) + .map_err(|error| StatementError::Error(crate::Error::from(error)))?; + + let in_txn_block = in_block(session); + if !in_txn_block && statement_needs_implicit_txn(state, &tasks) { + if !all_writes_bufferable(&tasks) { + return Err(StatementError::Error(unbufferable_joined_statement())); + } + let answer = run_implicit_statement( + exec, + PlannedStatement { + tasks, + plan_lease_scope: Arc::new(lease_scope), + sum_target_reads, + }, + session, + ) + .await?; + return Ok(AtomicRoute::Implicit(answer)); + } + + let (tasks, authorized) = + match try_edge_recon(state, exec.identity, in_txn_block, tasks, authorized) + .await + .map_err(StatementError::Error)? + { + EdgeRecon::Applied(applied) => { + meter_applied(exec, &applied); + return Ok(AtomicRoute::Calvin(applied)); + } + EdgeRecon::NotFired(tasks, authorized) => (tasks, authorized), + }; + + // An autocommit statement has no session read set to widen with. The + // reads to widen with are those materialized-sum settlement stamped. + let sum_read_vshards = read_vshards_of(&sum_target_reads).map_err(StatementError::Error)?; + match classify_dispatch(&tasks, &sum_read_vshards) { + DispatchClass::SingleShard { .. } => {} + DispatchClass::MultiShard { .. } => { + // Calvin applies the statement durably at statement time, + // outside any transaction buffer. Inside a block the per-task + // loop buffers each write, and COMMIT flushes the buffer through + // Calvin. A write the gate cannot buffer is refused. + if in_txn_block && !all_writes_bufferable(&tasks) { + return Err(StatementError::Error( + crate::Error::CrossShardInExplicitTransaction, + )); + } + let cross_shard_mode = session.sessions.cross_shard_txn_mode(session.session_id); + if !in_txn_block && cross_shard_mode == CrossShardTxnMode::Strict { + let plans = tasks.iter().map(|task| task.plan.clone()).collect(); + let apply_result = dispatch_authorized_tasks_to_calvin( + state, + authorized, + exec.tenant_id(), + cross_shard_mode, + TxnDispatchPosition::Autocommit, + &sum_target_reads, + None, + ) + .await + .map_err(StatementError::Error)?; + let applied = CalvinApplied { + apply_result, + plans, + database_id: exec.database_id, + }; + meter_applied(exec, &applied); + return Ok(AtomicRoute::Calvin(applied)); + } + // `BestEffortNonAtomic` falls through to the per-task loop. + } + } + + Ok(AtomicRoute::PerTask(AtomicStatement { + tasks, + lease_scope, + sum_target_reads, + })) +} + +/// Fold a statement Calvin applied into its answer: the RETURNING rows, when +/// a task carried them, and its one tag. The fold is the one pgwire renders. +pub(crate) fn calvin_answer( + exec: &StatementExec<'_>, + applied: CalvinApplied, +) -> Result { + let CalvinApplied { + apply_result, + plans, + database_id, + } = applied; + let plan_refs: Vec<_> = plans.iter().collect(); + let fold = fold_calvin_batch( + &plan_refs, + apply_result.as_ref(), + &CalvinFoldCtx { + projection: exec.output_schema, + state: exec.state, + tenant_id: exec.tenant_id(), + database_id, + auth: exec.scope.auth(), + }, + ) + .map_err(|error| match error { + CalvinFoldError::Verb(error) => StatementError::DmlFold(error), + CalvinFoldError::Shape(error) => StatementError::Error(error), + })?; + let mut warnings = Vec::new(); + let rows = match fold.rows { + Some(mut shaped) => { + if let Some(notice) = shaped.notice.take() { + warnings.push(notice); + } + vec![shaped] + } + None => Vec::new(), + }; + Ok(StatementAnswer { + rows, + warnings, + tag: fold.tag, + last_lsn: apply_result.map_or(0, |response| response.watermark_lsn.as_u64()), + }) +} + +/// Meter each task of a statement Calvin applied, as pgwire does. +fn meter_applied(exec: &StatementExec<'_>, applied: &CalvinApplied) { + if !exec.state.metering_config.enabled { + return; + } + for plan in &applied.plans { + meter_dispatch( + exec.state, + exec.scope, + &PlanMeteringInfo::extract(plan), + None, + ); + } +} diff --git a/nodedb/src/control/server/shared/statement_exec/dispatch.rs b/nodedb/src/control/server/shared/statement_exec/dispatch.rs new file mode 100644 index 000000000..d9fc36ff5 --- /dev/null +++ b/nodedb/src/control/server/shared/statement_exec/dispatch.rs @@ -0,0 +1,242 @@ +// SPDX-License-Identifier: BUSL-1.1 + +//! The single-task dispatch primitive a planned statement's loop calls. +//! +//! Decides HOW one already-planned task reaches an engine: a Control-Plane +//! orchestrator, Exchange resolution, then the gateway. The statement loop +//! decides what to do with the answers. + +use std::sync::Arc; + +use nodedb_physical::physical_plan::DocumentOp; +use nodedb_physical::physical_task::PhysicalTask; + +use crate::bridge::envelope::{PhysicalPlan, Response}; +use crate::control::gateway::core::QueryContext as GatewayQueryContext; +use crate::control::gateway::router::is_task_vshard_scoped; +use crate::control::security::audit::ArcAuditEmitter; +use crate::control::security::identity::AuthenticatedIdentity; +use crate::control::server::exchange::resolve::{Resolved, resolve_and_materialize}; +use crate::control::server::exchange::{DistributedReadCapture, ReadScope}; +use crate::control::server::shared::authorization::{AuthorizedTask, authorize_task_set}; +use crate::control::server::shared::clone_write::{ + CloneCheckedOutcome, InterceptAndAuthorizeParams, intercept_and_authorize, +}; +use crate::control::state::SharedState; +use crate::types::TraceId; + +/// One dispatched task's answer. +pub(crate) struct DispatchedTask { + pub(crate) response: Response, + /// The reads a distributed gather captured. + pub(crate) distributed_reads: Vec, +} + +impl DispatchedTask { + fn answered(response: Response) -> Self { + Self { + response, + distributed_reads: Vec::new(), + } + } +} + +/// Authorize one task with no clone-write check. Used only for the +/// Control-Plane orchestrated plans (`InsertSelect`, `Merge`, +/// `UpdateFromJoin`, a governed predicate resolution, array DDL, a cluster +/// array op), which are never clone-write shapes. +pub(crate) fn authorize_one_task( + state: &SharedState, + identity: &AuthenticatedIdentity, + task: &PhysicalTask, +) -> crate::Result { + let emitter = ArcAuditEmitter(Arc::clone(&state.audit)); + authorize_task_set( + identity, + std::slice::from_ref(task), + &state.permissions, + &state.roles, + &emitter, + ) + .map_err(crate::Error::from)? + .into_tasks() + .into_iter() + .next() + .ok_or_else(|| crate::Error::Internal { + detail: "authorization returned an empty capability set".into(), + }) +} + +/// Dispatch one `PhysicalTask` and return its answer. +/// +/// Multi-arm writes run through Control-Plane orchestrators. Everything else +/// resolves its Exchange nodes, then routes through +/// [`dispatch_task_via_gateway`]. +pub(crate) async fn dispatch_statement_task( + state: &Arc, + identity: &AuthenticatedIdentity, + mut task: PhysicalTask, +) -> crate::Result { + if let PhysicalPlan::Document(DocumentOp::InsertSelect { .. }) = &task.plan { + let authorized = authorize_one_task(state, identity, &task)?; + let response = + crate::control::insert_select::run_authorized_insert_select(state, authorized).await?; + return Ok(DispatchedTask::answered(response)); + } + + // Autocommit `MERGE` orchestrates on the Control Plane. + if let PhysicalPlan::Document(DocumentOp::Merge { + target_collection: _, + source_collection: _, + source_alias: _, + target_join_col: _, + source_join_col: _, + clauses: _, + returning: _, + resolved_inserts: None, + resolved_insert_identities: _, + source_rows: _, + rls_filters: _, + rls_write_check: _, + resolved_sum_targets: _, + declared_primary_key: _, + }) = &task.plan + { + let authorized = authorize_one_task(state, identity, &task)?; + let response = + crate::control::merge_orchestrator::run_authorized_merge(state, authorized).await?; + return Ok(DispatchedTask::answered(response)); + } + + // Autocommit `UPDATE ... FROM ` scans the source on its own core + // and ships it into the plan, since the source's vShard can live on a + // different core. + if let PhysicalPlan::Document(DocumentOp::UpdateFromJoin { + target_collection: _, + source_collection: _, + source_alias: _, + target_join_col: _, + source_join_col: _, + updates: _, + target_filters: _, + returning: _, + source_rows: None, + rls_filters: _, + rls_write_check: _, + resolved_sum_targets: _, + declared_primary_key: _, + }) = &task.plan + { + let authorized = authorize_one_task(state, identity, &task)?; + let response = + crate::control::update_from_join_orchestrator::run_authorized_update_from_join( + state, authorized, + ) + .await?; + return Ok(DispatchedTask::answered(response)); + } + + // A governed predicate resolves to a concrete row set before proposing. + if let Some(resolver) = crate::control::write_resolve::resolver_for_plan(&task.plan) { + let authorized = authorize_one_task(state, identity, &task)?; + let response = crate::control::write_resolve::run_authorized_write_resolve( + state, authorized, resolver, + ) + .await?; + return Ok(DispatchedTask::answered(response)); + } + + // Array DDL proposes a replicated catalog entry. + if crate::control::array_catalog::ddl::is_array_ddl(&task.plan) { + let authorized = authorize_one_task(state, identity, &task)?; + let response = + crate::control::array_catalog::ddl::run_authorized_array_ddl(state, authorized).await?; + return Ok(DispatchedTask::answered(response)); + } + + // Materialize catalog providers and resolve Exchange nodes before + // dispatch. Every statement read is strong, so every leg confirms its + // group where it is served. + let scope = ReadScope { + database_id: task.database_id, + tenant_id: task.tenant_id, + trace_id: TraceId::ZERO, + txn_id: task.txn_id, + linearizable: true, + }; + match resolve_and_materialize(state, identity, task.plan, scope).await? { + Resolved::Gathered(response, _watermarks, distributed_reads) => { + return Ok(DispatchedTask { + response, + distributed_reads, + }); + } + Resolved::Plan(resolved_plan) => { + task.plan = *resolved_plan; + } + // A materialized statement gathers the stream into one response. + Resolved::Stream(stream) => { + let response = + crate::control::server::exchange::gather::stream_to_response(stream).await?; + return Ok(DispatchedTask::answered(response)); + } + } + + let response = dispatch_task_via_gateway(state, identity, task).await?; + Ok(DispatchedTask::answered(response)) +} + +/// Dispatch one `PhysicalTask` through the gateway. +/// +/// The task takes the clone copy-on-write gate and authorization first. A +/// staged write and the other transaction meta-ops run on the core of the +/// task's own vShard. Every other task routes through +/// `Gateway::execute_response`, which handles cluster-aware routing, typed +/// `NotLeader` retry and plan caching. A `NotFound` verdict comes back as an +/// error status. +pub(crate) async fn dispatch_task_via_gateway( + state: &Arc, + identity: &AuthenticatedIdentity, + task: PhysicalTask, +) -> crate::Result { + let tenant_id = task.tenant_id; + let emitter = ArcAuditEmitter(Arc::clone(&state.audit)); + let checked = match intercept_and_authorize(InterceptAndAuthorizeParams { + state, + task, + identity, + tenant_id, + permissions: &state.permissions, + roles: &state.roles, + emitter: &emitter, + }) + .await? + { + CloneCheckedOutcome::Handled(response) => return Ok(response), + CloneCheckedOutcome::Proceed(checked) => checked, + }; + let database_id = checked.database_id(); + let txn_id = checked.txn_id(); + + let gateway = state.installed_gateway()?; + // The gateway routes a vShard-scoped meta-op to vShard 0. A write takes + // the durable route, a read the read route. + if is_task_vshard_scoped(checked.plan()) { + return crate::control::server::dispatch_utils::dispatch_authorized_task_by_class( + state, + checked, + TraceId::generate(), + ) + .await; + } + let gw_ctx = GatewayQueryContext { + tenant_id, + trace_id: TraceId::generate(), + database_id, + // The in-block transaction id lets local dispatch resolve the + // per-transaction staging overlay. + txn_id, + linearizable: true, + }; + gateway.execute_response(&gw_ctx, checked).await +} diff --git a/nodedb/src/control/server/shared/statement_exec/edge_recon.rs b/nodedb/src/control/server/shared/statement_exec/edge_recon.rs new file mode 100644 index 000000000..21a0ffb11 --- /dev/null +++ b/nodedb/src/control/server/shared/statement_exec/edge_recon.rs @@ -0,0 +1,85 @@ +// SPDX-License-Identifier: BUSL-1.1 + +//! The implicit-edge OLLP/Calvin gate. +//! +//! A dependent predicate (`BulkDelete`/`BulkUpdate`) on an edge-bearing +//! schemaless collection routes through the OLLP/Calvin coordinator, so the +//! implicit edge tasks derive from a pre-exec reconnaissance scan and commit +//! atomically with the document write. Without this gate such a delete +//! classifies as `SingleShard`, dispatches directly to the Data Plane, and +//! leaves its mirrored CSR edges dangling. + +use std::sync::Arc; + +use nodedb_physical::physical_task::PhysicalTask; + +use crate::bridge::envelope::{PhysicalPlan, Response}; +use crate::control::planner::calvin::{ + dispatch_authorized_dependent_edge_recon, plan_needs_implicit_edge_recon, +}; +use crate::control::security::identity::AuthenticatedIdentity; +use crate::control::server::shared::authorization::AuthorizedTaskSet; +use crate::control::state::SharedState; +use crate::types::{DatabaseId, TenantId}; + +/// A statement Calvin applied as one transaction, before any protocol +/// renders its answer. +pub(crate) struct CalvinApplied { + /// The applied response Calvin deposited, when a task deposits one. + pub(crate) apply_result: Option, + /// Every task's plan, in dispatch order. + pub(crate) plans: Vec, + /// The database the answer shapes against. + pub(crate) database_id: DatabaseId, +} + +/// What the gate did with a statement. +pub(crate) enum EdgeRecon { + /// The gate did not fire. The caller gets the tasks back. + NotFired(Vec, AuthorizedTaskSet), + /// The coordinator applied the statement. + Applied(CalvinApplied), +} + +/// Route `tasks` through the implicit-edge OLLP/Calvin coordinator when one +/// of them is a `BulkDelete`/`BulkUpdate` on an edge-bearing collection and +/// the statement runs outside a transaction block. +/// +/// Inside a block the gate does not fire: a multi-step OLLP inside the +/// client's transaction needs a two-phase commit across its boundary. A +/// catalog I/O error propagates. Misrouting on a real fault skips the edge +/// cleanup. +pub(crate) async fn try_edge_recon( + state: &Arc, + identity: &AuthenticatedIdentity, + in_txn_block: bool, + tasks: Vec, + authorized: AuthorizedTaskSet, +) -> crate::Result { + if in_txn_block { + return Ok(EdgeRecon::NotFired(tasks, authorized)); + } + let tenant_id: TenantId = identity.tenant_id; + let Some((_collection, database_id)) = + plan_needs_implicit_edge_recon(state, &tasks, tenant_id)? + else { + return Ok(EdgeRecon::NotFired(tasks, authorized)); + }; + // Captured before the coordinator consumes the tasks, so the answer can + // shape a RETURNING dependent write's rows and read a plain write's + // count from the applied response. + let plans: Vec = tasks.iter().map(|task| task.plan.clone()).collect(); + let recon = dispatch_authorized_dependent_edge_recon( + state, + authorized, + identity, + tenant_id, + database_id, + ) + .await?; + Ok(EdgeRecon::Applied(CalvinApplied { + apply_result: recon.apply_result, + plans, + database_id, + })) +} diff --git a/nodedb/src/control/server/shared/statement_exec/error.rs b/nodedb/src/control/server/shared/statement_exec/error.rs new file mode 100644 index 000000000..be11b01b6 --- /dev/null +++ b/nodedb/src/control/server/shared/statement_exec/error.rs @@ -0,0 +1,53 @@ +// SPDX-License-Identifier: BUSL-1.1 + +//! Why a planned statement stopped, before any protocol renders it. + +use crate::bridge::envelope::{ErrorCode, Response}; +use crate::control::server::response_shape::types::DmlFoldError; +use crate::types::TenantId; + +/// Why a statement stopped. Each protocol renders it in its own error shape. +pub(crate) enum StatementError { + /// A Control-Plane error. + Error(crate::Error), + /// A spent hard quota refused a task before it ran. + Quota(crate::Error), + /// A task names a tenant other than the statement's. + TenantIsolation, + /// The Data Plane refused a staged write. `None` when it gave no code. + Rejected(Option), + /// The Data Plane answered a dispatched task with an error status. + Response(Box), + /// Two tasks of the statement report verbs that cannot share one tag. + DmlFold(DmlFoldError), + /// A task's answer cannot be shaped. + Shape(nodedb_types::NodeDbError), +} + +impl StatementError { + /// The Control-Plane error a protocol with no finer error shape renders. + /// `tenant_id` is the statement's tenant. + pub(crate) fn into_error(self, tenant_id: TenantId) -> crate::Error { + match self { + Self::Error(error) | Self::Quota(error) => error, + Self::TenantIsolation => crate::Error::RejectedAuthz { + tenant_id, + resource: "tenant isolation violation".into(), + }, + Self::Rejected(Some(code)) => crate::Error::DataPlane(code), + Self::Rejected(None) => crate::Error::Internal { + detail: "unknown data plane error".into(), + }, + Self::Response(response) => match response.error_code { + Some(code) => crate::Error::DataPlane(*code), + None => crate::Error::Internal { + detail: "data plane returned an error status with no error code".into(), + }, + }, + Self::DmlFold(error) => crate::Error::Internal { + detail: error.to_string(), + }, + Self::Shape(error) => crate::Error::from(error), + } + } +} diff --git a/nodedb/src/control/server/shared/statement_exec/mod.rs b/nodedb/src/control/server/shared/statement_exec/mod.rs new file mode 100644 index 000000000..52b0928a6 --- /dev/null +++ b/nodedb/src/control/server/shared/statement_exec/mod.rs @@ -0,0 +1,21 @@ +// SPDX-License-Identifier: BUSL-1.1 + +//! One planned SQL statement, run the same way by every protocol: its atomic +//! routes, its per-task loop, the single-task dispatch primitive, and the +//! fold of its answer. Each protocol renders the answer in its own shape. + +pub mod answer; +pub mod atomic; +pub mod dispatch; +pub mod edge_recon; +pub mod error; +pub mod task_loop; + +pub(crate) use answer::StatementAnswer; +pub(crate) use atomic::{AtomicRoute, AtomicStatement, calvin_answer, route_atomic_statement}; +pub(crate) use dispatch::authorize_one_task; +pub(crate) use edge_recon::{EdgeRecon, try_edge_recon}; +pub(crate) use error::StatementError; +pub(crate) use task_loop::{ + PlannedStatement, StatementExec, run_implicit_statement, run_statement_loop, +}; diff --git a/nodedb/src/control/server/shared/statement_exec/task_loop.rs b/nodedb/src/control/server/shared/statement_exec/task_loop.rs new file mode 100644 index 000000000..0abfd8ae8 --- /dev/null +++ b/nodedb/src/control/server/shared/statement_exec/task_loop.rs @@ -0,0 +1,299 @@ +// SPDX-License-Identifier: BUSL-1.1 + +//! The per-task loop of a planned statement. +//! +//! A statement in a transaction (the client's block, or the implicit +//! transaction a statement runs in when [`statement_needs_implicit_txn`] +//! flags it) routes each task through the shared `txn_route`, the route +//! pgwire takes too: the write's trigger bodies join the transaction, a +//! shadowed clone takes its copy-on-write steps, and the write stages. An +//! autocommit statement dispatches each task directly. +//! +//! [`statement_needs_implicit_txn`]: crate::control::server::shared::txn_route::statement_needs_implicit_txn + +use std::sync::Arc; + +use nodedb_physical::physical_task::PhysicalTask; + +use crate::bridge::envelope::{ErrorCode, PhysicalPlan, Status}; +use crate::control::lease::QueryLeaseScope; +use crate::control::security::identity::AuthenticatedIdentity; +use crate::control::security::request_scope::RequestAuthScope; +use crate::control::sequence::SessionSequenceAccess; +use crate::control::server::response_shape::schema::OutputSchema; +use crate::control::server::response_shape::types::{describe_plan, replaced_write_outcome}; +use crate::control::server::shared::cluster_array_dispatch::{ + ClusterArrayShaped, execute_cluster_array, +}; +use crate::control::server::shared::metering::{PlanMeteringInfo, meter_dispatch}; +use crate::control::server::shared::quota_admission::admit_quota_for_dispatch; +use crate::control::server::shared::session::read_set::ReadSetEntry; +use crate::control::server::shared::session::staging_gate::StagingGateError; +use crate::control::server::shared::session::{ + DmlTxnCtx, ResponseReads, record_reads_for_response, +}; +use crate::control::server::shared::txn_route::{ + StatementEvents, TxnTaskContext, TxnTaskOutcome, route_txn_task, +}; +use crate::control::state::SharedState; +use crate::types::{DatabaseId, TenantId}; + +use super::answer::{StatementAnswer, StatementFold}; +use super::dispatch::{DispatchedTask, authorize_one_task, dispatch_statement_task}; +use super::error::StatementError; + +/// Who runs a statement, and what its answer shapes against. +pub(crate) struct StatementExec<'a> { + pub(crate) state: &'a Arc, + pub(crate) identity: &'a AuthenticatedIdentity, + /// The request's auth scope: RLS context, redaction, metering and quota. + pub(crate) scope: &'a RequestAuthScope<'a>, + /// The statement's announced output columns, when it announced any. + pub(crate) output_schema: Option<&'a OutputSchema>, + pub(crate) database_id: DatabaseId, + /// The sequence access the statement's Control-Plane computed columns + /// read: the registry, plus the session's `currval` map when it has one. + pub(crate) sequences: &'a SessionSequenceAccess<'a>, + /// The client's session, which notes its own committed writes. `None` + /// for a transport with no session. + pub(crate) client_session: Option>, +} + +impl StatementExec<'_> { + pub(crate) fn tenant_id(&self) -> TenantId { + self.identity.tenant_id + } +} + +/// A planned statement's tasks and what they carry into dispatch. +pub(crate) struct PlannedStatement { + pub(crate) tasks: Vec, + /// The statement's descriptor leases, kept on every task it buffers. + pub(crate) plan_lease_scope: Arc, + /// The row images the statement's cross-shard balances were settled + /// from. A statement in a transaction adds them to its read set. + pub(crate) sum_target_reads: Vec, +} + +/// Run every task of `statement` and fold their answers into one. +/// `txn` is the statement's transaction, `None` for an autocommit statement. +/// +/// The answer's tag is one [`crate::control::server::response_shape::types::StatementTag`] +/// folded over every task, the same fold pgwire renders as its command tag: +/// a count-bearing task contributes its reported count and verb, an opaque +/// task (buffered write, index or graph maintenance, computed-value payload, +/// derived balance or edge write) contributes nothing. Neither is ever +/// synthesised from the task count. +pub(crate) async fn run_statement_loop( + exec: &StatementExec<'_>, + statement: PlannedStatement, + txn: Option<&DmlTxnCtx<'_>>, +) -> Result { + let PlannedStatement { + tasks, + plan_lease_scope, + sum_target_reads, + } = statement; + if let Some(txn) = txn + && !sum_target_reads.is_empty() + { + txn.sessions + .record_read_entries(txn.session_id, sum_target_reads); + } + let state = exec.state; + let mut fold = StatementFold::for_tasks(&tasks); + // Checked once rather than per task: metering is disabled by default, so + // the per-task extraction below stays a no-op on the hot path. + let metering_enabled = state.metering_config.enabled; + let route_ctx = txn.map(|txn| TxnTaskContext { + state, + identity: exec.identity, + auth: exec.scope.auth(), + txn, + lease_scope: &plan_lease_scope, + fire_triggers: true, + }); + let mut statement_events = StatementEvents::default(); + + for task in tasks { + if task.tenant_id != exec.tenant_id() { + return Err(StatementError::TenantIsolation); + } + + // Cloned before routing consumes `task`: a staged write's rows and + // computed-value payload shape against it. + let plan = task.plan.clone(); + // Metering covers the direct dispatch below. A staged write meters + // itself inside the staging gate, and a buffered write meters at + // COMMIT. + let plan_metering_info = metering_enabled.then(|| PlanMeteringInfo::extract(&plan)); + + // A spent hard quota refuses the task before it runs. The charging + // call at the end of this loop is on the success path and never + // refuses. + if let Some(info) = &plan_metering_info { + admit_quota_for_dispatch(state, exec.scope, info).map_err(StatementError::Quota)?; + } + + // The task to dispatch below: the input task, or the one routing + // hands back. Every other routing outcome answers the task here. + let task = match route_ctx.as_ref() { + Some(route) => { + let routed = route_txn_task( + route, + task, + &mut statement_events, + |stage_task| async move { + dispatch_statement_task(state, exec.identity, stage_task) + .await + .map(|dispatched| dispatched.response) + }, + ) + .await; + match routed { + Ok(TxnTaskOutcome::Dispatch(routed_task)) => *routed_task, + Ok(TxnTaskOutcome::Buffered) => { + // Applied at COMMIT: no count and no verb yet. + fold.tag.fold_opaque(); + continue; + } + Ok(TxnTaskOutcome::Staged(outcome)) => { + fold.fold_staged(exec, &plan, outcome)?; + continue; + } + Ok(TxnTaskOutcome::CloneHandled(clone_response)) => { + fold.fold_answer(exec, &plan, clone_response.payload.as_ref())?; + continue; + } + Ok(TxnTaskOutcome::InsteadOf) => { + match replaced_write_outcome(describe_plan(&plan)) { + Some(outcome) => fold.fold_dml(&plan, outcome)?, + None => fold.tag.fold_opaque(), + } + continue; + } + Err(StagingGateError::Dispatch(error)) => { + return Err(StatementError::Error(error)); + } + Err(StagingGateError::Rejected { code }) => { + return Err(StatementError::Rejected(code)); + } + } + } + None => task, + }; + + // `ClusterArray` plans run entirely on the Control Plane through the + // `ArrayCoordinator`, never through the SPSC bridge or the trigger/DML + // machinery. A write in a transaction reshapes into per-shard + // `ArrayOp` tasks inside the staging gate above, so only reads + // (`Slice`/`Agg`) and autocommit `Put`/`Delete` arrive here. Metering + // is not applied here, matching pgwire's `ClusterArray` short-circuit. + if matches!(task.plan, PhysicalPlan::ClusterArray(_)) { + let authorized = + authorize_one_task(state, exec.identity, &task).map_err(StatementError::Error)?; + match execute_cluster_array(state, exec.scope.auth(), authorized, exec.output_schema) + .await + .map_err(StatementError::Error)? + { + ClusterArrayShaped::Rows(shaped) => { + fold.fold_shaped(shaped); + } + ClusterArrayShaped::Affected(outcome) => fold.fold_dml(&plan, outcome)?, + } + continue; + } + + let task_database_id = task.database_id; + let DispatchedTask { + response: task_response, + distributed_reads, + } = dispatch_statement_task(state, exec.identity, task) + .await + .map_err(StatementError::Error)?; + + // Track reads for commit-time conflict detection into the + // statement's transaction. Recorded BEFORE the error short-circuit so + // an absent-key point read (a `NotFound` from the Data Plane) is + // captured too: a "not found" is a validatable phantom observation. + // A committed write floors the session's later reads at the + // versions it stamped. + if let Some(client) = &exec.client_session { + client.sessions.note_own_write_response( + client.session_id, + task_database_id, + exec.tenant_id(), + &plan, + &task_response, + ); + } + let records_read = task_response.status == Status::Ok + || task_response.error_code.as_deref() == Some(&ErrorCode::NotFound); + if records_read && let Some(txn) = txn { + record_reads_for_response( + state, + txn.sessions, + txn.session_id, + exec.tenant_id(), + ResponseReads { + plan: &plan, + read_versions: &task_response.read_versions, + found: task_response.status == Status::Ok, + distributed_reads: &distributed_reads, + }, + ) + .await; + } + + if task_response.status == Status::Error { + return Err(StatementError::Response(Box::new(task_response))); + } + + // TRUNCATE RESTART IDENTITY. Autocommit only: a buffered truncate + // restarts its sequences at COMMIT. + if let Some((collection, true)) = plan.truncate_target() { + state.sequence_registry.restart_sequences_for_collection( + task_database_id.as_u64(), + exec.tenant_id().as_u64(), + collection.as_str(), + ); + } + + fold.last_lsn = task_response.watermark_lsn.as_u64(); + let task_rows = fold.fold_answer(exec, &plan, task_response.payload.as_ref())?; + + // Metered once per successfully dispatched task. + if let Some(info) = &plan_metering_info { + meter_dispatch(state, exec.scope, info, task_rows); + } + } + + if let Some(route) = route_ctx.as_ref() { + statement_events + .fire(route) + .await + .map_err(StatementError::Error)?; + } + + Ok(fold.finish()) +} + +/// Run a statement in an implicit transaction: it commits when every task +/// succeeds, and rolls back when one fails. `client` is the session the +/// statement arrived on. The transaction runs on a private session of its +/// own, so a transport with no session passes a detached one. +pub(crate) async fn run_implicit_statement( + exec: &StatementExec<'_>, + statement: PlannedStatement, + client: &DmlTxnCtx<'_>, +) -> Result { + crate::control::trigger::statement_txn::with_statement_txn_lifted( + exec.state, + exec.identity, + client, + true, + StatementError::Error, + async |txn: &DmlTxnCtx<'_>| run_statement_loop(exec, statement, Some(txn)).await, + ) + .await +} diff --git a/nodedb/src/control/server/shared/txn_route/joined.rs b/nodedb/src/control/server/shared/txn_route/joined.rs index 4d4e14530..54610c891 100644 --- a/nodedb/src/control/server/shared/txn_route/joined.rs +++ b/nodedb/src/control/server/shared/txn_route/joined.rs @@ -44,15 +44,78 @@ pub fn statement_fires_joined_body(state: &SharedState, tasks: &[PhysicalTask]) } /// Whether a statement outside a transaction block runs in an implicit -/// transaction. It does when it fires a joined body, and when it is a -/// `MERGE` into a collection an edge was ever written into: the MERGE then -/// stages each removed row with its node's edge tasks, and COMMIT applies -/// them together. +/// transaction. It does in three cases: +/// +/// - It fires a joined body. +/// - It is a `MERGE` into a collection an edge was ever written into. The +/// MERGE then stages each removed row with its node's edge tasks, and +/// COMMIT applies them together. +/// - It is an `INSERT ... SELECT`, `UPDATE ... FROM` or `MERGE` that writes +/// the source of a materialized sum whose target lives on another vShard. +/// The expansion ships that balance on an `ApplyBalanceDelta` task homed +/// on the target, and COMMIT applies it with the source rows. pub fn statement_needs_implicit_txn(state: &SharedState, tasks: &[PhysicalTask]) -> bool { statement_fires_joined_body(state, tasks) || tasks .iter() - .any(|task| merges_into_edge_bearing(state, task)) + .any(|task| merges_into_edge_bearing(state, task) || moves_cross_shard_sum(state, task)) +} + +/// Whether `task` is an unexpanded `INSERT ... SELECT`, `UPDATE ... FROM` or +/// `MERGE` whose written collection drives a materialized sum with a +/// cross-shard target. Its orchestrator applies on the written collection's +/// vShard alone, so it cannot move that target. A catalog read error reads +/// as a cross-shard sum, so the statement takes the transaction. +fn moves_cross_shard_sum(state: &SharedState, task: &PhysicalTask) -> bool { + let PhysicalPlan::Document(op) = &task.plan else { + return false; + }; + let written = match op { + DocumentOp::InsertSelect { + target_collection, .. + } + | DocumentOp::UpdateFromJoin { + target_collection, + source_rows: None, + .. + } + | DocumentOp::Merge { + target_collection, + resolved_inserts: None, + .. + } => target_collection.as_str(), + DocumentOp::UpdateFromJoin { .. } + | DocumentOp::Merge { .. } + | DocumentOp::PointGet { .. } + | DocumentOp::PointPut { .. } + | DocumentOp::PointInsert { .. } + | DocumentOp::PointDelete { .. } + | DocumentOp::PointUpdate { .. } + | DocumentOp::Upsert { .. } + | DocumentOp::BatchInsert { .. } + | DocumentOp::BulkUpdate { .. } + | DocumentOp::BulkDelete { .. } + | DocumentOp::Truncate { .. } + | DocumentOp::Scan { .. } + | DocumentOp::RangeScan { .. } + | DocumentOp::Register { .. } + | DocumentOp::IndexLookup { .. } + | DocumentOp::IndexedFetch { .. } + | DocumentOp::DropIndex { .. } + | DocumentOp::BackfillIndex { .. } + | DocumentOp::EstimateCount { .. } + | DocumentOp::MaterializeScan { .. } + | DocumentOp::ResolveWrite(_) + | DocumentOp::ResolvedWrite { .. } + | DocumentOp::ApplyBalanceDelta { .. } => return false, + }; + crate::control::planner::materialized_sum::drives_cross_shard_sum( + state, + written, + task.tenant_id, + task.database_id, + ) + .unwrap_or(true) } /// Whether `task` is a `MERGE` into an edge-bearing collection. A catalog diff --git a/nodedb/src/control/server/shared/txn_route/route.rs b/nodedb/src/control/server/shared/txn_route/route.rs index 281f8e7d4..7501c0d5f 100644 --- a/nodedb/src/control/server/shared/txn_route/route.rs +++ b/nodedb/src/control/server/shared/txn_route/route.rs @@ -28,6 +28,9 @@ use nodedb_physical::physical_task::PhysicalTask; use crate::bridge::envelope::{PhysicalPlan, Response}; use crate::control::lease::QueryLeaseScope; +use crate::control::planner::calvin::write_class::{ + plan_counts_toward_statement_tag, plans_have_user_write, +}; use crate::control::security::auth_context::AuthContext; use crate::control::security::identity::AuthenticatedIdentity; use crate::control::server::response_shape::types::{PlanKind, describe_plan}; @@ -182,17 +185,26 @@ where F: Fn(PhysicalTask) -> Fut, Fut: Future>, { + // A balance task the expansion appended stages like any op, but its count + // describes a row the statement never named. + let has_user_write = plans_have_user_write(ops.iter().map(|op| &op.plan)); let mut affected = 0usize; let mut returning_rows = Vec::new(); for op in ops { + let counts = plan_counts_toward_statement_tag(&op.plan, has_user_write); match route_point(ctx, op, expanded, dispatch).await? { TxnTaskOutcome::Staged(outcome) => { - affected += outcome.affected; + if counts { + affected += outcome.affected; + } returning_rows.extend(outcome.returning_rows); } TxnTaskOutcome::CloneHandled(resp) => { - affected += require_affected_count(resp.payload.as_ref()) + let changed = require_affected_count(resp.payload.as_ref()) .map_err(StagingGateError::Dispatch)? as usize; + if counts { + affected += changed; + } } TxnTaskOutcome::InsteadOf => {} TxnTaskOutcome::Buffered | TxnTaskOutcome::Dispatch(_) => { diff --git a/nodedb/src/control/server/shared/write_admission/admission_keys.rs b/nodedb/src/control/server/shared/write_admission/admission_keys.rs new file mode 100644 index 000000000..469be6e1f --- /dev/null +++ b/nodedb/src/control/server/shared/write_admission/admission_keys.rs @@ -0,0 +1,576 @@ +// SPDX-License-Identifier: BUSL-1.1 + +//! The lock keys the write-admission gate takes for one write. +//! +//! The keys come from the Calvin write-key extraction and its lock-request +//! expansion. An autocommit write therefore takes the keys a Calvin +//! transaction of the same plan takes: +//! +//! - A row write locks each row `Exclusive` and its collection `Intent`. +//! - A collection-wide write locks its collection `Exclusive`: a predicate +//! or bulk write, INSERT…SELECT, a truncate, a columnar update or delete, +//! an array write, a KV predicate write, and a CRDT snapshot import. +//! - A document write that claims a UNIQUE value locks that value +//! `Exclusive`. +//! +//! An append differs from its Calvin keys: a timeseries ingest or a columnar +//! insert locks the rows it names `Exclusive` and its collection `Intent`. +//! +//! - Two appends never contend at the gate. The data-group log orders them, +//! alike on every replica. +//! - Every Calvin writer of a columnar or timeseries collection locks it +//! `Exclusive`. So an append still orders against each Calvin writer. +//! - A contended append waits and never routes. A routed write applies +//! outside the data-group log. Each replica then installs it against its +//! own schema at its own point. +//! +//! A write no Calvin transaction sequences still takes keys: +//! +//! - A resolved write locks the rows it names, as a row write does. +//! - A text or spatial index write locks its collection `Intent`. +//! - A transaction batch takes the keys of every plan it carries. +//! - Any other write locks every collection it writes `Exclusive`. + +use std::collections::BTreeMap; + +use nodedb_cluster::calvin::types::EngineKeySet; +use nodedb_physical::physical_plan::{ + ColumnarOp, DocumentOp, KvOp, MetaOp, PhysicalPlan, TimeseriesOp, VectorOp, +}; + +use crate::control::cluster::calvin::scheduler::driver::helpers::expand_write_key_sets; +use crate::control::cluster::calvin::scheduler::lock_manager::{LockKey, LockMode}; +use crate::control::planner::calvin::submit::unique_claims::catalog_unique_claim_sets; +use crate::control::planner::calvin::tx_class::write_keys::{WriteKeys, add_plan_write_keys}; +use crate::control::planner::calvin::write_class::is_write_plan; +use crate::control::state::SharedState; +use crate::control::wal_replication::transaction_redo::collections::written_collections; +use crate::types::{DatabaseId, TenantId}; + +/// The moded lock request of one write. +#[derive(Debug, Default)] +pub(crate) struct AdmissionKeys { + /// Every key the write holds, each in its mode. + pub keys: BTreeMap, + /// Whether a contended write routes to the scheduler. A write a Calvin + /// transaction cannot sequence, and an append, waits instead. + pub sequenced: bool, +} + +/// The lock request of `plan`, a write-class plan of `tenant_id` in +/// `database_id` that no Calvin scheduler applies. +pub(crate) fn plan_admission_keys( + shared: &SharedState, + tenant_id: TenantId, + database_id: DatabaseId, + plan: &PhysicalPlan, +) -> crate::Result { + let (mut sets, sequenced) = plan_key_sets(plan)?; + if sequenced { + // A claim the plan does not carry locks the collection whole. The + // expansion merges each key's modes, so the strongest mode wins. + sets.extend(catalog_unique_claim_sets( + shared, + database_id, + tenant_id.as_u64(), + std::slice::from_ref(plan), + )?); + } + Ok(AdmissionKeys { + keys: expand_write_key_sets(&sets), + sequenced, + }) +} + +/// The lock request of a write that names only whole `collections`, each +/// locked `Exclusive`. A committed session transaction's redo and a synced +/// array op take it. +pub(crate) fn collection_admission_keys(collections: &[String]) -> AdmissionKeys { + let mut keys = WriteKeys::default(); + for collection in collections { + keys.whole_collection(collection); + } + AdmissionKeys { + keys: expand_write_key_sets(&keys.into_key_sets()), + sequenced: false, + } +} + +/// A Calvin-scheduled apply: the scheduler acquired its locks. +pub(super) fn is_calvin_apply(plan: &PhysicalPlan) -> bool { + matches!( + plan, + PhysicalPlan::Meta( + MetaOp::CalvinExecuteStatic { .. } + | MetaOp::CalvinExecutePassive { .. } + | MetaOp::CalvinExecuteActive { .. } + | MetaOp::CalvinDrop { .. } + | MetaOp::CalvinResolve { .. } + ) + ) +} + +/// The write key sets of `plan`, and whether it routes to the scheduler +/// when contended. +fn plan_key_sets(plan: &PhysicalPlan) -> crate::Result<(Vec, bool)> { + let mut keys = WriteKeys::default(); + match plan { + // An append: its rows, and its collection `Intent` (see the module + // docs). A timeseries row has no surrogate: its series names it. + PhysicalPlan::Timeseries(TimeseriesOp::Ingest { collection, .. }) => { + keys.rows(collection.as_str(), []); + } + PhysicalPlan::Columnar(ColumnarOp::Insert { + collection, + surrogates, + .. + }) => keys.rows( + collection.as_str(), + surrogates.iter().map(|surrogate| surrogate.as_u32()), + ), + PhysicalPlan::Meta(MetaOp::TransactionBatch { plans, .. }) => { + let mut sets = Vec::new(); + for inner in plans.iter().filter(|inner| is_write_plan(inner)) { + sets.extend(plan_key_sets(inner)?.0); + } + return Ok((sets, false)); + } + // An index write of rows its collection stores. No Calvin + // transaction writes the index alone, so the collection key orders + // it against a collection-wide writer. + PhysicalPlan::Text(_) | PhysicalPlan::Spatial(_) => { + for collection in written_collections(std::slice::from_ref(plan)) { + keys.rows(&collection, []); + } + } + PhysicalPlan::Document(_) + | PhysicalPlan::Kv(_) + | PhysicalPlan::Vector(_) + | PhysicalPlan::Graph(_) + | PhysicalPlan::Timeseries(_) + | PhysicalPlan::Columnar(_) + | PhysicalPlan::Crdt(_) + | PhysicalPlan::Array(_) + | PhysicalPlan::Meta(_) + | PhysicalPlan::Query(_) + | PhysicalPlan::ClusterArray(_) + | PhysicalPlan::ClusterEvent(_) => { + if is_write_plan(plan) && add_plan_write_keys(&mut keys, plan).is_ok() { + return Ok((keys.into_key_sets(), true)); + } + // The extraction refuses a write no Calvin transaction can + // sequence. The keys it added before refusing stay: they + // over-lock and never under-lock. + unsequenced_keys(&mut keys, plan); + } + } + Ok((keys.into_key_sets(), false)) +} + +/// Add the keys of a write no Calvin transaction sequences: the rows a +/// resolved write names, or every collection the write names, whole. +fn unsequenced_keys(keys: &mut WriteKeys, plan: &PhysicalPlan) { + match plan { + PhysicalPlan::Document(DocumentOp::ResolvedWrite { mutations, .. }) => { + for mutation in mutations { + let collection = mutation.collection().as_str(); + keys.rows(collection, [mutation.surrogate().as_u32()]); + keys.row_id(collection, mutation.document_id()); + } + } + PhysicalPlan::Kv(KvOp::ResolvedWrite { mutations, .. }) => { + for mutation in mutations { + keys.kv_keys(mutation.collection().as_str(), [mutation.key().to_vec()]); + } + } + PhysicalPlan::Vector(VectorOp::ResolvedDirectWrite { + collection, + mutations, + .. + }) => keys.vector_rows( + collection.as_str(), + mutations + .iter() + .map(|mutation| mutation.surrogate().as_u32()), + ), + PhysicalPlan::Document(_) + | PhysicalPlan::Kv(_) + | PhysicalPlan::Vector(_) + | PhysicalPlan::Graph(_) + | PhysicalPlan::Timeseries(_) + | PhysicalPlan::Columnar(_) + | PhysicalPlan::Crdt(_) + | PhysicalPlan::Array(_) + | PhysicalPlan::Meta(_) + | PhysicalPlan::Text(_) + | PhysicalPlan::Spatial(_) + | PhysicalPlan::Query(_) + | PhysicalPlan::ClusterArray(_) + | PhysicalPlan::ClusterEvent(_) => { + for collection in written_collections(std::slice::from_ref(plan)) { + keys.whole_collection(&collection); + } + } + } +} + +#[cfg(test)] +mod tests { + use std::sync::Arc; + + use nodedb_physical::physical_plan::{ColumnarInsertIntent, CrdtOp}; + use nodedb_physical::physical_plan::{DocumentResolvedMutation, KvResolvedMutation}; + + use crate::control::cluster::calvin::scheduler::lock_manager::{LockManager, TxnId}; + use nodedb_types::{QualifiedCollection, Surrogate}; + + use super::*; + + fn qualified(name: &str) -> QualifiedCollection { + QualifiedCollection::new(DatabaseId::DEFAULT, name) + } + + fn coll(name: &str) -> LockKey { + LockKey::Collection { + collection: Arc::from(qualified(name).as_str()), + } + } + + fn surrogate_key(name: &str, surrogate: u32) -> LockKey { + LockKey::Surrogate { + collection: Arc::from(qualified(name).as_str()), + surrogate, + } + } + + fn kv_key(name: &str, key: &[u8]) -> LockKey { + LockKey::Kv { + collection: Arc::from(qualified(name).as_str()), + key: Arc::from(key), + } + } + + /// The keys of `plan`, with no UNIQUE claim lookup. + fn keys_of(plan: &PhysicalPlan) -> (BTreeMap, bool) { + let (sets, sequenced) = plan_key_sets(plan).expect("key sets"); + (expand_write_key_sets(&sets), sequenced) + } + + fn doc_insert(id: &str, surrogate: u32) -> PhysicalPlan { + PhysicalPlan::Document(DocumentOp::PointInsert { + collection: qualified("docs"), + document_id: id.to_owned(), + value: Vec::new(), + if_absent: false, + surrogate: Surrogate::new(surrogate), + returning: None, + rls_filters: Vec::new(), + resolved_sum_targets: Vec::new(), + deferred_sum_targets: Vec::new(), + }) + } + + /// A point write locks its row `Exclusive`, by surrogate and by row id, + /// and its collection `Intent`. + #[test] + fn a_point_write_locks_its_row_and_announces_its_collection() { + let (keys, sequenced) = keys_of(&doc_insert("x", 17)); + assert!(sequenced); + assert_eq!(keys.get(&coll("docs")), Some(&LockMode::Intent)); + assert_eq!( + keys.get(&surrogate_key("docs", 17)), + Some(&LockMode::Exclusive) + ); + assert_eq!(keys.get(&kv_key("docs", b"x")), Some(&LockMode::Exclusive)); + assert_eq!(keys.len(), 3); + } + + /// A batch insert locks every row it inserts and its collection + /// `Intent`. + #[test] + fn a_batch_insert_locks_each_row() { + let plan = PhysicalPlan::Document(DocumentOp::BatchInsert { + collection: qualified("docs"), + documents: vec![("a".to_owned(), Vec::new()), ("b".to_owned(), Vec::new())], + surrogates: vec![Surrogate::new(1), Surrogate::new(2)], + returning: None, + rls_filters: Vec::new(), + resolved_sum_targets: Vec::new(), + deferred_sum_targets: Vec::new(), + }); + let (keys, sequenced) = keys_of(&plan); + assert!(sequenced); + assert_eq!(keys.get(&coll("docs")), Some(&LockMode::Intent)); + for (id, surrogate) in [("a", 1), ("b", 2)] { + assert_eq!( + keys.get(&surrogate_key("docs", surrogate)), + Some(&LockMode::Exclusive) + ); + assert_eq!( + keys.get(&kv_key("docs", id.as_bytes())), + Some(&LockMode::Exclusive) + ); + } + } + + /// Every collection-wide shape locks its collection `Exclusive` and + /// routes to the scheduler when contended. + #[test] + fn a_collection_wide_write_locks_its_collection_exclusive() { + let plans = [ + PhysicalPlan::Document(DocumentOp::Truncate { + collection: qualified("docs"), + restart_identity: false, + resolved_sum_targets: Vec::new(), + declared_primary_key: None, + }), + PhysicalPlan::Columnar(ColumnarOp::Delete { + collection: qualified("docs"), + filters: Vec::new(), + rls_write_check: nodedb_types::RlsWriteCheck::pending_injection(), + }), + PhysicalPlan::Timeseries(TimeseriesOp::Truncate { + collection: qualified("docs"), + restart_identity: false, + }), + PhysicalPlan::Kv(KvOp::Truncate { + collection: qualified("docs"), + restart_identity: false, + }), + PhysicalPlan::Crdt(CrdtOp::ImportSnapshot { + tenant_id: 1, + collection: qualified("docs"), + bytes: Vec::new(), + }), + ]; + for plan in plans { + let (keys, sequenced) = keys_of(&plan); + assert!(sequenced, "{plan:?}"); + assert_eq!( + keys.get(&coll("docs")), + Some(&LockMode::Exclusive), + "{plan:?}" + ); + } + } + + /// A resolved write no Calvin transaction sequences still locks the rows + /// it names, and waits rather than routes when contended. + #[test] + fn a_resolved_write_locks_the_rows_it_names() { + let doc = PhysicalPlan::Document(DocumentOp::ResolvedWrite { + mutations: vec![DocumentResolvedMutation::Delete { + collection: qualified("docs"), + document_id: "x".to_owned(), + surrogate: Surrogate::new(17), + pk_bytes: b"x".to_vec(), + precondition: None, + resolved_sum_targets: Vec::new(), + }], + response_payload: Vec::new(), + rls_write_check: nodedb_types::RlsWriteCheck::pending_injection(), + }); + let (keys, sequenced) = keys_of(&doc); + assert!(!sequenced); + assert_eq!( + keys.get(&surrogate_key("docs", 17)), + Some(&LockMode::Exclusive) + ); + assert_eq!(keys.get(&coll("docs")), Some(&LockMode::Intent)); + + let kv = PhysicalPlan::Kv(KvOp::ResolvedWrite { + mutations: vec![KvResolvedMutation::Put { + collection: qualified("kv"), + key: b"k".to_vec(), + value: Vec::new(), + ttl_ms: 0, + expire_at_ms: 0, + surrogate: Surrogate::new(5), + precondition: None, + }], + response_payload: Vec::new(), + rls_write_check: nodedb_types::RlsWriteCheck::pending_injection(), + }); + let (keys, sequenced) = keys_of(&kv); + assert!(!sequenced); + assert_eq!(keys.get(&kv_key("kv", b"k")), Some(&LockMode::Exclusive)); + assert_eq!(keys.get(&coll("kv")), Some(&LockMode::Intent)); + } + + /// A transaction batch takes the keys of every write it carries. + #[test] + fn a_transaction_batch_takes_the_keys_of_its_writes() { + let batch = PhysicalPlan::Meta(MetaOp::TransactionBatch { + plans: vec![doc_insert("x", 17), doc_insert("y", 18)], + txn_id: None, + }); + let (keys, sequenced) = keys_of(&batch); + assert!(!sequenced); + assert_eq!( + keys.get(&surrogate_key("docs", 17)), + Some(&LockMode::Exclusive) + ); + assert_eq!( + keys.get(&surrogate_key("docs", 18)), + Some(&LockMode::Exclusive) + ); + assert_eq!(keys.get(&coll("docs")), Some(&LockMode::Intent)); + } + + fn ts_ingest() -> PhysicalPlan { + PhysicalPlan::Timeseries(TimeseriesOp::Ingest { + collection: qualified("metrics"), + payload: b"metrics value=1.0 1000000000".to_vec(), + format: "ilp".to_owned(), + wal_lsn: None, + surrogates: Vec::new(), + provenance: None, + rls_write_check: nodedb_types::RlsWriteCheck::pending_injection(), + returning: None, + rls_filters: Vec::new(), + }) + } + + fn columnar_insert(intent: ColumnarInsertIntent) -> PhysicalPlan { + PhysicalPlan::Columnar(ColumnarOp::Insert { + collection: qualified("events"), + payload: Vec::new(), + format: "msgpack".to_owned(), + intent, + on_conflict_updates: Vec::new(), + surrogates: vec![Surrogate::new(4), Surrogate::new(5)], + schema_bytes: Vec::new(), + provenance: None, + wal_lsn: None, + rls_write_check: nodedb_types::RlsWriteCheck::pending_injection(), + returning: None, + rls_filters: Vec::new(), + }) + } + + /// A timeseries ingest locks its collection `Intent` and nothing else. + /// It waits rather than routes when contended. + #[test] + fn a_timeseries_ingest_announces_its_collection_and_never_routes() { + let (keys, sequenced) = keys_of(&ts_ingest()); + assert!(!sequenced); + assert_eq!(keys.get(&coll("metrics")), Some(&LockMode::Intent)); + assert_eq!(keys.len(), 1); + } + + /// A columnar insert of every intent locks its rows `Exclusive` and its + /// collection `Intent`. It waits rather than routes when contended. + #[test] + fn a_columnar_insert_locks_its_rows_and_never_routes() { + for intent in [ + ColumnarInsertIntent::Insert, + ColumnarInsertIntent::InsertIfAbsent, + ColumnarInsertIntent::Put, + ColumnarInsertIntent::InsertUnique, + ] { + let (keys, sequenced) = keys_of(&columnar_insert(intent)); + assert!(!sequenced, "{intent:?}"); + assert_eq!( + keys.get(&coll("events")), + Some(&LockMode::Intent), + "{intent:?}" + ); + for surrogate in [4, 5] { + assert_eq!( + keys.get(&surrogate_key("events", surrogate)), + Some(&LockMode::Exclusive), + "{intent:?}" + ); + } + assert_eq!(keys.len(), 3, "{intent:?}"); + } + } + + /// Two concurrent ingests of one collection hold their keys together. A + /// truncate of the collection cannot take it while either holds. + #[test] + fn concurrent_ingests_admit_together_and_fence_a_truncate() { + let mut table = LockManager::new(); + let (first, _) = keys_of(&ts_ingest()); + let (second, _) = keys_of(&ts_ingest()); + assert!(table.try_acquire(TxnId::new(TxnId::AUTOCOMMIT_EPOCH, 0), first)); + assert!(table.try_acquire(TxnId::new(TxnId::AUTOCOMMIT_EPOCH, 1), second)); + let truncate = PhysicalPlan::Timeseries(TimeseriesOp::Truncate { + collection: qualified("metrics"), + restart_identity: false, + }); + let (truncate_keys, _) = keys_of(&truncate); + assert_eq!( + truncate_keys.get(&coll("metrics")), + Some(&LockMode::Exclusive) + ); + assert!(!table.try_acquire(TxnId::new(TxnId::AUTOCOMMIT_EPOCH, 2), truncate_keys)); + } + + /// A fast-path point UPDATE of a balance target row and the Calvin + /// transaction that moves the row's balance lock the same row key + /// `Exclusive`. While the transaction holds its locks, from the grant + /// that stages the balance's post-image to the install, the UPDATE + /// cannot take the key. So it lands before the stage or after the + /// install, and the install never overwrites it. + #[test] + fn a_point_update_of_a_balance_target_is_fenced_by_the_balance_write() { + let balance = PhysicalPlan::Document(DocumentOp::ApplyBalanceDelta { + collection: qualified("accts"), + document_id: "00000009".to_owned(), + surrogate: Surrogate::new(9), + column: "balance".to_owned(), + delta: "5".to_owned(), + join_column: "account_id".to_owned(), + join_value: "acc".to_owned(), + declared_primary_key: None, + }); + let update = PhysicalPlan::Document(DocumentOp::PointUpdate { + collection: qualified("accts"), + document_id: "acc".to_owned(), + surrogate: Some(Surrogate::new(9)), + pk_bytes: b"acc".to_vec(), + updates: Vec::new(), + returning: None, + rls_filters: Vec::new(), + rls_write_check: nodedb_types::RlsWriteCheck::NoPolicyApplies, + resolved_sum_targets: Vec::new(), + declared_primary_key: None, + }); + + // The scheduler expands a transaction's write keys by the same rule. + let mut calvin_keys = WriteKeys::default(); + add_plan_write_keys(&mut calvin_keys, &balance).expect("a balance write has keys"); + let held = expand_write_key_sets(&calvin_keys.into_key_sets()); + let (requested, sequenced) = keys_of(&update); + assert!(sequenced); + assert_eq!( + held.get(&surrogate_key("accts", 9)), + Some(&LockMode::Exclusive) + ); + assert_eq!( + requested.get(&surrogate_key("accts", 9)), + Some(&LockMode::Exclusive) + ); + + let mut table = LockManager::new(); + assert!(table.try_acquire(TxnId::new(1, 0), held)); + assert!( + !table.try_acquire(TxnId::new(TxnId::AUTOCOMMIT_EPOCH, 0), requested), + "the UPDATE must not take the row while the balance write holds it" + ); + } + + /// A committed redo locks every collection it writes `Exclusive`. + #[test] + fn a_whole_collection_request_locks_each_collection_exclusive() { + let request = collection_admission_keys(&[ + qualified("a").as_str().to_owned(), + qualified("b").as_str().to_owned(), + ]); + assert!(!request.sequenced); + assert_eq!(request.keys.get(&coll("a")), Some(&LockMode::Exclusive)); + assert_eq!(request.keys.get(&coll("b")), Some(&LockMode::Exclusive)); + assert_eq!(request.keys.len(), 2); + } +} diff --git a/nodedb/src/control/server/shared/write_admission/gate.rs b/nodedb/src/control/server/shared/write_admission/gate.rs index 16a5a6ea6..45c487402 100644 --- a/nodedb/src/control/server/shared/write_admission/gate.rs +++ b/nodedb/src/control/server/shared/write_admission/gate.rs @@ -2,32 +2,45 @@ //! The write-admission gate. //! -//! Every write-class `PhysicalPlan` — regardless of transport or path — passes -//! through [`admit`] before it is enqueued to a Data-Plane core. The gate -//! decides one of four outcomes per write: +//! Every write-class `PhysicalPlan` passes through [`admit`] before it is +//! ordered: an autocommit write that applies on this node alone before its +//! enqueue, and a replicated write on its data-group leader before its +//! propose (see `leader_gate`). The gate decides one of five outcomes: //! -//! - [`WriteAdmission::FastPath`] — an uncontended POINT write whose exact -//! deterministic lock keys were acquired here. It carries a RAII -//! [`WriteAdmissionGuard`] the caller holds across the enqueue + response; the -//! guard releases the keys on drop. This is the normal autocommit path. +//! - [`WriteAdmission::FastPath`] — every lock key of the write was free and +//! is held now by the RAII [`WriteAdmissionGuard`]. A local write holds it +//! across its enqueue. A replicated write holds it until its leader starts +//! the entry's apply. The guard releases the keys on drop. //! - [`WriteAdmission::FastPathBlocking`] — a single-node POINT write for a //! vShard with no Calvin scheduler. There is no lock table to fence against, //! so the caller awaits a FIFO-fair per-key async order-lock before the WAL //! append + enqueue, serializing concurrent same-key writes in arrival order. -//! - [`WriteAdmission::RouteToCalvin`] — a point write whose keys are currently -//! held by a pending commit (acquire returned `Blocked`), OR any predicate / -//! bulk / multi-home write. The caller submits it through the deterministic -//! scheduler, which queues it FIFO behind the holder and applies it in order. +//! - [`WriteAdmission::RouteToCalvin`] — a key is held, and a Calvin +//! transaction can sequence the write, or the write homes on several +//! vShards. The caller submits it through the deterministic scheduler, which +//! queues it FIFO behind the holder and applies it in order. +//! - [`WriteAdmission::Wait`] — a key is held, and no Calvin transaction can +//! sequence the write. The caller awaits [`AdmissionWait::acquire`]. //! - [`WriteAdmission::ExemptRead`] — a non-write (read / meta op), or a //! Calvin-scheduled apply that already holds its locks. //! -//! The fence holds because the fast path and the scheduler share the SAME +//! Every write shape has a lock request (see `admission_keys`): its row keys +//! `Exclusive` plus its collection `Intent`, or its collection `Exclusive`. +//! +//! The fence holds because the gate and the scheduler share the SAME //! `Arc>` (via [`CalvinLocalState::lock_managers`]): a //! commit's lock validation calls `acquire` on the same key, is `Blocked`, and //! waits; whoever takes the OS mutex first wins, with no time-of-check / //! time-of-use gap. //! +//! The gate cannot deadlock with a Calvin transaction. It takes a write's keys +//! all at once with `try_acquire`, or none of them. A contended write routes +//! to the scheduler, which orders it by sequencer position, or waits holding +//! no key. A write that holds keys waits only for its own enqueue, which +//! never waits for a lock. +//! //! [`CalvinLocalState::lock_managers`]: crate::control::state::CalvinLocalState::lock_managers +//! [`AdmissionWait::acquire`]: super::wait::AdmissionWait::acquire use std::sync::atomic::{AtomicU64, Ordering}; use std::sync::{Arc, Mutex}; @@ -37,17 +50,17 @@ use tokio::sync::mpsc::UnboundedSender; use crate::bridge::envelope::PhysicalPlan; use crate::control::cluster::calvin::scheduler::driver::core::routing::{PlanRouting, plan_vshard}; use crate::control::cluster::calvin::scheduler::lock_manager::{LockKey, LockManager, TxnId}; -use crate::control::planner::calvin::is_dependent_predicate; use crate::control::state::SharedState; use crate::types::{DatabaseId, TenantId, VShardId}; -use nodedb_physical::physical_plan::MetaOp; -use super::lock_keys::{plan_lock_keys, plan_row_key}; +use super::admission_keys::{AdmissionKeys, is_calvin_apply, plan_admission_keys}; +use super::lock_keys::plan_row_key; use super::predicate::plan_is_write; +use super::wait::AdmissionWait; use super::write_order_lock::KeyedWriteOrderLock; /// Count of writes the gate routed to the deterministic scheduler instead of -/// the fast path (either a `Blocked` point write or a non-point write). Read by +/// the fast path (a contended write, or a write on several vShards). Read by /// the fence tests. static ROUTED_TO_CALVIN: AtomicU64 = AtomicU64::new(0); @@ -71,11 +84,11 @@ pub struct WriteTarget<'a> { /// The gate's decision for one write. See the module docs. pub enum WriteAdmission { - /// Uncontended point write admitted to the fast path. `guard` is `Some` when - /// real keys were acquired (a scheduler is active for the vShard) and `None` - /// when no lock manager is registered and the write carries no single point - /// key to serialize on (predicate / bulk / uncovered shapes — left unordered - /// for now). Either way the caller holds it across enqueue + response. + /// Uncontended write admitted to the fast path. `guard` is `Some` when + /// its keys were acquired. It is `None` when no Calvin scheduler runs for + /// the vShard and the write has no point key, or when the write names no + /// lock key. No Calvin transaction can then conflict with it, and the + /// vShard's write-order fence orders it against other writes. FastPath { guard: Option }, /// POINT write on a vShard with no Calvin scheduler registered. There is no /// lock table to fence against, but concurrent same-key writes must still @@ -91,6 +104,9 @@ pub enum WriteAdmission { }, /// Submit the write through the deterministic Calvin scheduler. RouteToCalvin, + /// A key is held, and no Calvin transaction can sequence the write. The + /// caller awaits [`AdmissionWait::acquire`] before it orders the write. + Wait(AdmissionWait), /// A non-write, or an already-locked Calvin apply — no fence needed. ExemptRead, } @@ -123,6 +139,21 @@ pub struct WriteAdmissionGuard { promotion_sender: Option>>, } +impl WriteAdmissionGuard { + /// The guard of the keys `txn` holds on `lock_manager`. + pub(super) fn new( + lock_manager: Arc>, + txn: TxnId, + promotion_sender: Option>>, + ) -> Self { + Self { + lock_manager, + txn, + promotion_sender, + } + } +} + impl Drop for WriteAdmissionGuard { fn drop(&mut self) { // Ordering is load-bearing: take the lock-manager mutex, release the @@ -152,94 +183,93 @@ impl Drop for WriteAdmissionGuard { } } -/// Admit a plan destined for a Data-Plane core. +/// Admit a write-class plan. /// /// Synchronous: never awaits and never parks — it only *chooses* the outcome; -/// any per-key await happens in the caller. `RouteToCalvin` is returned ONLY -/// when a deterministic scheduler is actually registered for the write's vShard. -/// With no scheduler (single-node / no-Calvin — the common case) a point write -/// returns `FastPathBlocking` carrying the global keyed order-lock so concurrent -/// same-key writes still serialize; every other shape fast-paths unfenced. -pub fn admit(shared: &SharedState, target: &WriteTarget<'_>) -> WriteAdmission { - // A Calvin-scheduled apply already holds its locks (acquired by the - // scheduler); it must never re-acquire at the gate. Defensive — these ops - // do not normally reach the gate. - if matches!( - target.plan, - PhysicalPlan::Meta( - MetaOp::CalvinExecuteStatic { .. } - | MetaOp::CalvinExecuteActive { .. } - | MetaOp::CalvinFlush { .. } - | MetaOp::CalvinDrop { .. } - | MetaOp::CalvinResolve { .. } - ) - ) { - return WriteAdmission::ExemptRead; - } +/// any await happens in the caller. `RouteToCalvin` and `Wait` are returned +/// only when a deterministic scheduler is registered for the write's vShard. +/// With no scheduler (single-node / no-Calvin) a point write returns +/// `FastPathBlocking` carrying the global keyed order-lock so concurrent +/// same-key writes still serialize, and every other shape takes the fast path +/// with no key: no Calvin transaction runs to conflict with it. +pub fn admit(shared: &SharedState, target: &WriteTarget<'_>) -> crate::Result { + admit_routed(shared, target, true) +} - if !plan_is_write(target.plan) { - return WriteAdmission::ExemptRead; +/// [`admit`], where `may_route` says whether a contended write may run +/// through the scheduler (see `calvin_route_keeps`). A contended write that +/// may not waits for its keys instead. +pub(crate) fn admit_routed( + shared: &SharedState, + target: &WriteTarget<'_>, + may_route: bool, +) -> crate::Result { + // A Calvin-scheduled apply already holds its locks (acquired by the + // scheduler); it must never re-acquire at the gate. + if is_calvin_apply(target.plan) || !plan_is_write(target.plan) { + return Ok(WriteAdmission::ExemptRead); } - // Only two write shapes participate in the fence: a single-home POINT write - // (Document / KV / Vector / single-home graph edge — a statically-known - // deterministic key) and a single-shard PREDICATE write (BulkUpdate / - // BulkDelete — its write set discovered by scheduler reconnaissance). Every - // other write — batch, INSERT..SELECT, upsert, CRDT, columnar / timeseries / - // spatial / array, and cross-home edges — has no Calvin lock representation - // and fast-paths unchanged. - let point_keys = plan_lock_keys(target.plan); - let is_predicate = is_dependent_predicate(target.plan); - let vshard = match &point_keys { - Some((v, _)) => *v, - None if is_predicate => match plan_vshard(target.plan) { - PlanRouting::Vshards(v) => match v.as_slice() { - [v] => *v, - _ => return WriteAdmission::FastPath { guard: None }, - }, - PlanRouting::ControlPlaneOnly | PlanRouting::NotAWrite | PlanRouting::Unroutable(_) => { - return WriteAdmission::FastPath { guard: None }; - } + let vshard = match plan_vshard(target.plan) { + PlanRouting::Vshards(homes) => match homes.as_slice() { + [home] => *home, + // Only the scheduler takes keys on several lock tables as one + // request, in sequencer order. + homes => return Ok(admit_multi_home(shared, homes)), }, - None => return WriteAdmission::FastPath { guard: None }, + // A write whose plan names no home orders on the vShard it targets. + PlanRouting::ControlPlaneOnly | PlanRouting::NotAWrite | PlanRouting::Unroutable(_) => { + target.vshard_id + } }; - // Availability gate: with no scheduler registered for this vShard there is no - // Calvin lock table to fence against. A POINT write still needs per-key - // arrival-order serialization so WAL-LSN order equals Data-Plane apply order - // per key — hand it the global keyed order-lock, on which concurrent same-key - // writers queue FIFO while distinct keys never contend. A predicate write has - // no single static point key here, so it stays unordered on the fast path - // (widening that coverage is a later unit). - let Some(lock_manager) = shared - .calvin - .lock_managers - .lock() - .unwrap_or_else(|p| p.into_inner()) - .get(&vshard.as_u32()) - .map(Arc::clone) - else { - return match point_keys.and_then(|_| plan_row_key(target.plan)) { + let Some(lock_manager) = lock_manager_of(shared, vshard) else { + return Ok(match plan_row_key(target.plan) { Some(key) => WriteAdmission::FastPathBlocking { key, keyed_lock: Arc::clone(&shared.write_order_locks), }, None => WriteAdmission::FastPath { guard: None }, - }; + }); }; - // Calvin IS running for this vShard. A predicate write has no static point - // key to acquire; the scheduler discovers its write set, so route it. - let Some((_v, keys)) = point_keys else { - ROUTED_TO_CALVIN.fetch_add(1, Ordering::Relaxed); - return WriteAdmission::RouteToCalvin; - }; + let mut request = + plan_admission_keys(shared, target.tenant_id, target.database_id, target.plan)?; + request.sequenced &= may_route; + let admission = admit_request(shared, vshard, lock_manager, request); + if matches!( + admission, + WriteAdmission::RouteToCalvin | WriteAdmission::Wait(_) + ) && tracing::enabled!(tracing::Level::DEBUG) + { + let plan: String = format!("{:?}", target.plan) + .chars() + .take(PLAN_SUMMARY_LEN) + .collect(); + tracing::debug!( + vshard_id = vshard.as_u32(), + routes = matches!(admission, WriteAdmission::RouteToCalvin), + %plan, + "the write gate routes or holds a contended write" + ); + } + Ok(admission) +} - // Point write: mint a holder id in the reserved band so it never collides - // with a real Calvin schedule position, then probe the exact keys WITHOUT - // blocking. `try_acquire` never enqueues a waiter on the contended path, so a - // routed write leaves no orphaned autocommit holder that a later `release` - // can promote to an unowned (never-released) lock. +/// How many characters of a contended write's plan the gate's debug line +/// names. +const PLAN_SUMMARY_LEN: usize = 240; + +/// Admit the lock request `request` on `vshard`, whose lock table is +/// `lock_manager`. +pub(crate) fn admit_request( + shared: &SharedState, + vshard: VShardId, + lock_manager: Arc>, + request: AdmissionKeys, +) -> WriteAdmission { + // A holder id in the reserved band never collides with a real Calvin + // schedule position. let txn = TxnId::new( TxnId::AUTOCOMMIT_EPOCH, shared @@ -247,34 +277,237 @@ pub fn admit(shared: &SharedState, target: &WriteTarget<'_>) -> WriteAdmission { .autocommit_lock_seq .fetch_add(1, Ordering::Relaxed), ); + // Every guard hands the scheduler waiters it promotes on drop back to + // the scheduler for dispatch. A lock manager and its promotion sender + // are registered together per vShard. + let promotion_sender = shared + .calvin + .promotion_senders + .lock() + .unwrap_or_else(|p| p.into_inner()) + .get(&vshard.as_u32()) + .cloned(); + let admission = probe(lock_manager, txn, request, promotion_sender); + if matches!(admission, WriteAdmission::RouteToCalvin) { + ROUTED_TO_CALVIN.fetch_add(1, Ordering::Relaxed); + } + admission +} + +/// Probe `lock_manager` for every key of `request` at once, WITHOUT +/// blocking. `try_acquire` never enqueues a waiter on the contended path, so +/// a routed or waiting write leaves no orphaned holder that a later `release` +/// can promote to an unowned lock. +fn probe( + lock_manager: Arc>, + txn: TxnId, + request: AdmissionKeys, + promotion_sender: Option>>, +) -> WriteAdmission { + let AdmissionKeys { keys, sequenced } = request; + if keys.is_empty() { + return WriteAdmission::FastPath { guard: None }; + } + // A write that waits probes again with the same request. + let retry_keys = (!sequenced).then(|| keys.clone()); let acquired = { - let mut lm = lock_manager.lock().unwrap_or_else(|p| p.into_inner()); - lm.try_acquire(txn, keys) + let mut table = lock_manager.lock().unwrap_or_else(|p| p.into_inner()); + let contention = + tracing::enabled!(tracing::Level::DEBUG).then(|| table.contention(txn, &keys)); + let acquired = table.try_acquire(txn, keys); + if !acquired { + tracing::debug!( + sequenced, + ?contention, + "a write found its keys held at the write gate" + ); + } + acquired }; - if acquired { - // Look up this vShard's promotion channel so the guard can hand any - // scheduler waiter it promotes on drop back to the scheduler for - // dispatch. `None` only if no scheduler is registered — but a registered - // lock manager without a promotion sender must not happen, since both - // are inserted together per vShard. - let promotion_sender = shared - .calvin - .promotion_senders - .lock() - .unwrap_or_else(|p| p.into_inner()) - .get(&vshard.as_u32()) - .cloned(); - WriteAdmission::FastPath { - guard: Some(WriteAdmissionGuard { + match (acquired, retry_keys) { + (true, _) => WriteAdmission::FastPath { + guard: Some(WriteAdmissionGuard::new( lock_manager, txn, promotion_sender, - }), - } - } else { - // A pending commit (or another fast-path write) holds a key: route behind - // it via the scheduler. Nothing was acquired or enqueued here. + )), + }, + // A holder or an earlier waiter has a key: the scheduler queues the + // write FIFO behind it. + (false, None) => WriteAdmission::RouteToCalvin, + (false, Some(keys)) => WriteAdmission::Wait(AdmissionWait::new( + lock_manager, + txn, + keys, + promotion_sender, + )), + } +} + +/// Admit a write homed on several vShards: the scheduler sequences it when +/// any home runs one. With no scheduler on any home, no Calvin transaction +/// conflicts with it. +fn admit_multi_home(shared: &SharedState, homes: &[VShardId]) -> WriteAdmission { + if homes + .iter() + .any(|home| lock_manager_of(shared, *home).is_some()) + { ROUTED_TO_CALVIN.fetch_add(1, Ordering::Relaxed); WriteAdmission::RouteToCalvin + } else { + WriteAdmission::FastPath { guard: None } + } +} + +/// The lock table of `vshard`'s Calvin scheduler on this node, if one runs. +pub(crate) fn lock_manager_of( + shared: &SharedState, + vshard: VShardId, +) -> Option>> { + shared + .calvin + .lock_managers + .lock() + .unwrap_or_else(|p| p.into_inner()) + .get(&vshard.as_u32()) + .map(Arc::clone) +} + +#[cfg(test)] +mod tests { + use std::collections::BTreeMap; + + use super::*; + use crate::control::cluster::calvin::scheduler::lock_manager::{AcquireOutcome, LockMode}; + + fn coll() -> LockKey { + LockKey::Collection { + collection: Arc::from("c"), + } + } + + fn row(surrogate: u32) -> LockKey { + LockKey::Surrogate { + collection: Arc::from("c"), + surrogate, + } + } + + fn row_write(surrogate: u32) -> BTreeMap { + BTreeMap::from([ + (row(surrogate), LockMode::Exclusive), + (coll(), LockMode::Intent), + ]) + } + + fn request(keys: BTreeMap, sequenced: bool) -> AdmissionKeys { + AdmissionKeys { keys, sequenced } + } + + fn autocommit(position: u32) -> TxnId { + TxnId::new(TxnId::AUTOCOMMIT_EPOCH, position) + } + + /// Two row writes of distinct rows hold the collection `Intent` together. + /// The guard passes each key in its own mode. + #[test] + fn row_writes_of_distinct_rows_admit_together() { + let lock_manager = Arc::new(Mutex::new(LockManager::new())); + let first = probe( + Arc::clone(&lock_manager), + autocommit(0), + request(row_write(1), true), + None, + ); + let second = probe( + Arc::clone(&lock_manager), + autocommit(1), + request(row_write(2), true), + None, + ); + assert!(matches!(first, WriteAdmission::FastPath { guard: Some(_) })); + assert!(matches!( + second, + WriteAdmission::FastPath { guard: Some(_) } + )); + drop((first, second)); + assert_eq!(lock_manager.lock().expect("table").lock_count(), 0); + } + + /// A Calvin truncate holds the collection `Exclusive`. A sequenced row + /// write routes to the scheduler, and an unsequenced one waits. + #[test] + fn a_contended_write_routes_when_sequenced_and_waits_otherwise() { + let lock_manager = Arc::new(Mutex::new(LockManager::new())); + let truncate = TxnId::new(3, 0); + assert_eq!( + lock_manager + .lock() + .expect("table") + .acquire(truncate, BTreeMap::from([(coll(), LockMode::Exclusive)])), + AcquireOutcome::Ready + ); + let routed = probe( + Arc::clone(&lock_manager), + autocommit(0), + request(row_write(1), true), + None, + ); + assert!(matches!(routed, WriteAdmission::RouteToCalvin)); + let waits = probe( + Arc::clone(&lock_manager), + autocommit(1), + request(row_write(1), false), + None, + ); + assert!(matches!(waits, WriteAdmission::Wait(_))); + assert_eq!( + lock_manager.lock().expect("table").holder_count(), + 1, + "a refused probe takes no key" + ); + } + + /// A Calvin transaction that needs a key a fast-path write holds waits + /// for the guard's drop, which promotes it. + #[test] + fn a_calvin_writer_waits_for_the_fast_path_guard() { + let lock_manager = Arc::new(Mutex::new(LockManager::new())); + let admitted = probe( + Arc::clone(&lock_manager), + autocommit(0), + request(row_write(1), true), + None, + ); + assert!(matches!( + admitted, + WriteAdmission::FastPath { guard: Some(_) } + )); + let truncate = TxnId::new(3, 0); + assert_eq!( + lock_manager + .lock() + .expect("table") + .acquire(truncate, BTreeMap::from([(coll(), LockMode::Exclusive)])), + AcquireOutcome::Blocked + ); + drop(admitted); + assert!( + lock_manager + .lock() + .expect("table") + .is_ready(truncate, &BTreeMap::from([(coll(), LockMode::Exclusive)])) + ); + } + + /// A write that names no lock key takes the fast path with no guard. + #[test] + fn a_write_with_no_keys_takes_no_guard() { + let lock_manager = Arc::new(Mutex::new(LockManager::new())); + let admission = probe(lock_manager, autocommit(0), AdmissionKeys::default(), None); + assert!(matches!( + admission, + WriteAdmission::FastPath { guard: None } + )); } } diff --git a/nodedb/src/control/server/shared/write_admission/holds.rs b/nodedb/src/control/server/shared/write_admission/holds.rs new file mode 100644 index 000000000..e1e108a13 --- /dev/null +++ b/nodedb/src/control/server/shared/write_admission/holds.rs @@ -0,0 +1,209 @@ +// SPDX-License-Identifier: BUSL-1.1 + +//! The lock keys replicated writes hold on their data-group leader. +//! +//! The leader's write gate takes a write's keys before its propose. The keys +//! stay held until the leader's apply loop starts the entry: its enqueue +//! returned, or it concluded without one. The apply loop enqueues a group's +//! writes on their cores in log order, and a core runs its queue in order. A +//! Calvin transaction granted a key after the release therefore stages on +//! the core after the write, and reads the write's effects. +//! +//! The keys are never held across a Data-Plane apply. A write that holds keys +//! waits only for the apply loop to start its entry, which never waits for a +//! lock. So no held key waits for a Calvin transaction to finish. + +use std::collections::{BTreeMap, HashMap}; +use std::sync::Mutex; + +use crate::types::VShardId; + +use super::gate::WriteAdmissionGuard; + +/// One write's keys, held for their `Drop`. +struct Held { + vshard: VShardId, + _guard: WriteAdmissionGuard, +} + +#[derive(Default)] +struct Inner { + /// Held keys by the `(group_id, log_index)` of the write's entry. + holds: BTreeMap<(u64, u64), Vec>, + /// Per group, the highest log index whose apply started on this node. + started: HashMap, +} + +/// Every replicated write's held keys on this node, by entry. +#[derive(Default)] +pub struct AdmissionHolds { + inner: Mutex, +} + +impl AdmissionHolds { + pub fn new() -> Self { + Self::default() + } + + /// Hold `guard` until the apply of entry `log_index` of `group_id` + /// starts on this node. An entry whose apply started already releases + /// it at once. + pub fn hold( + &self, + group_id: u64, + log_index: u64, + vshard: VShardId, + guard: WriteAdmissionGuard, + ) { + let held = Held { + vshard, + _guard: guard, + }; + // Dropped after the table's mutex: a guard's drop takes the lock + // table's mutex. + let _released = { + let mut inner = self.inner.lock().unwrap_or_else(|p| p.into_inner()); + if inner + .started + .get(&group_id) + .is_some_and(|started| *started >= log_index) + { + Some(held) + } else { + inner + .holds + .entry((group_id, log_index)) + .or_default() + .push(held); + None + } + }; + } + + /// The apply of entry `log_index` of `group_id` started on this node, + /// after every entry of the group below it. Release the keys every write + /// through it holds. + pub fn release_through(&self, group_id: u64, log_index: u64) { + let _released: Vec> = { + let mut inner = self.inner.lock().unwrap_or_else(|p| p.into_inner()); + let started = inner.started.entry(group_id).or_default(); + *started = (*started).max(log_index); + let through: Vec<(u64, u64)> = inner + .holds + .range((group_id, 0)..=(group_id, log_index)) + .map(|(key, _)| *key) + .collect(); + through + .into_iter() + .filter_map(|key| inner.holds.remove(&key)) + .collect() + }; + } + + /// The scheduler of `vshard` stopped on this node, and its lock table + /// with it. Drop every hold on that table. + pub fn release_vshard(&self, vshard: VShardId) { + let _released: Vec = { + let mut inner = self.inner.lock().unwrap_or_else(|p| p.into_inner()); + let mut released = Vec::new(); + inner.holds.retain(|_, held| { + let (gone, kept): (Vec, Vec) = std::mem::take(held) + .into_iter() + .partition(|h| h.vshard == vshard); + released.extend(gone); + *held = kept; + !held.is_empty() + }); + released + }; + } + + /// The number of entries whose writes hold keys. + #[cfg(test)] + fn held_entries(&self) -> usize { + self.inner + .lock() + .unwrap_or_else(|p| p.into_inner()) + .holds + .len() + } +} + +#[cfg(test)] +mod tests { + use std::collections::BTreeMap; + use std::sync::Arc; + + use super::*; + use crate::control::cluster::calvin::scheduler::lock_manager::{ + LockKey, LockManager, LockMode, TxnId, + }; + + fn guard(lock_manager: &Arc>, position: u32) -> WriteAdmissionGuard { + let txn = TxnId::new(TxnId::AUTOCOMMIT_EPOCH, position); + let key = LockKey::Surrogate { + collection: Arc::from("c"), + surrogate: position, + }; + assert!( + lock_manager + .lock() + .expect("table") + .try_acquire(txn, BTreeMap::from([(key, LockMode::Exclusive)])) + ); + WriteAdmissionGuard::new(Arc::clone(lock_manager), txn, None) + } + + fn held_keys(lock_manager: &Arc>) -> usize { + lock_manager.lock().expect("table").lock_count() + } + + /// A write's keys stay held until its entry starts, and no longer. + #[test] + fn keys_release_when_their_entry_starts() { + let lock_manager = Arc::new(Mutex::new(LockManager::new())); + let holds = AdmissionHolds::new(); + holds.hold(4, 10, VShardId::new(1), guard(&lock_manager, 1)); + holds.hold(4, 11, VShardId::new(1), guard(&lock_manager, 2)); + assert_eq!(held_keys(&lock_manager), 2); + holds.release_through(4, 10); + assert_eq!(held_keys(&lock_manager), 1); + assert_eq!(holds.held_entries(), 1); + holds.release_through(4, 11); + assert_eq!(held_keys(&lock_manager), 0); + } + + /// An entry that started before its proposer registered the hold + /// releases the keys at once. + #[test] + fn a_hold_on_a_started_entry_releases_at_once() { + let lock_manager = Arc::new(Mutex::new(LockManager::new())); + let holds = AdmissionHolds::new(); + holds.release_through(4, 12); + holds.hold(4, 12, VShardId::new(1), guard(&lock_manager, 1)); + assert_eq!(held_keys(&lock_manager), 0); + assert_eq!(holds.held_entries(), 0); + } + + /// Another group's start releases nothing. + #[test] + fn groups_release_apart() { + let lock_manager = Arc::new(Mutex::new(LockManager::new())); + let holds = AdmissionHolds::new(); + holds.hold(4, 10, VShardId::new(1), guard(&lock_manager, 1)); + holds.release_through(5, 20); + assert_eq!(held_keys(&lock_manager), 1); + } + + /// A stopped scheduler drops the holds on its table. + #[test] + fn a_stopped_vshard_drops_its_holds() { + let lock_manager = Arc::new(Mutex::new(LockManager::new())); + let holds = AdmissionHolds::new(); + holds.hold(4, 10, VShardId::new(1), guard(&lock_manager, 1)); + holds.hold(4, 10, VShardId::new(2), guard(&lock_manager, 2)); + holds.release_vshard(VShardId::new(1)); + assert_eq!(held_keys(&lock_manager), 1); + assert_eq!(holds.held_entries(), 1); + } +} diff --git a/nodedb/src/control/server/shared/write_admission/leader_gate.rs b/nodedb/src/control/server/shared/write_admission/leader_gate.rs new file mode 100644 index 000000000..27a8c5c3a --- /dev/null +++ b/nodedb/src/control/server/shared/write_admission/leader_gate.rs @@ -0,0 +1,329 @@ +// SPDX-License-Identifier: BUSL-1.1 + +//! The write gate a data-group leader runs on every proposal to its group. +//! +//! A replicated write applies on every replica in log order, with its +//! ordering decided once: on the leader, before the entry enters the log. +//! [`LeaderWriteGate`] admits the entry there with the same gate a local +//! write passes: +//! +//! - Uncontended: the leader proposes the entry, and its keys stay held until +//! the leader's apply loop starts the entry (see `holds`). +//! - Contended, and the write routes (see `admission_keys`): the propose +//! refuses with `RouteToSequencer`, and the proposer submits the write to +//! the Calvin sequencer. An append never routes. +//! - Contended otherwise: the gate waits holding no key, until the keys are +//! free or the proposer's deadline passes. A forwarded entry carries what +//! remains of that deadline. +//! +//! Followers apply the committed entry with no gate: the leader ordered it. + +use std::sync::{Arc, Weak}; + +use nodedb_cluster::{CalvinError, ClusterError, DataProposeGate, ProposeHold}; + +use crate::control::state::SharedState; +use crate::control::wal_replication::decode::decode_parsed_entry; +use crate::control::wal_replication::{ReplicatedEntry, ReplicatedWrite}; +use crate::types::VShardId; +use nodedb_types::QualifiedCollection; + +use super::admission_keys::collection_admission_keys; +use super::gate::{ + WriteAdmission, WriteAdmissionGuard, WriteTarget, admit_request, admit_routed, lock_manager_of, +}; +use super::route::calvin_route_keeps; + +/// The host side of [`DataProposeGate`]. Weak: `SharedState` outlives the +/// Raft loop that holds the gate only while the node runs. +pub struct LeaderWriteGate { + state: Weak, +} + +impl LeaderWriteGate { + pub fn new(state: &Arc) -> Self { + Self { + state: Arc::downgrade(state), + } + } +} + +#[async_trait::async_trait] +impl DataProposeGate for LeaderWriteGate { + async fn admit( + &self, + vshard_id: u32, + entry: &[u8], + deadline: tokio::time::Instant, + ) -> nodedb_cluster::Result>> { + // A node that is shutting down proposes nothing its gate must order. + let Some(state) = self.state.upgrade() else { + return Ok(None); + }; + let vshard = VShardId::new(vshard_id); + // Bytes that are not a replicated entry apply as nothing on any replica. + let Some(entry) = ReplicatedEntry::from_bytes(entry) else { + return Ok(None); + }; + admit_redo_stream(&state, &entry, deadline).map_err(gate_error)?; + let guard = match admit_entry(&state, vshard, &entry).map_err(gate_error)? { + WriteAdmission::ExemptRead + | WriteAdmission::FastPath { guard: None } + // No scheduler runs for the vShard here: the log alone orders + // the write. + | WriteAdmission::FastPathBlocking { .. } => return Ok(None), + WriteAdmission::FastPath { guard: Some(guard) } => guard, + WriteAdmission::RouteToCalvin => { + return Err(ClusterError::Calvin(CalvinError::RouteToSequencer)); + } + WriteAdmission::Wait(wait) => wait.acquire(deadline).await.map_err(gate_error)?, + }; + Ok(Some(Box::new(LandedHold { + state, + vshard, + guard, + }))) + } +} + +/// An admitted entry's keys, held until the entry lands in the log. +struct LandedHold { + state: Arc, + vshard: VShardId, + guard: WriteAdmissionGuard, +} + +impl ProposeHold for LandedHold { + fn landed(self: Box, group_id: u64, log_index: u64) { + let LandedHold { + state, + vshard, + guard, + } = *self; + state + .calvin + .admission_holds + .hold(group_id, log_index, vshard, guard); + } +} + +/// What the gate locks for one replicated entry. +enum EntryScope { + /// The entry writes no row, or its Calvin scheduler holds its locks. + Exempt, + /// The entry writes these collections whole. + Collections(Vec), + /// The entry carries a plan the gate locks as it locks a local write. + Plan, +} + +/// Admit the first chunk of a chunked redo stream against this node's open +/// stream bytes. Every replica takes the chunks the log holds, so the leader +/// is the one place a stream is refused for memory. Any other entry passes. +fn admit_redo_stream( + state: &SharedState, + entry: &ReplicatedEntry, + deadline: tokio::time::Instant, +) -> crate::Result<()> { + let ReplicatedWrite::RedoChunk { + stream, + index: 0, + len, + .. + } = &entry.write + else { + return Ok(()); + }; + state + .redo_chunks + .admit_stream(*stream, *len, deadline.into_std()) + .map_err(crate::Error::from) +} + +/// Admit `entry`, proposed to `vshard`'s group. +pub(crate) fn admit_entry( + state: &SharedState, + vshard: VShardId, + entry: &ReplicatedEntry, +) -> crate::Result { + let database_id = crate::types::DatabaseId::new(entry.database_id); + match entry_scope(&entry.write, database_id) { + EntryScope::Exempt => Ok(WriteAdmission::ExemptRead), + EntryScope::Collections(collections) => Ok(admit_collections(state, vshard, &collections)), + EntryScope::Plan => { + let Some((_, (tenant_id, _, plan, _))) = decode_parsed_entry(entry)? else { + return Ok(WriteAdmission::ExemptRead); + }; + // An entry the scheduler cannot apply as proposed waits for its + // keys rather than route. + let may_route = calvin_route_keeps( + &plan, + crate::event::EventSource::from(entry.event_source), + entry.restore_id, + ); + admit_routed( + state, + &WriteTarget { + tenant_id, + database_id, + vshard_id: vshard, + plan: &plan, + }, + may_route, + ) + } + } +} + +/// The lock scope of `write`, an entry of `database_id`. +fn entry_scope(write: &ReplicatedWrite, database_id: crate::types::DatabaseId) -> EntryScope { + match write { + // Calvin bookkeeping, a backup cut, a topic message, a surrogate + // binding and an array schema write no row. + ReplicatedWrite::CalvinReadResult { .. } + | ReplicatedWrite::CalvinReadTimeout { .. } + | ReplicatedWrite::CutBarrier { .. } + | ReplicatedWrite::TopicPublish { .. } + | ReplicatedWrite::SurrogateBind { .. } + | ReplicatedWrite::ArraySchema { .. } => EntryScope::Exempt, + // A Calvin transaction's redo carries its sequencer stamp, and its + // scheduler holds its locks. A session transaction's redo writes + // every collection it names. + ReplicatedWrite::TransactionRedo { + collections, + calvin, + .. + } => { + if calvin.is_some() { + EntryScope::Exempt + } else { + EntryScope::Collections(collections.clone()) + } + } + // A redo chunk and an abandon write no row: the stream's final entry + // takes the locks. + ReplicatedWrite::RedoChunk { .. } | ReplicatedWrite::RedoAbandon { .. } => { + EntryScope::Exempt + } + ReplicatedWrite::ArrayOp { array, .. } => EntryScope::Collections(vec![ + QualifiedCollection::new(database_id, array) + .as_str() + .to_owned(), + ]), + ReplicatedWrite::PointPut { .. } + | ReplicatedWrite::PointInsert { .. } + | ReplicatedWrite::PointDelete { .. } + | ReplicatedWrite::PointUpdate { .. } + | ReplicatedWrite::DocUpsert { .. } + | ReplicatedWrite::DocBatchInsert { .. } + | ReplicatedWrite::VectorInsert { .. } + | ReplicatedWrite::VectorBatchInsert { .. } + | ReplicatedWrite::VectorDelete { .. } + | ReplicatedWrite::SetVectorParams { .. } + | ReplicatedWrite::SparseInsert { .. } + | ReplicatedWrite::SparseDelete { .. } + | ReplicatedWrite::MultiVectorInsert { .. } + | ReplicatedWrite::MultiVectorDelete { .. } + | ReplicatedWrite::DeleteBySurrogate { .. } + | ReplicatedWrite::DirectUpsert { .. } + | ReplicatedWrite::CrdtApply { .. } + | ReplicatedWrite::ColumnarIngest { .. } + | ReplicatedWrite::TimeseriesIngest { .. } + | ReplicatedWrite::FtsIndex { .. } + | ReplicatedWrite::FtsDelete { .. } + | ReplicatedWrite::SpatialInsert { .. } + | ReplicatedWrite::SpatialDelete { .. } + | ReplicatedWrite::EdgePut { .. } + | ReplicatedWrite::EdgeDelete { .. } + | ReplicatedWrite::SetNodeLabels { .. } + | ReplicatedWrite::RemoveNodeLabels { .. } + | ReplicatedWrite::EdgePutBatch { .. } + | ReplicatedWrite::EdgeDeleteBatch { .. } + | ReplicatedWrite::KvPut { .. } + | ReplicatedWrite::KvDelete { .. } + | ReplicatedWrite::KvInsert { .. } + | ReplicatedWrite::KvInsertIfAbsent { .. } + | ReplicatedWrite::KvInsertOnConflictUpdate { .. } + | ReplicatedWrite::KvBatchPut { .. } + | ReplicatedWrite::KvExpire { .. } + | ReplicatedWrite::KvPersist { .. } + | ReplicatedWrite::KvIncr { .. } + | ReplicatedWrite::KvIncrFloat { .. } + | ReplicatedWrite::KvCas { .. } + | ReplicatedWrite::KvGetSet { .. } + | ReplicatedWrite::KvRegisterSortedIndex { .. } + | ReplicatedWrite::KvDropSortedIndex { .. } + | ReplicatedWrite::KvRegisterIndex { .. } + | ReplicatedWrite::KvDropIndex { .. } + | ReplicatedWrite::KvFieldSet { .. } + | ReplicatedWrite::KvTransfer { .. } + | ReplicatedWrite::KvTransferItem { .. } + | ReplicatedWrite::BulkDml { .. } + | ReplicatedWrite::ColumnarBulkDml { .. } + | ReplicatedWrite::InsertSelect { .. } + | ReplicatedWrite::CrdtImportCollection { .. } + | ReplicatedWrite::CrdtListInsert { .. } + | ReplicatedWrite::CrdtListDelete { .. } + | ReplicatedWrite::CrdtListMove { .. } + | ReplicatedWrite::CrdtDocUpsert { .. } + | ReplicatedWrite::CrdtDocDelete { .. } + | ReplicatedWrite::DocTruncate { .. } + | ReplicatedWrite::KvTruncate { .. } + | ReplicatedWrite::ConstraintChange { .. } + | ReplicatedWrite::ArrayCellPut { .. } + | ReplicatedWrite::ArrayCellDelete { .. } + | ReplicatedWrite::CrdtApplyFenced { .. } + | ReplicatedWrite::CrdtApplyAuthenticated { .. } + | ReplicatedWrite::DropVectorIndex { .. } + | ReplicatedWrite::ApplyBalanceDelta { .. } + | ReplicatedWrite::ColumnarBulkDmlResolved { .. } + | ReplicatedWrite::ColumnarTruncate { .. } + | ReplicatedWrite::TimeseriesTruncate { .. } + | ReplicatedWrite::KvResolvedWrite { .. } + | ReplicatedWrite::KvPredicateUpdate { .. } + | ReplicatedWrite::KvPredicateDelete { .. } + | ReplicatedWrite::DocumentResolvedWrite { .. } + | ReplicatedWrite::VectorDirectDelete { .. } + | ReplicatedWrite::VectorDirectTruncate { .. } + | ReplicatedWrite::VectorDirectUpdate { .. } + | ReplicatedWrite::VectorResolvedDirectWrite { .. } + | ReplicatedWrite::MergeApply { .. } + | ReplicatedWrite::UpdateFromJoinApply { .. } => EntryScope::Plan, + } +} + +/// Admit a write that locks each of `collections` whole. +fn admit_collections( + state: &SharedState, + vshard: VShardId, + collections: &[String], +) -> WriteAdmission { + match lock_manager_of(state, vshard) { + Some(lock_manager) => admit_request( + state, + vshard, + lock_manager, + collection_admission_keys(collections), + ), + None => WriteAdmission::FastPath { guard: None }, + } +} + +/// The cluster error a gate failure answers the proposer with. A deadline +/// keeps its class across the hop. Every other error crosses typed. +fn gate_error(error: crate::Error) -> ClusterError { + match error { + crate::Error::DeadlineExceeded { .. } => { + ClusterError::Calvin(CalvinError::AdmissionTimedOut) + } + other => { + let detail = format!("data-group write gate: {other}"); + ClusterError::ShardExecution { + error: Box::new( + crate::control::cluster::execution_error_wire::execution_error_to_typed(other), + ), + detail, + } + } + } +} diff --git a/nodedb/src/control/server/shared/write_admission/lock_keys.rs b/nodedb/src/control/server/shared/write_admission/lock_keys.rs index f0248182d..76af8e0a7 100644 --- a/nodedb/src/control/server/shared/write_admission/lock_keys.rs +++ b/nodedb/src/control/server/shared/write_admission/lock_keys.rs @@ -1,10 +1,12 @@ // SPDX-License-Identifier: BUSL-1.1 -//! Point-write lock-key extraction for the write-admission fast path. +//! Point-write key extraction for the per-vShard write order. //! -//! [`plan_lock_keys`] maps a plan to the deterministic lock keys the fast path -//! must hold. Returns `Some` only for single-vShard, single-identity point -//! writes; predicate/bulk/multi-home writes route to Calvin instead. +//! [`plan_lock_keys`] maps a single-vShard, single-identity point write to +//! the keys that name its row. The write-order fence reads them to decide +//! whether admission's guard already orders the row. [`plan_row_key`] names +//! the key the keyed order lock serializes on when no Calvin scheduler runs. +//! The gate's own lock request comes from `admission_keys`. use std::collections::BTreeSet; use std::sync::Arc; @@ -15,9 +17,8 @@ use crate::control::cluster::calvin::scheduler::lock_manager::LockKey; use crate::types::VShardId; use nodedb_physical::physical_plan::{DocumentOp, GraphOp, KvOp, VectorOp, VectorWriteTargets}; -/// The vShard and exact lock-key set a POINT write must hold on the fast path. -/// Returns `None` (routes to Calvin) for any plan that isn't a single-home, -/// single-identity point write. +/// The vShard and the row keys of a POINT write. `None` for any plan that +/// isn't a single-home, single-identity point write. pub(crate) fn plan_lock_keys(plan: &PhysicalPlan) -> Option<(VShardId, BTreeSet)> { // `plan_vshard` returns two vShards for a cross-home graph edge, which has no // single `(vShard, keys)` representation and is ineligible for the fast path. diff --git a/nodedb/src/control/server/shared/write_admission/mod.rs b/nodedb/src/control/server/shared/write_admission/mod.rs index 6d292ca23..fb5e24e62 100644 --- a/nodedb/src/control/server/shared/write_admission/mod.rs +++ b/nodedb/src/control/server/shared/write_admission/mod.rs @@ -1,21 +1,29 @@ // SPDX-License-Identifier: BUSL-1.1 //! Control-Plane write-admission gate: the single seam every write-class -//! `PhysicalPlan` passes through before it is enqueued to a Data-Plane core. -//! Uncontended point writes take the fast path holding per-vShard deterministic -//! locks; contended and bulk writes route to the Calvin scheduler. +//! `PhysicalPlan` passes through before it is ordered. A local write passes +//! it before its enqueue, a replicated write on its data-group leader before +//! its propose. An uncontended write holds its per-vShard deterministic locks; +//! a contended write routes to the Calvin scheduler or waits. +pub mod admission_keys; pub mod gate; +pub mod holds; +pub mod leader_gate; pub mod lock_keys; pub mod predicate; pub mod route; +pub mod wait; pub mod write_order_fence; pub mod write_order_lock; pub use gate::{WriteAdmission, WriteAdmissionGuard, WriteTarget, admit, cp_routed_to_calvin}; +pub use holds::AdmissionHolds; +pub use leader_gate::LeaderWriteGate; pub use predicate::{ all_writes_bufferable, plan_is_write, plan_requires_txn_buffering, plan_writes_user_data, }; -pub use route::{bare_ok_response, route_write_to_calvin}; +pub use route::{bare_ok_response, calvin_route_keeps, route_write_to_calvin}; +pub use wait::AdmissionWait; pub(crate) use write_order_fence::order_row_write; pub use write_order_fence::{WriteOrder, WriteOrderFence}; pub use write_order_lock::KeyedWriteOrderLock; diff --git a/nodedb/src/control/server/shared/write_admission/predicate/txn_buffering/classify.rs b/nodedb/src/control/server/shared/write_admission/predicate/txn_buffering/classify.rs index e46c854a0..2a878ca9e 100644 --- a/nodedb/src/control/server/shared/write_admission/predicate/txn_buffering/classify.rs +++ b/nodedb/src/control/server/shared/write_admission/predicate/txn_buffering/classify.rs @@ -359,8 +359,6 @@ pub fn plan_requires_txn_buffering(plan: &PhysicalPlan) -> bool { | MetaOp::DropTxnOverlay { .. } | MetaOp::MarkSavepoint { .. } | MetaOp::RollbackToSavepoint { .. } - | MetaOp::RecordCalvinWriteVersions { .. } - | MetaOp::CalvinFlush { .. } | MetaOp::CalvinDrop { .. } | MetaOp::CalvinResolve { .. } | MetaOp::ResolveTxn { .. } @@ -425,8 +423,7 @@ mod tests { use nodedb_types::timeseries::continuous_agg::{ContinuousAggregateDef, RefreshPolicy}; use nodedb_types::vector_distance::DistanceMetric; use nodedb_types::{ - DatabaseId, Lsn, QualifiedCollection, Surrogate, SystemTimeScope, TenantId, - VectorAnnOptions, + DatabaseId, QualifiedCollection, Surrogate, SystemTimeScope, TenantId, VectorAnnOptions, }; use std::collections::BTreeMap; @@ -1846,6 +1843,7 @@ mod tests { replace_mode: false, collections_to_clear: Vec::new(), group_vshards: Vec::new(), + version_floor: Vec::new(), }), PhysicalPlan::Meta(MetaOp::PurgeTenant { tenant_id: 1 }), PhysicalPlan::Meta(MetaOp::UnregisterCollection { @@ -1916,14 +1914,12 @@ mod tests { tenant_id: tenant(), plans: Vec::new(), epoch_system_ms: 0, - is_group_leader: false, versioned_reads: vec![VersionedReadEntry { engine: EngineTag::Kv, collection: QualifiedCollection::new(DatabaseId::DEFAULT, "c").to_string(), key: ReadKeyIdent::Predicate, - read_lsn: Lsn::ZERO, + read_version: nodedb_types::WriteVersion::ZERO, home_vshard: None, - served_by: 0, }], body_plans: Vec::new(), }), @@ -1945,7 +1941,6 @@ mod tests { plans: Vec::new(), injected_reads: BTreeMap::new(), epoch_system_ms: 0, - is_group_leader: false, }), PhysicalPlan::Meta(MetaOp::RebuildIndex { collection: QualifiedCollection::new(DatabaseId::DEFAULT, "c"), @@ -1974,17 +1969,6 @@ mod tests { txn_id: TxnId::new(1), savepoint: 1, }), - PhysicalPlan::Meta(MetaOp::RecordCalvinWriteVersions { - tenant_id: tenant(), - plans: Vec::new(), - }), - PhysicalPlan::Meta(MetaOp::CalvinFlush { - epoch: 0, - position: 0, - redo: Vec::new(), - collections: Vec::new(), - sum_targets: Vec::new(), - }), PhysicalPlan::Meta(MetaOp::CalvinDrop { epoch: 0, position: 0, diff --git a/nodedb/src/control/server/shared/write_admission/predicate/user_data_write.rs b/nodedb/src/control/server/shared/write_admission/predicate/user_data_write.rs index 6daa3787b..e0e510a74 100644 --- a/nodedb/src/control/server/shared/write_admission/predicate/user_data_write.rs +++ b/nodedb/src/control/server/shared/write_admission/predicate/user_data_write.rs @@ -9,14 +9,26 @@ //! of a backup that already holds every row the tenant wrote. use crate::bridge::envelope::PhysicalPlan; -use nodedb_physical::physical_plan::{CrdtOp, KvOp}; +use nodedb_physical::physical_plan::{CrdtOp, KvOp, MetaOp}; use super::plan_is_write::plan_is_write; /// Whether `plan` writes a tenant's rows: a write-class plan that is not a /// schema install. pub fn plan_writes_user_data(plan: &PhysicalPlan) -> bool { - plan_is_write(plan) && !installs_schema(plan) + plan_is_write(plan) && !installs_schema(plan) && !calvin_install_without_user_write(plan) +} + +/// The install of a Calvin slice that raises no write mark: a slice that +/// writes only derived rows or schema. +fn calvin_install_without_user_write(plan: &PhysicalPlan) -> bool { + matches!( + plan, + PhysicalPlan::Meta(MetaOp::ApplyTransactionRedo { + calvin: Some(install), + .. + }) if !install.user_write + ) } /// A write-class plan that installs or removes schema state on a replica and @@ -73,4 +85,28 @@ mod tests { }); assert!(plan_writes_user_data(&plan)); } + + fn calvin_install(user_write: bool) -> PhysicalPlan { + PhysicalPlan::Meta(MetaOp::ApplyTransactionRedo { + redo: Vec::new(), + collections: vec!["c".into()], + sum_targets: Vec::new(), + origin: nodedb_physical::physical_plan::RedoOrigin::Commit, + calvin: Some(nodedb_physical::physical_plan::CalvinInstall { + epoch: 1, + position: 0, + epoch_system_ms: 0, + reply: nodedb_physical::physical_plan::CalvinReplySpec::Count(Vec::new()), + user_write, + }), + }) + } + + /// A Calvin slice that only installs schema, such as a constraint set + /// the leader's write gate sequenced, raises no write mark at install. + #[test] + fn a_calvin_install_counts_only_when_its_slice_writes_user_rows() { + assert!(plan_writes_user_data(&calvin_install(true))); + assert!(!plan_writes_user_data(&calvin_install(false))); + } } diff --git a/nodedb/src/control/server/shared/write_admission/route.rs b/nodedb/src/control/server/shared/write_admission/route.rs index 2f63812ad..c20905d70 100644 --- a/nodedb/src/control/server/shared/write_admission/route.rs +++ b/nodedb/src/control/server/shared/write_admission/route.rs @@ -3,17 +3,18 @@ //! Submit a gate-rejected write through the deterministic Calvin scheduler. //! //! When [`admit`](super::admit) returns [`WriteAdmission::RouteToCalvin`](super::WriteAdmission), -//! the caller hands the single write here. Two shapes reach this path, each -//! routed through the existing Calvin entry point that builds a VALID write set -//! for it: +//! the caller hands the single write here. A data-group leader's gate routes a +//! replicated write here too, through its proposer. Two shapes reach this +//! path, each routed through the existing Calvin entry point that builds a +//! VALID write set for it: //! -//! - A **point write** whose key a pending commit holds (Document / KV / Vector / -//! single-home edge): submitted as a single-vshard -//! [`build_single_vshard_tx_class`] + [`submit_calvin_routed`]. Because it -//! targets one vshard, it uses the single-vshard opt-in rather than the strict -//! multi-vshard builder. The scheduler acquires its key on the SAME lock -//! table the gate probed, queues FIFO behind the holder, and applies it once -//! released. +//! - A **write with a static write set** whose key a pending commit holds (a +//! row write, a collection-wide write, an edge on several homes): submitted +//! as a single-vshard [`build_single_vshard_tx_class`] + +//! [`submit_calvin_routed`]. A write on one vshard uses the single-vshard +//! opt-in rather than the strict multi-vshard builder. The scheduler +//! acquires its keys on the SAME lock table the gate probed, queues FIFO +//! behind the holder, and applies it once released. //! - A **predicate write** (`BulkUpdate` / `BulkDelete` on a SINGLE //! collection): its write set is not statically known, so it goes through //! [`dispatch_dependent_edge_recon`], which runs the pre-exec @@ -42,6 +43,7 @@ use crate::control::planner::calvin::{ submit_calvin_routed, }; use crate::control::state::SharedState; +use crate::event::EventSource; use crate::types::{DatabaseId, RequestId, TenantId, VShardId}; use nodedb_physical::physical_plan::PhysicalPlan; use nodedb_physical::physical_task::{PhysicalTask, PostSetOp}; @@ -59,16 +61,29 @@ pub fn bare_ok_response(request_id: RequestId) -> Response { payload: crate::bridge::envelope::Payload::from_vec(Vec::new()), watermark_lsn: crate::types::Lsn::ZERO, error_code: None, - read_set_valid: None, - read_version_lsn: crate::types::Lsn::ZERO, + stage_vote: None, + read_versions: crate::types::ReadVersions::new(), write_set: Vec::new(), } } +/// Whether the scheduler applies `plan` exactly as its writer asked when the +/// gate routes it. A static write set carries `event_source`. The +/// reconnaissance of a predicate write applies it as a client write. No +/// route carries a restore id. A write the route cannot keep waits for its +/// keys at the gate instead. +pub fn calvin_route_keeps(plan: &PhysicalPlan, event_source: EventSource, restore_id: u64) -> bool { + restore_id == 0 && (matches!(event_source, EventSource::User) || !is_dependent_predicate(plan)) +} + /// Route one write to the deterministic scheduler and return the applied /// `Response`. `None` for a plain write with no RETURNING rows — the caller then /// synthesizes its normal command-tag response. /// +/// A static write set applies under `event_source` on every replica. The +/// gate routes a predicate write only when it is a client's (see +/// [`calvin_route_keeps`]). +/// /// Returns a boxed future (rather than an `async fn`) to break the async /// recursion cycle described in the module docs. pub fn route_write_to_calvin<'a>( @@ -77,6 +92,7 @@ pub fn route_write_to_calvin<'a>( database_id: DatabaseId, vshard_id: VShardId, plan: PhysicalPlan, + event_source: EventSource, ) -> Pin>> + Send + 'a>> { Box::pin(async move { let task = PhysicalTask { @@ -101,131 +117,19 @@ pub fn route_write_to_calvin<'a>( return Ok(recon.apply_result); } - // Point write: its key is known, so build a static TxClass and submit it. + // A static write set: build a static TxClass and submit it. // - // This write reaches here ONLY because `admit` returned `RouteToCalvin`: - // a pending commit already holds its key. A point write targets a single - // vshard, so it must be built with the single-vshard opt-in — it sequences + // This write reaches here ONLY because a gate returned `RouteToCalvin`: + // a pending commit already holds one of its keys. A write on a single + // vshard must be built with the single-vshard opt-in — it sequences // through the scheduler to serialize on the SAME shared per-vShard // `LockManager` the holder is on, rather than being rejected as a - // (spuriously) single-vshard multi-shard dispatch. - // A timeseries ingest resolves to its rows here, before it is - // sequenced: every replica resolves a sequenced write on its own. - let resolved = crate::control::write_resolve::resolve_tasks_for_log( - shared, - std::slice::from_ref(&task), - ) - .await?; - let tasks = resolved.unwrap_or_else(|| vec![task]); - // The submitter's resolve knows the rows the ingest stores and the - // lines it rejected. The client reads both from this answer, not - // from the scheduler's. - let counts = match tasks.first() { - Some(task) => crate::control::write_resolve::resolved_ingest_counts(&task.plan)?, - None => None, - }; - let tx_class = build_single_vshard_tx_class(&tasks, tenant_id, &[])?; - let applied = submit_calvin_routed(shared, tx_class).await?; - Ok(with_ingest_counts(applied, counts)) + // (spuriously) single-vshard multi-shard dispatch. The gate never + // routes an append (timeseries ingest, columnar insert), so no + // unresolved ingest reaches the builder. + let mut tx_class = + build_single_vshard_tx_class(std::slice::from_ref(&task), tenant_id, &[])?; + tx_class.set_event_source(event_source.wal_code()); + submit_calvin_routed(shared, tx_class).await }) } - -/// `applied`, carrying the resolved ingest's counts when there are any. The -/// apply's counts win: its install rejected the rows that conflict with the -/// schema at its position. The resolve's `counts` answer only when the -/// apply's answer carries none. An error answer stays as the scheduler gave -/// it. -fn with_ingest_counts(applied: Option, counts: Option>) -> Option { - let Some(counts) = counts else { - return applied; - }; - match applied { - Some(response) if response.status != crate::bridge::envelope::Status::Ok => Some(response), - Some(response) - if crate::control::write_resolve::carries_applied_ingest_counts( - response.payload.as_bytes(), - ) => - { - Some(response) - } - Some(response) => Some(Response { - payload: crate::bridge::envelope::Payload::from_vec(counts), - ..response - }), - None => { - let mut response = bare_ok_response(RequestId::new(0)); - response.payload = crate::bridge::envelope::Payload::from_vec(counts); - Some(response) - } - } -} - -#[cfg(test)] -mod tests { - use super::*; - - fn counts(accepted: u64, rejected: u64) -> Vec { - nodedb_types::json_to_msgpack(&serde_json::json!({ - "accepted": accepted, - "rejected": rejected, - "collection": "metrics", - })) - .expect("encode counts") - } - - fn decoded(response: &Response) -> serde_json::Value { - let json = crate::data::executor::response_codec::decode_payload_to_json( - response.payload.as_bytes(), - ); - sonic_rs::from_str(&json).expect("decode counts") - } - - /// A Calvin-routed timeseries ingest answers the rejected count its - /// submitter's resolve found, whether or not the scheduler answered. - #[test] - fn a_routed_ingest_answers_its_resolved_counts() { - let bare = with_ingest_counts(None, Some(counts(2, 1))).expect("an answer"); - assert_eq!(decoded(&bare)["rejected"], serde_json::json!(1)); - - let applied = with_ingest_counts( - Some(bare_ok_response(RequestId::new(3))), - Some(counts(2, 1)), - ) - .expect("an answer"); - assert_eq!(applied.request_id, RequestId::new(3)); - assert_eq!(decoded(&applied)["accepted"], serde_json::json!(2)); - assert_eq!(decoded(&applied)["rejected"], serde_json::json!(1)); - } - - /// The Calvin completion hands back the flush's answer. When it carries - /// the install's counts, the client gets the apply's count: here the - /// install rejected one row the resolve accepted. - #[test] - fn a_routed_ingest_answers_the_apply_counts_when_the_flush_carries_them() { - use crate::engine::timeseries::install_counts::{TsInstallCount, TsInstallCounts}; - let mut flushed = bare_ok_response(RequestId::new(4)); - flushed.payload = crate::bridge::envelope::Payload::from_vec( - TsInstallCounts::new(vec![TsInstallCount { - collection: "metrics".into(), - accepted: 1, - rejected: 2, - }]) - .to_bytes() - .expect("encode install counts"), - ); - let applied = with_ingest_counts(Some(flushed), Some(counts(2, 1))).expect("an answer"); - assert_eq!(decoded(&applied)["accepted"], serde_json::json!(1)); - assert_eq!(decoded(&applied)["rejected"], serde_json::json!(2)); - assert_eq!( - decoded(&applied)["collection"], - serde_json::json!("metrics") - ); - } - - /// A routed write that is not a resolved ingest keeps the scheduler's - /// answer. - #[test] - fn a_routed_non_ingest_keeps_the_scheduler_answer() { - assert!(with_ingest_counts(None, None).is_none()); - } -} diff --git a/nodedb/src/control/server/shared/write_admission/wait.rs b/nodedb/src/control/server/shared/write_admission/wait.rs new file mode 100644 index 000000000..b33cc5842 --- /dev/null +++ b/nodedb/src/control/server/shared/write_admission/wait.rs @@ -0,0 +1,161 @@ +// SPDX-License-Identifier: BUSL-1.1 + +//! A contended write no Calvin transaction can sequence: it waits for its +//! keys at the gate. +//! +//! The waiter holds no key while it waits. It probes the lock table with an +//! all-or-nothing `try_acquire` after every release, so it never holds one +//! key while it waits for another. A Calvin transaction therefore never +//! waits on a waiting writer, and the two cannot deadlock. + +use std::collections::BTreeMap; +use std::sync::{Arc, Mutex}; + +use tokio::sync::mpsc::UnboundedSender; + +use crate::control::cluster::calvin::scheduler::lock_manager::{ + LockKey, LockManager, LockMode, TxnId, +}; +use crate::types::RequestId; + +use super::gate::WriteAdmissionGuard; + +/// The keys of a contended write, and the lock table it waits on. +pub struct AdmissionWait { + lock_manager: Arc>, + txn: TxnId, + keys: BTreeMap, + promotion_sender: Option>>, +} + +impl AdmissionWait { + pub(super) fn new( + lock_manager: Arc>, + txn: TxnId, + keys: BTreeMap, + promotion_sender: Option>>, + ) -> Self { + Self { + lock_manager, + txn, + keys, + promotion_sender, + } + } + + /// Wait until every key is free, take them all, and return the guard + /// that holds them. + /// + /// Returns [`crate::Error::DeadlineExceeded`] when the keys stay held + /// until `deadline`. The write then holds nothing. + pub async fn acquire( + self, + deadline: tokio::time::Instant, + ) -> crate::Result { + let signal = self + .lock_manager + .lock() + .unwrap_or_else(|p| p.into_inner()) + .release_signal(); + loop { + // Enabled before the probe, so a release between the probe and + // the wait still wakes this waiter. + let released = signal.notified(); + tokio::pin!(released); + released.as_mut().enable(); + let acquired = { + let mut lock_manager = self.lock_manager.lock().unwrap_or_else(|p| p.into_inner()); + lock_manager.try_acquire(self.txn, self.keys.clone()) + }; + if acquired { + return Ok(WriteAdmissionGuard::new( + self.lock_manager, + self.txn, + self.promotion_sender, + )); + } + if tokio::time::timeout_at(deadline, released).await.is_err() { + let contention = self + .lock_manager + .lock() + .unwrap_or_else(|p| p.into_inner()) + .contention(self.txn, &self.keys); + tracing::warn!( + ?contention, + "a contended write reached its deadline waiting for its keys" + ); + return Err(crate::Error::DeadlineExceeded { + request_id: RequestId::new(0), + }); + } + } + } +} + +#[cfg(test)] +mod tests { + use std::time::Duration; + + use super::*; + + fn coll() -> LockKey { + LockKey::Collection { + collection: Arc::from("c"), + } + } + + fn waiter(lock_manager: &Arc>) -> AdmissionWait { + AdmissionWait::new( + Arc::clone(lock_manager), + TxnId::new(TxnId::AUTOCOMMIT_EPOCH, 1), + BTreeMap::from([(coll(), LockMode::Exclusive)]), + None, + ) + } + + /// The waiter takes its keys once the Calvin holder releases them. + #[tokio::test] + async fn a_waiter_takes_its_keys_after_the_holder_releases() { + let lock_manager = Arc::new(Mutex::new(LockManager::new())); + let holder = TxnId::new(7, 0); + assert!( + lock_manager + .lock() + .expect("lock table") + .try_acquire(holder, BTreeMap::from([(coll(), LockMode::Intent)])) + ); + let wait = tokio::spawn( + waiter(&lock_manager).acquire(tokio::time::Instant::now() + Duration::from_secs(10)), + ); + tokio::task::yield_now().await; + assert!( + !wait.is_finished(), + "the waiter waits while the key is held" + ); + lock_manager.lock().expect("lock table").release(holder); + let guard = wait.await.expect("join").expect("the keys free up"); + drop(guard); + assert_eq!(lock_manager.lock().expect("lock table").lock_count(), 0); + } + + /// The waiter gives up at its deadline and holds nothing. + #[tokio::test] + async fn a_waiter_gives_up_at_its_deadline() { + let lock_manager = Arc::new(Mutex::new(LockManager::new())); + let holder = TxnId::new(7, 0); + assert!( + lock_manager + .lock() + .expect("lock table") + .try_acquire(holder, BTreeMap::from([(coll(), LockMode::Exclusive)])) + ); + let outcome = waiter(&lock_manager) + .acquire(tokio::time::Instant::now() + Duration::from_millis(20)) + .await; + assert!(matches!( + outcome, + Err(crate::Error::DeadlineExceeded { .. }) + )); + assert_eq!(lock_manager.lock().expect("lock table").holder_count(), 1); + } +} diff --git a/nodedb/src/control/server/shuffle/aggregator_hook.rs b/nodedb/src/control/server/shuffle/aggregator_hook.rs index 9883b0f89..6119ef186 100644 --- a/nodedb/src/control/server/shuffle/aggregator_hook.rs +++ b/nodedb/src/control/server/shuffle/aggregator_hook.rs @@ -66,8 +66,7 @@ const SIDE_PRODUCER: u8 = 0; /// SINGLE-SIDED aggregate sibling of [`RegistryShuffleConsumer`]. Holds the /// node's [`SharedState`] so it can reach the shuffle receiver registry (for the /// staged inbox), the SPSC dispatcher + request tracker (to run the merge + -/// finalize on the Data Plane), and the network tuning (deadline / result-byte -/// ceilings). +/// finalize on the Data Plane), and the network tuning (result-byte ceiling). pub struct RegistryShuffleAggregator { state: Arc, } @@ -107,6 +106,7 @@ impl RegistryShuffleAggregator { // deadline; on expiry surface a deterministic DeadlineExceeded rather // than hanging. let deadline_ms = req.deadline_remaining_ms.max(1); + let hop_deadline = Instant::now() + Duration::from_millis(deadline_ms); if tokio::time::timeout(Duration::from_millis(deadline_ms), inbox.wait_finalized()) .await .is_err() @@ -160,9 +160,9 @@ impl RegistryShuffleAggregator { // bounded-collect path the join consume hook uses; the plan is built // locally (node-local path, never wire-encoded), so we build the // `Request` directly rather than round-tripping through `plan_bytes`. - let deadline = Duration::from_millis(deadline_ms).min(Duration::from_secs( - self.state.tuning.network.default_deadline_secs, - )); + // The hop runs on what is left of the budget the coordinator sent. + // This node's default is no ceiling on it. + let deadline = hop_deadline.saturating_duration_since(Instant::now()); let request_id = self.state.next_request_id(); let request = Request { @@ -171,7 +171,7 @@ impl RegistryShuffleAggregator { database_id: DatabaseId::from(req.database_id), vshard_id: crate::types::VShardId::new(0), plan, - deadline: Instant::now() + deadline, + deadline: hop_deadline, priority: Priority::Normal, trace_id: nodedb_types::TraceId(req.trace_id), consistency: ReadConsistency::Strong, @@ -184,6 +184,7 @@ impl RegistryShuffleAggregator { wal_lsn: None, resolved_now_ms: None, commit_hlc: None, + entry_version: None, admission: crate::bridge::envelope::Admission::Exempt( crate::bridge::envelope::ExemptReason::Read, ), diff --git a/nodedb/src/control/server/shuffle/consumer_hook.rs b/nodedb/src/control/server/shuffle/consumer_hook.rs index a6f11b563..e4cc0f988 100644 --- a/nodedb/src/control/server/shuffle/consumer_hook.rs +++ b/nodedb/src/control/server/shuffle/consumer_hook.rs @@ -61,8 +61,8 @@ const SIDE_PROBE: u8 = 1; /// /// Holds the node's [`SharedState`] so it can reach the shuffle receiver /// registry (for the staged inboxes), the SPSC dispatcher + request tracker (to -/// run the grace join on the Data Plane), and the network tuning (deadline / -/// result-byte ceilings). +/// run the grace join on the Data Plane), and the network tuning (result-byte +/// ceiling). pub struct RegistryShuffleConsumer { state: Arc, } @@ -110,6 +110,7 @@ impl RegistryShuffleConsumer { // individually — is capped; on expiry we surface a deterministic // DeadlineExceeded rather than hanging. let deadline_ms = req.deadline_remaining_ms.max(1); + let hop_deadline = Instant::now() + Duration::from_millis(deadline_ms); let wait_both = async { build_inbox.wait_finalized().await; probe_inbox.wait_finalized().await; @@ -162,9 +163,9 @@ impl RegistryShuffleConsumer { // plan is built locally (it carries node-local paths and must never be // wire-encoded), so we build the `Request` directly rather than // round-tripping through `plan_bytes`. - let deadline = Duration::from_millis(deadline_ms).min(Duration::from_secs( - self.state.tuning.network.default_deadline_secs, - )); + // The hop runs on what is left of the budget the coordinator sent. + // This node's default is no ceiling on it. + let deadline = hop_deadline.saturating_duration_since(Instant::now()); let request_id = self.state.next_request_id(); let request = Request { @@ -173,7 +174,7 @@ impl RegistryShuffleConsumer { database_id: DatabaseId::from(req.database_id), vshard_id: crate::types::VShardId::new(0), plan, - deadline: Instant::now() + deadline, + deadline: hop_deadline, priority: Priority::Normal, trace_id: nodedb_types::TraceId(req.trace_id), consistency: ReadConsistency::Strong, @@ -186,6 +187,7 @@ impl RegistryShuffleConsumer { wal_lsn: None, resolved_now_ms: None, commit_hlc: None, + entry_version: None, admission: crate::bridge::envelope::Admission::Exempt( crate::bridge::envelope::ExemptReason::Read, ), diff --git a/nodedb/src/control/server/shuffle/fanout.rs b/nodedb/src/control/server/shuffle/fanout.rs index 730c3346e..e5ab2de01 100644 --- a/nodedb/src/control/server/shuffle/fanout.rs +++ b/nodedb/src/control/server/shuffle/fanout.rs @@ -78,13 +78,12 @@ pub struct ShuffleFanoutSink { keys: Vec, /// `part -> PartState`, one entry per part in `0..num_parts`. parts: HashMap, - /// Max per-collection read-version LSN observed across every scanned frame - /// fed to this sink (each frame's scanned collection `coll_write_lsn` at read - /// time). A producer scans exactly one collection, so this single max-folded - /// value is that collection's observed read version — the sound comparand the - /// coordinator uses for cross-shard OCC read validation of an in-transaction - /// distributed aggregate. - max_read_version_lsn: u64, + /// The read versions observed across every scanned frame fed to this sink, + /// one per vShard. A producer scans exactly one collection, so the fold + /// holds that collection's observed versions: the comparand the + /// coordinator uses for cross-shard OCC read validation of an + /// in-transaction distributed aggregate. + read_versions: crate::types::ReadVersions, } /// Parameters for [`ShuffleFanoutSink::new`]. @@ -148,16 +147,16 @@ impl ShuffleFanoutSink { producer_count, keys, parts, - max_read_version_lsn: 0, + read_versions: crate::types::ReadVersions::new(), } } - /// The max per-collection read-version LSN observed across every scanned - /// frame fed to this sink (`0` if no frame carried one). The producer hook - /// reads this after the streaming scan completes and reports it on the + /// The read versions observed across every scanned frame fed to this + /// sink (empty if no frame carried one). The producer hook reads them + /// after the streaming scan completes and reports them on the /// `ShuffleProduceResponse` so the coordinator can validate the read. - pub fn observed_read_version_lsn(&self) -> u64 { - self.max_read_version_lsn + pub fn observed_read_versions(&self) -> &crate::types::ReadVersions { + &self.read_versions } /// The `ShufflePushRequest` opener for `part` (shared shape for remote + @@ -412,16 +411,17 @@ impl nodedb_cluster::ChunkSink for &mut ShuffleFanoutSink { &mut self, payload: Vec, _watermark_lsn: u64, - read_version_lsn: u64, + read_versions: Vec, ) -> nodedb_cluster::Result<()> { - // The streaming executor calls this per scan batch. Max-fold this frame's - // per-collection read version so the producer can report the observed - // read version for cross-shard OCC validation (NOT the core-global - // watermark). Then hash-partition the batch and buffer/flush per part. A + // The streaming executor calls this per scan batch. Fold this frame's + // read versions so the producer can report what it observed for + // cross-shard OCC validation (NOT the core-global watermark). Then + // hash-partition the batch and buffer/flush per part. A // routing/flush failure is surfaced as a cluster error so the streaming // executor terminates the scan; the producer hook then `finish`es the // fan-out with the error so consumers fail fast (never a silent drop). - self.max_read_version_lsn = self.max_read_version_lsn.max(read_version_lsn); + self.read_versions + .merge(&crate::types::ReadVersions::from_wire(&read_versions)); self.route_chunk(payload) .await .map_err(|e| nodedb_cluster::ClusterError::Storage { diff --git a/nodedb/src/control/server/shuffle/producer_hook.rs b/nodedb/src/control/server/shuffle/producer_hook.rs index 9be9b4442..06c622530 100644 --- a/nodedb/src/control/server/shuffle/producer_hook.rs +++ b/nodedb/src/control/server/shuffle/producer_hook.rs @@ -56,9 +56,8 @@ impl RegistryShuffleProducer { } /// Run the produce, returning a [`ShuffleProduceResponse`] whose `error` is a - /// terminal error (or `None` on clean produce) and whose `read_version_lsn` is - /// the max per-collection read version the local scan observed (`0` on - /// failure). + /// terminal error (or `None` on clean produce) and whose `read_versions` are + /// the read versions the local scan observed (empty on failure). /// /// Factored out of the trait method so the trait impl stays a thin shim and /// every early-exit error path is `?`-propagated here, then mapped to the @@ -72,7 +71,7 @@ impl RegistryShuffleProducer { message: "shuffle produce requires a cluster transport (single-node mode)" .into(), }), - read_version_lsn: 0, + read_versions: Vec::new(), }; }; @@ -129,12 +128,12 @@ impl RegistryShuffleProducer { // (validation reject, decode, deadline, or data-plane stream error). let scan_outcome = executor.execute_plan_streaming(exec_req, &mut sink).await; - // Capture the max per-collection read version the scan observed BEFORE - // `finish` consumes the sink. On a clean produce this is the scanned - // collection's `coll_write_lsn` at read time — the comparand the - // coordinator max-folds across producers for cross-shard OCC read - // validation of an in-transaction distributed aggregate. - let observed_read_version = sink.observed_read_version_lsn(); + // Capture the read versions the scan observed BEFORE `finish` consumes + // the sink. On a clean produce they are the scanned collection's + // versions at read time: the comparand the coordinator folds across + // producers for cross-shard OCC read validation of an in-transaction + // distributed aggregate. + let observed_read_versions = sink.observed_read_versions().to_wire(); // Finalize the fan-out: flush residuals (clean path) and `End` EVERY part // for this side — with the scan error if any — so each receiver's barrier @@ -148,20 +147,21 @@ impl RegistryShuffleProducer { code: 0, message: format!("shuffle produce fan-out finalize failed: {e}"), })), - read_version_lsn: 0, + read_versions: Vec::new(), }; } - // On a failed scan the read version is meaningless — report 0 (mirroring - // `ExecuteResponse::err`). On a clean produce report the observed max. - let read_version_lsn = if scan_outcome.is_none() { - observed_read_version + // On a failed scan the read versions are meaningless: report none + // (mirroring `ExecuteResponse::err`). On a clean produce report what + // the scan observed. + let read_versions = if scan_outcome.is_none() { + observed_read_versions } else { - 0 + Vec::new() }; ShuffleProduceResponse { error: scan_outcome, - read_version_lsn, + read_versions, } } } diff --git a/nodedb/src/control/server/surrogate_exchange/authority.rs b/nodedb/src/control/server/surrogate_exchange/authority.rs index 71e870ecc..fc0420d10 100644 --- a/nodedb/src/control/server/surrogate_exchange/authority.rs +++ b/nodedb/src/control/server/surrogate_exchange/authority.rs @@ -78,7 +78,8 @@ pub(crate) async fn assign_many_at_home( home.as_u32(), ReplicatedWrite::SurrogateBind { identities }, ); - propose_replicated_entry(state, proposer, entry).await?; + let deadline = crate::control::wal_replication::statement_propose_deadline(state); + propose_replicated_entry(state, proposer, entry, deadline).await?; } pks.iter() .map(|pk| { diff --git a/nodedb/src/control/server/sync/async_dispatch/delta/compensation.rs b/nodedb/src/control/server/sync/async_dispatch/delta/compensation.rs index 7fc11a1b9..6029411a7 100644 --- a/nodedb/src/control/server/sync/async_dispatch/delta/compensation.rs +++ b/nodedb/src/control/server/sync/async_dispatch/delta/compensation.rs @@ -195,6 +195,7 @@ fn compensation_hint_for_code(code: &ErrorCode) -> CompensationHint { | ErrorCode::CrdtFrontierMismatch { .. } | ErrorCode::CollectionDraining { .. } | ErrorCode::RetryableRefusal { .. } + | ErrorCode::CoreFailStopped { .. } | ErrorCode::TransactionRollback { .. } => CompensationHint::Retry { retry_after_ms: 0 }, other @ (ErrorCode::SyncRejected { .. } | ErrorCode::SyncNotApplied { .. } diff --git a/nodedb/src/control/server/sync/raft_dispatch/durability_test_support.rs b/nodedb/src/control/server/sync/raft_dispatch/durability_test_support.rs index 35cedebc2..4e36ec249 100644 --- a/nodedb/src/control/server/sync/raft_dispatch/durability_test_support.rs +++ b/nodedb/src/control/server/sync/raft_dispatch/durability_test_support.rs @@ -18,7 +18,7 @@ use crate::control::security::audit::NoopAuditEmitter; use crate::control::security::identity::AuthenticatedIdentity; use crate::control::server::shared::authorization::{AuthorizedTask, authorize_task_set}; use crate::control::state::SharedState; -use crate::types::{DatabaseId, Lsn, TenantId, VShardId}; +use crate::types::{DatabaseId, ReadVersions, TenantId, VShardId}; use crate::wal::WalManager; pub(super) const COLLECTION: &str = "docs"; @@ -48,7 +48,7 @@ pub(super) fn fixture() -> (Arc, CoreChannelDataSide, tempfile::Tem /// answers each proposal with the payload `applied`. pub(super) fn applying_proposer() -> Arc { Arc::new(|_vshard, _key, _data, _deadline| { - Box::pin(async { Ok((b"applied".to_vec(), Lsn::ZERO)) }) + Box::pin(async { Ok((b"applied".to_vec(), ReadVersions::new())) }) }) } diff --git a/nodedb/src/control/server/sync/raft_dispatch/propose.rs b/nodedb/src/control/server/sync/raft_dispatch/propose.rs index 0461af376..f2001f6ee 100644 --- a/nodedb/src/control/server/sync/raft_dispatch/propose.rs +++ b/nodedb/src/control/server/sync/raft_dispatch/propose.rs @@ -109,9 +109,9 @@ pub(crate) async fn propose_sync_write( for (attempt, backoff_ms) in BACKOFF_MS.iter().enumerate() { match proposer(vshard_id, idempotency_key, data.clone(), deadline).await { - // The committed log index rides alongside the payload; the sync-ack - // path only needs the payload bytes. - Ok((p, _committed_version)) => { + // The write's versions ride alongside the payload. The sync-ack + // path needs only the payload bytes. + Ok((p, _write_versions)) => { payload = Some(p); break; } diff --git a/nodedb/src/control/server/sync/raft_dispatch/response.rs b/nodedb/src/control/server/sync/raft_dispatch/response.rs index ed9f36a4d..8be27caf8 100644 --- a/nodedb/src/control/server/sync/raft_dispatch/response.rs +++ b/nodedb/src/control/server/sync/raft_dispatch/response.rs @@ -72,8 +72,8 @@ fn ok_response(state: &SharedState, payload: Vec) -> Response { payload: payload.into(), watermark_lsn: Lsn::ZERO, error_code: None, - read_set_valid: None, - read_version_lsn: Lsn::ZERO, + stage_vote: None, + read_versions: crate::types::ReadVersions::new(), write_set: Vec::new(), } } diff --git a/nodedb/src/control/server/sync/refusal.rs b/nodedb/src/control/server/sync/refusal.rs index ecf8e3d96..e97ccf6d8 100644 --- a/nodedb/src/control/server/sync/refusal.rs +++ b/nodedb/src/control/server/sync/refusal.rs @@ -25,6 +25,8 @@ pub(super) fn retryable_refusal_reason(error: &crate::Error) -> Option<&str> { crate::Error::RetryableRefusal { reason } => Some(reason), crate::Error::DataPlane(code) => match code { ErrorCode::RetryableRefusal { reason } => Some(reason), + // The fail-stopped core applied nothing: a restart serves the frame. + ErrorCode::CoreFailStopped { detail, .. } => Some(detail), ErrorCode::DeadlineExceeded | ErrorCode::RejectedConstraint { .. } | ErrorCode::RejectedPrevalidation { .. } @@ -353,6 +355,7 @@ fn is_indeterminate_code(code: &ErrorCode) -> bool { | ErrorCode::DispatchCapacity { .. } | ErrorCode::ConflictRetry | ErrorCode::RetryableRefusal { .. } + | ErrorCode::CoreFailStopped { .. } | ErrorCode::OllpRetryRequired | ErrorCode::CrdtFrontierMismatch { .. } | ErrorCode::CollectionDraining { .. } diff --git a/nodedb/src/control/server/wal_dispatch/core.rs b/nodedb/src/control/server/wal_dispatch/core.rs index b7ef91567..bb54d0d86 100644 --- a/nodedb/src/control/server/wal_dispatch/core.rs +++ b/nodedb/src/control/server/wal_dispatch/core.rs @@ -259,6 +259,7 @@ mod tests { collections: Vec::new(), sum_targets: Vec::new(), origin: nodedb_physical::physical_plan::RedoOrigin::Commit, + calvin: None, }, ); diff --git a/nodedb/src/control/shutdown/data_plane.rs b/nodedb/src/control/shutdown/data_plane.rs index 812b49f04..ccf3fc4fa 100644 --- a/nodedb/src/control/shutdown/data_plane.rs +++ b/nodedb/src/control/shutdown/data_plane.rs @@ -274,6 +274,7 @@ mod tests { wal_lsn: None, resolved_now_ms: None, commit_hlc: None, + entry_version: None, admission: Admission::Exempt(ExemptReason::Read), } } @@ -287,8 +288,8 @@ mod tests { payload: Payload::empty(), watermark_lsn: Lsn::ZERO, error_code: None, - read_set_valid: None, - read_version_lsn: Lsn::ZERO, + stage_vote: None, + read_versions: crate::types::ReadVersions::new(), write_set: Vec::new(), } } diff --git a/nodedb/src/control/state/calvin_ack_result.rs b/nodedb/src/control/state/calvin_ack_result.rs index 959ddb183..965e814f8 100644 --- a/nodedb/src/control/state/calvin_ack_result.rs +++ b/nodedb/src/control/state/calvin_ack_result.rs @@ -23,7 +23,7 @@ pub const ACK_ROWS_FRAME_BUDGET: u64 = (nodedb_cluster::rpc_codec::MAX_RPC_PAYLO /// The `RETURNING` answer of a participant's slice. #[derive(Debug, Clone, PartialEq, Eq, zerompk::ToMessagePack, zerompk::FromMessagePack)] pub enum AckReturning { - /// The encoded rows, as the flush answered them. + /// The encoded rows, as the install answered them. Rows { rows: Vec }, /// The rows took `bytes`, past `limit`. OverLimit { bytes: u64, limit: u64 }, @@ -52,7 +52,7 @@ pub struct CalvinAckResult { /// The plain answer of a primary-write slice. #[derive(Debug, Clone, PartialEq, Eq, zerompk::ToMessagePack, zerompk::FromMessagePack)] pub enum AckReply { - /// The encoded answer, as the flush answered it. + /// The encoded answer, as the install answered it. Payload { payload: Vec }, /// The install stored the writes, and rendering the answer failed. Failed { detail: String }, @@ -68,7 +68,7 @@ pub struct AckSlice { } impl CalvinAckResult { - /// The report of a flush that answered `response` for `slice`. `limit` + /// The report of an install that answered `response` for `slice`. `limit` /// bounds the rows it carries. /// /// A primary-write slice always reports its answer: its rows, or its @@ -164,8 +164,8 @@ mod tests { payload: crate::bridge::envelope::Payload::from_vec(payload), watermark_lsn: crate::types::Lsn::ZERO, error_code: None, - read_set_valid: None, - read_version_lsn: crate::types::Lsn::ZERO, + stage_vote: None, + read_versions: crate::types::ReadVersions::new(), write_set: Vec::new(), } } diff --git a/nodedb/src/control/state/calvin_counters.rs b/nodedb/src/control/state/calvin_counters.rs index 4eb519b05..acdc0fb8a 100644 --- a/nodedb/src/control/state/calvin_counters.rs +++ b/nodedb/src/control/state/calvin_counters.rs @@ -6,23 +6,30 @@ use std::sync::Arc; use std::sync::atomic::AtomicU64; /// Node-global Calvin observability counters, incremented by the per-vShard -/// schedulers (which hold `Arc`) and read by tests and metrics +/// schedulers and the data-group apply loops, and read by tests and metrics /// without reaching into the `!Send` Data-Plane state. All start at 0 and stay /// 0 in single-node / no-Calvin deployments. pub struct CalvinCounters { - /// Count of committed Calvin applies whose write versions were recorded - /// into the per-core write-version index, incremented once per apply for - /// which a scheduler dispatched a write-version record op. + /// Count of committed Calvin slices whose stamped redo installed on this + /// node. The install records the slice's write versions into the + /// per-core write-version index. pub write_versions_recorded: Arc, /// Count of committed Calvin applies whose participant reported that its /// slice of the transaction's reads was no longer current at apply time. /// Observation only — the apply still committed. pub read_set_validation_failures: Arc, - /// Count of staged Calvin transactions the scheduler resolved to COMMIT - /// by dispatching a flush of their commit-pending buffer to base. + /// Count of staged Calvin transactions the scheduler resolved to COMMIT. + /// Each commit's stamped redo installs its staged writes. pub commits_flushed: Arc, /// Count of staged Calvin transactions the scheduler resolved to ABORT /// by dispatching a drop of their commit-pending buffer, mirroring /// [`CalvinCounters::commits_flushed`]. pub commits_dropped: Arc, + /// Count of stamped Calvin redo copies that applied on this node and + /// installed nothing: their position was applied, or another copy held + /// its claim. + pub redo_copies_skipped: Arc, + /// Count of restages this node's leaders dispatched: a committed Calvin + /// slice staged again after the leader's own stage failed. + pub stage_restages: Arc, } diff --git a/nodedb/src/control/state/calvin_cuts.rs b/nodedb/src/control/state/calvin_cuts.rs index 5c052b1c9..72fefb9b9 100644 --- a/nodedb/src/control/state/calvin_cuts.rs +++ b/nodedb/src/control/state/calvin_cuts.rs @@ -208,6 +208,20 @@ impl CalvinCuts { .collect() } + /// The vShards of `vshards` whose scheduler runs here and has not passed + /// exactly the marker `hlc`. A vShard with no scheduler here never lags. + pub fn lagging_among(&self, hlc: u64, vshards: &[u32]) -> Vec { + let registered = self.registered.lock().unwrap_or_else(|p| p.into_inner()); + let recent = self.recent.lock().unwrap_or_else(|p| p.into_inner()); + let passed = recent.get(&hlc).map(|marker| &marker.passed); + vshards + .iter() + .copied() + .filter(|vshard_id| registered.contains(vshard_id)) + .filter(|vshard_id| passed.is_none_or(|passed| !passed.contains(vshard_id))) + .collect() + } + /// Wait until every scheduler passed the marker `hlc`, or `deadline`. /// Returns the vShards still lagging, empty once every one passed. pub async fn await_passed(&self, hlc: u64, deadline: tokio::time::Instant) -> Vec { @@ -310,4 +324,24 @@ mod tests { let other: BTreeSet = [3].into_iter().collect(); assert_eq!(cuts.await_vshards_passed(80, &other, soon()).await, None); } + + /// A group's barrier waits only on the schedulers of its vShards that + /// run here, and only until each passed exactly the cut's marker. + #[test] + fn a_group_lags_on_its_own_registered_vshards() { + let cuts = CalvinCuts::default(); + cuts.register(1); + cuts.register(2); + assert_eq!(cuts.lagging_among(90, &[1, 2, 9]), vec![1, 2]); + cuts.note_passed(1, 90); + cuts.note_passed(2, 91); + assert_eq!( + cuts.lagging_among(90, &[1, 2, 9]), + vec![2], + "vShard 2 passed another marker; vShard 9 runs no scheduler here" + ); + cuts.note_passed(2, 90); + assert!(cuts.lagging_among(90, &[1, 2, 9]).is_empty()); + assert!(cuts.lagging_among(95, &[]).is_empty(), "no vShard, no wait"); + } } diff --git a/nodedb/src/control/state/calvin_local.rs b/nodedb/src/control/state/calvin_local.rs index c8d0e0bb3..c612b64e3 100644 --- a/nodedb/src/control/state/calvin_local.rs +++ b/nodedb/src/control/state/calvin_local.rs @@ -7,9 +7,12 @@ use std::collections::BTreeMap; use std::sync::atomic::{AtomicU32, AtomicU64}; use std::sync::{Arc, Mutex, OnceLock}; -use crate::control::cluster::calvin::scheduler::SequencerProposer; use crate::control::cluster::calvin::scheduler::lock::HotKeyTable; use crate::control::cluster::calvin::scheduler::lock_manager::{LockManager, TxnId}; +use crate::control::cluster::calvin::scheduler::{ + CalvinAppliedLedgers, CalvinInboxes, CalvinReadResults, CaughtUpRegistry, SequencerProposer, +}; +use crate::control::server::shared::write_admission::AdmissionHolds; use super::calvin_apply_sidecar::CalvinApplySidecar; use super::calvin_bases::CalvinBases; @@ -29,7 +32,7 @@ pub struct CalvinLocalState { /// session reads. 0 until a scheduler applies an epoch. pub last_applied_epoch: Arc, /// Node-global Calvin observability counters (write versions recorded, - /// read-set validation failures, commits flushed/dropped). + /// read-set validation failures, commits installed/dropped). pub counters: CalvinCounters, /// Local, in-process sidecar carrying the applied Data-Plane /// [`Response`](crate::bridge::envelope::Response) (affected-count and any @@ -73,13 +76,32 @@ pub struct CalvinLocalState { /// with [`TxnId::AUTOCOMMIT_EPOCH`] to mint holder identities that never /// collide with a real Calvin `(epoch, position)` schedule position. pub autocommit_lock_seq: AtomicU32, + /// The lock keys replicated writes hold on this data-group leader, from + /// their propose until the apply loop starts their entries. + pub admission_holds: AdmissionHolds, /// The backup cut markers each local scheduler passed. pub cuts: CalvinCuts, + /// Each ordered cut's barrier per data group: where this node proposed + /// it while it led the group, and where this node applied it. + pub cut_barriers: crate::control::backup::cut_order::CutBarriers, /// Where this node's Calvin state of each vShard is whole from. pub bases: CalvinBases, /// Hands this node's sequencer entries to the sequencer Raft group. Set /// once the schedulers start; unset on a node that runs none. pub sequencer_proposer: OnceLock>, + /// Which Calvin positions this node's replica of each vShard applied. + /// Boot recovery fills it before the data-group apply loop starts. + pub applied: CalvinAppliedLedgers, + /// Whether each running scheduler reached its rebuild target. Startup + /// reads it before it opens the client gateway. + pub caught_up: CaughtUpRegistry, + /// The inbox of each running scheduler: the data-group apply loop tells + /// it how each stamped Calvin redo concluded. + pub inboxes: CalvinInboxes, + /// Dependent-read barrier events of each vShard that wait for its + /// scheduler: read results and timeouts the data-group apply loop + /// applied before the scheduler took them. + pub read_results: CalvinReadResults, } impl CalvinLocalState { @@ -92,15 +114,23 @@ impl CalvinLocalState { read_set_validation_failures: Arc::new(AtomicU64::new(0)), commits_flushed: Arc::new(AtomicU64::new(0)), commits_dropped: Arc::new(AtomicU64::new(0)), + redo_copies_skipped: Arc::new(AtomicU64::new(0)), + stage_restages: Arc::new(AtomicU64::new(0)), }, apply_results: CalvinApplySidecar::default(), lock_managers: Arc::new(Mutex::new(BTreeMap::new())), hot_key_table: Arc::new(Mutex::new(HotKeyTable::new())), promotion_senders: Arc::new(Mutex::new(BTreeMap::new())), autocommit_lock_seq: AtomicU32::new(0), + admission_holds: AdmissionHolds::new(), cuts: CalvinCuts::default(), + cut_barriers: crate::control::backup::cut_order::CutBarriers::new(), bases: CalvinBases::default(), sequencer_proposer: OnceLock::new(), + applied: CalvinAppliedLedgers::default(), + caught_up: CaughtUpRegistry::default(), + inboxes: CalvinInboxes::default(), + read_results: CalvinReadResults::default(), } } } diff --git a/nodedb/src/control/state/fields.rs b/nodedb/src/control/state/fields.rs index d5a6327cd..29891b91a 100644 --- a/nodedb/src/control/state/fields.rs +++ b/nodedb/src/control/state/fields.rs @@ -117,23 +117,12 @@ pub struct SharedState { >, /// Per-Raft-group apply watermark registry for commit-wait and drain paths. pub group_watchers: Arc, - /// Serializes this node's attempts to acquire the replicated descriptor - /// preparation lease, and a local DDL through its post-apply. An async - /// proposer holds it across the post-apply, so it is a tokio mutex. - pub metadata_ddl_lock: tokio::sync::Mutex<()>, - /// Replicated preparation owner: its token, its node, and the local - /// monotonic apply time. The token and node are persisted in - /// `SystemCatalog` and seeded at boot. - pub metadata_ddl_owner: Mutex>, - /// Most recent fenced DDL token applied while its owner remained current. - /// Starts at 0 on boot. - pub metadata_ddl_applied_token: AtomicU64, + /// This node's state of the replicated descriptor preparation lease. + pub metadata_ddl: super::metadata_ddl::MetadataDdlState, /// Apply-progress steps this node made on metadata entries: each batch /// sub-entry and each purge-reclaim step (a core acknowledgement, a closed /// scan). A proposer on a long apply reads it to tell slow from stalled. pub metadata_apply_progress: AtomicU64, - /// Per-node uniqueness component for descriptor-preparation lease tokens. - pub metadata_ddl_token_seq: AtomicU64, /// Node-local table of in-flight `DdlPendingPropose` records. Persisted in /// `SystemCatalog` and seeded at boot. pub pending_ddl: crate::control::pending_ddl::PendingDdlTable, @@ -173,6 +162,8 @@ pub struct SharedState { /// linearizable read, replica freshness for a bounded-staleness one. /// Set by `start_raft`. pub raft_read_gate: OnceLock>, + /// This node's Raft groups, for a leadership move. Set by `start_raft`. + pub multi_raft: OnceLock>>, /// This node's cluster-epoch state. Set once by `start_raft`. pub cluster_epoch: OnceLock>, /// Query Raft group statuses for observability. Set by `start_raft`. @@ -294,6 +285,9 @@ pub struct SharedState { /// setup. Every committed redo that carries a cross-shard key records it /// here as it applies. pub cross_shard_dedup: OnceLock>, + /// Open chunked redo streams: the chunks every data-group apply holds + /// until each stream's final entry. + pub redo_chunks: Arc, /// Kafka bridge producer manager. pub kafka_manager: crate::event::kafka::KafkaManager, /// Definition sync fanout: broadcasts `DefinitionSync` (0x70) frames to diff --git a/nodedb/src/control/state/init.rs b/nodedb/src/control/state/init.rs index 540a0291f..f1fbf5f35 100644 --- a/nodedb/src/control/state/init.rs +++ b/nodedb/src/control/state/init.rs @@ -106,6 +106,14 @@ impl SharedState { // as the production constructor wires it. let stream_registry = Arc::new(crate::event::cdc::StreamRegistry::new()); + let redo_chunks = Arc::new( + crate::control::wal_replication::transaction_redo::RedoChunkStore::new( + Arc::clone(&wal), + crate::control::wal_replication::transaction_redo::chunks::RedoChunkLimits::from_tuning( + &TuningConfig::default().calvin, + ), + ), + ); let hlc_clock = wal.hlc_clock(); let state = Arc::new(Self { outcome_floor: dispatcher.outcome_floor(), @@ -130,11 +138,8 @@ impl SharedState { ) .0, group_watchers: Arc::new(nodedb_cluster::GroupAppliedWatchers::new()), - metadata_ddl_lock: tokio::sync::Mutex::new(()), - metadata_ddl_owner: std::sync::Mutex::new(None), - metadata_ddl_applied_token: std::sync::atomic::AtomicU64::new(0), + metadata_ddl: super::MetadataDdlState::new(), metadata_apply_progress: std::sync::atomic::AtomicU64::new(0), - metadata_ddl_token_seq: std::sync::atomic::AtomicU64::new(1), pending_ddl: crate::control::pending_ddl::PendingDdlTable::new(), metadata_apply_wedge: std::sync::Arc::default(), sequencer_halt: std::sync::Arc::default(), @@ -150,6 +155,7 @@ impl SharedState { raft_applied_index_sink: std::sync::OnceLock::new(), raft_apply_gates: std::sync::OnceLock::new(), raft_read_gate: std::sync::OnceLock::new(), + multi_raft: std::sync::OnceLock::new(), cluster_epoch: std::sync::OnceLock::new(), raft_status_fn: std::sync::OnceLock::new(), cluster_observer: std::sync::OnceLock::new(), @@ -293,6 +299,7 @@ impl SharedState { cross_shard_dlq: None, cross_shard_metrics: None, cross_shard_dedup: std::sync::OnceLock::new(), + redo_chunks, kafka_manager: crate::event::kafka::KafkaManager::new(shutdown.raw_receiver()), definition_sync_fanout: std::sync::Arc::new( crate::control::server::sync::definition_fanout::DefinitionSyncFanout::new(), diff --git a/nodedb/src/control/state/init_prod/open.rs b/nodedb/src/control/state/init_prod/open.rs index 25284d425..256000b9c 100644 --- a/nodedb/src/control/state/init_prod/open.rs +++ b/nodedb/src/control/state/init_prod/open.rs @@ -112,6 +112,14 @@ impl SharedState { session_handles, } = super::stores::session_controls(auth_config, &rate_limit_config); + let redo_chunks = Arc::new( + crate::control::wal_replication::transaction_redo::RedoChunkStore::new( + Arc::clone(&wal), + crate::control::wal_replication::transaction_redo::chunks::RedoChunkLimits::from_tuning( + &tuning.calvin, + ), + ), + ); let hlc_clock = wal.hlc_clock(); let state = Arc::new(Self { outcome_floor: dispatcher.outcome_floor(), @@ -182,6 +190,7 @@ impl SharedState { cross_shard_dlq: None, cross_shard_metrics: None, cross_shard_dedup: std::sync::OnceLock::new(), + redo_chunks, kafka_manager: crate::event::kafka::KafkaManager::new(shutdown.raw_receiver()), definition_sync_fanout: std::sync::Arc::new( crate::control::server::sync::definition_fanout::DefinitionSyncFanout::new(), @@ -200,11 +209,8 @@ impl SharedState { ) .0, group_watchers: Arc::new(nodedb_cluster::GroupAppliedWatchers::new()), - metadata_ddl_lock: tokio::sync::Mutex::new(()), - metadata_ddl_owner: std::sync::Mutex::new(None), - metadata_ddl_applied_token: std::sync::atomic::AtomicU64::new(0), + metadata_ddl: crate::control::state::MetadataDdlState::new(), metadata_apply_progress: std::sync::atomic::AtomicU64::new(0), - metadata_ddl_token_seq: std::sync::atomic::AtomicU64::new(1), pending_ddl: crate::control::pending_ddl::PendingDdlTable::new(), metadata_apply_wedge: std::sync::Arc::default(), sequencer_halt: std::sync::Arc::default(), @@ -220,6 +226,7 @@ impl SharedState { raft_applied_index_sink: std::sync::OnceLock::new(), raft_apply_gates: std::sync::OnceLock::new(), raft_read_gate: std::sync::OnceLock::new(), + multi_raft: std::sync::OnceLock::new(), cluster_epoch: std::sync::OnceLock::new(), raft_status_fn: std::sync::OnceLock::new(), cluster_observer: std::sync::OnceLock::new(), diff --git a/nodedb/src/control/state/metadata_ddl.rs b/nodedb/src/control/state/metadata_ddl.rs new file mode 100644 index 000000000..12da8ad24 --- /dev/null +++ b/nodedb/src/control/state/metadata_ddl.rs @@ -0,0 +1,43 @@ +// SPDX-License-Identifier: BUSL-1.1 + +//! This node's state of the replicated descriptor preparation lease. + +use std::sync::Mutex; +use std::sync::atomic::AtomicU64; + +use crate::control::metadata_proposer::DdlPrepareOwner; + +/// This node's state of the replicated descriptor preparation lease. +pub struct MetadataDdlState { + /// Serializes this node's attempts to acquire the replicated descriptor + /// preparation lease, and a local DDL through its post-apply. An async + /// proposer holds it across the post-apply, so it is a tokio mutex. + pub lock: tokio::sync::Mutex<()>, + /// Replicated preparation owner: its token, its node, and the local + /// monotonic apply time. The token and node are persisted in + /// `SystemCatalog` and seeded at boot. + pub owner: Mutex>, + /// Most recent fenced DDL token applied while its owner remained current. + /// Starts at 0 on boot. + pub applied_token: AtomicU64, + /// Per-node uniqueness component for descriptor-preparation lease tokens. + /// Starts at 1. + pub token_seq: AtomicU64, +} + +impl MetadataDdlState { + pub fn new() -> Self { + Self { + lock: tokio::sync::Mutex::new(()), + owner: Mutex::new(None), + applied_token: AtomicU64::new(0), + token_seq: AtomicU64::new(1), + } + } +} + +impl Default for MetadataDdlState { + fn default() -> Self { + Self::new() + } +} diff --git a/nodedb/src/control/state/methods.rs b/nodedb/src/control/state/methods.rs index ee95a44fe..fb48065f9 100644 --- a/nodedb/src/control/state/methods.rs +++ b/nodedb/src/control/state/methods.rs @@ -43,6 +43,19 @@ impl SharedState { .ok_or_else(|| raft_not_started("the raw async raft proposer")) } + /// The propose phase alone: it returns once the data-group leader holds + /// the entry in its log, with the wait for this node's apply of it. It + /// takes no vShard admission slot. Calvin bookkeeping entries, which no + /// write admission orders, propose through it. + pub(in crate::control) fn async_raft_submit( + &self, + ) -> crate::Result<&Arc> { + self.async_raft_proposer_pair + .get() + .map(|pair| &pair.submit) + .ok_or_else(|| raft_not_started("the async raft submit")) + } + /// Handle for proposing to the metadata Raft group. `start_raft` /// installs it before any listener opens. pub fn metadata_raft_handle( @@ -70,14 +83,19 @@ impl SharedState { }) } - /// Install both Raft proposal handles atomically during cluster startup. + /// Install the Raft proposal handles atomically during cluster startup. pub(in crate::control) fn install_async_raft_proposer_pair( &self, sequenced: Arc, raw: Arc, + submit: Arc, ) -> crate::Result<()> { self.async_raft_proposer_pair - .set(super::proposer_pair::AsyncRaftProposerPair { sequenced, raw }) + .set(super::proposer_pair::AsyncRaftProposerPair { + sequenced, + raw, + submit, + }) .map_err(|_| crate::Error::Internal { detail: "async raft proposer already installed".into(), }) diff --git a/nodedb/src/control/state/mod.rs b/nodedb/src/control/state/mod.rs index 595b78ddc..602132929 100644 --- a/nodedb/src/control/state/mod.rs +++ b/nodedb/src/control/state/mod.rs @@ -13,6 +13,7 @@ mod fields; mod init; mod init_prod; mod init_variants; +mod metadata_ddl; mod methods; mod methods_audit; mod methods_lease; @@ -38,5 +39,6 @@ pub use self::calvin_cuts::{CalvinCuts, cut_instant_hook}; pub use self::calvin_local::CalvinLocalState; pub use self::fields::SharedState; pub use self::init_prod::DataPlaneHandles; +pub use self::metadata_ddl::MetadataDdlState; pub use self::tenant_request::TenantRequestGuard; pub use self::tenant_write::{TenantWriteMark, TenantWriteMarks, TenantWriteOrigin}; diff --git a/nodedb/src/control/state/proposer_pair.rs b/nodedb/src/control/state/proposer_pair.rs index 5d3f21d75..0d19fcfbd 100644 --- a/nodedb/src/control/state/proposer_pair.rs +++ b/nodedb/src/control/state/proposer_pair.rs @@ -1,6 +1,6 @@ // SPDX-License-Identifier: BUSL-1.1 -//! The two Raft proposal handles `start_raft` installs together. +//! The Raft proposal handles `start_raft` installs together. use std::sync::Arc; @@ -8,4 +8,6 @@ use std::sync::Arc; pub(super) struct AsyncRaftProposerPair { pub(super) sequenced: Arc, pub(super) raw: Arc, + /// The propose phase both proposers are built from. + pub(super) submit: Arc, } diff --git a/nodedb/src/control/state/tenant_marks.rs b/nodedb/src/control/state/tenant_marks.rs index a83a95d6c..794208dd8 100644 --- a/nodedb/src/control/state/tenant_marks.rs +++ b/nodedb/src/control/state/tenant_marks.rs @@ -29,10 +29,9 @@ pub type GroupMarkEntry = (u64, u64, u8, String, u64); /// The apply path that recorded a mark. #[derive(Debug, Clone, Copy, PartialEq, Eq)] pub enum MarkSite { - /// A committed data-group entry. + /// A committed data-group entry, a committed Calvin slice's install + /// included. ReplicatedApply, - /// A committed Calvin transaction's install. - CalvinFlush, /// A write a RESTORE re-issued. Restore, } @@ -42,7 +41,6 @@ impl MarkSite { pub fn as_str(self) -> &'static str { match self { Self::ReplicatedApply => "replicated apply", - Self::CalvinFlush => "calvin flush", Self::Restore => "restore re-issue", } } @@ -51,7 +49,6 @@ impl MarkSite { pub fn code(self) -> u8 { match self { Self::ReplicatedApply => 0, - Self::CalvinFlush => 1, Self::Restore => RESTORE_SITE_CODE, } } @@ -59,7 +56,6 @@ impl MarkSite { /// The site a persisted or wire code names. pub fn from_code(code: u8) -> Self { match code { - 1 => Self::CalvinFlush, RESTORE_SITE_CODE => Self::Restore, _ => Self::ReplicatedApply, } @@ -293,7 +289,7 @@ mod tests { fn a_mark_only_rises() { let marks = TenantMarks::default(); marks.raise(1, 7, 100, MarkSite::ReplicatedApply, Some("docs")); - marks.raise(1, 7, 50, MarkSite::CalvinFlush, None); + marks.raise(1, 7, 50, MarkSite::ReplicatedApply, None); let mark = marks.get(1, 7).expect("mark"); assert_eq!(mark.hlc, 100); assert_eq!(mark.collection.as_deref(), Some("docs")); @@ -339,7 +335,7 @@ mod tests { let dir = tempfile::tempdir().expect("tempdir"); let catalog = SystemCatalog::open(&dir.path().join("system.redb")).expect("catalog"); let marks = TenantMarks::default(); - marks.raise(3, 1, 900, MarkSite::CalvinFlush, Some("orders")); + marks.raise(3, 1, 900, MarkSite::ReplicatedApply, Some("orders")); marks.raise_restore(4, 1, 950, Some("orders"), 5); marks.persist(&catalog).expect("persist"); @@ -348,7 +344,7 @@ mod tests { reloaded.get(3, 1), Some(GroupMark { hlc: 900, - site: MarkSite::CalvinFlush, + site: MarkSite::ReplicatedApply, collection: Some("orders".to_owned()), restore_id: 0, }) diff --git a/nodedb/src/control/state/test_core.rs b/nodedb/src/control/state/test_core.rs index 04137eb56..ff8007515 100644 --- a/nodedb/src/control/state/test_core.rs +++ b/nodedb/src/control/state/test_core.rs @@ -29,8 +29,8 @@ pub(crate) async fn acknowledge_every_request( payload: Payload::empty(), watermark_lsn: Lsn::ZERO, error_code: None, - read_set_valid: None, - read_version_lsn: Lsn::ZERO, + stage_vote: None, + read_versions: crate::types::ReadVersions::new(), write_set: Vec::new(), }, }) diff --git a/nodedb/src/control/system_txn/scope.rs b/nodedb/src/control/system_txn/scope.rs index 2e76e5bd4..3df345650 100644 --- a/nodedb/src/control/system_txn/scope.rs +++ b/nodedb/src/control/system_txn/scope.rs @@ -28,10 +28,6 @@ impl SystemTxnScope { sessions.ensure_session(addr); let session_id = SessionId::from(addr); - let snapshot_lsn = { - let next = state.wal.next_lsn(); - crate::types::Lsn::new(next.as_u64().saturating_sub(1)) - }; let snapshot_epoch = state .calvin .last_applied_epoch @@ -41,7 +37,7 @@ impl SystemTxnScope { // around the transaction, and COMMIT applies it with the writes. crate::control::server::shared::session::ddl_buffer::activate(); sessions - .begin(session_id, snapshot_lsn, snapshot_epoch) + .begin(session_id, snapshot_epoch) .map_err(|detail| crate::Error::BadRequest { detail: detail.to_owned(), })?; diff --git a/nodedb/src/control/system_txn/tests.rs b/nodedb/src/control/system_txn/tests.rs index a4f061f60..94631f368 100644 --- a/nodedb/src/control/system_txn/tests.rs +++ b/nodedb/src/control/system_txn/tests.rs @@ -111,8 +111,8 @@ async fn answer_all(state: Arc, mut side: CoreChannelDataSide, stop payload: Payload::empty(), watermark_lsn: Lsn::ZERO, error_code, - read_set_valid: None, - read_version_lsn: Lsn::ZERO, + stage_vote: None, + read_versions: crate::types::ReadVersions::new(), write_set: Vec::new(), }; side.response_tx @@ -214,6 +214,7 @@ async fn a_body_plan_carries_only_stageable_forms() { &mut tasks, TenantId::new(1), DatabaseId::DEFAULT, + None, crate::types::TraceId::ZERO, ) .await diff --git a/nodedb/src/control/update_from_join_orchestrator/expand_staged_update_from_join.rs b/nodedb/src/control/update_from_join_orchestrator/expand_staged_update_from_join.rs index a5f9d253d..b165f24c1 100644 --- a/nodedb/src/control/update_from_join_orchestrator/expand_staged_update_from_join.rs +++ b/nodedb/src/control/update_from_join_orchestrator/expand_staged_update_from_join.rs @@ -19,7 +19,9 @@ use nodedb_physical::physical_task::{PhysicalTask, PostSetOp}; /// Resolve one in-transaction `UpdateFromJoin` task into the concrete, /// surrogate-carrying `PointPut` tasks its matched target rows expand to. -/// `task.txn_id` must be the active transaction. Caller stages + buffers each op. +/// `task.txn_id` must be the active transaction. The ops carry no sum or +/// period-lock resolution: the caller runs the statement pass on them, then +/// stages + buffers each one. pub(crate) async fn resolve_and_emit_update_from_join_ops( state: &SharedState, tenant_id: TenantId, @@ -72,23 +74,6 @@ pub(crate) async fn resolve_and_emit_update_from_join_ops( )? .vshard(); - // A join-column rewrite debits the target left and credits the one joined — - // resolving post-images alone would leave the abandoned target overstated. - let sum_bodies: Vec<&[u8]> = resolved - .iter() - .flat_map(|(_, _, body, old_body)| [body.as_slice(), old_body.as_slice()]) - .collect(); - let resolved_sum_targets = - crate::control::planner::materialized_sum::resolve_sum_targets_for_bodies( - state, - &sum_bodies, - target_collection.as_str(), - tenant_id, - task.database_id, - crate::types::TraceId::ZERO, - ) - .await?; - let mut out: Vec = Vec::with_capacity(resolved.len()); for (_doc_id, surrogate_u32, body, _old_body) in resolved { // The wire surrogate is never absent: every matched `UPDATE ... FROM` @@ -109,7 +94,10 @@ pub(crate) async fn resolve_and_emit_update_from_join_ops( // The op owns the statement's projection; expanded puts answer no client. returning: None, rls_filters: Vec::new(), - resolved_sum_targets: resolved_sum_targets.clone(), + // Filled by the caller's sum and period-lock pass. It reads + // the stored row, so a join-column rewrite debits the target + // the row leaves as well as crediting the one it joins. + resolved_sum_targets: Vec::new(), }), post_set_op: PostSetOp::None, txn_id: task.txn_id, diff --git a/nodedb/src/control/vshard_admission.rs b/nodedb/src/control/vshard_admission.rs index 5c6923c03..6092b2fbd 100644 --- a/nodedb/src/control/vshard_admission.rs +++ b/nodedb/src/control/vshard_admission.rs @@ -134,8 +134,14 @@ impl VShardAdmissionSequencer { let queued = self.reserve(slot, vshard_id)?; let mut active = tokio::time::timeout_at(deadline, slot.active.lock()) .await - .map_err(|_| crate::Error::DeadlineExceeded { - request_id: crate::types::RequestId::new(0), + .map_err(|_| { + tracing::warn!( + vshard_id = vshard_id.as_u32(), + "a write waited for its vShard's admission slot until its deadline" + ); + crate::Error::DeadlineExceeded { + request_id: crate::types::RequestId::new(0), + } })?; let proposed = submit().await?; if proposed.at.is_some() { @@ -167,7 +173,7 @@ impl Default for VShardAdmissionSequencer { } /// Install raw and admission-sequenced proposal handles, both built from -/// `submit`, in one atomic set. +/// `submit`, and `submit` itself, in one atomic set. pub(crate) fn install_async_raft_proposer( shared: &SharedState, submit: Arc, @@ -176,9 +182,9 @@ pub(crate) fn install_async_raft_proposer( Arc::clone(&shared.vshard_admission_sequencer), Arc::clone(&submit), ); - let raw = raw_async_raft_proposer(submit); + let raw = raw_async_raft_proposer(Arc::clone(&submit)); let expected_raw = Arc::clone(&raw); - shared.install_async_raft_proposer_pair(sequenced, raw)?; + shared.install_async_raft_proposer_pair(sequenced, raw, submit)?; let installed_raw = shared.raw_async_raft_proposer()?; if !Arc::ptr_eq(installed_raw, &expected_raw) { return Err(crate::Error::Internal { @@ -305,7 +311,12 @@ mod tests { use tokio::sync::{Barrier, Notify}; use super::*; - use crate::types::Lsn; + use crate::types::ReadVersions; + + /// The versions a test proposer answers proposal `key` with. + fn applied_at(key: u64) -> ReadVersions { + ReadVersions::single(VShardId::new(0), nodedb_types::WriteVersion::logged(0, key)) + } fn test_deadline() -> tokio::time::Instant { tokio::time::Instant::now() + std::time::Duration::from_secs(30) @@ -536,7 +547,7 @@ mod tests { entered.notify_one(); release.notified().await; active.fetch_sub(1, Ordering::SeqCst); - Ok((data, Lsn::new(key))) + Ok((data, applied_at(key))) }) }) }; @@ -557,13 +568,13 @@ mod tests { release.notify_one(); assert_eq!( first.await.expect("first joins").expect("first success"), - (vec![1], Lsn::new(11)) + (vec![1], applied_at(11)) ); entered.notified().await; release.notify_one(); assert_eq!( second.await.expect("second joins").expect("second success"), - (vec![2], Lsn::new(12)) + (vec![2], applied_at(12)) ); assert_eq!(maximum.load(Ordering::SeqCst), 1); } @@ -577,7 +588,7 @@ mod tests { let proposed = Arc::clone(&proposed); Box::pin(async move { proposed.lock().expect("proposed log").push(key); - Ok((data, Lsn::new(key))) + Ok((data, applied_at(key))) }) }) }; @@ -658,11 +669,11 @@ mod tests { .await .expect("second joins") .expect("second proposes"), - (vec![22], Lsn::new(22)) + (vec![22], applied_at(22)) ); assert_eq!( third.await.expect("third joins").expect("third proposes"), - (vec![23], Lsn::new(23)) + (vec![23], applied_at(23)) ); assert_eq!( *proposed.lock().expect("proposed log"), @@ -696,7 +707,7 @@ mod tests { detail: format!("test apply gate closed: {e}"), })? .forget(); - Ok((data, Lsn::new(key))) + Ok((data, applied_at(key))) }), }) }) @@ -744,10 +755,10 @@ mod tests { first.await.expect("first joins").expect("first applies"), second.await.expect("second joins").expect("second applies"), ]; - results.sort_by_key(|(_, lsn)| lsn.as_u64()); + results.sort_by_key(|(_, versions)| versions.of(VShardId::new(0))); assert_eq!( results, - vec![(vec![1], Lsn::new(11)), (vec![2], Lsn::new(12))] + vec![(vec![1], applied_at(11)), (vec![2], applied_at(12))] ); assert_eq!(sequencer.slots[5].capacity.available_permits(), 4); } diff --git a/nodedb/src/control/wal_replication/decode/entry.rs b/nodedb/src/control/wal_replication/decode/entry.rs index 5878670ed..7bfdbb723 100644 --- a/nodedb/src/control/wal_replication/decode/entry.rs +++ b/nodedb/src/control/wal_replication/decode/entry.rs @@ -55,6 +55,15 @@ fn decode_with_database(data: &[u8]) -> crate::Result e, None => return Ok(None), }; + decode_parsed_entry(&entry) +} + +/// Decodes an entry already parsed from its bytes into its database and its +/// plan, without binding any identity. `None` for an array CRDT op or array +/// schema, which the distributed applier handles before any plan exists. +pub fn decode_parsed_entry( + entry: &ReplicatedEntry, +) -> crate::Result> { // Array CRDT variants are handled by the distributed applier before this call. match &entry.write { ReplicatedWrite::ArrayOp { .. } | ReplicatedWrite::ArraySchema { .. } => { @@ -206,6 +215,10 @@ fn to_physical_plan( detail: "CalvinReadResult reached to_physical_plan (should have been intercepted)" .into(), }), + ReplicatedWrite::CalvinReadTimeout { .. } => Err(crate::Error::Internal { + detail: "CalvinReadTimeout reached to_physical_plan (should have been intercepted)" + .into(), + }), // The apply loop applies these through `transaction_redo`, which stamps // the redo with the entry's Raft coordinates. ReplicatedWrite::TransactionRedo { .. } => Err(crate::Error::Internal { @@ -220,5 +233,12 @@ fn to_physical_plan( ReplicatedWrite::SurrogateBind { .. } => Err(crate::Error::Internal { detail: "SurrogateBind reached to_physical_plan (should have been intercepted)".into(), }), + // The apply loop holds a redo stream's chunks itself; they reach no core. + ReplicatedWrite::RedoChunk { .. } => Err(crate::Error::Internal { + detail: "RedoChunk reached to_physical_plan (should have been intercepted)".into(), + }), + ReplicatedWrite::RedoAbandon { .. } => Err(crate::Error::Internal { + detail: "RedoAbandon reached to_physical_plan (should have been intercepted)".into(), + }), } } diff --git a/nodedb/src/control/wal_replication/decode/mod.rs b/nodedb/src/control/wal_replication/decode/mod.rs index bcbd9f2ec..7dfdce432 100644 --- a/nodedb/src/control/wal_replication/decode/mod.rs +++ b/nodedb/src/control/wal_replication/decode/mod.rs @@ -38,6 +38,6 @@ mod transaction_redo; mod vector; mod vector_direct; -pub use entry::{decode_replicated_entry, from_replicated_entry}; +pub use entry::{decode_parsed_entry, decode_replicated_entry, from_replicated_entry}; pub use entry_graph::edge_write_plan; -pub use transaction_redo::transaction_redo_payload; +pub use transaction_redo::{ChunkedRedo, DecodedTransactionRedo, decode_transaction_redo}; diff --git a/nodedb/src/control/wal_replication/decode/transaction_redo.rs b/nodedb/src/control/wal_replication/decode/transaction_redo.rs index 323379836..5fb824508 100644 --- a/nodedb/src/control/wal_replication/decode/transaction_redo.rs +++ b/nodedb/src/control/wal_replication/decode/transaction_redo.rs @@ -1,27 +1,103 @@ // SPDX-License-Identifier: BUSL-1.1 //! Decode a committed `ReplicatedWrite::TransactionRedo` entry back into the -//! payload every replica applies. +//! payload every replica applies, or into the stream a chunked body names. //! //! The apply loop intercepts these entries before the generic decode: the //! apply stamps the redo with the entry's Raft coordinates, which the generic //! `from_replicated_entry` result has no place for. +use nodedb_physical::physical_plan::RedoOrigin; use nodedb_types::Surrogate; use super::super::transaction_redo::TransactionRedoPayload; -use super::super::types::ReplicatedWrite; +use super::super::types::{ + CalvinRedoMeta, RedoBody, RedoContent, ReplicatedEventSource, ReplicatedWrite, +}; use crate::control::surrogate::CarriedIdentity; +use crate::wal::RedoStreamId; -/// The payload `write` carries, or an error when `write` is another variant. -pub fn transaction_redo_payload(write: &ReplicatedWrite) -> crate::Result { +/// A decoded `TransactionRedo` entry. +#[derive(Debug, Clone)] +pub enum DecodedTransactionRedo { + /// The entry carries the whole payload. + Inline(Box), + /// The entry names the stream whose chunks carry the content. + Chunked(ChunkedRedo), +} + +/// The final entry of a chunked redo: its stream and the fields the entry +/// itself carries. +#[derive(Debug, Clone)] +pub struct ChunkedRedo { + pub stream: RedoStreamId, + /// How many chunks the stream holds. + pub count: u32, + /// How many bytes the chunks hold in all. + pub len: u64, + collections: Vec, + event_source: ReplicatedEventSource, + origin: RedoOrigin, + calvin: Option, +} + +impl ChunkedRedo { + /// The payload of the stream's assembled `content`. + pub fn payload(&self, content: RedoContent) -> TransactionRedoPayload { + payload_of( + content, + &self.collections, + self.event_source, + self.origin, + self.calvin.clone(), + ) + } + + /// Every collection the transaction wrote. + pub fn collections(&self) -> &[String] { + &self.collections + } + + /// The Calvin meta of a committed Calvin slice's redo. + pub fn calvin(&self) -> Option<&CalvinRedoMeta> { + self.calvin.as_ref() + } +} + +fn payload_of( + content: RedoContent, + collections: &[String], + event_source: ReplicatedEventSource, + origin: RedoOrigin, + calvin: Option, +) -> TransactionRedoPayload { + TransactionRedoPayload { + redo: content.redo, + collections: collections.to_vec(), + sum_targets: content.sum_targets, + identities: content + .identities + .into_iter() + .map(|identity| CarriedIdentity { + collection: identity.collection, + pk_bytes: identity.pk_bytes, + surrogate: Surrogate::new(identity.surrogate), + }) + .collect(), + event_source: event_source.into(), + origin, + calvin, + } +} + +/// What `write` carries, or an error when `write` is another variant. +pub fn decode_transaction_redo(write: &ReplicatedWrite) -> crate::Result { let ReplicatedWrite::TransactionRedo { - redo, + body, collections, - sum_targets, - identities, event_source, origin, + calvin, } = write else { return Err(crate::Error::Internal { @@ -29,20 +105,23 @@ pub fn transaction_redo_payload(write: &ReplicatedWrite) -> crate::Result DecodedTransactionRedo::Inline(Box::new(payload_of( + (**content).clone(), + collections, + *event_source, + *origin, + calvin.clone(), + ))), + RedoBody::Chunked { stream, count, len } => DecodedTransactionRedo::Chunked(ChunkedRedo { + stream: *stream, + count: *count, + len: *len, + collections: collections.clone(), + event_source: *event_source, + origin: *origin, + calvin: calvin.clone(), + }), }) } @@ -50,11 +129,13 @@ pub fn transaction_redo_payload(write: &ReplicatedWrite) -> crate::Result TransactionRedoPayload { TransactionRedoPayload { @@ -68,8 +149,6 @@ mod tests { epoch: 11, position: 2, vshard_id: 7, - collections: Vec::new(), - sum_targets: Vec::new(), }), cross_shard_applied: None, row_sources: Vec::new(), @@ -88,7 +167,6 @@ mod tests { sum_targets: vec![RedoSumTargets { collection: "entries".into(), resolved: vec![ResolvedSumTarget::new("accounts", "a1", Surrogate::new(9))], - deferred: vec!["audit".into()], }], identities: vec![CarriedIdentity { collection: "entries".into(), @@ -97,6 +175,16 @@ mod tests { }], event_source: EventSource::Trigger, origin: RedoOrigin::Restore, + calvin: None, + } + } + + fn inline(write: &ReplicatedWrite) -> TransactionRedoPayload { + match decode_transaction_redo(write).expect("payload decodes") { + DecodedTransactionRedo::Inline(payload) => *payload, + DecodedTransactionRedo::Chunked(chunked) => { + panic!("expected an inline body, got stream {:?}", chunked.stream) + } } } @@ -116,7 +204,7 @@ mod tests { assert_eq!(decoded_entry.vshard_id, 7); assert_eq!(decoded_entry.idempotency_key, entry.idempotency_key); - let decoded = transaction_redo_payload(&decoded_entry.write).expect("payload decodes"); + let decoded = inline(&decoded_entry.write); assert_eq!(decoded.redo, original.redo); assert_eq!(decoded.collections, original.collections); assert_eq!(decoded.sum_targets, original.sum_targets); @@ -132,6 +220,77 @@ mod tests { assert_eq!(named, vec!["accounts", "entries"]); } + /// A committed Calvin slice's meta travels unchanged through the entry, + /// so every replica renders the same reply. + #[test] + fn a_calvin_slice_meta_round_trips_through_the_raft_bytes() { + let mut original = payload(); + let meta = CalvinRedoMeta { + epoch_system_ms: 1_700_000_000_000, + reply: nodedb_physical::physical_plan::CalvinReplySpec::Count(vec![1, 2]), + primary_write: true, + user_write: true, + returning: true, + }; + original.calvin = Some(meta.clone()); + let entry = transaction_redo_entry( + TenantId::new(1), + DatabaseId::new(5), + VShardId::new(7), + &original, + ); + let decoded_entry = ReplicatedEntry::from_bytes(&entry.to_bytes()).expect("entry decodes"); + + let decoded = inline(&decoded_entry.write); + assert_eq!(decoded.calvin, Some(meta)); + assert_eq!(decoded.redo.calvin_stamp, original.redo.calvin_stamp); + } + + #[test] + fn a_chunked_final_entry_round_trips_and_rebuilds_the_payload() { + let mut original = payload(); + original.redo.ops[0].payload = vec![9; 200_000]; + let RedoProposal::Chunked { + stream, + chunks, + last, + } = session_redo_proposal( + RedoEntryTarget { + tenant_id: TenantId::new(1), + database_id: DatabaseId::new(5), + vshard_id: VShardId::new(7), + }, + &original, + 64 * 1024, + ) + .expect("build") + else { + panic!("a 200 KB redo travels chunked"); + }; + let decoded_last = ReplicatedEntry::from_bytes(&last.to_bytes()).expect("entry decodes"); + let DecodedTransactionRedo::Chunked(chunked) = + decode_transaction_redo(&decoded_last.write).expect("decodes") + else { + panic!("the final entry names its stream"); + }; + assert_eq!(chunked.stream, stream); + assert_eq!(chunked.count as usize, chunks.len()); + assert_eq!(chunked.collections(), original.collections.as_slice()); + let joined: Vec = chunks + .iter() + .flat_map(|chunk| match &chunk.write { + ReplicatedWrite::RedoChunk { bytes, .. } => bytes.clone(), + other => panic!("expected a chunk, got {other:?}"), + }) + .collect(); + let rebuilt = chunked.payload(RedoContent::from_bytes(&joined).expect("content")); + assert_eq!(rebuilt.redo, original.redo); + assert_eq!(rebuilt.sum_targets, original.sum_targets); + assert_eq!(rebuilt.identities, original.identities); + assert_eq!(rebuilt.event_source, EventSource::Trigger); + assert_eq!(rebuilt.origin, RedoOrigin::Restore); + } + #[test] fn apply_plan_carries_the_committed_redo_bytes_unchanged() { let original = payload(); @@ -153,6 +312,6 @@ mod tests { collection: "kv".into(), restart_identity: false, }; - assert!(transaction_redo_payload(&write).is_err()); + assert!(decode_transaction_redo(&write).is_err()); } } diff --git a/nodedb/src/control/wal_replication/encode/mod.rs b/nodedb/src/control/wal_replication/encode/mod.rs index 5ac14a582..dcad00f0b 100644 --- a/nodedb/src/control/wal_replication/encode/mod.rs +++ b/nodedb/src/control/wal_replication/encode/mod.rs @@ -24,4 +24,8 @@ mod transaction_redo; mod vector; pub use entry::to_replicated_entry; -pub use transaction_redo::transaction_redo_entry; +pub use transaction_redo::{ + CHUNK_ENTRY_ENVELOPE_BYTES, CalvinEntryStamps, RedoEntryTarget, RedoProposal, + calvin_redo_proposal, chunk_entries, redo_abandon_entry, session_redo_proposal, + transaction_redo_entry, +}; diff --git a/nodedb/src/control/wal_replication/encode/transaction_redo.rs b/nodedb/src/control/wal_replication/encode/transaction_redo.rs index 44573c789..b91255b5d 100644 --- a/nodedb/src/control/wal_replication/encode/transaction_redo.rs +++ b/nodedb/src/control/wal_replication/encode/transaction_redo.rs @@ -1,45 +1,451 @@ // SPDX-License-Identifier: BUSL-1.1 -//! Encode a committed transaction's redo as a `ReplicatedWrite::TransactionRedo` -//! entry for its vShard's data-group log. +//! Encode a committed transaction's redo as `ReplicatedWrite::TransactionRedo` +//! entries for its vShard's data-group log. //! //! A redo is not built from a `PhysicalPlan`, so it has no arm in -//! `to_replicated_entry`: the commit that resolved it encodes it here. +//! `to_replicated_entry`: the commit that resolved it encodes it here. A redo +//! whose entry passes the redo entry limit travels as `RedoChunk` entries of +//! one stream, then a final entry whose body names the stream. use super::super::transaction_redo::TransactionRedoPayload; use super::super::types::{ - ReplicatedEntry, ReplicatedEventSource, ReplicatedIdentity, ReplicatedWrite, + CollectionIncarnation, RedoBody, RedoContent, ReplicatedEntry, ReplicatedEventSource, + ReplicatedIdentity, ReplicatedWrite, }; use crate::types::{DatabaseId, TenantId, VShardId}; +use crate::wal::RedoStreamId; -/// The Raft entry that carries `payload` to every replica of `vshard_id`. -pub fn transaction_redo_entry( - tenant_id: TenantId, - database_id: DatabaseId, - vshard_id: VShardId, +/// Room a chunk entry keeps for its envelope beside the chunk bytes: the +/// routing fields, the stream header, and the stamps the proposer adds. +pub const CHUNK_ENTRY_ENVELOPE_BYTES: usize = 4 * 1024; + +/// Room an inline entry keeps for its fixed fields and the stamps the +/// proposer adds after the size check: the commit HLC and the metadata floor. +const ENTRY_STAMP_BYTES: usize = 1024; + +/// Room each named collection's incarnation and wire framing take, beside +/// its name. +const COLLECTION_ENTRY_BYTES: usize = 64; + +/// Where a committed redo's entries go. +#[derive(Debug, Clone, Copy)] +pub struct RedoEntryTarget { + pub tenant_id: TenantId, + pub database_id: DatabaseId, + pub vshard_id: VShardId, +} + +/// How one committed redo travels through its vShard's data-group log. +pub enum RedoProposal { + /// One entry carries the whole redo. + Inline(ReplicatedEntry), + /// `chunks` carry the redo in order, then `last` names their stream. + Chunked { + stream: RedoStreamId, + chunks: Vec, + last: ReplicatedEntry, + }, +} + +/// The content of `payload`: the part a chunked redo splits. +fn content_of(payload: &TransactionRedoPayload) -> RedoContent { + RedoContent { + redo: payload.redo.clone(), + sum_targets: payload.sum_targets.clone(), + identities: payload + .identities + .iter() + .map(|identity| ReplicatedIdentity { + collection: identity.collection.clone(), + pk_bytes: identity.pk_bytes.clone(), + surrogate: identity.surrogate.as_u32(), + }) + .collect(), + } +} + +/// The `TransactionRedo` entry of `payload` with `body`. +fn redo_entry( + target: RedoEntryTarget, payload: &TransactionRedoPayload, + body: RedoBody, ) -> ReplicatedEntry { ReplicatedEntry::new( - tenant_id.as_u64(), - database_id.as_u64(), - vshard_id.as_u32(), + target.tenant_id.as_u64(), + target.database_id.as_u64(), + target.vshard_id.as_u32(), ReplicatedWrite::TransactionRedo { - redo: payload.redo.clone(), + body, collections: payload.collections.clone(), - sum_targets: payload.sum_targets.clone(), - identities: payload - .identities - .iter() - .map(|identity| ReplicatedIdentity { - collection: identity.collection.clone(), - pk_bytes: identity.pk_bytes.clone(), - surrogate: identity.surrogate.as_u32(), - }) - .collect(), event_source: ReplicatedEventSource::from(payload.event_source), origin: payload.origin, + calvin: payload.calvin.clone(), }, ) .with_event_source(payload.event_source) .naming(payload.collections.iter().map(String::as_str)) } + +/// The one Raft entry that carries `payload` to every replica of `vshard_id`. +pub fn transaction_redo_entry( + tenant_id: TenantId, + database_id: DatabaseId, + vshard_id: VShardId, + payload: &TransactionRedoPayload, +) -> ReplicatedEntry { + redo_entry( + RedoEntryTarget { + tenant_id, + database_id, + vshard_id, + }, + payload, + RedoBody::Inline(Box::new(content_of(payload))), + ) +} + +/// The entries of a session COMMIT's redo. The redo travels inline when its +/// entry fits `max_entry_bytes`, else chunked under a stream the final +/// entry's idempotency key names. +pub fn session_redo_proposal( + target: RedoEntryTarget, + payload: &TransactionRedoPayload, + max_entry_bytes: usize, +) -> crate::Result { + redo_proposal(target, payload, max_entry_bytes, |last| { + RedoStreamId::Session { + vshard: target.vshard_id.as_u32(), + idempotency_key: last.idempotency_key, + } + }) +} + +/// What the leader stamps on a committed Calvin slice's final entry. Every +/// re-proposal of the slice carries the same stamps. +#[derive(Debug, Clone)] +pub struct CalvinEntryStamps { + /// The slice's commit HLC: every replica records it on the tenant's + /// write mark. + pub write_hlc: u64, + /// The RESTORE that re-issued the slice, `0` for any other. + pub restore_id: u64, + /// The metadata index the coordinator planned the transaction against. + pub metadata_floor: u64, + /// The incarnation the coordinator planned against for each collection + /// the slice writes. + pub incarnations: Vec, +} + +/// The entries of a committed Calvin slice's redo, for one proposal +/// attempt. The redo travels inline when its entry fits `max_entry_bytes`, +/// else chunked under `stream`. +pub fn calvin_redo_proposal( + target: RedoEntryTarget, + payload: &TransactionRedoPayload, + stamps: &CalvinEntryStamps, + stream: RedoStreamId, + max_entry_bytes: usize, +) -> crate::Result { + let proposal = redo_proposal(target, payload, max_entry_bytes, |_| stream)?; + let stamp = |mut last: ReplicatedEntry| { + last.write_hlc = stamps.write_hlc; + last.restore_id = stamps.restore_id; + last.metadata_floor = stamps.metadata_floor; + last.incarnations = stamps.incarnations.clone(); + last + }; + Ok(match proposal { + RedoProposal::Inline(last) => RedoProposal::Inline(stamp(last)), + RedoProposal::Chunked { + stream, + chunks, + last, + } => RedoProposal::Chunked { + stream, + chunks, + last: stamp(last), + }, + }) +} + +/// The entries of `payload`: inline when its entry fits `max_entry_bytes`, +/// else chunks under the stream `stream_of` names for the final entry. +fn redo_proposal( + target: RedoEntryTarget, + payload: &TransactionRedoPayload, + max_entry_bytes: usize, + stream_of: impl FnOnce(&ReplicatedEntry) -> RedoStreamId, +) -> crate::Result { + let content = content_of(payload); + let bytes = content.to_bytes()?; + // The entry around the content: the collections it names twice, once in + // the write and once with their incarnations, and the fixed fields. + let envelope = payload + .collections + .iter() + .map(|collection| 2 * collection.len() + COLLECTION_ENTRY_BYTES) + .sum::() + .saturating_add(ENTRY_STAMP_BYTES); + let mut last = redo_entry(target, payload, RedoBody::Inline(Box::new(content))); + if bytes.len().saturating_add(envelope) <= max_entry_bytes { + return Ok(RedoProposal::Inline(last)); + } + let stream = stream_of(&last); + let chunks = chunk_entries( + target, + stream, + &bytes, + max_entry_bytes, + payload.event_source, + )?; + let count = u32::try_from(chunks.len()).map_err(|_| crate::Error::Internal { + detail: format!( + "a redo of {} bytes splits into too many chunks", + bytes.len() + ), + })?; + last.write = ReplicatedWrite::TransactionRedo { + body: RedoBody::Chunked { + stream, + count, + len: bytes.len() as u64, + }, + collections: payload.collections.clone(), + event_source: ReplicatedEventSource::from(payload.event_source), + origin: payload.origin, + calvin: payload.calvin.clone(), + }; + Ok(RedoProposal::Chunked { + stream, + chunks, + last, + }) +} + +/// The `RedoChunk` entries that carry `bytes` under `stream`, each encoded +/// within `max_entry_bytes`. +pub fn chunk_entries( + target: RedoEntryTarget, + stream: RedoStreamId, + bytes: &[u8], + max_entry_bytes: usize, + event_source: crate::event::EventSource, +) -> crate::Result> { + let room = max_entry_bytes + .checked_sub(CHUNK_ENTRY_ENVELOPE_BYTES) + .filter(|room| *room > 0) + .ok_or_else(|| crate::Error::Config { + detail: format!( + "tuning.calvin.max_redo_entry_bytes {max_entry_bytes} leaves no room for a \ + redo chunk beside its {CHUNK_ENTRY_ENVELOPE_BYTES}-byte envelope" + ), + })?; + let len = bytes.len() as u64; + Ok((0u32..) + .zip(bytes.chunks(room)) + .map(|(index, part)| { + ReplicatedEntry::new( + target.tenant_id.as_u64(), + target.database_id.as_u64(), + target.vshard_id.as_u32(), + ReplicatedWrite::RedoChunk { + stream, + index, + len, + bytes: part.to_vec(), + }, + ) + .with_event_source(event_source) + }) + .collect()) +} + +/// The entry that drops `stream` on every replica. +pub fn redo_abandon_entry(target: RedoEntryTarget, stream: RedoStreamId) -> ReplicatedEntry { + ReplicatedEntry::new( + target.tenant_id.as_u64(), + target.database_id.as_u64(), + target.vshard_id.as_u32(), + ReplicatedWrite::RedoAbandon { stream }, + ) +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::event::EventSource; + use crate::wal::{RedoRecord, RedoSubRecord}; + use nodedb_physical::physical_plan::RedoOrigin; + + fn payload(op_bytes: usize) -> TransactionRedoPayload { + TransactionRedoPayload { + redo: RedoRecord { + version: 1, + ops: vec![RedoSubRecord { + record_type: nodedb_wal::record::RecordType::Put as u32, + payload: vec![7; op_bytes], + }], + calvin_stamp: None, + cross_shard_applied: None, + row_sources: Vec::new(), + publishes: Vec::new(), + row_changes: Vec::new(), + }, + collections: vec!["orders".into()], + sum_targets: Vec::new(), + identities: Vec::new(), + event_source: EventSource::User, + origin: RedoOrigin::Commit, + calvin: None, + } + } + + fn target() -> RedoEntryTarget { + RedoEntryTarget { + tenant_id: TenantId::new(1), + database_id: DatabaseId::DEFAULT, + vshard_id: VShardId::new(3), + } + } + + /// The payload of a committed Calvin slice. + fn calvin_payload(op_bytes: usize) -> TransactionRedoPayload { + let mut payload = payload(op_bytes); + payload.calvin = Some(super::super::super::types::CalvinRedoMeta { + epoch_system_ms: 7, + reply: nodedb_physical::physical_plan::CalvinReplySpec::Count(Vec::new()), + primary_write: true, + user_write: true, + returning: false, + }); + payload + } + + fn calvin_stamps() -> CalvinEntryStamps { + CalvinEntryStamps { + write_hlc: 41, + restore_id: 0, + metadata_floor: 9, + incarnations: vec![CollectionIncarnation { + collection: "orders".into(), + incarnation: nodedb_types::Hlc::ZERO, + }], + } + } + + fn calvin_stream() -> RedoStreamId { + RedoStreamId::Calvin { + vshard: 3, + epoch: 5, + position: 1, + attempt: 2, + } + } + + /// The final entry of a Calvin slice's redo carries the leader's stamps + /// and the slice's meta, inline or chunked. Chunks carry no stamp. + #[test] + fn a_calvin_proposal_stamps_its_final_entry() { + let inline = calvin_redo_proposal( + target(), + &calvin_payload(100), + &calvin_stamps(), + calvin_stream(), + 64 * 1024, + ) + .expect("build"); + let RedoProposal::Inline(last) = inline else { + panic!("a small redo travels inline"); + }; + assert_eq!(last.write_hlc, 41); + assert_eq!(last.metadata_floor, 9); + assert_eq!(last.incarnations, calvin_stamps().incarnations); + assert!(matches!( + last.write, + ReplicatedWrite::TransactionRedo { + calvin: Some(_), + .. + } + )); + + let chunked = calvin_redo_proposal( + target(), + &calvin_payload(300_000), + &calvin_stamps(), + calvin_stream(), + 64 * 1024, + ) + .expect("build"); + let RedoProposal::Chunked { + stream, + chunks, + last, + } = chunked + else { + panic!("a 300 KB redo travels chunked"); + }; + assert_eq!(stream, calvin_stream(), "the attempt names its stream"); + assert_eq!(last.write_hlc, 41); + assert!(chunks.iter().all(|chunk| chunk.write_hlc == 0)); + assert!(matches!( + last.write, + ReplicatedWrite::TransactionRedo { + body: RedoBody::Chunked { stream: named, .. }, + calvin: Some(_), + .. + } if named == calvin_stream() + )); + } + + #[test] + fn a_redo_within_the_limit_travels_inline() { + let proposal = session_redo_proposal(target(), &payload(100), 64 * 1024).expect("build"); + assert!(matches!(proposal, RedoProposal::Inline(_))); + } + + #[test] + fn a_redo_past_the_limit_travels_in_chunks_that_fit_the_limit() { + let limit = 64 * 1024; + let proposal = session_redo_proposal(target(), &payload(300_000), limit).expect("build"); + let RedoProposal::Chunked { + stream, + chunks, + last, + } = proposal + else { + panic!("a 300 KB redo must travel chunked"); + }; + assert_eq!( + stream, + RedoStreamId::Session { + vshard: 3, + idempotency_key: last.idempotency_key + } + ); + assert!(chunks.len() >= 5); + for chunk in &chunks { + assert!(chunk.encode().expect("encode").len() + ENTRY_STAMP_BYTES <= limit); + } + let ReplicatedWrite::TransactionRedo { + body: RedoBody::Chunked { count, len, .. }, + .. + } = last.write + else { + panic!("the final entry names its stream"); + }; + assert_eq!(count as usize, chunks.len()); + let joined: Vec = chunks + .iter() + .flat_map(|chunk| match &chunk.write { + ReplicatedWrite::RedoChunk { bytes, .. } => bytes.clone(), + other => panic!("expected a chunk, got {other:?}"), + }) + .collect(); + assert_eq!(joined.len() as u64, len); + assert_eq!( + RedoContent::from_bytes(&joined).expect("content").redo, + payload(300_000).redo + ); + } +} diff --git a/nodedb/src/control/wal_replication/mod.rs b/nodedb/src/control/wal_replication/mod.rs index b232b0a52..bf0e1cd61 100644 --- a/nodedb/src/control/wal_replication/mod.rs +++ b/nodedb/src/control/wal_replication/mod.rs @@ -19,12 +19,14 @@ pub use decode::{decode_replicated_entry, from_replicated_entry}; pub use encode::to_replicated_entry; pub(crate) use propose::{ propose_replicated_entry, stamp_collection_incarnations, stamp_metadata_floor, + statement_propose_deadline, }; pub use replicable_write::ReplicableWrite; pub use types::{ - AppliedWait, AsyncRaftProposer, AsyncRaftSubmit, CollectionIncarnation, ConstraintChangeOp, - ProposedAt, ProposedWrite, RaftAppliedIndexSink, RaftCompactor, RaftProposer, ReplicatedEntry, - ReplicatedEventSource, ReplicatedIdentity, ReplicatedSumTarget, ReplicatedWrite, + AppliedOutput, AppliedWait, AsyncRaftProposer, AsyncRaftSubmit, CollectionIncarnation, + ConstraintChangeOp, ProposedAt, ProposedWrite, RaftAppliedIndexSink, RaftCompactor, + RaftProposer, ReplicatedEntry, ReplicatedEventSource, ReplicatedIdentity, ReplicatedSumTarget, + ReplicatedWrite, }; pub use crate::control::distributed_applier::{ diff --git a/nodedb/src/control/wal_replication/propose.rs b/nodedb/src/control/wal_replication/propose.rs index 1a57507d9..6f85d6503 100644 --- a/nodedb/src/control/wal_replication/propose.rs +++ b/nodedb/src/control/wal_replication/propose.rs @@ -19,6 +19,14 @@ const FIRST_BACKOFF: std::time::Duration = std::time::Duration::from_millis(10); /// Longest wait between two re-proposals. const MAX_BACKOFF: std::time::Duration = std::time::Duration::from_millis(200); +/// The deadline a proposal made now carries: the running statement's +/// deadline, or the node default outside a statement. +pub(crate) fn statement_propose_deadline(state: &SharedState) -> tokio::time::Instant { + tokio::time::Instant::from_std(crate::control::server::shared::session::statement_deadline( + state.tuning.network.default_deadline_secs, + )) +} + /// Stamp this node's metadata floor on `entry`: the catalog the write was /// planned against, including the batch being applied now (see /// `AppliedIndexWatcher::floor`). Every replica applies the entry only once @@ -46,20 +54,21 @@ pub(crate) fn stamp_collection_incarnations( Ok(()) } -/// Propose `entry` via `proposer` and return the Data Plane apply payload bytes -/// together with the write's per-collection version (as an -/// [`crate::types::Lsn`]): the written collection's `coll_write_lsn` after the -/// write, stamped by the applying replica from the WAL LSN it minted for the -/// entry's redo record. `Lsn::ZERO` when the write's plan names no single user -/// collection. See [`AsyncRaftProposer`] for why this is a WAL LSN and never the -/// Raft log index. +/// Propose `entry` via `proposer` and return the Data Plane apply payload +/// bytes and the versions the write stamped. Each version is the data-group +/// log position of the entry that applied the write, so it is the same on +/// every replica. A write whose plan names no single user collection stamps +/// none. /// /// An edge write is not proposed. It runs as a Calvin transaction /// (`planner::calvin::edge_sequencing`), and its applied payload and read /// version come back the same way. /// -/// Re-proposes the same payload until the statement deadline while the group -/// has no leader to take it: +/// `deadline` is the caller's deadline. Every attempt, each attempt's wait at +/// the leader's write gate, and each wait for the local apply end at it. +/// +/// Re-proposes the same payload until `deadline` while the group has no +/// leader to take it: /// - [`crate::Error::RetryableLeaderChange`]: a new leader's election no-op /// overwrote the previous leader's entry; /// - [`crate::Error::NoLeader`]: the group is electing, or a leadership @@ -73,13 +82,14 @@ pub(crate) async fn propose_replicated_entry( state: &SharedState, proposer: &Arc, mut entry: ReplicatedEntry, -) -> crate::Result<(Vec, crate::types::Lsn)> { + deadline: tokio::time::Instant, +) -> crate::Result { // An edge write runs as a Calvin transaction, never as a data-group // entry, so every edge version of a collection takes a Calvin ordinal. if let Some(response) = crate::control::planner::calvin::sequence_replicated_edge_write(state, &entry).await? { - return Ok((response.payload.to_vec(), response.read_version_lsn)); + return Ok((response.payload.to_vec(), response.read_versions)); } // The write's commit instant. Stamped once, before the first propose, so // every re-proposal and every replica's apply carries the same value. @@ -91,10 +101,6 @@ pub(crate) async fn propose_replicated_entry( let data = entry.encode()?; let vshard_id = entry.vshard_id; - // The statement deadline. Every attempt, and each attempt's wait for the - // local apply, ends at this one instant. - let deadline = tokio::time::Instant::now() - + std::time::Duration::from_secs(state.tuning.network.default_deadline_secs); let mut backoff = FIRST_BACKOFF; let mut attempt: u32 = 0; let payload = loop { diff --git a/nodedb/src/control/wal_replication/transaction_redo/abandon.rs b/nodedb/src/control/wal_replication/transaction_redo/abandon.rs new file mode 100644 index 000000000..c0d0dc723 --- /dev/null +++ b/nodedb/src/control/wal_replication/transaction_redo/abandon.rs @@ -0,0 +1,145 @@ +// SPDX-License-Identifier: BUSL-1.1 + +//! Retry the abandon of a session redo stream until the stream closes. +//! +//! A session proposes the abandon of its stream once. When that proposal +//! fails, the stream joins the store's abandon queue. One task per node +//! retries each queued abandon while the stream is open on this node. +//! +//! The stream closes when its abandon or final entry applies, when its group +//! applies an entry of a later term, or when this node leaves the group. So +//! the proposer's term bounds the retries, not a clock. A stream that opens +//! here only after its abandon left the queue closes at its group's next +//! term. +//! +//! The task stops only at shutdown. Each queued stream still open then gets +//! a capture. + +use std::collections::BTreeMap; +use std::sync::{Arc, Weak}; +use std::time::Duration; + +use crate::control::shutdown::{ShutdownPhase, spawn_loop}; +use crate::control::state::SharedState; +use crate::control::wal_replication::encode::{RedoEntryTarget, redo_abandon_entry}; +use crate::control::wal_replication::propose_replicated_entry; +use crate::wal::RedoStreamId; + +/// How long one abandon proposal waits for its apply on this node. +const ATTEMPT_BUDGET: Duration = Duration::from_secs(5); + +/// Wait before the first retry. Each later wait doubles it, up to +/// [`MAX_BACKOFF`]. +const FIRST_BACKOFF: Duration = Duration::from_millis(50); + +const MAX_BACKOFF: Duration = Duration::from_secs(2); + +/// Propose the abandon of `stream` to `target`'s data group, and wait until +/// it applied on this node. +pub(super) async fn propose_abandon( + state: &SharedState, + target: RedoEntryTarget, + stream: RedoStreamId, +) -> crate::Result<()> { + let proposer = state.async_raft_proposer()?; + let entry = redo_abandon_entry(target, stream); + let deadline = tokio::time::Instant::now() + ATTEMPT_BUDGET; + propose_replicated_entry(state, proposer, entry, deadline) + .await + .map(|_| ()) +} + +/// Spawn the task that retries `shared`'s queued abandons. `start_raft` calls +/// it once the async raft proposer is installed. A second call spawns +/// nothing. +pub(crate) fn spawn_redo_abandoner(shared: &Arc) { + if !shared.redo_chunks.claim_abandoner() { + tracing::warn!("redo abandoner already running; start_raft appears to have run twice"); + return; + } + let weak = Arc::downgrade(shared); + let wake = shared.redo_chunks.abandon_wake(); + spawn_loop( + &shared.loop_registry, + &shared.shutdown, + "redo_abandoner", + ShutdownPhase::DrainingControlPlane, + move |mut shutdown| async move { + // The last error of each queued abandon that has not applied. + let mut failed: BTreeMap = BTreeMap::new(); + let mut backoff = FIRST_BACKOFF; + loop { + let Some(state) = weak.upgrade() else { + return; + }; + let due = state.redo_chunks.due_abandons(); + failed.retain(|stream, _| due.iter().any(|abandon| abandon.stream == *stream)); + for abandon in &due { + tokio::select! { + biased; + _ = shutdown.wait_cancelled() => { + give_up(&weak, &failed); + return; + } + result = propose_abandon(&state, abandon.target, abandon.stream) => { + match result { + Ok(()) => { + state.redo_chunks.settle_abandon(&abandon.stream); + failed.remove(&abandon.stream); + } + Err(error) => { + tracing::debug!( + stream = ?abandon.stream, + %error, + "redo stream abandon did not apply; retrying" + ); + failed.insert(abandon.stream, error); + } + } + } + } + } + drop(state); + if failed.is_empty() { + backoff = FIRST_BACKOFF; + tokio::select! { + biased; + _ = shutdown.wait_cancelled() => { + give_up(&weak, &failed); + return; + } + _ = wake.notified() => {} + } + } else { + tokio::select! { + biased; + _ = shutdown.wait_cancelled() => { + give_up(&weak, &failed); + return; + } + _ = tokio::time::sleep(backoff) => {} + _ = wake.notified() => {} + } + backoff = backoff.saturating_mul(2).min(MAX_BACKOFF); + } + } + }, + ); +} + +/// Record every queued abandon whose stream is still open: no retry runs +/// after this, so the stream closes only at its group's next term. +fn give_up(state: &Weak, failed: &BTreeMap) { + let Some(state) = state.upgrade() else { + return; + }; + for abandon in state.redo_chunks.due_abandons() { + tracing::warn!( + stream = ?abandon.stream, + group_id = abandon.group_id, + term = abandon.term, + "redo stream abandon given up at shutdown; the stream closes at its group's next term" + ); + crate::diag::redo_abandon_given_up(&abandon, failed.get(&abandon.stream)); + } +} diff --git a/nodedb/src/control/wal_replication/transaction_redo/apply.rs b/nodedb/src/control/wal_replication/transaction_redo/apply.rs index cab003697..9b90502f0 100644 --- a/nodedb/src/control/wal_replication/transaction_redo/apply.rs +++ b/nodedb/src/control/wal_replication/transaction_redo/apply.rs @@ -87,7 +87,6 @@ pub(crate) async fn enqueue_transaction_redo( partition: target.vshard_id.as_u32(), epoch: position.epoch, index: position.log_index, - base: 0, }, ); stamped.apply_plan()? diff --git a/nodedb/src/control/wal_replication/transaction_redo/chunks/abandon.rs b/nodedb/src/control/wal_replication/transaction_redo/chunks/abandon.rs new file mode 100644 index 000000000..442341ace --- /dev/null +++ b/nodedb/src/control/wal_replication/transaction_redo/chunks/abandon.rs @@ -0,0 +1,124 @@ +// SPDX-License-Identifier: BUSL-1.1 + +//! The session streams whose abandon proposal failed. +//! +//! The session proposes an abandon once. When that proposal fails, the +//! stream joins this queue, and the abandoner retries it while the stream is +//! open on this node. The stream closes once its proposer's term ends, so +//! the term bounds the retries. + +use std::sync::Arc; +use std::sync::atomic::Ordering; + +use tokio::sync::Notify; + +use crate::control::wal_replication::encode::RedoEntryTarget; +use crate::wal::RedoStreamId; + +use super::store::RedoChunkStore; + +/// One queued abandon whose stream is open on this node. +#[derive(Debug, Clone, Copy)] +pub struct DueAbandon { + pub stream: RedoStreamId, + /// Where the abandon entry goes. + pub target: RedoEntryTarget, + /// The data group whose log carries the stream. + pub group_id: u64, + /// The Raft term of the stream's first chunk entry. + pub term: u64, + /// The stream's declared byte length. + pub len: u64, +} + +impl RedoChunkStore { + /// Queue the abandon of `stream` for the abandoner, and wake it. + pub fn queue_abandon(&self, stream: RedoStreamId, target: RedoEntryTarget) { + self.lock().abandons.insert(stream, target); + self.abandon_wake.notify_one(); + } + + /// The queued abandons whose stream is open. Drops every queued abandon + /// whose stream closed. + pub fn due_abandons(&self) -> Vec { + let mut state = self.lock(); + let state = &mut *state; + let streams = &state.streams; + state + .abandons + .retain(|stream, _| streams.contains_key(stream)); + state + .abandons + .iter() + .filter_map(|(stream, target)| { + streams.get(stream).map(|open| DueAbandon { + stream: *stream, + target: *target, + group_id: open.group_id, + term: open.term, + len: open.len, + }) + }) + .collect() + } + + /// Remove `stream` from the queue: its abandon applied on this node. + pub fn settle_abandon(&self, stream: &RedoStreamId) { + self.lock().abandons.remove(stream); + } + + /// Wakes the abandoner when an abandon is queued. + pub fn abandon_wake(&self) -> Arc { + Arc::clone(&self.abandon_wake) + } + + /// Claim the one abandoner task of this store. `false` when one runs. + pub fn claim_abandoner(&self) -> bool { + !self.abandoner.swap(true, Ordering::AcqRel) + } +} + +#[cfg(test)] +mod tests { + use super::super::store::tests::{chunk, session, wal}; + use super::super::store::{RedoChunkLimits, RedoChunkStore}; + use super::*; + use crate::types::{DatabaseId, TenantId, VShardId}; + + fn target() -> RedoEntryTarget { + RedoEntryTarget { + tenant_id: TenantId::new(1), + database_id: DatabaseId::DEFAULT, + vshard_id: VShardId::new(2), + } + } + + #[tokio::test] + async fn a_queued_abandon_is_due_while_its_stream_is_open() { + let dir = tempfile::tempdir().expect("tempdir"); + let store = RedoChunkStore::new( + wal(&dir), + RedoChunkLimits { + max_entry_bytes: 64 * 1024, + max_open_bytes: 1 << 20, + }, + ); + store + .apply_chunk(chunk(session(1), 0, 4, b"ab")) + .await + .expect("chunk"); + store.queue_abandon(session(1), target()); + // A stream that never opened here is not due. + store.queue_abandon(session(2), target()); + let due = store.due_abandons(); + assert_eq!(due.len(), 1); + assert_eq!(due[0].stream, session(1)); + assert_eq!((due[0].group_id, due[0].term, due[0].len), (1, 3, 4)); + // A later term of the group closes the stream, and its abandon with it. + store.end_terms_before(1, 4); + assert!(store.due_abandons().is_empty()); + assert!(store.lock().abandons.is_empty()); + assert!(store.claim_abandoner()); + assert!(!store.claim_abandoner()); + } +} diff --git a/nodedb/src/control/wal_replication/transaction_redo/chunks/carry.rs b/nodedb/src/control/wal_replication/transaction_redo/chunks/carry.rs new file mode 100644 index 000000000..54e58daff --- /dev/null +++ b/nodedb/src/control/wal_replication/transaction_redo/chunks/carry.rs @@ -0,0 +1,174 @@ +// SPDX-License-Identifier: BUSL-1.1 + +//! Carry a group's open streams in its Raft snapshot. +//! +//! A follower that installs a snapshot cut between a stream's chunks and its +//! final entry never applies those chunks. The snapshot carries every open +//! stream of the group, so the final entry finds the same stream on every +//! replica. + +use crate::types::{DatabaseId, Lsn, TenantId, VShardId}; +use crate::wal::manager::{NO_APPLY_KEY, RedoChunkPlacement}; +use crate::wal::{CarriedRedoStream, RedoChunkHeader}; + +use super::error::RedoChunkError; +use super::store::RedoChunkStore; +use super::stream::OpenStream; + +impl RedoChunkStore { + /// Every open stream of `group_id`, for the group's snapshot. The caller + /// holds the group's apply fence. + pub fn capture_group(&self, group_id: u64) -> Vec { + self.lock() + .streams + .values() + .filter(|open| open.group_id == group_id) + .map(|open| CarriedRedoStream { + stream: open.stream, + term: open.term, + len: open.len, + chunks: open.chunks.clone(), + }) + .collect() + } + + /// Replace the open and parked streams of `group_id` with `carried`, + /// from the group's installed snapshot. The carried chunks are durable + /// in this node's WAL before they count, so a restart rebuilds them. + /// Runs after the install marker: boot drops the group's earlier streams + /// there. + pub async fn install_group( + &self, + group_id: u64, + carried: Vec, + ) -> Result<(), RedoChunkError> { + let (replaced, released) = { + let mut state = self.lock(); + let streams = std::mem::take(&mut state.streams); + let (replaced, kept): (Vec<_>, Vec<_>) = streams + .into_iter() + .partition(|(_, open)| open.group_id == group_id); + state.streams = kept.into_iter().collect(); + // The snapshot covers every final entry a parked stream waits + // for, so no re-apply needs its records. + let parked = std::mem::take(&mut state.parked); + let (released, kept): (Vec<_>, Vec<_>) = parked + .into_iter() + .partition(|parked| parked.group_id == group_id); + state.parked = kept; + self.note_open(&state); + (replaced, released) + }; + // The floor holds release outside the store lock. + drop(replaced); + drop(released); + let mut installed = Vec::with_capacity(carried.len()); + let mut last = None; + for stream in carried { + let placement = RedoChunkPlacement { + tenant_id: TenantId::new(0), + vshard_id: VShardId::new(stream.stream.vshard()), + database_id: DatabaseId::DEFAULT, + }; + let mut first: Option = None; + for (index, bytes) in (0u32..).zip(&stream.chunks) { + let header = RedoChunkHeader { + stream: stream.stream, + group_id, + term: stream.term, + len: stream.len, + index, + }; + let lsns = self + .wal + .append_redo_chunk(NO_APPLY_KEY, placement, header, bytes) + .map_err(|e| RedoChunkError::storage(stream.stream, e))?; + first.get_or_insert(lsns.first); + last = Some((stream.stream, lsns.last)); + } + if let Some(first) = first { + installed.push((stream, first)); + } + } + if let Some((stream, lsn)) = last { + self.wal + .wait_durable(lsn) + .await + .map_err(|e| RedoChunkError::storage(stream, e))?; + } + let mut state = self.lock(); + for (carried, first) in installed { + let held = carried.chunks.iter().map(|chunk| chunk.len() as u64).sum(); + let hold = self.wal.floor_holds().hold(first); + state.streams.insert( + carried.stream, + OpenStream { + stream: carried.stream, + group_id, + term: carried.term, + len: carried.len, + chunks: carried.chunks, + held, + hold, + }, + ); + } + self.note_open(&state); + Ok(()) + } +} + +#[cfg(test)] +mod tests { + use super::super::store::tests::{chunk, session, wal}; + use super::super::store::{RedoChunkLimits, RedoChunkStore}; + + fn limits() -> RedoChunkLimits { + RedoChunkLimits { + max_entry_bytes: 64 * 1024, + max_open_bytes: 1 << 20, + } + } + + #[tokio::test] + async fn a_carried_stream_replaces_the_followers_streams_and_survives_a_rebuild() { + let leader_dir = tempfile::tempdir().expect("tempdir"); + let leader = RedoChunkStore::new(wal(&leader_dir), limits()); + leader + .apply_chunk(chunk(session(1), 0, 5, b"abc")) + .await + .expect("chunk"); + leader + .apply_chunk(chunk(session(1), 1, 5, b"de")) + .await + .expect("chunk"); + let carried = leader.capture_group(1); + assert_eq!(carried.len(), 1); + + let follower_dir = tempfile::tempdir().expect("tempdir"); + let follower_wal = wal(&follower_dir); + let follower = RedoChunkStore::new(std::sync::Arc::clone(&follower_wal), limits()); + follower + .apply_chunk(chunk(session(9), 0, 2, b"zz")) + .await + .expect("stale chunk"); + follower_wal + .appender(0) + .append_snapshot_installed(1) + .expect("install marker"); + follower.install_group(1, carried).await.expect("install"); + assert!(follower.take_for_final(&session(9)).is_none()); + + let rebuilt = RedoChunkStore::new(std::sync::Arc::clone(&follower_wal), limits()); + rebuilt + .rebuild(&follower_wal.replay().expect("replay")) + .expect("rebuild"); + assert!(rebuilt.take_for_final(&session(9)).is_none()); + let open = rebuilt.take_for_final(&session(1)).expect("carried stream"); + assert_eq!(open.assemble(2, 5).expect("assembles"), b"abcde".to_vec()); + let live = follower + .take_for_final(&session(1)) + .expect("installed stream"); + assert_eq!(live.assemble(2, 5).expect("assembles"), b"abcde".to_vec()); + } +} diff --git a/nodedb/src/control/wal_replication/transaction_redo/chunks/error.rs b/nodedb/src/control/wal_replication/transaction_redo/chunks/error.rs new file mode 100644 index 000000000..bbb633d01 --- /dev/null +++ b/nodedb/src/control/wal_replication/transaction_redo/chunks/error.rs @@ -0,0 +1,93 @@ +// SPDX-License-Identifier: BUSL-1.1 + +//! Why a chunked redo stream refused an entry. + +use crate::bridge::envelope::ErrorCode; +use crate::wal::RedoStreamId; + +/// A chunked redo stream refused an entry, or its storage failed. +/// +/// Every variant but [`Self::Storage`] is a refusal: the log alone decides +/// it, so every replica refuses the same entry, and the entry applied +/// nothing. A refusal reaches the proposer as a retryable refusal. +#[derive(Debug, thiserror::Error)] +pub enum RedoChunkError { + #[error( + "redo stream {stream:?}: this node holds {open} bytes of open redo streams, and \ + {len} more pass the cap of {cap} (tuning.calvin.max_open_redo_bytes); retry once \ + open streams finish" + )] + StoreFull { + stream: RedoStreamId, + open: u64, + len: u64, + cap: u64, + }, + #[error("redo stream {stream:?} is not open on this node")] + NotOpen { stream: RedoStreamId }, + #[error("redo stream {stream:?}: chunk {index} arrived, chunk {expected} was next")] + OutOfOrder { + stream: RedoStreamId, + index: u32, + expected: u32, + }, + #[error("redo stream {stream:?}: a chunk of term {term} reached a stream of term {open_term}")] + TermChanged { + stream: RedoStreamId, + term: u64, + open_term: u64, + }, + #[error("redo stream {stream:?}: a chunk declares {declared} bytes, the stream {len}")] + LengthChanged { + stream: RedoStreamId, + declared: u64, + len: u64, + }, + #[error("redo stream {stream:?}: chunks hold {held} bytes, past the declared {len}")] + Overflow { + stream: RedoStreamId, + held: u64, + len: u64, + }, + #[error( + "redo stream {stream:?}: the final entry names {count} chunks of {len} bytes, the \ + stream holds {held_count} chunks of {held} bytes" + )] + Mismatch { + stream: RedoStreamId, + count: u32, + len: u64, + held_count: usize, + held: u64, + }, + #[error("redo stream {stream:?}: {source}")] + Storage { + stream: RedoStreamId, + source: Box, + }, +} + +impl RedoChunkError { + /// Whether the log decides this outcome alike on every replica. + pub fn is_refusal(&self) -> bool { + !matches!(self, Self::Storage { .. }) + } + + pub(super) fn storage(stream: RedoStreamId, source: crate::Error) -> Self { + Self::Storage { + stream, + source: Box::new(source), + } + } +} + +impl From for crate::Error { + fn from(error: RedoChunkError) -> Self { + match error { + RedoChunkError::Storage { source, .. } => *source, + refusal => crate::Error::DataPlane(ErrorCode::RetryableRefusal { + reason: refusal.to_string(), + }), + } + } +} diff --git a/nodedb/src/control/wal_replication/transaction_redo/chunks/leave.rs b/nodedb/src/control/wal_replication/transaction_redo/chunks/leave.rs new file mode 100644 index 000000000..7f88dd464 --- /dev/null +++ b/nodedb/src/control/wal_replication/transaction_redo/chunks/leave.rs @@ -0,0 +1,168 @@ +// SPDX-License-Identifier: BUSL-1.1 + +//! Drop the streams of the data groups this node stops hosting. +//! +//! A node that leaves a group applies no later entry of it. No final entry, +//! abandon, or later term then closes the group's streams, so they close +//! here. Their floor holds release with them. +//! +//! A remount of the group resumes its old log, which can still name a +//! dropped stream. So each group that loses a stream owes a snapshot +//! install first (see [`super::owed`]). The debt is durable before any +//! floor hold releases: a checkpoint can truncate the stream's records once +//! the hold is gone. +//! +//! No WAL record marks the drop. A boot can rebuild a dropped stream from +//! its records, and the next membership pass drops it again. + +use std::collections::BTreeSet; + +use crate::control::security::catalog::SystemCatalog; + +use super::store::RedoChunkStore; + +impl RedoChunkStore { + /// Drop every open and parked stream of a data group `hosted` rejects, + /// once each such group owes a snapshot install in `catalog`. Returns + /// how many open streams dropped. + /// + /// The caller holds the lock that orders group mounts, so a group that + /// mounts concurrently keeps its streams, and a remount sees the debt. + /// When the debt does not persist, every stream stays. + pub fn drop_unhosted_groups( + &self, + hosted: impl Fn(u64) -> bool, + catalog: &SystemCatalog, + ) -> crate::Result { + let leaving: BTreeSet = { + let state = self.lock(); + state + .streams + .values() + .map(|open| open.group_id) + .chain(state.parked.iter().map(|parked| parked.group_id)) + .filter(|&group_id| !hosted(group_id)) + .collect() + }; + if leaving.is_empty() { + return Ok(0); + } + self.record_owed(catalog, &leaving)?; + // A stream of a group outside `leaving` opened after the scan. It + // drops on the next pass, once its group owes a snapshot. + let drops = |group_id: u64| leaving.contains(&group_id) && !hosted(group_id); + let (dropped, released) = { + let mut state = self.lock(); + let streams = std::mem::take(&mut state.streams); + let (dropped, kept): (Vec<_>, Vec<_>) = streams + .into_iter() + .partition(|(_, open)| drops(open.group_id)); + state.streams = kept.into_iter().collect(); + let parked = std::mem::take(&mut state.parked); + let (released, kept): (Vec<_>, Vec<_>) = parked + .into_iter() + .partition(|parked| drops(parked.group_id)); + state.parked = kept; + self.note_open(&state); + (dropped, released) + }; + let count = dropped.len(); + // The floor holds release outside the store lock. + drop(dropped); + drop(released); + Ok(count) + } +} + +#[cfg(test)] +mod tests { + use super::super::store::tests::{chunk, session, wal}; + use super::super::store::{RedoChunkLimits, RedoChunkStore}; + use super::*; + + fn limits() -> RedoChunkLimits { + RedoChunkLimits { + max_entry_bytes: 64 * 1024, + max_open_bytes: 1 << 20, + } + } + + fn catalog(dir: &tempfile::TempDir) -> SystemCatalog { + SystemCatalog::open(&dir.path().join("system.redb")).expect("catalog") + } + + #[tokio::test] + async fn leaving_a_group_drops_its_streams_and_releases_their_floor_holds() { + let dir = tempfile::tempdir().expect("tempdir"); + let catalog = catalog(&dir); + let store = RedoChunkStore::new(wal(&dir), limits()); + // Group 1: one open stream and one parked stream. + store + .apply_chunk(chunk(session(1), 0, 4, b"ab")) + .await + .expect("chunk"); + store + .apply_chunk(chunk(session(2), 0, 2, b"cd")) + .await + .expect("chunk"); + let failed = store.take_for_final(&session(2)).expect("open stream"); + store.park(failed); + // Group 9 stays hosted. + let mut kept = chunk(session(3), 0, 4, b"ef"); + kept.group_id = 9; + store.apply_chunk(kept).await.expect("chunk"); + assert_eq!(store.open_streams(), 2); + + let dropped = store + .drop_unhosted_groups(|group_id| group_id == 9, &catalog) + .expect("drop"); + assert_eq!(dropped, 1); + assert_eq!(store.open_streams(), 1); + assert!(store.take_for_final(&session(1)).is_none()); + // Only the hosted group's stream holds the floor. + assert!(store.wal.floor_holds().lowest().is_some()); + drop( + store + .take_for_final(&session(3)) + .expect("the hosted stream stays"), + ); + assert_eq!(store.wal.floor_holds().lowest(), None); + } + + /// A remount of a group that lost a stream resumes a log that can name + /// the stream. The group owes a snapshot install, durably, and a hosted + /// group owes nothing. + #[tokio::test] + async fn dropping_a_groups_streams_records_its_snapshot_requirement() { + let dir = tempfile::tempdir().expect("tempdir"); + let catalog = catalog(&dir); + let store = RedoChunkStore::new(wal(&dir), limits()); + assert_eq!( + store + .drop_unhosted_groups(|_| false, &catalog) + .expect("drop"), + 0 + ); + assert!( + store.owed_groups().is_empty(), + "a group with no stream owes nothing" + ); + + store + .apply_chunk(chunk(session(1), 0, 4, b"ab")) + .await + .expect("chunk"); + let failed = store.take_for_final(&session(1)).expect("open stream"); + store.park(failed); + let mut kept = chunk(session(3), 0, 4, b"ef"); + kept.group_id = 9; + store.apply_chunk(kept).await.expect("chunk"); + + store + .drop_unhosted_groups(|group_id| group_id == 9, &catalog) + .expect("drop"); + assert!(store.owes_snapshot(1), "a parked stream's group owes one"); + assert!(!store.owes_snapshot(9)); + assert_eq!(catalog.load_redo_snapshot_owed().expect("load"), vec![1]); + } +} diff --git a/nodedb/src/control/wal_replication/transaction_redo/chunks/mod.rs b/nodedb/src/control/wal_replication/transaction_redo/chunks/mod.rs new file mode 100644 index 000000000..16bcd0dc3 --- /dev/null +++ b/nodedb/src/control/wal_replication/transaction_redo/chunks/mod.rs @@ -0,0 +1,28 @@ +// SPDX-License-Identifier: BUSL-1.1 + +//! Chunked transaction redo: the streams a redo past the entry limit travels +//! in, held on every replica until the stream's final entry. +//! +//! - [`store`]: the node's open streams, leader admission, and stream close. +//! - [`stream`]: one open stream and its assembly. +//! - [`rebuild`]: boot rebuild from the WAL. +//! - [`carry`]: the open streams a group snapshot carries. +//! - [`leave`]: the drop of a left group's streams. +//! - [`owed`]: the groups whose replica here owes a snapshot install. +//! - [`abandon`]: the session streams whose abandon proposal failed. +//! - [`error`]: why a stream refused an entry. + +pub mod abandon; +pub mod carry; +pub mod error; +pub mod leave; +pub mod owed; +pub mod rebuild; +pub mod store; +pub mod stream; + +pub use abandon::DueAbandon; +pub use error::RedoChunkError; +pub use rebuild::RebuildStats; +pub use store::{ChunkApply, RedoChunkLimits, RedoChunkStore}; +pub use stream::OpenStream; diff --git a/nodedb/src/control/wal_replication/transaction_redo/chunks/owed.rs b/nodedb/src/control/wal_replication/transaction_redo/chunks/owed.rs new file mode 100644 index 000000000..a3ee4317c --- /dev/null +++ b/nodedb/src/control/wal_replication/transaction_redo/chunks/owed.rs @@ -0,0 +1,118 @@ +// SPDX-License-Identifier: BUSL-1.1 + +//! The data groups whose replica on this node owes a snapshot install. +//! +//! A node that leaves a group drops the group's streams (see +//! [`super::leave`]). A remount resumes the group's old log. That log can +//! hold a later chunk or the final entry of a dropped stream. Here it +//! refuses, while every other replica installs the redo. So the drop marks +//! the group owed, durably, before the streams release their WAL floor +//! holds: +//! +//! - The group's snapshot requirement holds on this node while it is owed. +//! Its replica refuses log entries, and its leader sends a snapshot. The +//! snapshot carries the group's open streams. +//! - A refusal of a chunk or final entry of an owed group holds the group's +//! durable floor instead of passing the entry. +//! - A snapshot install of the group settles the debt. +//! +//! The set is a leaf lock: no other lock is taken while it is held. + +use std::collections::BTreeSet; +use std::sync::MutexGuard; + +use crate::control::security::catalog::SystemCatalog; + +use super::store::RedoChunkStore; + +impl RedoChunkStore { + fn owed_set(&self) -> MutexGuard<'_, BTreeSet> { + self.owed.lock().unwrap_or_else(|p| p.into_inner()) + } + + /// Load the owed groups the catalog saved, at boot. + pub fn load_owed(&self, groups: Vec) { + self.owed_set().extend(groups); + } + + /// Whether this node's replica of `group_id` owes a snapshot install. + pub fn owes_snapshot(&self, group_id: u64) -> bool { + self.owed_set().contains(&group_id) + } + + /// Every group whose replica here owes a snapshot install. + pub fn owed_groups(&self) -> Vec { + self.owed_set().iter().copied().collect() + } + + /// Record that the replicas of `groups` here owe a snapshot install. + /// Durable before it takes effect. A group owed already is not written + /// again. + pub fn record_owed( + &self, + catalog: &SystemCatalog, + groups: &BTreeSet, + ) -> crate::Result<()> { + let fresh: Vec = { + let owed = self.owed_set(); + groups + .iter() + .copied() + .filter(|group_id| !owed.contains(group_id)) + .collect() + }; + catalog.save_redo_snapshot_owed(&fresh)?; + self.owed_set().extend(fresh); + Ok(()) + } + + /// Settle the debt of `group_id`: a snapshot of the group installed + /// here, with the group's open streams. + pub fn settle_owed(&self, catalog: &SystemCatalog, group_id: u64) -> crate::Result<()> { + if !self.owes_snapshot(group_id) { + return Ok(()); + } + catalog.remove_redo_snapshot_owed(group_id)?; + self.owed_set().remove(&group_id); + Ok(()) + } +} + +#[cfg(test)] +mod tests { + use super::super::store::tests::wal; + use super::super::store::{RedoChunkLimits, RedoChunkStore}; + use super::*; + + fn limits() -> RedoChunkLimits { + RedoChunkLimits { + max_entry_bytes: 64 * 1024, + max_open_bytes: 1 << 20, + } + } + + #[test] + fn an_owed_group_survives_a_restart_until_its_snapshot_installs() { + let dir = tempfile::tempdir().expect("tempdir"); + let catalog = SystemCatalog::open(&dir.path().join("system.redb")).expect("catalog"); + let store = RedoChunkStore::new(wal(&dir), limits()); + + store + .record_owed(&catalog, &BTreeSet::from([3, 8])) + .expect("record"); + assert!(store.owes_snapshot(3)); + assert!(!store.owes_snapshot(4)); + + let restarted_dir = tempfile::tempdir().expect("tempdir"); + let restarted = RedoChunkStore::new(wal(&restarted_dir), limits()); + restarted.load_owed(catalog.load_redo_snapshot_owed().expect("load")); + assert_eq!(restarted.owed_groups(), vec![3, 8]); + + restarted.settle_owed(&catalog, 3).expect("settle"); + restarted + .settle_owed(&catalog, 4) + .expect("settle a group never owed"); + assert_eq!(restarted.owed_groups(), vec![8]); + assert_eq!(catalog.load_redo_snapshot_owed().expect("load"), vec![8]); + } +} diff --git a/nodedb/src/control/wal_replication/transaction_redo/chunks/rebuild.rs b/nodedb/src/control/wal_replication/transaction_redo/chunks/rebuild.rs new file mode 100644 index 000000000..a80695305 --- /dev/null +++ b/nodedb/src/control/wal_replication/transaction_redo/chunks/rebuild.rs @@ -0,0 +1,274 @@ +// SPDX-License-Identifier: BUSL-1.1 + +//! Rebuild the open streams from the WAL at boot. +//! +//! A chunk entry's records are durable before its entry concludes, and its +//! key rides its last piece, so a re-delivered chunk entry is skipped as a +//! repeat. The store must hold that chunk again from its records. Boot reads +//! every `RedoChunk` record in WAL order and keeps a stream open unless: +//! +//! - a `Closed` record names it (an abandon); +//! - a later snapshot install of its group replaced the group's streams; +//! - a session stream's final entry key is on a record (the final applied); +//! - a stamped transaction redo names a Calvin stream's position. +//! +//! A stream keeps its chunks from index `0` up to the first chunk with a +//! missing piece. That chunk's entry never concluded, so it applies again. + +use std::collections::{BTreeMap, HashSet}; + +use nodedb_wal::WalRecord; +use nodedb_wal::record::RecordType; + +use crate::types::Lsn; +use crate::wal::{RedoChunkHeader, RedoChunkRecord, RedoRecord, RedoStreamId}; + +use super::store::RedoChunkStore; +use super::stream::OpenStream; + +/// What a boot rebuild found. +#[derive(Debug, Clone, Copy, Default, PartialEq, Eq)] +pub struct RebuildStats { + /// Streams the store holds open again. + pub open: usize, + /// Streams the WAL shows closed. + pub closed: usize, +} + +/// The pieces of one chunk read so far. +struct ChunkPieces { + pieces: u32, + got: BTreeMap>, + first: u64, +} + +/// One stream read so far. +struct Building { + group_id: u64, + term: u64, + len: u64, + chunks: BTreeMap, +} + +impl Building { + fn new(header: &RedoChunkHeader) -> Self { + Self { + group_id: header.group_id, + term: header.term, + len: header.len, + chunks: BTreeMap::new(), + } + } + + /// The whole chunks from index `0`, the first record's LSN, and the + /// bytes they hold. `None` when chunk `0` is not whole. + fn whole_prefix(self) -> Option<(Vec>, u64, u64)> { + let mut chunks = Vec::new(); + let mut first = u64::MAX; + let mut held = 0u64; + for (index, chunk) in self.chunks { + let whole = index as usize == chunks.len() + && chunk.got.len() == chunk.pieces as usize + && chunk.got.keys().copied().eq(0..chunk.pieces); + if !whole { + break; + } + first = first.min(chunk.first); + let bytes: Vec = chunk.got.into_values().flatten().collect(); + held += bytes.len() as u64; + chunks.push(bytes); + } + (!chunks.is_empty()).then_some((chunks, first, held)) + } +} + +impl RedoChunkStore { + /// Rebuild the open streams from `records`, the whole WAL in LSN order. + /// Runs at boot, before the data-group apply loop starts. + pub fn rebuild(&self, records: &[WalRecord]) -> crate::Result { + let mut building: BTreeMap = BTreeMap::new(); + let mut keys: HashSet = HashSet::new(); + let mut redo_records: Vec<&WalRecord> = Vec::new(); + let mut closed = 0usize; + for record in records { + let key = record.apply_key(); + if key != 0 { + keys.insert(key); + } + match RecordType::from_raw(record.logical_record_type()) { + Some(RecordType::RedoChunk) => { + match RedoChunkRecord::from_bytes(&record.payload)? { + RedoChunkRecord::Piece(piece) => { + let stream = building + .entry(piece.header.stream) + .or_insert_with(|| Building::new(&piece.header)); + let chunk = + stream + .chunks + .entry(piece.header.index) + .or_insert(ChunkPieces { + pieces: piece.pieces, + got: BTreeMap::new(), + first: record.header.lsn, + }); + chunk.first = chunk.first.min(record.header.lsn); + chunk.got.insert(piece.piece, piece.bytes); + } + RedoChunkRecord::Closed { stream } => { + closed += usize::from(building.remove(&stream).is_some()); + } + } + } + Some(RecordType::SnapshotInstalled) => { + if let Ok(raw) = <[u8; 8]>::try_from(record.payload.as_slice()) { + let group_id = u64::from_le_bytes(raw); + let before = building.len(); + building.retain(|_, stream| stream.group_id != group_id); + closed += before - building.len(); + } + } + Some(RecordType::TransactionRedo) => redo_records.push(record), + _ => {} + } + } + let applied_positions = if building + .keys() + .any(|stream| matches!(stream, RedoStreamId::Calvin { .. })) + { + stamped_positions(&redo_records)? + } else { + HashSet::new() + }; + let mut state = self.lock(); + let mut open = 0usize; + for (stream, built) in building { + let finished = match stream { + RedoStreamId::Session { + idempotency_key, .. + } => keys.contains(&idempotency_key), + RedoStreamId::Calvin { + vshard, + epoch, + position, + .. + } => applied_positions.contains(&(vshard, epoch, position)), + }; + if finished { + closed += 1; + continue; + } + let (group_id, term, len) = (built.group_id, built.term, built.len); + let Some((chunks, first, held)) = built.whole_prefix() else { + continue; + }; + let hold = self.wal.floor_holds().hold(Lsn::new(first)); + state.streams.insert( + stream, + OpenStream { + stream, + group_id, + term, + len, + chunks, + held, + hold, + }, + ); + open += 1; + } + self.note_open(&state); + Ok(RebuildStats { open, closed }) + } +} + +/// The `(vshard, epoch, position)` every stamped transaction redo names. +fn stamped_positions(records: &[&WalRecord]) -> crate::Result> { + let mut positions = HashSet::new(); + for record in records { + if let Some(stamp) = RedoRecord::from_bytes(&record.payload)?.calvin_stamp { + positions.insert((stamp.vshard_id, stamp.epoch, stamp.position)); + } + } + Ok(positions) +} + +#[cfg(test)] +mod tests { + use super::super::store::tests::{chunk, session, wal}; + use super::super::store::{RedoChunkLimits, RedoChunkStore}; + use super::*; + + fn limits() -> RedoChunkLimits { + RedoChunkLimits { + max_entry_bytes: 64 * 1024, + max_open_bytes: 1 << 20, + } + } + + #[tokio::test] + async fn boot_rebuilds_an_open_stream() { + let dir = tempfile::tempdir().expect("tempdir"); + let wal = wal(&dir); + let live = RedoChunkStore::new(std::sync::Arc::clone(&wal), limits()); + let open = session(41); + let finished = session(42); + let abandoned = session(43); + for (index, part) in [&b"abc"[..], b"de"].into_iter().enumerate() { + live.apply_chunk(chunk(open, index as u32, 5, part)) + .await + .expect("chunk"); + } + live.apply_chunk(chunk(finished, 0, 2, b"xy")) + .await + .expect("chunk"); + live.apply_chunk(chunk(abandoned, 0, 2, b"zz")) + .await + .expect("chunk"); + // The finished stream's final entry appended a record under its key. + wal.appender(42) + .append_proposal_applied( + crate::types::TenantId::new(1), + crate::types::VShardId::new(2), + crate::types::DatabaseId::DEFAULT, + ) + .expect("final record"); + live.abandon(abandoned, 77, chunk(abandoned, 0, 0, b"").placement) + .await + .expect("abandon"); + wal.sync().expect("sync"); + + let rebuilt = RedoChunkStore::new(std::sync::Arc::clone(&wal), limits()); + let stats = rebuilt + .rebuild(&wal.replay().expect("replay")) + .expect("rebuild"); + assert_eq!(stats, RebuildStats { open: 1, closed: 2 }); + let stream = rebuilt.take_for_final(&open).expect("open stream rebuilt"); + assert_eq!(stream.assemble(2, 5).expect("assembles"), b"abcde".to_vec()); + assert!(rebuilt.take_for_final(&finished).is_none()); + assert!(rebuilt.take_for_final(&abandoned).is_none()); + } + + #[tokio::test] + async fn a_snapshot_install_drops_the_groups_earlier_streams() { + let dir = tempfile::tempdir().expect("tempdir"); + let wal = wal(&dir); + let live = RedoChunkStore::new(std::sync::Arc::clone(&wal), limits()); + live.apply_chunk(chunk(session(5), 0, 2, b"ab")) + .await + .expect("chunk"); + wal.appender(0) + .append_snapshot_installed(1) + .expect("install marker"); + live.apply_chunk(chunk(session(6), 0, 2, b"cd")) + .await + .expect("chunk"); + wal.sync().expect("sync"); + + let rebuilt = RedoChunkStore::new(std::sync::Arc::clone(&wal), limits()); + rebuilt + .rebuild(&wal.replay().expect("replay")) + .expect("rebuild"); + assert!(rebuilt.take_for_final(&session(5)).is_none()); + assert!(rebuilt.take_for_final(&session(6)).is_some()); + } +} diff --git a/nodedb/src/control/wal_replication/transaction_redo/chunks/store.rs b/nodedb/src/control/wal_replication/transaction_redo/chunks/store.rs new file mode 100644 index 000000000..5bdcef794 --- /dev/null +++ b/nodedb/src/control/wal_replication/transaction_redo/chunks/store.rs @@ -0,0 +1,597 @@ +// SPDX-License-Identifier: BUSL-1.1 + +//! The node's open chunked redo streams. +//! +//! Every replica decides each chunk and final entry from the log alone: the +//! chunk order, the stream term and the declared lengths. No replica refuses +//! an entry for its own memory. The group leader bounds memory instead: its +//! write gate admits a stream's first chunk only while this node's open and +//! admitted stream bytes, plus the new stream's length, stay within +//! `tuning.calvin.max_open_redo_bytes`. A node then holds at most that cap +//! per data group it hosts. +//! +//! A stream closes when its final entry takes it, when its proposer abandons +//! it, when its group applies an entry of a later term, when its Calvin +//! position applies through another copy, or when this node stops hosting +//! its group. + +use std::collections::{BTreeMap, BTreeSet, HashMap}; +use std::sync::atomic::{AtomicBool, AtomicUsize, Ordering}; +use std::sync::{Arc, Mutex, MutexGuard}; +use std::time::Instant; + +use nodedb_types::config::tuning::CalvinTuning; +use tokio::sync::Notify; + +use crate::control::wal_replication::encode::RedoEntryTarget; +use crate::types::Lsn; +use crate::wal::manager::{RedoChunkPlacement, WalFloorHold}; +use crate::wal::{RedoChunkHeader, RedoStreamId, WalManager}; + +use super::error::RedoChunkError; +use super::stream::OpenStream; + +/// The size bounds of chunked redo. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub struct RedoChunkLimits { + /// Largest encoded redo entry a proposer puts in a data-group log. + pub max_entry_bytes: usize, + /// Most bytes of open streams the leader admits on its node. + pub max_open_bytes: u64, +} + +impl RedoChunkLimits { + pub fn from_tuning(tuning: &CalvinTuning) -> Self { + Self { + max_entry_bytes: tuning.max_redo_entry_bytes, + max_open_bytes: tuning.max_open_redo_bytes, + } + } +} + +/// A first chunk the leader admitted that has not applied yet. +#[derive(Debug, Clone, Copy)] +pub(super) struct Pending { + pub(super) len: u64, + /// The proposer's deadline. Past it the admission counts no more. + pub(super) until: Instant, +} + +/// The floor of a stream whose final entry failed on this node. +pub(super) struct ParkedStream { + pub(super) group_id: u64, + /// Keeps the floor until the stream drops. + pub(super) _hold: WalFloorHold, +} + +#[derive(Default)] +pub(super) struct StoreState { + pub(super) streams: BTreeMap, + pub(super) pending: HashMap, + /// Streams whose final entry failed on this node. The next boot rebuilds + /// them from the records their floors keep. + pub(super) parked: Vec, + /// Session streams whose abandon proposal failed, and the target the + /// abandon entry goes to. The abandoner retries each while it is open. + pub(super) abandons: BTreeMap, +} + +/// One committed chunk entry, as the apply loop hands it over. +#[derive(Debug, Clone)] +pub struct ChunkApply { + pub group_id: u64, + /// The Raft term of the chunk entry. + pub term: u64, + /// The chunk entry's idempotency key. + pub apply_key: u64, + pub placement: RedoChunkPlacement, + pub stream: RedoStreamId, + pub index: u32, + /// The byte length of the whole stream. + pub len: u64, + pub bytes: Vec, +} + +/// The node's open chunked redo streams. +pub struct RedoChunkStore { + pub(super) wal: Arc, + limits: RedoChunkLimits, + pub(super) state: Mutex, + /// How many streams are open: the per-entry term check skips the lock + /// when none is. + open_count: AtomicUsize, + /// Wakes the abandoner when a failed abandon is queued. + pub(super) abandon_wake: Arc, + /// Whether an abandoner task runs for this store. + pub(super) abandoner: AtomicBool, + /// The data groups whose replica here owes a snapshot install (see + /// [`super::owed`]). A leaf lock, apart from `state`. + pub(super) owed: Mutex>, +} + +impl RedoChunkStore { + pub fn new(wal: Arc, limits: RedoChunkLimits) -> Self { + Self { + wal, + limits, + state: Mutex::new(StoreState::default()), + open_count: AtomicUsize::new(0), + abandon_wake: Arc::new(Notify::new()), + abandoner: AtomicBool::new(false), + owed: Mutex::new(BTreeSet::new()), + } + } + + pub fn limits(&self) -> RedoChunkLimits { + self.limits + } + + pub(super) fn lock(&self) -> MutexGuard<'_, StoreState> { + self.state.lock().unwrap_or_else(|p| p.into_inner()) + } + + /// Record how many streams `state` holds open. + pub(super) fn note_open(&self, state: &StoreState) { + self.open_count + .store(state.streams.len(), Ordering::Release); + } + + /// How many streams are open. + pub fn open_streams(&self) -> usize { + self.open_count.load(Ordering::Acquire) + } + + /// The declared bytes of every open stream. + pub fn open_bytes(&self) -> u64 { + self.lock().streams.values().map(|open| open.len).sum() + } + + /// Admit the first chunk of `stream`, declaring `len` bytes, on this + /// group leader. The admission counts until the chunk applies here or + /// `until` passes. A re-proposal of an admitted or open stream passes. + pub fn admit_stream( + &self, + stream: RedoStreamId, + len: u64, + until: Instant, + ) -> Result<(), RedoChunkError> { + let now = Instant::now(); + let mut state = self.lock(); + state.pending.retain(|_, pending| pending.until > now); + if state.streams.contains_key(&stream) { + return Ok(()); + } + if let Some(pending) = state.pending.get_mut(&stream) { + pending.until = pending.until.max(until); + return Ok(()); + } + let open = state.streams.values().map(|open| open.len).sum::() + + state + .pending + .values() + .map(|pending| pending.len) + .sum::(); + let cap = self.limits.max_open_bytes; + if open.saturating_add(len) > cap { + return Err(RedoChunkError::StoreFull { + stream, + open, + len, + cap, + }); + } + state.pending.insert(stream, Pending { len, until }); + Ok(()) + } + + /// Apply one chunk: check it against its stream, append its records + /// keyed with the entry's key, wait until they are durable, then hold it. + pub async fn apply_chunk(&self, chunk: ChunkApply) -> Result<(), RedoChunkError> { + self.check_chunk(&chunk)?; + let header = RedoChunkHeader { + stream: chunk.stream, + group_id: chunk.group_id, + term: chunk.term, + len: chunk.len, + index: chunk.index, + }; + let lsns = self + .wal + .append_redo_chunk(chunk.apply_key, chunk.placement, header, &chunk.bytes) + .map_err(|e| RedoChunkError::storage(chunk.stream, e))?; + self.wal + .wait_durable(lsns.last) + .await + .map_err(|e| RedoChunkError::storage(chunk.stream, e))?; + self.hold_chunk(chunk, lsns.first); + Ok(()) + } + + /// Refuse a chunk the stream cannot take next. + fn check_chunk(&self, chunk: &ChunkApply) -> Result<(), RedoChunkError> { + let stream = chunk.stream; + let bytes = chunk.bytes.len() as u64; + let state = self.lock(); + let Some(open) = state.streams.get(&stream) else { + if chunk.index != 0 { + return Err(RedoChunkError::NotOpen { stream }); + } + if bytes > chunk.len { + return Err(RedoChunkError::Overflow { + stream, + held: bytes, + len: chunk.len, + }); + } + return Ok(()); + }; + let expected = u32::try_from(open.chunks.len()).unwrap_or(u32::MAX); + if chunk.index != expected { + return Err(RedoChunkError::OutOfOrder { + stream, + index: chunk.index, + expected, + }); + } + if open.group_id != chunk.group_id || open.term != chunk.term { + return Err(RedoChunkError::TermChanged { + stream, + term: chunk.term, + open_term: open.term, + }); + } + if open.len != chunk.len { + return Err(RedoChunkError::LengthChanged { + stream, + declared: chunk.len, + len: open.len, + }); + } + if open.held.saturating_add(bytes) > open.len { + return Err(RedoChunkError::Overflow { + stream, + held: open.held.saturating_add(bytes), + len: open.len, + }); + } + Ok(()) + } + + /// Hold a durable chunk whose first record is at `first`. + fn hold_chunk(&self, chunk: ChunkApply, first: Lsn) { + let mut state = self.lock(); + let held = chunk.bytes.len() as u64; + match state.streams.get_mut(&chunk.stream) { + Some(open) => { + open.held += held; + open.chunks.push(chunk.bytes); + } + None => { + state.pending.remove(&chunk.stream); + let hold = self.wal.floor_holds().hold(first); + state.streams.insert( + chunk.stream, + OpenStream { + stream: chunk.stream, + group_id: chunk.group_id, + term: chunk.term, + len: chunk.len, + chunks: vec![chunk.bytes], + held, + hold, + }, + ); + } + } + self.note_open(&state); + } + + /// Take `stream` for its final entry. The final entry decides it from + /// the log, so it leaves the store whatever the final entry's outcome. + pub fn take_for_final(&self, stream: &RedoStreamId) -> Option { + let mut state = self.lock(); + let taken = state.streams.remove(stream); + self.note_open(&state); + taken + } + + /// Keep the records of a taken stream whose final entry failed on this + /// node, so the next boot rebuilds the stream for the entry's re-apply. + pub fn park(&self, open: OpenStream) { + let group_id = open.group_id; + let hold = open.into_hold(); + self.lock().parked.push(ParkedStream { + group_id, + _hold: hold, + }); + } + + /// Drop `stream` for its abandon entry, and make the drop durable under + /// the entry's key. + pub async fn abandon( + &self, + stream: RedoStreamId, + apply_key: u64, + placement: RedoChunkPlacement, + ) -> Result<(), RedoChunkError> { + { + let mut state = self.lock(); + state.streams.remove(&stream); + state.pending.remove(&stream); + self.note_open(&state); + } + let lsn = self + .wal + .append_redo_stream_closed(apply_key, placement, stream) + .map_err(|e| RedoChunkError::storage(stream, e))?; + self.wal + .wait_durable(lsn) + .await + .map_err(|e| RedoChunkError::storage(stream, e)) + } + + /// Drop every stream of `group_id` opened in a term before `term`: the + /// group applies an entry of `term`, so no earlier term adds an entry. + pub fn end_terms_before(&self, group_id: u64, term: u64) { + if self.open_streams() == 0 { + return; + } + let mut state = self.lock(); + state + .streams + .retain(|_, open| open.group_id != group_id || open.term >= term); + self.note_open(&state); + } + + /// Drop every stream of the Calvin position `(epoch, position)` of + /// `vshard`. The position applied through one copy of its redo, so no + /// other attempt's stream installs. The Calvin apply path calls it once + /// the position's install is durable. + pub fn drop_calvin_position(&self, vshard: u32, epoch: u64, position: u32) { + if self.open_streams() == 0 { + return; + } + let mut state = self.lock(); + state.streams.retain(|stream, _| { + !matches!( + stream, + RedoStreamId::Calvin { vshard: v, epoch: e, position: p, .. } + if *v == vshard && *e == epoch && *p == position + ) + }); + self.note_open(&state); + } + + /// Drop every Calvin stream whose position `applied` names. Boot calls + /// it once the applied ledgers recovered: a position that applied + /// through one copy installs no other. + pub fn drop_applied_calvin_streams(&self, applied: impl Fn(u32, u64, u32) -> bool) { + if self.open_streams() == 0 { + return; + } + let mut state = self.lock(); + state.streams.retain(|stream, _| { + !matches!( + stream, + RedoStreamId::Calvin { vshard, epoch, position, .. } + if applied(*vshard, *epoch, *position) + ) + }); + self.note_open(&state); + } +} + +#[cfg(test)] +pub(super) mod tests { + use super::*; + use crate::types::{DatabaseId, TenantId, VShardId}; + + pub(in super::super) fn wal(dir: &tempfile::TempDir) -> Arc { + Arc::new(WalManager::open_for_testing(&dir.path().join("wal")).expect("open wal")) + } + + pub(in super::super) fn session(key: u64) -> RedoStreamId { + RedoStreamId::Session { + vshard: 2, + idempotency_key: key, + } + } + + pub(in super::super) fn chunk( + stream: RedoStreamId, + index: u32, + len: u64, + bytes: &[u8], + ) -> ChunkApply { + ChunkApply { + group_id: 1, + term: 3, + apply_key: 100 + u64::from(index), + placement: RedoChunkPlacement { + tenant_id: TenantId::new(1), + vshard_id: VShardId::new(2), + database_id: DatabaseId::DEFAULT, + }, + stream, + index, + len, + bytes: bytes.to_vec(), + } + } + + fn limits(max_open_bytes: u64) -> RedoChunkLimits { + RedoChunkLimits { + max_entry_bytes: 64 * 1024, + max_open_bytes, + } + } + + #[tokio::test] + async fn chunks_reassemble_in_order() { + let dir = tempfile::tempdir().expect("tempdir"); + let store = RedoChunkStore::new(wal(&dir), limits(1 << 20)); + let stream = session(7); + for (index, part) in [&b"abc"[..], b"def", b"g"].into_iter().enumerate() { + store + .apply_chunk(chunk(stream, index as u32, 7, part)) + .await + .expect("chunk applies"); + } + assert_eq!(store.open_streams(), 1); + assert_eq!(store.open_bytes(), 7); + let open = store.take_for_final(&stream).expect("open stream"); + assert_eq!(open.assemble(3, 7).expect("assembles"), b"abcdefg".to_vec()); + assert_eq!(store.open_streams(), 0); + // The floor holds the stream's first record until the stream drops. + assert!(store.wal.floor_holds().lowest().is_some()); + drop(open); + assert_eq!(store.wal.floor_holds().lowest(), None); + } + + #[tokio::test] + async fn a_missing_chunk_refuses_the_final_entry() { + let dir = tempfile::tempdir().expect("tempdir"); + let store = RedoChunkStore::new(wal(&dir), limits(1 << 20)); + let stream = session(8); + store + .apply_chunk(chunk(stream, 0, 6, b"abc")) + .await + .expect("first chunk"); + let skipped = store.apply_chunk(chunk(stream, 2, 6, b"ghi")).await; + assert!(matches!( + skipped, + Err(RedoChunkError::OutOfOrder { + index: 2, + expected: 1, + .. + }) + )); + let open = store.take_for_final(&stream).expect("open stream"); + let refused = open.assemble(2, 6); + assert!(matches!(refused, Err(RedoChunkError::Mismatch { .. }))); + let error = crate::Error::from(refused.expect_err("refused")); + assert!(matches!( + error, + crate::Error::DataPlane(crate::bridge::envelope::ErrorCode::RetryableRefusal { .. }) + )); + // A later chunk of the taken stream finds no stream. + assert!(matches!( + store.apply_chunk(chunk(stream, 1, 6, b"def")).await, + Err(RedoChunkError::NotOpen { .. }) + )); + } + + #[tokio::test] + async fn a_new_stream_past_the_open_byte_cap_is_refused() { + let dir = tempfile::tempdir().expect("tempdir"); + let store = RedoChunkStore::new(wal(&dir), limits(10)); + let until = Instant::now() + std::time::Duration::from_secs(60); + store + .admit_stream(session(1), 6, until) + .expect("first stream fits"); + // A re-proposal of an admitted stream passes. + store + .admit_stream(session(1), 6, until) + .expect("re-proposal passes"); + let refused = store.admit_stream(session(2), 5, until); + assert!(matches!( + refused, + Err(RedoChunkError::StoreFull { + open: 6, + len: 5, + cap: 10, + .. + }) + )); + // The admitted stream applies, then finishes, and frees its bytes. + store + .apply_chunk(chunk(session(1), 0, 6, b"abcdef")) + .await + .expect("chunk"); + assert!(store.admit_stream(session(2), 5, until).is_err()); + drop(store.take_for_final(&session(1))); + store + .admit_stream(session(2), 5, until) + .expect("room after the first stream closed"); + // An admission past its deadline counts no more. + let other_dir = tempfile::tempdir().expect("tempdir"); + let store = RedoChunkStore::new(wal(&other_dir), limits(10)); + store + .admit_stream(session(3), 10, Instant::now()) + .expect("fits"); + store + .admit_stream(session(4), 10, until) + .expect("the expired admission frees its bytes"); + } + + #[tokio::test] + async fn a_later_term_drops_the_groups_earlier_streams() { + let dir = tempfile::tempdir().expect("tempdir"); + let store = RedoChunkStore::new(wal(&dir), limits(1 << 20)); + store + .apply_chunk(chunk(session(1), 0, 4, b"ab")) + .await + .expect("chunk"); + let mut other_group = chunk(session(2), 0, 4, b"ab"); + other_group.group_id = 9; + store.apply_chunk(other_group).await.expect("chunk"); + store.end_terms_before(1, 3); + assert_eq!(store.open_streams(), 2); + store.end_terms_before(1, 4); + assert_eq!(store.open_streams(), 1); + assert!(store.take_for_final(&session(2)).is_some()); + } + + #[tokio::test] + async fn an_applied_calvin_position_drops_every_attempt() { + let dir = tempfile::tempdir().expect("tempdir"); + let store = RedoChunkStore::new(wal(&dir), limits(1 << 20)); + for attempt in 0..2 { + let stream = RedoStreamId::Calvin { + vshard: 2, + epoch: 5, + position: 1, + attempt, + }; + store + .apply_chunk(chunk(stream, 0, 2, b"ab")) + .await + .expect("chunk"); + } + store.drop_calvin_position(2, 5, 1); + assert_eq!(store.open_streams(), 0); + } + + /// Boot drops the streams of every position its ledgers hold applied, + /// and keeps the rest. + #[tokio::test] + async fn boot_drops_the_streams_of_applied_calvin_positions() { + let dir = tempfile::tempdir().expect("tempdir"); + let store = RedoChunkStore::new(wal(&dir), limits(1 << 20)); + let calvin = |position| RedoStreamId::Calvin { + vshard: 2, + epoch: 5, + position, + attempt: 0, + }; + for position in [1, 2] { + store + .apply_chunk(chunk(calvin(position), 0, 2, b"ab")) + .await + .expect("chunk"); + } + store + .apply_chunk(chunk(session(7), 0, 2, b"ab")) + .await + .expect("chunk"); + + store.drop_applied_calvin_streams(|vshard, epoch, position| { + (vshard, epoch, position) == (2, 5, 1) + }); + + assert_eq!(store.open_streams(), 2); + assert!(store.take_for_final(&calvin(1)).is_none()); + assert!(store.take_for_final(&calvin(2)).is_some()); + assert!(store.take_for_final(&session(7)).is_some()); + } +} diff --git a/nodedb/src/control/wal_replication/transaction_redo/chunks/stream.rs b/nodedb/src/control/wal_replication/transaction_redo/chunks/stream.rs new file mode 100644 index 000000000..29996111c --- /dev/null +++ b/nodedb/src/control/wal_replication/transaction_redo/chunks/stream.rs @@ -0,0 +1,56 @@ +// SPDX-License-Identifier: BUSL-1.1 + +//! One open chunked redo stream on this node. + +use crate::wal::RedoStreamId; +use crate::wal::manager::WalFloorHold; + +use super::error::RedoChunkError; + +/// The chunks one stream holds, and the WAL floor that keeps their records. +#[derive(Debug)] +pub struct OpenStream { + pub(super) stream: RedoStreamId, + /// The data group whose log carries the stream. + pub(super) group_id: u64, + /// The Raft term of the stream's first chunk entry. + pub(super) term: u64, + /// The byte length every chunk declares for the whole stream. + pub(super) len: u64, + /// The chunks, in index order. + pub(super) chunks: Vec>, + /// The bytes `chunks` hold. + pub(super) held: u64, + /// Keeps the stream's `RedoChunk` records until the stream closes. + pub(super) hold: WalFloorHold, +} + +impl OpenStream { + pub fn stream(&self) -> RedoStreamId { + self.stream + } + + /// The stream's bytes, when it holds exactly `count` chunks of `len` + /// bytes in all, as its final entry names. + pub fn assemble(&self, count: u32, len: u64) -> Result, RedoChunkError> { + if self.chunks.len() != count as usize || self.held != len || self.len != len { + return Err(RedoChunkError::Mismatch { + stream: self.stream, + count, + len, + held_count: self.chunks.len(), + held: self.held, + }); + } + let mut bytes = Vec::with_capacity(self.held as usize); + for chunk in &self.chunks { + bytes.extend_from_slice(chunk); + } + Ok(bytes) + } + + /// The floor guard, once the bytes are no longer needed. + pub(super) fn into_hold(self) -> WalFloorHold { + self.hold + } +} diff --git a/nodedb/src/control/wal_replication/transaction_redo/mod.rs b/nodedb/src/control/wal_replication/transaction_redo/mod.rs index d6c49df7f..62a4091fd 100644 --- a/nodedb/src/control/wal_replication/transaction_redo/mod.rs +++ b/nodedb/src/control/wal_replication/transaction_redo/mod.rs @@ -4,14 +4,23 @@ //! //! - [`payload`]: the redo a commit hands to the data-group log. //! - [`apply`]: the one path that appends and applies it on a node. +//! - [`chunks`]: the streams a redo past the entry limit travels in. +//! - [`propose`]: the proposal of a committed redo, inline or chunked. +//! - [`abandon`]: the retry of a failed stream abandon. //! - [`collections`] / [`sum_targets`]: what the payload derives from the //! commit's buffered plans. +pub mod abandon; pub mod apply; +pub mod chunks; pub mod collections; pub mod payload; +pub mod propose; pub mod sum_targets; +pub(crate) use abandon::spawn_redo_abandoner; pub use apply::RedoTarget; pub(crate) use apply::{enqueue_transaction_redo, record_cross_shard_key}; -pub use payload::TransactionRedoPayload; +pub use chunks::{RedoChunkError, RedoChunkStore}; +pub use payload::{CalvinSlice, TransactionRedoPayload}; +pub(crate) use propose::propose_transaction_redo; diff --git a/nodedb/src/control/wal_replication/transaction_redo/payload.rs b/nodedb/src/control/wal_replication/transaction_redo/payload.rs index 9885faeb6..0f8796fa6 100644 --- a/nodedb/src/control/wal_replication/transaction_redo/payload.rs +++ b/nodedb/src/control/wal_replication/transaction_redo/payload.rs @@ -3,10 +3,13 @@ //! One committed transaction's redo for one vShard, as a commit hands it to //! the data-group log and as every replica applies it. -use nodedb_physical::physical_plan::{MetaOp, PhysicalPlan, RedoOrigin, RedoSumTargets}; +use nodedb_physical::physical_plan::{ + CalvinInstall, MetaOp, PhysicalPlan, RedoOrigin, RedoSumTargets, +}; use crate::control::state::SharedState; use crate::control::surrogate::{CarriedIdentity, collect_plan_identities}; +use crate::control::wal_replication::types::CalvinRedoMeta; use crate::event::EventSource; use crate::types::{DatabaseId, TenantId}; use crate::wal::RedoRecord; @@ -30,6 +33,21 @@ pub struct TransactionRedoPayload { pub event_source: EventSource, /// Which commit-boundary checks every replica's apply runs. pub origin: RedoOrigin, + /// Set when the redo is a committed Calvin slice. The record's + /// `calvin_stamp` names its `(epoch, position)`. + pub calvin: Option, +} + +/// What a committed Calvin slice's redo carries beside its record, derived +/// from the slice's local plans when it stages. +#[derive(Debug, Clone)] +pub struct CalvinSlice { + pub collections: Vec, + pub sum_targets: Vec, + pub identities: Vec, + pub event_source: EventSource, + pub origin: RedoOrigin, + pub meta: CalvinRedoMeta, } impl TransactionRedoPayload { @@ -55,9 +73,33 @@ impl TransactionRedoPayload { )?, event_source, origin: RedoOrigin::Commit, + calvin: None, }) } + /// The payload of a committed Calvin slice whose resolved redo is `redo`, + /// built on the vShard's data-group leader. The record's `calvin_stamp` + /// must name the slice's `(epoch, position)`. + pub fn from_calvin(redo: RedoRecord, slice: CalvinSlice) -> Self { + let CalvinSlice { + collections, + sum_targets, + identities, + event_source, + origin, + meta, + } = slice; + Self { + redo, + collections, + sum_targets, + identities, + event_source, + origin, + calvin: Some(meta), + } + } + /// The Data-Plane plan that applies this redo. pub fn apply_plan(&self) -> crate::Result { Ok(PhysicalPlan::Meta(MetaOp::ApplyTransactionRedo { @@ -65,6 +107,111 @@ impl TransactionRedoPayload { collections: self.collections.clone(), sum_targets: self.sum_targets.clone(), origin: self.origin, + calvin: self.calvin_install()?, + })) + } + + /// The Calvin install the apply plan carries: the slice's stamp and the + /// meta beside it. A Calvin meta on a record with no stamp names no + /// position, so it is refused. + fn calvin_install(&self) -> crate::Result> { + let Some(meta) = &self.calvin else { + return Ok(None); + }; + let stamp = self + .redo + .calvin_stamp + .as_ref() + .ok_or_else(|| crate::Error::Internal { + detail: "a Calvin redo entry carries no calvin_stamp, so its install names no \ + (epoch, position); the entry is refused" + .into(), + })?; + Ok(Some(CalvinInstall { + epoch: stamp.epoch, + position: stamp.position, + epoch_system_ms: meta.epoch_system_ms, + reply: meta.reply.clone(), + user_write: meta.user_write, })) } } + +#[cfg(test)] +mod tests { + use super::*; + use crate::wal::CalvinStamp; + use nodedb_physical::physical_plan::CalvinReplySpec; + + fn payload( + stamp: Option, + calvin: Option, + ) -> TransactionRedoPayload { + TransactionRedoPayload { + redo: RedoRecord { + version: 1, + ops: Vec::new(), + calvin_stamp: stamp, + cross_shard_applied: None, + row_sources: Vec::new(), + publishes: Vec::new(), + row_changes: Vec::new(), + }, + collections: vec!["orders".into()], + sum_targets: Vec::new(), + identities: Vec::new(), + event_source: EventSource::User, + origin: RedoOrigin::Commit, + calvin, + } + } + + fn meta() -> CalvinRedoMeta { + CalvinRedoMeta { + epoch_system_ms: 1_700, + reply: CalvinReplySpec::Count(vec![7]), + primary_write: true, + user_write: true, + returning: false, + } + } + + fn install_of(payload: &TransactionRedoPayload) -> crate::Result> { + match payload.apply_plan()? { + PhysicalPlan::Meta(MetaOp::ApplyTransactionRedo { calvin, .. }) => Ok(calvin), + other => panic!("apply plan must be ApplyTransactionRedo, got {other:?}"), + } + } + + #[test] + fn a_session_redo_installs_without_a_calvin_install() { + let plain = payload(None, None); + assert_eq!(install_of(&plain).expect("plan"), None); + } + + #[test] + fn a_calvin_redo_installs_at_its_stamped_position() { + let stamp = CalvinStamp { + epoch: 9, + position: 4, + vshard_id: 2, + }; + let calvin = payload(Some(stamp), Some(meta())); + assert_eq!( + install_of(&calvin).expect("plan"), + Some(CalvinInstall { + epoch: 9, + position: 4, + epoch_system_ms: 1_700, + reply: CalvinReplySpec::Count(vec![7]), + user_write: true, + }) + ); + } + + #[test] + fn a_calvin_meta_without_a_stamp_is_refused() { + let unstamped = payload(None, Some(meta())); + assert!(install_of(&unstamped).is_err()); + } +} diff --git a/nodedb/src/control/wal_replication/transaction_redo/propose.rs b/nodedb/src/control/wal_replication/transaction_redo/propose.rs new file mode 100644 index 000000000..fc1c608c7 --- /dev/null +++ b/nodedb/src/control/wal_replication/transaction_redo/propose.rs @@ -0,0 +1,79 @@ +// SPDX-License-Identifier: BUSL-1.1 + +//! Propose a committed session redo through its vShard's data-group log. +//! +//! A redo whose entry fits `tuning.calvin.max_redo_entry_bytes` travels as +//! one entry. A larger redo travels as chunk entries, proposed in order, each +//! applied on this node before the next, and then a final entry that names +//! their stream. A proposal that stops before its final entry applies +//! proposes an abandon, so every replica drops the stream's chunks. A failed +//! abandon proposal goes to the abandoner, which retries it while the stream +//! is open (see [`super::abandon`]). + +use crate::control::state::SharedState; +use crate::control::wal_replication::encode::{ + RedoEntryTarget, RedoProposal, session_redo_proposal, +}; +use crate::control::wal_replication::propose_replicated_entry; +use crate::wal::RedoStreamId; + +use super::abandon::propose_abandon; +use super::apply::RedoTarget; +use super::payload::TransactionRedoPayload; + +/// Propose `payload` to `target`'s data group and wait until it applied on +/// this node. Returns what [`propose_replicated_entry`] returns for the entry +/// that installs the redo. +pub(crate) async fn propose_transaction_redo( + state: &SharedState, + target: RedoTarget, + payload: &TransactionRedoPayload, + deadline: tokio::time::Instant, +) -> crate::Result { + let proposer = state.async_raft_proposer()?; + let entry_target = RedoEntryTarget { + tenant_id: target.tenant_id, + database_id: target.database_id, + vshard_id: target.vshard_id, + }; + let proposal = session_redo_proposal( + entry_target, + payload, + state.redo_chunks.limits().max_entry_bytes, + )?; + let (stream, chunks, last) = match proposal { + RedoProposal::Inline(entry) => { + return propose_replicated_entry(state, proposer, entry, deadline).await; + } + RedoProposal::Chunked { + stream, + chunks, + last, + } => (stream, chunks, last), + }; + for chunk in chunks { + if let Err(error) = propose_replicated_entry(state, proposer, chunk, deadline).await { + abandon(state, entry_target, stream).await; + return Err(error); + } + } + let installed = propose_replicated_entry(state, proposer, last, deadline).await; + if installed.is_err() { + abandon(state, entry_target, stream).await; + } + installed +} + +/// Propose the abandon of `stream` once. A stream the final entry already +/// took is gone, so the abandon then drops nothing. A failed proposal queues +/// the abandon for the abandoner, so the session's error returns now. +async fn abandon(state: &SharedState, target: RedoEntryTarget, stream: RedoStreamId) { + if let Err(error) = propose_abandon(state, target, stream).await { + tracing::warn!( + ?stream, + %error, + "redo stream abandon did not apply; the abandoner retries it while the stream is open" + ); + state.redo_chunks.queue_abandon(stream, target); + } +} diff --git a/nodedb/src/control/wal_replication/transaction_redo/sum_targets.rs b/nodedb/src/control/wal_replication/transaction_redo/sum_targets.rs index dcce2dc28..159880744 100644 --- a/nodedb/src/control/wal_replication/transaction_redo/sum_targets.rs +++ b/nodedb/src/control/wal_replication/transaction_redo/sum_targets.rs @@ -9,9 +9,15 @@ use std::collections::BTreeMap; use nodedb_physical::physical_plan::{DocumentOp, PhysicalPlan, RedoSumTargets, ResolvedSumTarget}; -/// Every resolved and deferred sum target `plans` carry, one entry per source -/// collection, in collection order. A target resolved by several plans is -/// listed once. +/// Every sum target `plans` fold on the source's own core, one entry per +/// source collection, in collection order. A target resolved by several plans +/// is listed once. +/// +/// The install folds every row of a collection against this one merged +/// table. So a target a plan defers onto its own `ApplyBalanceDelta` task +/// never enters it. Otherwise a sibling row of the same collection, a DELETE +/// next to an INSERT on the same account, folds that target on the source's +/// core as well as the shipped task moving it. pub fn redo_sum_targets(plans: &[PhysicalPlan]) -> Vec { let mut by_collection: BTreeMap = BTreeMap::new(); for plan in plans { @@ -21,7 +27,11 @@ pub fn redo_sum_targets(plans: &[PhysicalPlan]) -> Vec { let Some((collection, resolved, deferred)) = document_sum_targets(op) else { continue; }; - if resolved.is_empty() && deferred.is_empty() { + let mut folded = resolved + .iter() + .filter(|target| !shipped_on_own_task(target, deferred)) + .peekable(); + if folded.peek().is_none() { continue; } let entry = by_collection @@ -29,22 +39,25 @@ pub fn redo_sum_targets(plans: &[PhysicalPlan]) -> Vec { .or_insert_with(|| RedoSumTargets { collection: collection.to_string(), resolved: Vec::new(), - deferred: Vec::new(), }); - for target in resolved { + for target in folded { if !entry.resolved.contains(target) { entry.resolved.push(target.clone()); } } - for target in deferred { - if !entry.deferred.contains(target) { - entry.deferred.push(target.clone()); - } - } } by_collection.into_values().collect() } +/// Whether `target` names a target collection its plan defers onto its own +/// `ApplyBalanceDelta` task. +fn shipped_on_own_task(target: &ResolvedSumTarget, deferred: &[String]) -> bool { + target + .target_collection + .as_deref() + .is_some_and(|collection| deferred.iter().any(|d| d == collection)) +} + type SumTargetSlots<'a> = (&'a str, &'a [ResolvedSumTarget], &'a [String]); /// The source collection and sum-target slots of one document write. @@ -159,6 +172,54 @@ mod tests { assert_eq!(targets[0].resolved, vec![target]); } + fn insert_deferring(target: ResolvedSumTarget, deferred: &str) -> PhysicalPlan { + PhysicalPlan::Document(DocumentOp::PointInsert { + collection: QualifiedCollection::new(DatabaseId::DEFAULT, "entries"), + document_id: "e2".into(), + value: Vec::new(), + if_absent: false, + surrogate: Surrogate::new(2), + returning: None, + rls_filters: Vec::new(), + resolved_sum_targets: vec![target], + deferred_sum_targets: vec![deferred.to_string()], + }) + } + + /// An insert that ships its balance on its own task lends no resolution + /// to the merged table. A delete of the same account in the same + /// transaction ships its own balance too, and the merged table must not + /// tell the install to fold it on the source's core as well. + #[test] + fn a_target_shipped_on_its_own_task_is_not_folded_by_a_sibling_row() { + let shipped = ResolvedSumTarget::new("accounts", "a1", Surrogate::new(9)); + let plans = vec![ + insert_deferring(shipped, "accounts"), + PhysicalPlan::Document(DocumentOp::PointDelete { + collection: QualifiedCollection::new(DatabaseId::DEFAULT, "entries"), + document_id: "e1".into(), + surrogate: Some(Surrogate::new(1)), + pk_bytes: b"e1".to_vec(), + returning: None, + rls_filters: Vec::new(), + rls_write_check: nodedb_types::RlsWriteCheck::NoPolicyApplies, + resolved_sum_targets: Vec::new(), + }), + ]; + assert!(redo_sum_targets(&plans).is_empty()); + } + + /// A deferral names one target collection. The same insert still folds a + /// target it does not defer. + #[test] + fn a_deferral_keeps_the_targets_it_does_not_name() { + let kept = ResolvedSumTarget::new("audit_totals", "a1", Surrogate::new(7)); + let plan = insert_deferring(kept.clone(), "accounts"); + let targets = redo_sum_targets(&[plan]); + assert_eq!(targets.len(), 1); + assert_eq!(targets[0].resolved, vec![kept]); + } + #[test] fn a_write_with_no_binding_contributes_nothing() { let plan = PhysicalPlan::Document(DocumentOp::PointDelete { diff --git a/nodedb/src/control/wal_replication/types/aliases.rs b/nodedb/src/control/wal_replication/types/aliases.rs index 23fa6d5bd..b9bbd5def 100644 --- a/nodedb/src/control/wal_replication/types/aliases.rs +++ b/nodedb/src/control/wal_replication/types/aliases.rs @@ -13,15 +13,9 @@ pub type RaftProposer = /// Type alias for the asynchronous Raft propose callback with leader forwarding. /// /// Takes `(vshard_id, idempotency_key, serialized_entry, deadline)` and returns, on -/// success, the Data Plane apply payload bytes together with the write's -/// per-collection version: the written collection's `coll_write_lsn` AFTER the -/// write, as the replica that applied the entry recorded it. It is a WAL LSN, -/// minted by the apply path's own WAL append — NOT the Raft log index, which is -/// a per-group counter sharing no scale with the WAL LSNs the version index is -/// otherwise fed. Callers that need the write's version (e.g. a -/// read-your-writes floor for cross-shard OCC, which validates in the WAL-LSN -/// domain) read it here without a second lookup; it is `Lsn::ZERO` for a write -/// whose plan names no single user collection, which records no such version. +/// success, an [`AppliedOutput`]: the Data Plane apply payload bytes and the +/// write's versions. A write's version is the data-group log position of the +/// entry that applied it, so every replica records the same version. /// The `idempotency_key` matches the one embedded in the /// serialized `ReplicatedEntry`; the proposer registers the tracker waiter with /// this key so apply-side mismatch detection can surface `RetryableLeaderChange` @@ -38,13 +32,17 @@ pub type AsyncRaftProposer = dyn Fn( tokio::time::Instant, ) -> std::pin::Pin< Box< - dyn std::future::Future< - Output = std::result::Result<(Vec, crate::types::Lsn), crate::Error>, - > + Send, + dyn std::future::Future> + + Send, >, > + Send + Sync; +/// What this node's apply of a proposed write returns: the Data Plane apply +/// payload and the versions the write stamped, one per written vShard. A +/// write that names no single user collection stamps none. +pub type AppliedOutput = (Vec, crate::types::ReadVersions); + /// Where a proposed entry landed in its data group's Raft log. #[derive(Debug, Clone, Copy, PartialEq, Eq)] pub struct ProposedAt { @@ -55,11 +53,7 @@ pub struct ProposedAt { /// The wait for this node's apply of a proposed entry: the result an /// [`AsyncRaftProposer`] returns. pub type AppliedWait = std::pin::Pin< - Box< - dyn std::future::Future< - Output = std::result::Result<(Vec, crate::types::Lsn), crate::Error>, - > + Send, - >, + Box> + Send>, >; /// A proposal the group's leader accepted into its log. diff --git a/nodedb/src/control/wal_replication/types/mod.rs b/nodedb/src/control/wal_replication/types/mod.rs index 7af91c755..1080445d2 100644 --- a/nodedb/src/control/wal_replication/types/mod.rs +++ b/nodedb/src/control/wal_replication/types/mod.rs @@ -17,20 +17,23 @@ //! - [`replicated_write`]: the [`ReplicatedWrite`] enum itself. //! - [`replicated_entry`]: [`ReplicatedEntry`] (routing envelope) + (de)serialization. //! - [`transaction_redo_wire`]: wire types of the committed-transaction redo entry. +//! - [`redo_chunk_wire`]: a redo body, inline or chunked across entries. mod aliases; +mod redo_chunk_wire; mod replicated_entry; mod replicated_write; mod transaction_redo_wire; mod wire_shapes; pub use aliases::{ - AppliedWait, AsyncRaftProposer, AsyncRaftSubmit, ProposedAt, ProposedWrite, + AppliedOutput, AppliedWait, AsyncRaftProposer, AsyncRaftSubmit, ProposedAt, ProposedWrite, RaftAppliedIndexSink, RaftCompactor, RaftProposer, }; +pub use redo_chunk_wire::{RedoBody, RedoContent}; pub use replicated_entry::{CollectionIncarnation, ReplicatedEntry}; pub use replicated_write::ReplicatedWrite; -pub use transaction_redo_wire::{ReplicatedEventSource, ReplicatedIdentity}; +pub use transaction_redo_wire::{CalvinRedoMeta, ReplicatedEventSource, ReplicatedIdentity}; pub use wire_shapes::{ BalanceDeltaFields, ColumnarResolvedRow, ConstraintChangeOp, DocumentResolvedMutationWire, KvResolvedMutationWire, ReplicatedBatchEdge, ReplicatedSumTarget, diff --git a/nodedb/src/control/wal_replication/types/redo_chunk_wire.rs b/nodedb/src/control/wal_replication/types/redo_chunk_wire.rs new file mode 100644 index 000000000..1687fc269 --- /dev/null +++ b/nodedb/src/control/wal_replication/types/redo_chunk_wire.rs @@ -0,0 +1,65 @@ +// SPDX-License-Identifier: BUSL-1.1 + +//! Wire shapes of a committed redo body: inline, or chunked across entries. + +use nodedb_physical::physical_plan::RedoSumTargets; + +use super::transaction_redo_wire::ReplicatedIdentity; +use crate::wal::RedoStreamId; + +/// The row-sized part of a committed redo: what a chunked redo splits. +#[derive( + Debug, + Clone, + PartialEq, + serde::Serialize, + serde::Deserialize, + zerompk::ToMessagePack, + zerompk::FromMessagePack, +)] +pub struct RedoContent { + /// The resolved post-images. + pub redo: crate::wal::RedoRecord, + /// Materialized-sum resolution the document writes fold into their + /// targets, keyed by source collection. + pub sum_targets: Vec, + /// Identities every replica binds before the apply. + pub identities: Vec, +} + +impl RedoContent { + pub fn to_bytes(&self) -> crate::Result> { + zerompk::to_msgpack_vec(self).map_err(|e| crate::Error::Serialization { + format: "msgpack".into(), + detail: format!("redo content encode: {e}"), + }) + } + + pub fn from_bytes(bytes: &[u8]) -> crate::Result { + zerompk::from_msgpack(bytes).map_err(|e| crate::Error::Serialization { + format: "msgpack".into(), + detail: format!("redo content decode: {e}"), + }) + } +} + +/// Where a `TransactionRedo` entry's content is. +#[derive( + Debug, + Clone, + serde::Serialize, + serde::Deserialize, + zerompk::ToMessagePack, + zerompk::FromMessagePack, +)] +pub enum RedoBody { + /// The entry carries the content. + Inline(Box), + /// Earlier `RedoChunk` entries of `stream` carry the content: `count` + /// chunks of `len` bytes in all, encoded as one [`RedoContent`]. + Chunked { + stream: RedoStreamId, + count: u32, + len: u64, + }, +} diff --git a/nodedb/src/control/wal_replication/types/replicated_write.rs b/nodedb/src/control/wal_replication/types/replicated_write.rs index 814b58832..b63fb766b 100644 --- a/nodedb/src/control/wal_replication/types/replicated_write.rs +++ b/nodedb/src/control/wal_replication/types/replicated_write.rs @@ -5,11 +5,13 @@ use super::aliases::{ default_columnar_ingest_format, default_columnar_insert_intent, default_ivf_cells, default_ivf_nprobe, default_pq_m, }; +use super::redo_chunk_wire::RedoBody; use super::transaction_redo_wire::{ReplicatedEventSource, ReplicatedIdentity}; use super::wire_shapes::{ ColumnarResolvedRow, ConstraintChangeOp, DocumentResolvedMutationWire, KvResolvedMutationWire, ReplicatedBatchEdge, ReplicatedSumTarget, }; +use crate::wal::RedoStreamId; use nodedb_physical::physical_plan::document::MergeClauseOp; use nodedb_physical::physical_plan::{ ColumnarInsertIntent, CrdtWriteVerb, KvCounterShape, UpdateValue, VectorDirectWriteIntent, @@ -1029,24 +1031,22 @@ pub enum ReplicatedWrite { }, /// One committed transaction's resolved post-images for one vShard. /// - /// Every replica, the proposer included, appends `redo` to its own WAL - /// and applies it through the WAL replay arms, in Raft log order. The - /// record is resolved once, on the proposer; no replica re-derives it. - /// `redo.calvin_stamp` names the Calvin `(epoch, position)` the record - /// applies, when a Calvin flush produced it. + /// Every replica, the proposer included, appends the body's redo to its + /// own WAL and applies it through the WAL replay arms, in Raft log order. + /// The record is resolved once, on the proposer; no replica re-derives + /// it. A chunked body assembles from the `RedoChunk` entries of its + /// stream. TransactionRedo { - redo: crate::wal::RedoRecord, + body: RedoBody, /// Every collection the transaction wrote, for the collection-floor /// write versions. collections: Vec, - /// Materialized-sum resolution the document writes fold into their - /// targets, keyed by source collection. - sum_targets: Vec, - /// Identities every replica binds before the apply. - identities: Vec, event_source: ReplicatedEventSource, /// Which commit-boundary checks every replica's apply runs. origin: nodedb_physical::physical_plan::RedoOrigin, + /// Set for a committed Calvin slice. The redo's `calvin_stamp` names + /// its `(epoch, position)`, and the slice's scheduler holds its locks. + calvin: Option, }, /// A backup's consistent cut through this group's log. It writes no /// data. Every entry before it applies before the backup snapshots, and @@ -1086,6 +1086,30 @@ pub enum ReplicatedWrite { SurrogateBind { identities: Vec, }, + /// One chunk of a chunked redo's content. Every replica appends it to its + /// WAL and holds it until the stream's final `TransactionRedo` entry. + /// `len` is the byte length of the whole stream. + RedoChunk { + stream: RedoStreamId, + index: u32, + len: u64, + bytes: Vec, + }, + /// The proposer of `stream` gave up before its final entry. Every replica + /// drops the stream's held chunks. + RedoAbandon { + stream: RedoStreamId, + }, + /// The dependent-read barrier of Calvin txn `(epoch, position)` on this + /// vShard waited past its timeout. The vShard's data-group leader + /// proposes it. Every replica applies it in log order: a read result + /// below it in the log counts, one above it does not. A barrier still + /// missing a read result at this entry votes abort on every replica. + CalvinReadTimeout { + epoch: u64, + position: u32, + tenant_id: u64, + }, } #[cfg(test)] @@ -1235,6 +1259,23 @@ mod tests { constraint_version: 1, constraints: vec![vec![1, 2, 3]], }, + ReplicatedWrite::RedoChunk { + stream: RedoStreamId::Session { + vshard: 0, + idempotency_key: 9, + }, + index: 0, + len: 3, + bytes: vec![1, 2, 3], + }, + ReplicatedWrite::RedoAbandon { + stream: RedoStreamId::Calvin { + vshard: 0, + epoch: 4, + position: 1, + attempt: 0, + }, + }, ]; for write in writes { diff --git a/nodedb/src/control/wal_replication/types/transaction_redo_wire.rs b/nodedb/src/control/wal_replication/types/transaction_redo_wire.rs index ec9c736b8..8b715617a 100644 --- a/nodedb/src/control/wal_replication/types/transaction_redo_wire.rs +++ b/nodedb/src/control/wal_replication/types/transaction_redo_wire.rs @@ -2,8 +2,39 @@ //! Wire shapes carried by [`super::ReplicatedWrite::TransactionRedo`]. +use nodedb_physical::physical_plan::CalvinReplySpec; + use crate::event::EventSource; +/// What a committed Calvin slice's redo carries beyond the redo record. +/// The record's `calvin_stamp` names the slice's `(epoch, position)`. +#[derive( + Debug, + Clone, + PartialEq, + serde::Serialize, + serde::Deserialize, + zerompk::ToMessagePack, + zerompk::FromMessagePack, +)] +pub struct CalvinRedoMeta { + /// The epoch's deterministic instant. + pub epoch_system_ms: i64, + /// The reply the leader's stage decided. Every replica renders it after + /// its install. + pub reply: CalvinReplySpec, + /// Whether the slice writes a row the statement names, not only a + /// derived row. Only a primary slice deposits the statement's reply. + pub primary_write: bool, + /// Whether the install raises the tenant's write mark: a primary write + /// that changes a tenant row. A slice that only installs schema raises + /// none. + pub user_write: bool, + /// Whether the slice answers rows (a RETURNING write), not a count. + /// Every replica reads it to build the same `CompletionAck` result. + pub returning: bool, +} + /// One `(collection, primary key) → surrogate` identity a committed /// transaction's writes carry. Every replica binds it into its own catalog /// before the redo applies, so a later write by primary key on any replica diff --git a/nodedb/src/control/write_resolve/mod.rs b/nodedb/src/control/write_resolve/mod.rs index 040817136..4a7f34f34 100644 --- a/nodedb/src/control/write_resolve/mod.rs +++ b/nodedb/src/control/write_resolve/mod.rs @@ -27,7 +27,7 @@ pub use run::{MAX_WRITE_RESOLVE_RETRIES, run_authorized_write_resolve, run_write pub use select::resolver_for_plan; pub use timeseries::TimeseriesWriteResolver; pub(crate) use timeseries::{ - carries_applied_ingest_counts, is_unresolved_ingest, rejected_lines, resolve_for_log, - resolve_ingest_plan, resolve_tasks_for_log, resolved_ingest_counts, + is_unresolved_ingest, rejected_lines, resolve_for_log, resolve_ingest_plan, + resolve_tasks_for_log, }; pub use vector::VectorWriteResolver; diff --git a/nodedb/src/control/write_resolve/propose.rs b/nodedb/src/control/write_resolve/propose.rs index b8b1d371c..16e7b697a 100644 --- a/nodedb/src/control/write_resolve/propose.rs +++ b/nodedb/src/control/write_resolve/propose.rs @@ -42,8 +42,9 @@ pub(super) async fn propose_resolved( ), })?; - match propose_replicated_entry(state, proposer, entry).await { - Ok((payload, write_version)) => { + let deadline = crate::control::wal_replication::statement_propose_deadline(state); + match propose_replicated_entry(state, proposer, entry, deadline).await { + Ok((payload, write_versions)) => { let request_id = RequestId::new(state.request_id_counter.fetch_add(1, Ordering::Relaxed)); let response = Response { @@ -52,10 +53,11 @@ pub(super) async fn propose_resolved( attempt: 1, partial: false, payload: payload.into(), - watermark_lsn: write_version, + // A write carries no read watermark. + watermark_lsn: crate::types::Lsn::ZERO, error_code: None, - read_set_valid: None, - read_version_lsn: write_version, + stage_vote: None, + read_versions: write_versions, write_set: Vec::new(), }; Ok(ProposeOutcome::Applied(response)) diff --git a/nodedb/src/control/write_resolve/timeseries.rs b/nodedb/src/control/write_resolve/timeseries.rs index 06be8d213..3ee4104ed 100644 --- a/nodedb/src/control/write_resolve/timeseries.rs +++ b/nodedb/src/control/write_resolve/timeseries.rs @@ -14,7 +14,6 @@ use nodedb_types::RlsWriteCheck; use crate::bridge::envelope::{PhysicalPlan, Status}; use crate::control::maintenance::clone_materializer::dispatch_local; use crate::control::state::SharedState; -use crate::engine::timeseries::install_counts::TsInstallCounts; use crate::engine::timeseries::resolved_ingest::{ RESOLVED_INGEST_FORMAT, ResolveBase, ResolvedTsBatch, TsDriftPolicy, }; @@ -247,44 +246,6 @@ pub(crate) fn rejected_lines(plan: &PhysicalPlan) -> crate::Result { } } -/// The count payload of the resolved timeseries ingest `plan`, in the shape -/// an install answers: the rows it stores and the lines its resolve -/// rejected. `None` for every other plan, and for an ingest with -/// `RETURNING`, which answers rows. -pub(crate) fn resolved_ingest_counts(plan: &PhysicalPlan) -> crate::Result>> { - let PhysicalPlan::Timeseries(TimeseriesOp::Ingest { - collection, - payload, - format, - returning: None, - .. - }) = plan - else { - return Ok(None); - }; - if format != RESOLVED_INGEST_FORMAT { - return Ok(None); - } - let batch = ResolvedTsBatch::from_bytes(payload)?; - let counts = serde_json::json!({ - "accepted": batch.rows.len(), - "rejected": batch.rejected, - "collection": collection.as_str(), - }); - nodedb_types::json_to_msgpack(&counts) - .map(Some) - .map_err(|e| crate::Error::Codec { - detail: format!("timeseries ingest counts: {e}"), - }) -} - -/// Whether an applied answer's `payload` carries its timeseries installs' -/// counts. Such a payload has the shape an install answers: the rows the -/// apply stored, and the lines and rows it rejected. -pub(crate) fn carries_applied_ingest_counts(payload: &[u8]) -> bool { - TsInstallCounts::from_payload(payload).is_some() -} - /// Whether `plan` is a timeseries ingest that has not resolved its rows. pub(crate) fn is_unresolved_ingest(plan: &PhysicalPlan) -> bool { matches!( diff --git a/nodedb/src/data/executor/array_checkpoint.rs b/nodedb/src/data/executor/array_checkpoint.rs index 1bd1f445c..ff234e7bf 100644 --- a/nodedb/src/data/executor/array_checkpoint.rs +++ b/nodedb/src/data/executor/array_checkpoint.rs @@ -7,7 +7,7 @@ //! `ArrayStore::memtable` is a plain in-memory `Memtable` (see //! `engine::array::memtable`). Every `INSERT INTO ARRAY` / `DELETE FROM ARRAY` //! lands there and advances the core watermark (`dispatch::array::mutate` -//! calls `note_write_lsn` with the Control Plane's `wal_lsn`). The periodic +//! calls `note_write` with the Control Plane's `wal_lsn`). The periodic //! checkpoint reports those writes as durable, and the manager then removes //! the `ArrayPut` / `ArrayDelete` records below that LSN. The checkpoint must //! therefore flush every memtable first. An explicit `NDARRAY_FLUSH` and the @@ -275,6 +275,7 @@ mod tests { wal_lsn: None, resolved_now_ms: None, commit_hlc: None, + entry_version: None, admission: crate::bridge::envelope::Admission::Admitted, })) .expect("push request"); diff --git a/nodedb/src/data/executor/cascade_journal_tests.rs b/nodedb/src/data/executor/cascade_journal_tests.rs index 64a6c4da7..ad2265130 100644 --- a/nodedb/src/data/executor/cascade_journal_tests.rs +++ b/nodedb/src/data/executor/cascade_journal_tests.rs @@ -133,6 +133,7 @@ fn request(plan: PhysicalPlan, wal_lsn: Option) -> Request { wal_lsn, resolved_now_ms: None, commit_hlc: None, + entry_version: None, admission: Admission::Admitted, } } diff --git a/nodedb/src/data/executor/columnar_checkpoint/load.rs b/nodedb/src/data/executor/columnar_checkpoint/load.rs index 907f57da6..5543baf6c 100644 --- a/nodedb/src/data/executor/columnar_checkpoint/load.rs +++ b/nodedb/src/data/executor/columnar_checkpoint/load.rs @@ -312,6 +312,7 @@ mod tests { wal_lsn: None, resolved_now_ms: None, commit_hlc: None, + entry_version: None, admission: crate::bridge::envelope::Admission::Exempt( crate::bridge::envelope::ExemptReason::Read, ), diff --git a/nodedb/src/data/executor/columnar_checkpoint/write.rs b/nodedb/src/data/executor/columnar_checkpoint/write.rs index bdf89a45a..e08c17243 100644 --- a/nodedb/src/data/executor/columnar_checkpoint/write.rs +++ b/nodedb/src/data/executor/columnar_checkpoint/write.rs @@ -40,7 +40,7 @@ impl CoreLoop { /// applied before the export and replayed again after it duplicates the /// row. Every columnar record that mutates an engine therefore notes its /// LSN after applying: `execute_columnar_insert`, `execute_columnar_update` - /// and `execute_columnar_delete` call `note_collection_write_lsn`, which + /// and `execute_columnar_delete` call `note_collection_write`, which /// records the LSN as applied. /// /// A record whose live execution affected ZERO rows notes no LSN and so diff --git a/nodedb/src/data/executor/core_loop/accessors.rs b/nodedb/src/data/executor/core_loop/accessors.rs index 9a50285c3..0c7ee2892 100644 --- a/nodedb/src/data/executor/core_loop/accessors.rs +++ b/nodedb/src/data/executor/core_loop/accessors.rs @@ -173,6 +173,22 @@ impl CoreLoop { self.quarantine_registry = Some(registry); } + /// Scope this core's fail points to node `node_id`, and the fail points + /// of the event producer it holds. Called once by the server bootstrap + /// before the core recovers. + pub fn set_fail_scope(&mut self, node_id: u64) { + self.fail_scope = nodedb_types::fail_point::FailScope::Node(node_id); + #[cfg(feature = "failpoints")] + if let Some(producer) = self.events.producer.as_mut() { + producer.set_fail_scope(self.fail_scope); + } + } + + /// The scope this core's fail points evaluate in. + pub fn fail_scope(&self) -> nodedb_types::fail_point::FailScope { + self.fail_scope + } + /// Install the shared set of collections some Event Plane consumer /// reads. Called once by the server bootstrap before the core recovers. pub fn set_event_interest(&mut self, interest: Arc) { diff --git a/nodedb/src/data/executor/core_loop/calvin_fence.rs b/nodedb/src/data/executor/core_loop/calvin_fence.rs deleted file mode 100644 index d6d49dca4..000000000 --- a/nodedb/src/data/executor/core_loop/calvin_fence.rs +++ /dev/null @@ -1,371 +0,0 @@ -// SPDX-License-Identifier: BUSL-1.1 - -//! Calvin key ownership on a core. -//! -//! A staged Calvin transaction owns the rows it staged from its stage until -//! its flush or drop. Its flush installs images the stage computed, so a -//! write that lands on one of those rows in between would be overwritten. -//! A write to an owned row therefore waits here, in arrival order. -//! -//! - A document or KV point write waits only when a staged transaction staged -//! its row, or truncated its collection. -//! - Every other write, and a committed session transaction's redo, waits -//! when a staged transaction writes its collection. -//! - A CRDT constraint-set install writes no row and never waits. -//! -//! When the owner resolves, each waiting write that no other owner fences -//! runs, in arrival order. A write whose WAL record sits below the redo record -//! of a flush it waited on is refused with `RetryableRefusal`: restart replay -//! applies records in LSN order, so it would install that write before the -//! flush. The refusal cancels its record, and the client retries. A write -//! still waiting at its request deadline is refused the same way. - -use std::collections::VecDeque; -use std::time::Instant; - -use nodedb_physical::physical_plan::{CrdtOp, DocumentOp, KvOp, MetaOp, PhysicalPlan}; -use nodedb_types::{RowIdentity, TenantId}; - -use super::CoreLoop; -use crate::bridge::envelope::{ErrorCode, Response}; -use crate::control::server::shared::write_admission::plan_is_write; -use crate::control::wal_replication::transaction_redo::collections::written_collections; -use crate::data::executor::handlers::control::calvin_synthetic_txn_id; -use crate::data::executor::handlers::transaction::stage_write::kv_row_identity; -use crate::data::executor::task::ExecutionTask; -use crate::types::Lsn; - -/// `(epoch, position, vshard)` of a staged Calvin transaction. -type Owner = (u64, u32, u32); - -/// A write waiting for the Calvin transaction that owns its rows. -pub(in crate::data::executor) struct ParkedWrite { - task: ExecutionTask, - owner: Owner, - /// The highest redo LSN among the flushes this write waited on. - passed_flush_lsn: Option, -} - -/// The writes waiting on this core, in arrival order, and the owners that -/// resolved since the core last released writes. -#[derive(Default)] -pub(in crate::data::executor) struct CalvinFence { - parked: VecDeque, - /// Each owner that flushed, with its redo LSN, or dropped, with `None`. - resolved: Vec<(Owner, Option)>, -} - -#[cfg(test)] -impl CalvinFence { - /// Number of waiting writes. - pub(in crate::data::executor) fn len(&self) -> usize { - self.parked.len() - } -} - -impl CalvinFence { - /// Record that the staged transaction `owner` flushed at `flush_lsn`, or - /// dropped when `flush_lsn` is `None`. The core releases its waiting - /// writes once the resolving request answered. - pub(in crate::data::executor) fn note_resolved( - &mut self, - owner: (u64, u32, u32), - flush_lsn: Option, - ) { - if !self.parked.is_empty() { - self.resolved.push((owner, flush_lsn)); - } - } -} - -/// What a write targets, as the fence compares it with staged state. -enum FenceTarget { - /// Every row of a collection. - Collection(String), - /// One document row, by surrogate. - Surrogate(String, u32), - /// One KV row, by its overlay identity. - Row(String, RowIdentity), -} - -impl FenceTarget { - fn collection(&self) -> &str { - match self { - Self::Collection(collection) - | Self::Surrogate(collection, _) - | Self::Row(collection, _) => collection, - } - } -} - -/// The targets of `plan`, or `None` when it writes no base state. -fn fence_targets(plan: &PhysicalPlan) -> Option> { - if let PhysicalPlan::Meta(MetaOp::ApplyTransactionRedo { collections, .. }) = plan { - return Some( - collections - .iter() - .cloned() - .map(FenceTarget::Collection) - .collect(), - ); - } - // A constraint-set install replaces the per-core CRDT validator's set for - // its collection, fenced by its own constraint version. It writes no row, - // so no flush can overwrite it. Parking it holds its data group's - // apply loop, and every later write in that group, behind the flush. - if matches!( - plan, - PhysicalPlan::Meta( - MetaOp::CalvinExecuteStatic { .. } - | MetaOp::CalvinExecuteActive { .. } - | MetaOp::CalvinFlush { .. } - | MetaOp::CalvinDrop { .. } - | MetaOp::CalvinResolve { .. } - ) | PhysicalPlan::Crdt(CrdtOp::SetConstraints { .. } | CrdtOp::DropConstraints { .. }) - ) || !plan_is_write(plan) - { - return None; - } - let point = match plan { - PhysicalPlan::Document( - DocumentOp::PointPut { - collection, - surrogate, - .. - } - | DocumentOp::PointInsert { - collection, - surrogate, - .. - }, - ) => vec![FenceTarget::Surrogate( - collection.as_str().to_string(), - surrogate.as_u32(), - )], - // A key unbound in this database names no row to fence. - PhysicalPlan::Document( - DocumentOp::PointUpdate { - collection, - surrogate, - .. - } - | DocumentOp::PointDelete { - collection, - surrogate, - .. - }, - ) => surrogate - .iter() - .map(|s| FenceTarget::Surrogate(collection.as_str().to_string(), s.as_u32())) - .collect(), - PhysicalPlan::Kv( - KvOp::Put { - collection, key, .. - } - | KvOp::Insert { - collection, key, .. - } - | KvOp::InsertIfAbsent { - collection, key, .. - } - | KvOp::Expire { - collection, key, .. - } - | KvOp::Persist { - collection, key, .. - }, - ) => vec![FenceTarget::Row( - collection.as_str().to_string(), - kv_row_identity(key), - )], - PhysicalPlan::Kv(KvOp::Delete { - collection, keys, .. - }) => keys - .iter() - .map(|key| FenceTarget::Row(collection.as_str().to_string(), kv_row_identity(key))) - .collect(), - _ => written_collections(std::slice::from_ref(plan)) - .into_iter() - .map(FenceTarget::Collection) - .collect(), - }; - Some(point) -} - -impl CoreLoop { - /// Hold `task` when a staged Calvin transaction owns a row it writes. - /// Returns the task when nothing owns its rows. - pub(in crate::data::executor) fn park_if_calvin_owned( - &mut self, - task: ExecutionTask, - ) -> Option { - match self.calvin_owner_of(&task) { - Some(owner) => { - // Raft fixed this write's order. Parking it holds its data - // group's applied index behind the owner's flush, so the - // capture names the write and its owner. - if matches!( - task.request.admission, - crate::bridge::envelope::Admission::Exempt( - crate::bridge::envelope::ExemptReason::AlreadyOrdered - ) - ) { - crate::diag::replicated_write_parked( - self.core_id, - task.plan() - .named_collections() - .first() - .copied() - .unwrap_or(""), - owner, - ); - } - self.calvin.fence.parked.push_back(ParkedWrite { - task, - owner, - passed_flush_lsn: None, - }); - None - } - None => Some(task), - } - } - - /// The earliest staged Calvin transaction that owns a row `task` writes. - fn calvin_owner_of(&self, task: &ExecutionTask) -> Option { - if self.calvin.commit_pending.is_empty() { - return None; - } - let targets = fence_targets(task.plan())?; - let database_id = task.request.database_id; - let tenant_id = task.request.tenant_id; - let mut owners: Vec = self.calvin.commit_pending.keys().copied().collect(); - owners.sort_unstable(); - owners - .into_iter() - .find(|owner| self.owns_any(*owner, database_id, tenant_id, &targets)) - } - - /// Whether the staged transaction `owner` owns any of `targets`. - fn owns_any( - &self, - owner: Owner, - database_id: crate::types::DatabaseId, - tenant_id: TenantId, - targets: &[FenceTarget], - ) -> bool { - let Some(pending) = self.calvin.commit_pending.get(&owner) else { - return false; - }; - if pending.tenant_id != tenant_id { - return false; - } - let written = written_collections(&pending.plans); - let overlay = calvin_synthetic_txn_id(owner.0, owner.1, owner.2) - .ok() - .and_then(|txn_id| self.txn_overlays.get(&txn_id)); - targets.iter().any(|target| { - if !written.iter().any(|c| c == target.collection()) { - return false; - } - let coll_key = (database_id, tenant_id, target.collection().to_string()); - match target { - FenceTarget::Collection(_) => true, - FenceTarget::Surrogate(_, surrogate) => overlay.is_none_or(|overlay| { - overlay.is_truncated(&coll_key) - || overlay.get(&coll_key, *surrogate).is_some() - || overlay.get_ttl(&coll_key, *surrogate).is_some() - }), - FenceTarget::Row(_, identity) => overlay.is_none_or(|overlay| { - overlay.is_truncated(&coll_key) - || overlay.surrogate_for_doc_id(&coll_key, identity).is_some() - }), - } - }) - } - - /// Run every waiting write no owner fences any more, once an owner - /// resolved. - pub(in crate::data::executor) fn release_resolved_calvin_owners(&mut self) { - let resolved = std::mem::take(&mut self.calvin.fence.resolved); - if resolved.is_empty() { - return; - } - for (owner, flush_lsn) in resolved { - let Some(lsn) = flush_lsn else { - continue; - }; - for parked in &mut self.calvin.fence.parked { - if parked.owner == owner { - parked.passed_flush_lsn = parked.passed_flush_lsn.max(Some(lsn)); - } - } - } - let parked = std::mem::take(&mut self.calvin.fence.parked); - for mut write in parked { - if let Some(next) = self.calvin_owner_of(&write.task) { - write.owner = next; - self.calvin.fence.parked.push_back(write); - continue; - } - let below_flush = write - .passed_flush_lsn - .is_some_and(|flush| write.task.wal_lsn().is_some_and(|lsn| lsn < flush)); - if below_flush { - let response = self.response_error( - &write.task, - ErrorCode::RetryableRefusal { - reason: "a Calvin transaction installed this row at a later log \ - position while the write waited" - .into(), - }, - ); - self.send_parked_response(response); - } else { - self.run_task(write.task); - } - } - } - - /// Refuse every waiting write whose request deadline passed. - pub(in crate::data::executor) fn expire_calvin_parked(&mut self) { - if self.calvin.fence.parked.is_empty() { - return; - } - // no-determinism: the request deadline bounds a wait; the refusal - // cancels the write's record, so no replica applies it. - let now = Instant::now(); - let parked = std::mem::take(&mut self.calvin.fence.parked); - for write in parked { - if now > write.task.request.deadline { - self.drop_journal_group(write.task.request_id().as_u64()); - let response = self.response_error( - &write.task, - ErrorCode::RetryableRefusal { - reason: format!( - "the write waited past its deadline for Calvin transaction \ - {}/{} that owns its row", - write.owner.0, write.owner.1 - ), - }, - ); - self.send_parked_response(response); - } else { - self.calvin.fence.parked.push_back(write); - } - } - } - - fn send_parked_response(&mut self, response: Response) { - if let Err(e) = self - .response_tx - .try_push(crate::bridge::dispatch::BridgeResponse { inner: response }) - { - tracing::warn!( - core = self.core_id, - error = %e, - "failed to send a parked write's refusal: response queue full" - ); - } - } -} diff --git a/nodedb/src/data/executor/core_loop/calvin_state.rs b/nodedb/src/data/executor/core_loop/calvin_state.rs index 212f96085..8e5b152f1 100644 --- a/nodedb/src/data/executor/core_loop/calvin_state.rs +++ b/nodedb/src/data/executor/core_loop/calvin_state.rs @@ -4,50 +4,29 @@ use std::collections::HashMap; -use super::calvin_fence::CalvinFence; use super::commit_pending::PendingCommit; -/// Staged Calvin transactions, the writes that wait on them, and the leader -/// flag of the transaction executing now. +/// Staged Calvin transactions this core holds. pub(in crate::data::executor) struct CalvinCoreState { /// Staged Calvin transactions awaiting the global verdict, keyed by /// `(epoch, position, vshard)`. `CalvinExecuteStatic` validates a /// transaction, stages its plans into the synthetic overlay, and inserts - /// its entry here without mutating base. The verdict-driven `CalvinFlush` - /// installs the transaction's redo record, and `CalvinDrop` discards the - /// staged state. Nothing staged here is observable in the base engines - /// until a flush. + /// its entry here without mutating base. The install of the slice's + /// stamped redo entry consumes the entry, and `CalvinDrop` discards it. + /// Nothing staged here is observable in the base engines until the + /// install. /// /// The vShard is part of the key because vShards round-robin onto cores. /// Several vShard slices of one multi-participant transaction share the /// same `(epoch, position)` and can land on the same core. Each slice - /// stages and flushes on its own. + /// stages and installs on its own. pub(in crate::data::executor) commit_pending: HashMap<(u64, u32, u32), PendingCommit>, - - /// Whether this node leads the data group of the Calvin transaction - /// executing now. - /// - /// `execute_calvin_execute_active` sets it from the scheduler-stamped, - /// per-node `is_group_leader` around its OLLP verification, then restores - /// the resting value `true`. A shard that runs a bulk DML directly has no - /// replication followers, so it must run OLLP drift verification. - /// - /// OLLP determinism: the verification emits `OllpRetryRequired` only when - /// this is `true`. Every replica stages the carried - /// `ollp_predicted_surrogates` set verbatim, so all replicas write the same - /// surrogate set whatever their local scans read. - pub(in crate::data::executor) ollp_is_group_leader: bool, - - /// Writes that wait for the staged transaction owning their rows. - pub(in crate::data::executor) fence: CalvinFence, } impl CalvinCoreState { pub(in crate::data::executor) fn new() -> Self { Self { commit_pending: HashMap::new(), - ollp_is_group_leader: true, - fence: CalvinFence::default(), } } } diff --git a/nodedb/src/data/executor/core_loop/commit_pending.rs b/nodedb/src/data/executor/core_loop/commit_pending.rs index 59f46f60b..0cc1abf45 100644 --- a/nodedb/src/data/executor/core_loop/commit_pending.rs +++ b/nodedb/src/data/executor/core_loop/commit_pending.rs @@ -7,14 +7,14 @@ use crate::types::TenantId; use nodedb_physical::physical_plan::PhysicalPlan; /// A Calvin transaction staged for commit, held between the -/// validate-and-stage step and the verdict-driven flush-or-drop. +/// validate-and-stage step and its install or drop. /// /// `CalvinExecuteStatic` and `CalvinExecuteActive` stage the plans into the /// synthetic overlay and insert this entry WITHOUT mutating base or firing /// side effects. `CalvinResolve` resolves the overlay into the transaction's -/// redo record. A verdict-driven `CalvinFlush` installs that record and -/// answers with `reply`. A `CalvinDrop` discards the entry. -/// Nothing here is observable in the base engines until a flush. +/// redo record. The install of the slice's stamped redo entry consumes the +/// entry and answers with `reply`. A `CalvinDrop` discards the entry. +/// Nothing here is observable in the base engines until the install. pub(in crate::data::executor) struct PendingCommit { /// The staged write plans `CalvinResolve` resolves against the overlay. pub plans: Vec, diff --git a/nodedb/src/data/executor/core_loop/event_emit.rs b/nodedb/src/data/executor/core_loop/event_emit.rs index 47ccea942..e50265e79 100644 --- a/nodedb/src/data/executor/core_loop/event_emit.rs +++ b/nodedb/src/data/executor/core_loop/event_emit.rs @@ -125,8 +125,13 @@ impl CoreLoop { } /// Set the Event Plane producer (called after open, before event loop). + /// The producer's fail points take this core's scope. pub fn set_event_producer(&mut self, producer: crate::event::bus::EventProducer) { self.events.producer = Some(producer); + #[cfg(feature = "failpoints")] + if let Some(producer) = self.events.producer.as_mut() { + producer.set_fail_scope(self.fail_scope); + } } /// Emit a write event carrying any [`crate::event::types::RowId`]. diff --git a/nodedb/src/data/executor/core_loop/event_emit_engines.rs b/nodedb/src/data/executor/core_loop/event_emit_engines.rs index 9f7697678..8cc1103c4 100644 --- a/nodedb/src/data/executor/core_loop/event_emit_engines.rs +++ b/nodedb/src/data/executor/core_loop/event_emit_engines.rs @@ -90,7 +90,7 @@ impl CoreLoop { /// The node-label write is already durable at its WAL LSN by the time this /// runs (the record was appended in the Control Plane before dispatch), so we /// advance the core watermark to it — exactly as every other write chokepoint - /// does via `note_write_lsn` — before emitting. That makes the forward + /// does via `note_write` — before emitting. That makes the forward /// event's LSN equal the WAL record's LSN the Event-Plane replay uses, /// satisfying watermark dedup. pub(in crate::data::executor) fn emit_graph_label_event( @@ -150,7 +150,7 @@ impl CoreLoop { /// surrogates from the plan. /// /// The caller MUST have advanced the core watermark to this edge's WAL LSN - /// (via `note_edge_write_lsn`) before calling, so the forward event's LSN + /// (via `note_edge_write`) before calling, so the forward event's LSN /// matches the WAL-replay reconstruction. pub(in crate::data::executor) fn emit_graph_edge_event( &mut self, diff --git a/nodedb/src/data/executor/core_loop/fail_stop.rs b/nodedb/src/data/executor/core_loop/fail_stop.rs index 5f81ddcff..c3474732c 100644 --- a/nodedb/src/data/executor/core_loop/fail_stop.rs +++ b/nodedb/src/data/executor/core_loop/fail_stop.rs @@ -16,8 +16,9 @@ //! records the node-wide marker that //! `/healthz`, the native `STATUS` and the `nodedb_data_plane_core_fail_stopped` //! gauge read, and latches the core. A latched core refuses every request it -//! dequeues with `RetryableRefusal`: it applied nothing, so the funnel +//! dequeues with `CoreFailStopped`: it applied nothing, so the funnel //! cancels the request's record and another replica or a restart serves it. +//! A Calvin scheduler reads the code and halts instead of voting. //! The latch never clears. use tracing::{error, warn}; @@ -140,7 +141,7 @@ impl CoreLoop { let Some((cause, _)) = self.fail_stop.cause() else { return 0; }; - let reason = format!( + let detail = format!( "core {} is fail-stopped ({}): its state is unknown until restart", self.core_id, cause.label() @@ -150,8 +151,9 @@ impl CoreLoop { self.drop_journal_group(task.request_id().as_u64()); let response = self.response_error( &task, - ErrorCode::RetryableRefusal { - reason: reason.clone(), + ErrorCode::CoreFailStopped { + core_id: self.core_id, + detail: detail.clone(), }, ); if let Err(e) = self diff --git a/nodedb/src/data/executor/core_loop/index_value_versions.rs b/nodedb/src/data/executor/core_loop/index_value_versions.rs index 023f7f616..e6bd0330e 100644 --- a/nodedb/src/data/executor/core_loop/index_value_versions.rs +++ b/nodedb/src/data/executor/core_loop/index_value_versions.rs @@ -3,19 +3,19 @@ //! Per-core, per-index write-VALUE version substrate. //! //! Sibling of [`super::write_index::WriteVersionIndex`]. That index records, -//! per written key/collection, the WAL LSN of the last committed write; this -//! one records — per `(database, tenant, collection, field)` secondary-index -//! dimension — the last committed-write LSN against each distinct indexed -//! VALUE. A later change validates an index-range read against the max LSN over -//! the read's value range instead of the coarse collection floor. +//! per written key/collection, the version of the last committed write; this +//! one records — per `(vshard, database, tenant, collection, field)` +//! secondary-index dimension — the last committed-write version against each +//! distinct indexed VALUE. An index-range read validates against the max +//! version over its value range instead of the coarse collection version. //! -//! Presence of the outer `(db, tenant, collection, field)` key is the -//! monotonic "tracked" flag: once any autocommit write records a value for a -//! field, that field stays tracked (horizon GC keeps the — now possibly empty -//! — inner map). An untracked field returns `None` (the validator falls back -//! to the collection floor); a tracked field with no in-range value returns -//! `Some(ZERO)`. A tracked field MUST have had every autocommit write's values -//! recorded — completeness is the recording chokepoints' contract. +//! Presence of the outer dimension key is the monotonic "tracked" flag: once +//! any committed write records a value for a field, that field stays tracked +//! (horizon GC keeps the, possibly empty, inner map). An untracked field +//! answers [`IndexVersion::Untracked`] (the validator falls back to the +//! collection version). A tracked field MUST have had every committed +//! write's values recorded: completeness is the recording chokepoints' +//! contract. //! //! Lives on the `!Send` core: plain `HashMap`/`BTreeMap`, no atomics/locks. @@ -23,59 +23,84 @@ use std::borrow::Borrow; use std::collections::{BTreeMap, HashMap}; use std::hash::{Hash, Hasher}; -use nodedb_types::{DatabaseId, TenantId}; +use nodedb_types::{DatabaseId, TenantId, WriteVersion}; -use crate::types::Lsn; +use crate::types::VShardId; use super::CoreLoop; - -/// Horizon retain window for per-value entries, in LSNs. Mirrors the per-key -/// index so both substrates age out together. -const RETAIN_WINDOW: u64 = 16_384; +use super::write_index::RETAIN_WINDOW; +use super::write_index::WriteStamp; /// Hard upper bound on total per-value entries across every tracked dimension. const MAX_INDEX_VALUE_ENTRIES: usize = 65_536; /// Per-index dimension key: one secondary-index `field` of a -/// `(database, tenant, collection)`. +/// `(database, tenant, collection)` on one vShard. #[derive(Debug, Clone, PartialEq, Eq)] pub struct IndexKey { + pub vshard: VShardId, pub db: DatabaseId, pub tenant: TenantId, pub collection: Box, pub field: Box, } +/// What the substrate knows about the values a read observed. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum IndexVersion { + /// No committed write recorded a value of the dimension. + Untracked, + /// The dimension is tracked. The highest version of the values the read + /// covers, `None` when none of them has a recorded version. + Tracked(Option), +} + /// The identity of an index dimension expressed as borrowable parts, so a /// `HashMap` keyed by owned [`IndexKey`] can be probed with borrowed `&str`s /// without allocating an owned key per lookup. trait IndexDim { - fn parts(&self) -> (DatabaseId, TenantId, &str, &str); + fn parts(&self) -> (VShardId, DatabaseId, TenantId, &str, &str); } impl IndexDim for IndexKey { - fn parts(&self) -> (DatabaseId, TenantId, &str, &str) { - (self.db, self.tenant, &self.collection, &self.field) + fn parts(&self) -> (VShardId, DatabaseId, TenantId, &str, &str) { + ( + self.vshard, + self.db, + self.tenant, + &self.collection, + &self.field, + ) } } -/// Borrowed probe into the per-index map — holds `&str`s, allocates nothing. -struct IndexDimRef<'a> { - db: DatabaseId, - tenant: TenantId, - collection: &'a str, - field: &'a str, +/// One index dimension, borrowed: the probe into the per-index map. Holds +/// `&str`s and allocates nothing. +#[derive(Debug, Clone, Copy)] +pub struct IndexDimRef<'a> { + pub vshard: VShardId, + pub db: DatabaseId, + pub tenant: TenantId, + pub collection: &'a str, + pub field: &'a str, } impl IndexDim for IndexDimRef<'_> { - fn parts(&self) -> (DatabaseId, TenantId, &str, &str) { - (self.db, self.tenant, self.collection, self.field) + fn parts(&self) -> (VShardId, DatabaseId, TenantId, &str, &str) { + ( + self.vshard, + self.db, + self.tenant, + self.collection, + self.field, + ) } } impl Hash for dyn IndexDim + '_ { fn hash(&self, state: &mut H) { - let (db, tenant, collection, field) = self.parts(); + let (vshard, db, tenant, collection, field) = self.parts(); + vshard.hash(state); db.hash(state); tenant.hash(state); collection.hash(state); @@ -106,75 +131,50 @@ impl Hash for IndexKey { /// Per-core, per-index write-VALUE version index. #[derive(Default)] pub struct IndexValueVersionIndex { - per_index: HashMap, Lsn>>, + per_index: HashMap, WriteVersion>>, } impl IndexValueVersionIndex { - /// Record a committed write of `value` on `(db, tenant, collection, field)` - /// at `lsn`. Marks the dimension tracked if new; advances the value slot to - /// `max(current, lsn)`. - pub fn record( - &mut self, - db: DatabaseId, - tenant: TenantId, - collection: &str, - field: &str, - value: &str, - lsn: Lsn, - ) { + /// Record a committed write of `value` on `dim` at `version`. Marks the + /// dimension tracked if new; advances the value slot to + /// `max(current, version)`. + pub fn record(&mut self, dim: IndexDimRef<'_>, value: &str, version: WriteVersion) { let key = IndexKey { - db, - tenant, - collection: Box::from(collection), - field: Box::from(field), + vshard: dim.vshard, + db: dim.db, + tenant: dim.tenant, + collection: Box::from(dim.collection), + field: Box::from(dim.field), }; let values = self.per_index.entry(key).or_default(); - let slot = values.entry(Box::from(value)).or_insert(Lsn::ZERO); - if lsn > *slot { - *slot = lsn; - } + let slot = values.entry(Box::from(value)).or_default(); + *slot = (*slot).max(version); } - /// Max committed-write LSN for an exact indexed value. `None` == the - /// `(collection, field)` dimension is UNtracked (validator falls back to the - /// collection floor). `Some(ZERO)` == tracked but this value has no recorded - /// write on this core (never invalidated → always current). - pub(in crate::data::executor) fn eq_max_lsn( + /// The recorded version of an exact indexed value. + pub(in crate::data::executor) fn eq_version( &self, - db: DatabaseId, - tenant: TenantId, - collection: &str, - field: &str, + dim: IndexDimRef<'_>, value: &str, - ) -> Option { - let values = self.per_index.get(&IndexDimRef { - db, - tenant, - collection, - field, - } as &dyn IndexDim)?; - Some(values.get(value).copied().unwrap_or(Lsn::ZERO)) + ) -> IndexVersion { + match self.dimension(dim) { + None => IndexVersion::Untracked, + Some(values) => IndexVersion::Tracked(values.get(value).copied()), + } } - /// Max committed-write LSN over the inclusive value range `[lo, hi]` (a - /// `None` bound is open). `None` == untracked dimension; `Some(ZERO)` == - /// tracked with no in-range recorded write. - pub(in crate::data::executor) fn range_max_lsn( + /// The highest recorded version over the inclusive value range `[lo, hi]` + /// (a `None` bound is open). + pub(in crate::data::executor) fn range_version( &self, - db: DatabaseId, - tenant: TenantId, - collection: &str, - field: &str, + dim: IndexDimRef<'_>, lo: Option<&str>, hi: Option<&str>, - ) -> Option { + ) -> IndexVersion { use std::ops::Bound; - let values = self.per_index.get(&IndexDimRef { - db, - tenant, - collection, - field, - } as &dyn IndexDim)?; + let Some(values) = self.dimension(dim) else { + return IndexVersion::Untracked; + }; let lo_b = match lo { Some(s) => Bound::Included(s), None => Bound::Unbounded, @@ -183,73 +183,91 @@ impl IndexValueVersionIndex { Some(s) => Bound::Included(s), None => Bound::Unbounded, }; - let mut max = Lsn::ZERO; - for (_v, lsn) in values.range::((lo_b, hi_b)) { - if *lsn > max { - max = *lsn; - } - } - Some(max) + IndexVersion::Tracked( + values + .range::((lo_b, hi_b)) + .map(|(_value, version)| *version) + .max(), + ) } - /// Test accessor: the recorded LSN for a `(db, tenant, collection, field)` - /// dimension's `value`, or `None` if the dimension or value is untracked. + fn dimension(&self, dim: IndexDimRef<'_>) -> Option<&BTreeMap, WriteVersion>> { + self.per_index.get(&dim as &dyn IndexDim) + } + + /// Test accessor: the recorded version of a dimension's `value`, or + /// `None` if the dimension or value is untracked. #[cfg(test)] - pub(in crate::data::executor) fn value_lsn( + pub(in crate::data::executor) fn value_version( &self, - db: DatabaseId, - tenant: TenantId, - collection: &str, - field: &str, + dim: IndexDimRef<'_>, value: &str, - ) -> Option { - self.per_index - .get(&IndexDimRef { - db, - tenant, - collection, - field, - } as &dyn IndexDim) + ) -> Option { + self.dimension(dim) .and_then(|values| values.get(value).copied()) } - /// Horizon GC against `watermark`, mirroring the per-key index. Evicts value - /// entries below `watermark - RETAIN_WINDOW`, keeping the (possibly empty) - /// inner map so the dimension stays tracked. A count backstop drops the - /// lowest-LSN entries under a TOTAL order so tied-LSN eviction is - /// replica-identical. - pub fn gc(&mut self, watermark: Lsn) { - let floor = watermark.as_u64().saturating_sub(RETAIN_WINDOW); + /// Forget every dimension of `vshard`: a snapshot install replaced its + /// rows. + pub(in crate::data::executor) fn drop_vshard(&mut self, vshard: VShardId) { + self.per_index.retain(|key, _| key.vshard != vshard); + } + + /// Horizon GC against each vShard's `latest` version, mirroring the + /// per-key index. Evicts value entries more than the retain window below + /// their vShard's latest version, keeping the (possibly empty) inner map + /// so the dimension stays tracked. A count backstop drops the lowest + /// versions under a TOTAL order so tied-version eviction is + /// replica-identical. Returns the highest version evicted per vShard. + pub fn gc( + &mut self, + latest: &HashMap, + ) -> HashMap { + let mut evicted: HashMap = HashMap::new(); let mut total = 0usize; - for values in self.per_index.values_mut() { - values.retain(|_, lsn| lsn.as_u64() >= floor); + for (key, values) in self.per_index.iter_mut() { + let latest = latest.get(&key.vshard).copied(); + values.retain(|_, version| { + let keep = latest + .and_then(|latest| latest.distance_above(*version)) + .is_none_or(|distance| distance <= RETAIN_WINDOW); + if !keep { + let slot = evicted.entry(key.vshard).or_default(); + *slot = (*slot).max(*version); + } + keep + }); total += values.len(); } if total > MAX_INDEX_VALUE_ENTRIES { let overflow = total - MAX_INDEX_VALUE_ENTRIES; - let mut all: Vec<(Lsn, IndexKey, Box)> = self + let mut all: Vec<(WriteVersion, IndexKey, Box)> = self .per_index .iter() .flat_map(|(k, values)| { values .iter() - .map(move |(v, lsn)| (*lsn, k.clone(), v.clone())) + .map(move |(v, version)| (*version, k.clone(), v.clone())) }) .collect(); all.sort_by(|a, b| { a.0.cmp(&b.0) + .then_with(|| a.1.vshard.cmp(&b.1.vshard)) .then_with(|| a.1.db.as_u64().cmp(&b.1.db.as_u64())) .then_with(|| a.1.tenant.as_u64().cmp(&b.1.tenant.as_u64())) .then_with(|| a.1.collection.cmp(&b.1.collection)) .then_with(|| a.1.field.cmp(&b.1.field)) .then_with(|| a.2.cmp(&b.2)) }); - for (_lsn, key, value) in all.into_iter().take(overflow) { + for (version, key, value) in all.into_iter().take(overflow) { + let slot = evicted.entry(key.vshard).or_default(); + *slot = (*slot).max(version); if let Some(values) = self.per_index.get_mut(&key) { values.remove(&value); } } } + evicted } } @@ -264,12 +282,21 @@ impl CoreLoop { tenant: TenantId, collection: &str, tuples: &[(String, String)], - lsn: Lsn, + stamp: WriteStamp, ) { + if tuples.is_empty() { + return; + } + let version = self.write_version_of(stamp); for (field, value) in tuples { - self.write_index - .index_values - .record(db, tenant, collection, field, value, lsn); + let dim = IndexDimRef { + vshard: stamp.vshard, + db, + tenant, + collection, + field, + }; + self.write_index.index_values.record(dim, value, version); } } } @@ -286,8 +313,17 @@ mod tests { TenantId::new(1) } + fn vshard() -> VShardId { + VShardId::new(3) + } + + fn at(index: u64) -> WriteVersion { + WriteVersion::logged(0, index) + } + fn key(field: &str) -> IndexKey { IndexKey { + vshard: vshard(), db: db(), tenant: tenant(), collection: Box::from("orders"), @@ -295,69 +331,72 @@ mod tests { } } + fn email() -> IndexDimRef<'static> { + IndexDimRef { + vshard: vshard(), + db: db(), + tenant: tenant(), + collection: "orders", + field: "email", + } + } + + fn record(index: &mut IndexValueVersionIndex, value: &str, version: WriteVersion) { + index.record(email(), value, version); + } + + fn latest(version: WriteVersion) -> HashMap { + HashMap::from([(vshard(), version)]) + } + #[test] - fn record_tracks_dimension_and_stores_value_lsn() { + fn record_tracks_dimension_and_stores_value_version() { let mut index = IndexValueVersionIndex::default(); - index.record(db(), tenant(), "orders", "email", "a@b.c", Lsn::new(10)); + record(&mut index, "a@b.c", at(10)); let values = index .per_index .get(&key("email")) .expect("dimension tracked after record"); - assert_eq!(values.get("a@b.c"), Some(&Lsn::new(10))); + assert_eq!(values.get("a@b.c"), Some(&at(10))); } #[test] fn record_is_monotonic_max_per_value() { let mut index = IndexValueVersionIndex::default(); - index.record(db(), tenant(), "orders", "email", "a@b.c", Lsn::new(10)); - index.record(db(), tenant(), "orders", "email", "a@b.c", Lsn::new(30)); - // A later record with a LOWER LSN must not regress the slot. - index.record(db(), tenant(), "orders", "email", "a@b.c", Lsn::new(20)); + record(&mut index, "a@b.c", at(10)); + record(&mut index, "a@b.c", at(30)); + record(&mut index, "a@b.c", at(20)); let values = index.per_index.get(&key("email")).expect("tracked"); - assert_eq!(values.get("a@b.c"), Some(&Lsn::new(30))); + assert_eq!(values.get("a@b.c"), Some(&at(30))); } #[test] - fn gc_horizon_evicts_but_keeps_dimension_tracked() { + fn another_vshard_does_not_track_the_dimension() { let mut index = IndexValueVersionIndex::default(); - index.record(db(), tenant(), "orders", "email", "a@b.c", Lsn::new(1)); - // Watermark far above the entry + retain window → entry evicted. - index.gc(Lsn::new(RETAIN_WINDOW + 100)); - let values = index - .per_index - .get(&key("email")) - .expect("dimension stays tracked after gc"); - assert!(values.is_empty(), "evicted value must not remain"); + record(&mut index, "a@b.c", at(10)); + let other = IndexDimRef { + vshard: VShardId::new(4), + ..email() + }; + assert_eq!(index.eq_version(other, "a@b.c"), IndexVersion::Untracked); } #[test] - fn gc_horizon_retains_entries_above_floor() { + fn gc_evicts_old_values_keeps_dimension_tracked_and_reports_the_bound() { let mut index = IndexValueVersionIndex::default(); - let watermark = Lsn::new(RETAIN_WINDOW + 100); - // Below floor (evicted) and above floor (retained). - index.record(db(), tenant(), "orders", "email", "old", Lsn::new(1)); - index.record( - db(), - tenant(), - "orders", - "email", - "new", - Lsn::new(watermark.as_u64()), - ); - index.gc(watermark); + record(&mut index, "old", at(1)); + record(&mut index, "new", at(RETAIN_WINDOW + 100)); + let evicted = index.gc(&latest(at(RETAIN_WINDOW + 100))); let values = index.per_index.get(&key("email")).expect("tracked"); assert_eq!(values.get("old"), None); - assert_eq!(values.get("new"), Some(&watermark)); + assert_eq!(values.get("new"), Some(&at(RETAIN_WINDOW + 100))); + assert_eq!(evicted.get(&vshard()), Some(&at(1))); } #[test] fn gc_count_backstop_is_insert_order_independent() { - // Overflow the count backstop by one, with all entries tied at the same - // LSN so eviction order falls entirely to the total order over - // (db, tenant, collection, field, value). The surviving set must be the - // same regardless of insertion order. let n = MAX_INDEX_VALUE_ENTRIES + 1; - let lsn = Lsn::new(1_000_000); + let version = at(1_000_000); let build = |ascending: bool| { let mut index = IndexValueVersionIndex::default(); @@ -366,12 +405,9 @@ mod tests { order.reverse(); } for i in order { - let value = format!("{i:08}"); - index.record(db(), tenant(), "orders", "email", &value, lsn); + record(&mut index, &format!("{i:08}"), version); } - // Watermark keeps every entry above the horizon floor, so only the - // count backstop fires. - index.gc(lsn); + index.gc(&latest(version)); index }; @@ -379,11 +415,10 @@ mod tests { let desc = build(false); assert_eq!(asc.per_index, desc.per_index); - // The lowest value under the total order is dropped, the next survives. let dropped = format!("{:08}", 0); let survivor = format!("{:08}", 1); let values = asc.per_index.get(&key("email")).expect("tracked"); assert_eq!(values.get(dropped.as_str()), None); - assert_eq!(values.get(survivor.as_str()), Some(&lsn)); + assert_eq!(values.get(survivor.as_str()), Some(&version)); } } diff --git a/nodedb/src/data/executor/core_loop/mod.rs b/nodedb/src/data/executor/core_loop/mod.rs index cf3575947..afdbb8fcc 100644 --- a/nodedb/src/data/executor/core_loop/mod.rs +++ b/nodedb/src/data/executor/core_loop/mod.rs @@ -3,7 +3,6 @@ mod accessors; pub(in crate::data::executor) mod apply_scope; mod bitemporal_time; -pub(in crate::data::executor) mod calvin_fence; pub(in crate::data::executor) mod calvin_state; pub(in crate::data::executor) mod checkpoint_floors; mod columnar_schema_seed; diff --git a/nodedb/src/data/executor/core_loop/open.rs b/nodedb/src/data/executor/core_loop/open.rs index b453fba4c..c5018d34f 100644 --- a/nodedb/src/data/executor/core_loop/open.rs +++ b/nodedb/src/data/executor/core_loop/open.rs @@ -96,6 +96,9 @@ impl CoreLoop { Ok(Self { core_id, + // A core opened without a node fires only actions armed for every + // node. The server bootstrap scopes it with `set_fail_scope`. + fail_scope: nodedb_types::fail_point::FailScope::Any, request_rx, response_tx, task_queue: PriorityQueues::new(), diff --git a/nodedb/src/data/executor/core_loop/pressure/fixtures.rs b/nodedb/src/data/executor/core_loop/pressure/fixtures.rs index ef9da6963..4d61bfeda 100644 --- a/nodedb/src/data/executor/core_loop/pressure/fixtures.rs +++ b/nodedb/src/data/executor/core_loop/pressure/fixtures.rs @@ -91,6 +91,7 @@ pub(super) fn make_stub_request(id: u64) -> Request { wal_lsn: None, resolved_now_ms: None, commit_hlc: None, + entry_version: None, admission: Admission::Exempt(ExemptReason::Read), } } diff --git a/nodedb/src/data/executor/core_loop/priority_queues.rs b/nodedb/src/data/executor/core_loop/priority_queues.rs index e5abaf441..7febb77c9 100644 --- a/nodedb/src/data/executor/core_loop/priority_queues.rs +++ b/nodedb/src/data/executor/core_loop/priority_queues.rs @@ -359,6 +359,7 @@ mod tests { wal_lsn: None, resolved_now_ms: None, commit_hlc: None, + entry_version: None, admission: crate::bridge::envelope::Admission::Exempt( crate::bridge::envelope::ExemptReason::Read, ), diff --git a/nodedb/src/data/executor/core_loop/response.rs b/nodedb/src/data/executor/core_loop/response.rs index c1aecc230..05d2251ab 100644 --- a/nodedb/src/data/executor/core_loop/response.rs +++ b/nodedb/src/data/executor/core_loop/response.rs @@ -42,9 +42,9 @@ impl CoreLoop { partial: false, payload: Payload::empty(), watermark_lsn: self.watermark, - read_version_lsn: self.read_version_lsn(task), + read_versions: self.read_versions(task), error_code: None, - read_set_valid: None, + stage_vote: None, write_set: Vec::new(), } } @@ -61,9 +61,9 @@ impl CoreLoop { partial: false, payload: Payload::from_vec(payload), watermark_lsn: self.watermark, - read_version_lsn: self.read_version_lsn(task), + read_versions: self.read_versions(task), error_code: None, - read_set_valid: None, + stage_vote: None, write_set: Vec::new(), } } @@ -106,56 +106,110 @@ impl CoreLoop { partial: true, payload: Payload::from_vec(payload), watermark_lsn: self.watermark, - read_version_lsn: self.read_version_lsn(task), + read_versions: self.read_versions(task), error_code: None, - read_set_valid: None, + stage_vote: None, write_set: Vec::new(), } } - /// Per-collection read-version LSN for `task`'s plan (distinct from the - /// core-global `watermark`). A graph read scoped to one collection reports - /// that collection's version: its edges are the collection's rows. On a write response this is the POST-write - /// version. A resolved KV write's version is the max across its - /// mutations, which may span two collections. - pub(in crate::data::executor) fn read_version_lsn( + /// The write versions `task` observed, one per vShard (distinct from the + /// core-global `watermark`). + /// + /// A write reports the POST-write version of each collection it wrote, on + /// its own vShard. A graph read reports every vShard's latest version on + /// this core: its edges live on their endpoints' vShards. An array read + /// reports the array's version on the vShard its task covers, held or + /// not. Any other read reports its + /// collection's version on the collection's home vShard, where the read + /// validates. A core reports only vShards it holds versions of. + pub(in crate::data::executor) fn read_versions( &self, task: &ExecutionTask, - ) -> crate::types::Lsn { + ) -> crate::types::ReadVersions { + let mut versions = crate::types::ReadVersions::new(); + let plan = task.plan(); + if task.wal_lsn().is_none() + && let crate::bridge::envelope::PhysicalPlan::Graph(_) = plan + { + for (vshard, latest) in self.write_index.latest_by_vshard() { + versions.note(vshard, latest); + } + return versions; + } + if task.wal_lsn().is_none() + && matches!( + plan, + crate::bridge::envelope::PhysicalPlan::Array(_) + | crate::bridge::envelope::PhysicalPlan::ClusterArray(_) + ) + && let Some(array) = plan.collection() + { + // The task's vShard is the one the read covers: the array's own + // vShard, or the shard a cluster slice leg reads. It is reported + // even when no write of it reached this core, at the bound its + // state holds, so a later first write of it makes the read stale. + let vshard = task.request.vshard_id; + versions.note( + vshard, + self.write_index.collection_current( + task.request.database_id, + task.request.tenant_id, + vshard, + array, + ), + ); + return versions; + } + let mut note = |collection: &str| { + let vshard = match task.wal_lsn() { + Some(_) => task.request.vshard_id, + None => match nodedb_types::CollectionKey::from_qualified_str( + task.request.database_id, + collection, + ) { + Ok(key) => key.vshard(), + Err(_) => return, + }, + }; + if self.write_index.holds(vshard) { + versions.note( + vshard, + self.write_index.collection_current( + task.request.database_id, + task.request.tenant_id, + vshard, + collection, + ), + ); + } + }; if let crate::bridge::envelope::PhysicalPlan::Kv( nodedb_physical::physical_plan::KvOp::ResolvedWrite { mutations, .. }, - ) = task.plan() + ) = plan { - return mutations - .iter() - .map(|m| self.collection_read_version(task, m.collection().as_str())) - .max() - .unwrap_or(crate::types::Lsn::ZERO); + for mutation in mutations { + note(mutation.collection().as_str()); + } + } else if let Some(collection) = plan.collection() { + note(collection); } - task.plan() - .collection() - .or_else(|| graph_read_collection(task.plan())) - .map(|c| self.collection_read_version(task, c)) - .unwrap_or(crate::types::Lsn::ZERO) - } - - /// One collection's recorded write LSN on this core, or `Lsn::ZERO` when - /// it has none. - fn collection_read_version(&self, task: &ExecutionTask, collection: &str) -> crate::types::Lsn { - self.write_index - .collection_write_lsn(&super::write_index::CollKey { - db: task.request.database_id, - tenant: task.request.tenant_id, - collection: Box::from(collection), - }) - .unwrap_or(crate::types::Lsn::ZERO) + versions } + /// An error answer to `task`. A `NotFound` answer is a read that found + /// no row, which a transaction validates, so it carries the versions the + /// read observed like any answered read. pub(in crate::data::executor) fn response_error( &self, task: &ExecutionTask, error_code: impl Into, ) -> Response { + let error_code = error_code.into(); + let read_versions = match error_code { + ErrorCode::NotFound => self.read_versions(task), + _ => crate::types::ReadVersions::new(), + }; Response { request_id: task.request_id(), status: Status::Error, @@ -163,9 +217,9 @@ impl CoreLoop { partial: false, payload: Payload::empty(), watermark_lsn: self.watermark, - read_version_lsn: crate::types::Lsn::ZERO, - error_code: Some(Box::new(error_code.into())), - read_set_valid: None, + read_versions, + error_code: Some(Box::new(error_code)), + stage_vote: None, write_set: Vec::new(), } } @@ -310,23 +364,6 @@ impl CoreLoop { } } -/// The collection a collection-scoped graph read walks. Graph plans report no -/// collection to routing (an edge homes on its endpoints), but a scoped read -/// still observes that one collection's edges. -fn graph_read_collection(plan: &crate::bridge::envelope::PhysicalPlan) -> Option<&str> { - use nodedb_physical::physical_plan::GraphOp; - match plan { - crate::bridge::envelope::PhysicalPlan::Graph( - GraphOp::Hop { collection, .. } - | GraphOp::Neighbors { collection, .. } - | GraphOp::NeighborsMulti { collection, .. } - | GraphOp::Path { collection, .. } - | GraphOp::Subgraph { collection, .. }, - ) => collection.as_ref().map(|c| c.as_str()), - _ => None, - } -} - #[cfg(test)] mod tests { use super::super::tests::{make_core_with_dir, make_default_task}; diff --git a/nodedb/src/data/executor/core_loop/state.rs b/nodedb/src/data/executor/core_loop/state.rs index 19fefc4e2..612fb50d3 100644 --- a/nodedb/src/data/executor/core_loop/state.rs +++ b/nodedb/src/data/executor/core_loop/state.rs @@ -38,7 +38,8 @@ use super::priority_queues::PriorityQueues; /// This type is intentionally `!Send` — pinned to a single core. pub struct CoreLoop { pub(in crate::data::executor) core_id: usize, - + /// The node this core runs on: the scope its fail points evaluate in. + pub(in crate::data::executor) fail_scope: nodedb_types::fail_point::FailScope, /// SPSC channel: receives requests from Control Plane. pub(in crate::data::executor) request_rx: Consumer, @@ -429,7 +430,7 @@ pub struct CoreLoop { /// /// Set to `Some(ms)` by `execute_calvin_execute_static`, /// `execute_calvin_execute_active` and `execute_calvin_resolve` while they - /// stage or resolve a transaction's plans, and by `execute_calvin_flush` + /// stage or resolve a transaction's plans, and by `install_calvin_redo` /// while it renders the reply, then restored immediately after. Engine handlers that need "current time" (bitemporal sys_from, /// KV TTL expire_at, timeseries system_ms) call /// `self.epoch_system_ms.unwrap_or_else()` so that @@ -478,8 +479,8 @@ pub struct CoreLoop { pub(in crate::data::executor) ts_resolve_holds: HashMap<(DatabaseId, TenantId, String), std::collections::HashSet>, - /// Per-core last-write-LSN version index (per key + per collection), - /// advanced by every committed write-apply. Type + GC in `write_index.rs`. + /// Per-core write-version index (per key + per collection, per vShard), + /// advanced by every committed write-apply. Type + GC in `write_index/`. pub(in crate::data::executor) write_index: super::write_index::WriteVersionIndex, /// The version keys the current record's apply forces. diff --git a/nodedb/src/data/executor/core_loop/tick.rs b/nodedb/src/data/executor/core_loop/tick.rs index e09a7a8c8..69cddf6ee 100644 --- a/nodedb/src/data/executor/core_loop/tick.rs +++ b/nodedb/src/data/executor/core_loop/tick.rs @@ -58,14 +58,10 @@ impl CoreLoop { }; self.io_metrics.record_wait(tier, wait_ns); - // A write to a row a staged Calvin transaction owns waits for it. // A plain REINDEX starts its rebuilds and waits for their cutovers. - if let Some(task) = self.park_if_calvin_owned(qt.task) - && let Some(task) = self.hold_plain_reindex(task) - { + if let Some(task) = self.hold_plain_reindex(qt.task) { self.run_task(task); } - self.release_resolved_calvin_owners(); true } @@ -101,8 +97,8 @@ impl CoreLoop { watermark_lsn: self.watermark, // The task never started, so nothing it would write ran. error_code: Some(Box::new(ErrorCode::ExpiredBeforeExecution)), - read_set_valid: None, - read_version_lsn: crate::types::Lsn::ZERO, + stage_vote: None, + read_versions: crate::types::ReadVersions::new(), write_set: Vec::new(), } } else { @@ -128,7 +124,10 @@ impl CoreLoop { #[cfg(feature = "failpoints")] if task.wal_lsn().is_some() { for collection in task.plan().named_collections() { - crate::fail_point!(&format!("core::after_apply::{collection}")); + crate::fail_point!( + self.fail_scope, + &format!("core::after_apply::{collection}") + ); } } if let Some(group) = journal { @@ -137,7 +136,10 @@ impl CoreLoop { // is stored and its parts are not journalled yet. #[cfg(feature = "failpoints")] for collection in task.plan().named_collections() { - crate::fail_point!(&format!("core::after_capture::{collection}")); + crate::fail_point!( + self.fail_scope, + &format!("core::after_capture::{collection}") + ); } } task.state = TaskState::Completed; @@ -184,7 +186,6 @@ impl CoreLoop { // Adjust SPSC read depth based on current memory pressure. self.apply_spsc_pressure(); self.drain_requests(); - self.expire_calvin_parked(); let mut processed = 0; while !self.task_queue.is_empty() { // A fail-stopped core serves nothing, including the rest of the @@ -271,6 +272,7 @@ mod tests { wal_lsn: None, resolved_now_ms: None, commit_hlc: None, + entry_version: None, admission: crate::bridge::envelope::Admission::Admitted, } } @@ -337,7 +339,7 @@ mod tests { assert!( matches!( resp.inner.error_code.as_deref(), - Some(ErrorCode::RetryableRefusal { .. }) + Some(ErrorCode::CoreFailStopped { .. }) ), "{:?}", resp.inner.error_code diff --git a/nodedb/src/data/executor/core_loop/write_index.rs b/nodedb/src/data/executor/core_loop/write_index.rs deleted file mode 100644 index 2ac9f7d67..000000000 --- a/nodedb/src/data/executor/core_loop/write_index.rs +++ /dev/null @@ -1,1359 +0,0 @@ -// SPDX-License-Identifier: BUSL-1.1 - -//! Per-core last-write-LSN version index. -//! -//! Records, for every committed write applied on this Data-Plane core, the WAL -//! LSN of the write against the written key (`last_write_lsn`) and against the -//! written collection (`coll_write_lsn`). The WAL LSN is allocated in the -//! Control Plane at wal-dispatch time and threaded onto the write task; the -//! apply chokepoints on this core feed it here. -//! -//! This is the shard-local write-version substrate the optimistic-concurrency -//! commit path validates a transaction's read-set against (see -//! [`CoreLoop::read_set_still_current`]). Because the index lives on the -//! `!Send` core it is a plain `HashMap`: no atomics, no locks, no cross-core -//! sharing. -//! -//! The per-key map is bounded: horizon GC (run from the periodic maintenance -//! hook) evicts entries far below the core watermark and enforces a hard -//! entry-count backstop. The per-collection map is bounded by the number of -//! live collections and is never LSN-GC'd. - -use std::collections::HashMap; - -use nodedb_types::calvin::{ReadKeyIdent, VersionedReadEntry}; -use nodedb_types::{DatabaseId, TenantId}; - -use crate::types::Lsn; - -use super::CoreLoop; - -/// Row identity type, re-exported from its plane-neutral home -/// ([`crate::types::KeyRepr`]) so Data-Plane call sites can keep referring to -/// it through this module. Read keys and write keys share this one namespace. -pub use crate::types::KeyRepr; - -/// Horizon retain window for per-key entries, in LSNs. Horizon GC evicts any -/// `last_write_lsn` entry whose LSN is more than this far below the core -/// watermark. Sized in the same order of magnitude as the idempotency-cache -/// cap (16,384 entries): a bounded recent-write history — enough to validate -/// in-flight transactions — not an unbounded write log. -const RETAIN_WINDOW: u64 = 16_384; - -/// Hard upper bound on the `last_write_lsn` entry count. When horizon GC leaves -/// more entries than this (a burst of distinct keys all inside the retain -/// window), the lowest-LSN (oldest) entries are dropped until the map is back -/// within bound. -const MAX_KEY_ENTRIES: usize = 65_536; - -/// Fully-qualified per-key version-index key. Scoped by `(database, tenant)` -/// exactly like the write path, so two tenants (or databases) never alias. -#[derive(Debug, Clone, PartialEq, Eq, Hash)] -pub struct WriteKey { - pub db: DatabaseId, - pub tenant: TenantId, - pub collection: Box, - pub key: KeyRepr, -} - -/// Fully-qualified per-collection version-index key. -#[derive(Debug, Clone, PartialEq, Eq, Hash)] -pub struct CollKey { - pub db: DatabaseId, - pub tenant: TenantId, - pub collection: Box, -} - -/// Per-core last-write-LSN version index. -#[derive(Default)] -pub struct WriteVersionIndex { - /// Last committed-write LSN per written key. - last_write_lsn: HashMap, - /// Last committed-write LSN per written collection (the phantom-safe floor: - /// a predicate reader validates against this when it owns no per-key entry). - coll_write_lsn: HashMap, - /// Per-secondary-index-dimension write-VALUE versions — the finer-grained - /// sibling of `coll_write_lsn` an index-range read validates against. - pub(in crate::data::executor) index_values: super::index_value_versions::IndexValueVersionIndex, -} - -impl WriteVersionIndex { - pub fn new() -> Self { - Self::default() - } - - /// Record a committed write at `lsn`. - /// - /// Always advances the collection floor `coll_write_lsn[collection]` to the - /// max of its current value and `lsn`. When `key` is `Some`, also advances - /// the per-key version `last_write_lsn[key]` monotonically. Advancing the - /// core watermark is the caller's responsibility (see - /// [`CoreLoop::note_write_lsn`]). - pub fn note_write_lsn( - &mut self, - db: DatabaseId, - tenant: TenantId, - collection: &str, - key: Option, - lsn: Lsn, - ) { - let coll_key = CollKey { - db, - tenant, - collection: Box::from(collection), - }; - let slot = self.coll_write_lsn.entry(coll_key).or_insert(Lsn::ZERO); - if lsn > *slot { - *slot = lsn; - } - - if let Some(key) = key { - let write_key = WriteKey { - db, - tenant, - collection: Box::from(collection), - key, - }; - let slot = self.last_write_lsn.entry(write_key).or_insert(Lsn::ZERO); - if lsn > *slot { - *slot = lsn; - } - } - } - - /// Current per-key version, if recorded. - pub(crate) fn key_write_lsn(&self, key: &WriteKey) -> Option { - self.last_write_lsn.get(key).copied() - } - - /// Current per-collection floor version, if recorded. - pub(crate) fn collection_write_lsn(&self, key: &CollKey) -> Option { - self.coll_write_lsn.get(key).copied() - } - - /// Whether a previously observed read is still current against this - /// core's recorded write versions. - /// - /// A `Point` read is current iff no write to that exact key has been - /// recorded since the read (`last_write_lsn <= read_lsn`); a key with no - /// recorded write has never been written on this core since the read, so - /// it is treated as version zero and is always current. A `Predicate` - /// read is current iff no write to the collection has been recorded - /// since the read, checked the same way against the collection floor. - pub(crate) fn read_is_valid( - &self, - db: DatabaseId, - tenant: TenantId, - collection: &str, - key: &ReadKeyIdent, - read_lsn: Lsn, - ) -> bool { - // Collection-floor check shared by `Predicate` and the fallback for - // untracked secondary-index dimensions: a read is current iff no write - // to the collection has been recorded since it. `IndexEq` / `IndexRange` - // consult the per-value substrate first and only fall back here when the - // `(collection, field)` dimension is untracked. - let collection_floor_current = || { - let coll_key = CollKey { - db, - tenant, - collection: Box::from(collection), - }; - self.collection_write_lsn(&coll_key).unwrap_or(Lsn::ZERO) <= read_lsn - }; - - match key { - ReadKeyIdent::Point(repr) => { - let write_key = WriteKey { - db, - tenant, - collection: Box::from(collection), - key: repr.clone(), - }; - self.key_write_lsn(&write_key).unwrap_or(Lsn::ZERO) <= read_lsn - } - ReadKeyIdent::Predicate => collection_floor_current(), - ReadKeyIdent::IndexEq { field, value } => { - match self - .index_values - .eq_max_lsn(db, tenant, collection, field, value) - { - Some(max) => max <= read_lsn, - None => collection_floor_current(), - } - } - ReadKeyIdent::IndexRange { field, lo, hi } => { - match self.index_values.range_max_lsn( - db, - tenant, - collection, - field, - lo.as_deref(), - hi.as_deref(), - ) { - Some(max) => max <= read_lsn, - None => collection_floor_current(), - } - } - } - } - - /// Horizon garbage-collect the per-key map against `watermark`. - /// - /// Evicts every entry whose LSN falls below `watermark - RETAIN_WINDOW`, - /// then, if more than [`MAX_KEY_ENTRIES`] remain, drops the lowest-LSN - /// entries until back within bound. The per-collection map is bounded by - /// the live-collection count and is intentionally left untouched. - pub fn gc(&mut self, watermark: Lsn) { - let floor = watermark.as_u64().saturating_sub(RETAIN_WINDOW); - self.last_write_lsn.retain(|_, lsn| lsn.as_u64() >= floor); - - if self.last_write_lsn.len() > MAX_KEY_ENTRIES { - let overflow = self.last_write_lsn.len() - MAX_KEY_ENTRIES; - // Drop the `overflow` oldest (lowest-LSN) entries. - let mut by_lsn: Vec<(Lsn, WriteKey)> = self - .last_write_lsn - .iter() - .map(|(k, lsn)| (*lsn, k.clone())) - .collect(); - // TOTAL order so tied-LSN eviction is replica-identical: a plain - // `sort_by_key(lsn)` over a `HashMap`-collected Vec would let the - // dropped set depend on hash-iteration layout, diverging across - // replicas. `DatabaseId`/`TenantId` lack `Ord` (compare via - // `as_u64()`); `KeyRepr` is `Ord`. - by_lsn.sort_by(|a, b| { - a.0.cmp(&b.0) - .then_with(|| a.1.db.as_u64().cmp(&b.1.db.as_u64())) - .then_with(|| a.1.tenant.as_u64().cmp(&b.1.tenant.as_u64())) - .then_with(|| a.1.collection.cmp(&b.1.collection)) - .then_with(|| a.1.key.cmp(&b.1.key)) - }); - for (_, key) in by_lsn.into_iter().take(overflow) { - self.last_write_lsn.remove(&key); - } - } - - self.index_values.gc(watermark); - } -} - -impl CoreLoop { - /// Record a committed write into the per-core version index and advance the - /// core watermark monotonically. - /// - /// Called once per written key at every Data-Plane apply chokepoint, using - /// the WAL LSN the Control Plane allocated at wal-dispatch and threaded onto - /// the write task. `key` is `None` for engines whose per-key identity is - /// internal (columnar / timeseries / array / spatial / FTS) — those record - /// only the collection floor. - /// - /// In the install pass of a committed-redo apply the version waits in the - /// apply scope. The apply publishes it once the record settled, so a - /// rolled-back install leaves no version and no watermark behind. - pub(in crate::data::executor) fn note_write_lsn( - &mut self, - db: DatabaseId, - tenant: TenantId, - collection: &str, - key: Option, - lsn: Lsn, - ) { - if let Some(scope) = self.redo_apply.scope.as_mut() { - scope.defer_write_version(db, tenant, collection, key, lsn); - return; - } - self.publish_write_version(db, tenant, collection, key, lsn); - } - - /// Record a write version and advance the core watermark monotonically. - /// - /// Every applied write reaches here, so this is also where the core notes - /// the write's LSN as applied: a checkpoint's replay stamp names it. - pub(in crate::data::executor) fn publish_write_version( - &mut self, - db: DatabaseId, - tenant: TenantId, - collection: &str, - key: Option, - lsn: Lsn, - ) { - self.floors.applied_prefix.note_applied(lsn); - self.write_index - .note_write_lsn(db, tenant, collection, key, lsn); - if lsn > self.watermark { - self.watermark = lsn; - } - } - - /// Record a committed write's collection floor only (no per-key entry), - /// if a WAL LSN was threaded onto `task`. Shared by the columnar-family - /// write handlers (columnar / timeseries / array / spatial / FTS) whose - /// per-key identity is internal — a predicate reader validates against the - /// collection floor when it owns no per-key version. - pub(in crate::data::executor) fn note_collection_write_lsn( - &mut self, - task: &super::super::task::ExecutionTask, - collection: &str, - ) { - if let Some(lsn) = task.wal_lsn() { - self.note_write_lsn( - task.request.database_id, - task.request.tenant_id, - collection, - None, - lsn, - ); - } - } - - /// Run horizon GC on the per-core version index. Invoked from the periodic - /// maintenance hook — no dedicated timer. - pub(in crate::data::executor) fn gc_write_index(&mut self) { - self.write_index.gc(self.watermark); - } - - /// Whether this shard's slice of a transaction's LSN-versioned read-set was - /// still current against the local write versions. - /// - /// Filters the read-set to the entries homed on this request's vShard — - /// the only reads this core holds versions for — then checks each against - /// the per-core write-version index via - /// [`WriteVersionIndex::read_is_valid`]. Short-circuits on the first entry - /// that is no longer current. An empty or fully-remote slice is vacuously - /// current (`true`). The `(database, tenant)` scope mirrors the write-version - /// recorder so a read validates against the same key space it was recorded - /// in. - /// - /// An entry's home is its `home_vshard` when set (a cross-shard graph read - /// names the key vShard it read edges on), else its collection's vShard, - /// by the same collection-in-database function the scheduler routes plans - /// with. A homed entry with no collection observed every collection on its - /// vShard, so it is current only while the core watermark has not passed - /// its read version. - pub(in crate::data::executor) fn read_set_still_current( - &self, - task: &super::super::task::ExecutionTask, - tid: u64, - versioned_reads: &[VersionedReadEntry], - ) -> bool { - let db = task.request.database_id; - let tenant = TenantId::new(tid); - let local_vshard = task.request.vshard_id.as_u32(); - versioned_reads.iter().all(|entry| { - if let Some(home) = entry.home_vshard { - if home != local_vshard { - return true; - } - if entry.collection.is_empty() { - return self.watermark <= entry.read_lsn; - } - return self.write_index.read_is_valid( - db, - tenant, - &entry.collection, - &entry.key, - entry.read_lsn, - ); - } - // An entry carries the plan's database-qualified name. One that does - // not de-qualify cannot be homed or validated, so the read set fails - // closed. - match nodedb_types::CollectionKey::from_qualified_str(db, &entry.collection) { - Err(_) => false, - Ok(key) if key.vshard().as_u32() != local_vshard => true, - Ok(_) => self.write_index.read_is_valid( - db, - tenant, - &entry.collection, - &entry.key, - entry.read_lsn, - ), - } - }) - } - - /// Record a committed document/vector write's version, keyed by the - /// written row's cross-engine surrogate, if a WAL LSN was threaded onto - /// `task`. Shared by every per-surrogate write chokepoint (point put, - /// point insert, point delete, bulk update, bulk delete). - pub(in crate::data::executor) fn note_surrogate_write_lsn( - &mut self, - task: &super::super::task::ExecutionTask, - tid: u64, - collection: &str, - surrogate: u32, - ) { - if let Some(lsn) = task.wal_lsn() { - self.note_write_lsn( - task.request.database_id, - TenantId::new(tid), - collection, - Some(KeyRepr::Surrogate(surrogate)), - lsn, - ); - } - } - - /// Record a committed WAL-replay write's version. A no-op when - /// `record_lsn == 0` (no durable LSN was recorded for this write); `key` - /// is `None` for collection-only entries (e.g. truncate) and `Some` for - /// per-key/per-surrogate entries, exactly like [`Self::note_write_lsn`]. - /// - /// Shared by every WAL replay chokepoint (KV, document, document-vector): - /// unlike the live write path, replay only has the raw `(database_id, - /// tenant_id, record_lsn)` off the WAL record header, not an - /// `ExecutionTask` — hence the separate `u64`-typed entry point rather - /// than reusing [`Self::note_surrogate_write_lsn`] / - /// [`Self::note_collection_write_lsn`]. - pub(in crate::data::executor) fn note_replay_write_lsn( - &mut self, - database_id: u64, - tenant_id: u64, - collection: &str, - key: Option, - record_lsn: u64, - ) { - if record_lsn != 0 { - self.note_write_lsn( - DatabaseId::new(database_id), - TenantId::new(tenant_id), - collection, - key, - Lsn::new(record_lsn), - ); - } - } -} - -#[cfg(test)] -pub(crate) mod tests { - use std::time::{Duration, Instant}; - - use nodedb_bridge::buffer::RingBuffer; - use nodedb_physical::physical_plan::DocumentOp; - use nodedb_types::calvin::EngineTag; - use nodedb_types::{QualifiedCollection, Surrogate}; - - use super::*; - use crate::bridge::dispatch::{BridgeRequest, BridgeResponse}; - use crate::bridge::envelope::{PhysicalPlan, Priority, Request, Status}; - use crate::data::executor::core_loop::CoreLoop; - use crate::data::executor::handlers::graph::EdgePutParams; - use crate::data::executor::handlers::kv::crud::KvWriteParams; - use crate::data::executor::handlers::point::put::PointPutExec; - use crate::data::executor::task::ExecutionTask; - use crate::types::{ReadConsistency, RequestId, TraceId, VShardId}; - - fn db() -> DatabaseId { - DatabaseId::DEFAULT - } - - fn tenant() -> TenantId { - TenantId::new(1) - } - - #[test] - fn point_read_is_valid_when_key_never_written() { - let index = WriteVersionIndex::new(); - let key = ReadKeyIdent::Point(KeyRepr::Surrogate(7)); - assert!(index.read_is_valid(db(), tenant(), "orders", &key, Lsn::new(10))); - } - - #[test] - fn point_read_is_valid_when_write_at_or_before_read_lsn() { - let mut index = WriteVersionIndex::new(); - index.note_write_lsn( - db(), - tenant(), - "orders", - Some(KeyRepr::Surrogate(7)), - Lsn::new(10), - ); - - let key = ReadKeyIdent::Point(KeyRepr::Surrogate(7)); - assert!(index.read_is_valid(db(), tenant(), "orders", &key, Lsn::new(10))); - assert!(index.read_is_valid(db(), tenant(), "orders", &key, Lsn::new(20))); - } - - #[test] - fn point_read_is_invalid_when_write_after_read_lsn() { - let mut index = WriteVersionIndex::new(); - index.note_write_lsn( - db(), - tenant(), - "orders", - Some(KeyRepr::Surrogate(7)), - Lsn::new(20), - ); - - let key = ReadKeyIdent::Point(KeyRepr::Surrogate(7)); - assert!(!index.read_is_valid(db(), tenant(), "orders", &key, Lsn::new(10))); - } - - #[test] - fn predicate_read_is_valid_when_collection_never_written() { - let index = WriteVersionIndex::new(); - let key = ReadKeyIdent::Predicate; - assert!(index.read_is_valid(db(), tenant(), "orders", &key, Lsn::new(10))); - } - - #[test] - fn predicate_read_is_valid_when_floor_at_or_before_read_lsn() { - let mut index = WriteVersionIndex::new(); - index.note_write_lsn(db(), tenant(), "orders", None, Lsn::new(10)); - - let key = ReadKeyIdent::Predicate; - assert!(index.read_is_valid(db(), tenant(), "orders", &key, Lsn::new(10))); - assert!(index.read_is_valid(db(), tenant(), "orders", &key, Lsn::new(20))); - } - - #[test] - fn predicate_read_is_invalid_when_floor_after_read_lsn() { - let mut index = WriteVersionIndex::new(); - index.note_write_lsn(db(), tenant(), "orders", None, Lsn::new(20)); - - let key = ReadKeyIdent::Predicate; - assert!(!index.read_is_valid(db(), tenant(), "orders", &key, Lsn::new(10))); - } - - #[test] - fn untracked_index_dimension_falls_back_to_collection_floor() { - // A `(collection, field)` never recorded in the per-value substrate is - // untracked → `eq_max_lsn`/`range_max_lsn` return `None` → the validator - // falls back to the collection floor, producing the SAME verdict as a - // `Predicate` read for every floor position. - let index_eq = ReadKeyIdent::IndexEq { - field: "email".to_string(), - value: "a@b.c".to_string(), - }; - let index_range = ReadKeyIdent::IndexRange { - field: "age".to_string(), - lo: Some("18".to_string()), - hi: None, - }; - let predicate = ReadKeyIdent::Predicate; - - // Floor below the read LSN (current), at it (current), and above it - // (stale) — the untracked index variants track `Predicate` in every case. - for (floor, read_lsn) in [(5u64, 10u64), (10, 10), (20, 10)] { - let mut index = WriteVersionIndex::new(); - index.note_write_lsn(db(), tenant(), "orders", None, Lsn::new(floor)); - let want = - index.read_is_valid(db(), tenant(), "orders", &predicate, Lsn::new(read_lsn)); - assert_eq!( - index.read_is_valid(db(), tenant(), "orders", &index_eq, Lsn::new(read_lsn)), - want, - "untracked IndexEq must match Predicate (floor {floor}, read {read_lsn})" - ); - assert_eq!( - index.read_is_valid(db(), tenant(), "orders", &index_range, Lsn::new(read_lsn)), - want, - "untracked IndexRange must match Predicate (floor {floor}, read {read_lsn})" - ); - } - } - - #[test] - fn index_eq_disjoint_write_does_not_abort() { - // Read of email = "a@b.c" at read_lsn 10; a later write to a DIFFERENT - // value on the same dimension must NOT abort the read (the coarse - // collection floor would have). - let mut index = WriteVersionIndex::new(); - // A real write to a disjoint value advances BOTH the collection floor - // (which the coarse `Predicate` path would abort on) AND records the - // per-value entry — so this proves the per-value check reduces the abort, - // not merely that nothing was written. - index.note_write_lsn(db(), tenant(), "orders", None, Lsn::new(20)); - index - .index_values - .record(db(), tenant(), "orders", "email", "z@z.z", Lsn::new(20)); - let key = ReadKeyIdent::IndexEq { - field: "email".to_string(), - value: "a@b.c".to_string(), - }; - assert!(index.read_is_valid(db(), tenant(), "orders", &key, Lsn::new(10))); - } - - #[test] - fn index_range_disjoint_write_does_not_abort() { - // Range [10, 20] read at read_lsn 10; a write to out-of-range "50" must - // not abort. - let mut index = WriteVersionIndex::new(); - // Advance the collection floor too (the coarse path aborts on it) so this - // proves range validation reduces the abort, not just an empty index. - index.note_write_lsn(db(), tenant(), "orders", None, Lsn::new(20)); - index - .index_values - .record(db(), tenant(), "orders", "age", "50", Lsn::new(20)); - let key = ReadKeyIdent::IndexRange { - field: "age".to_string(), - lo: Some("10".to_string()), - hi: Some("20".to_string()), - }; - assert!(index.read_is_valid(db(), tenant(), "orders", &key, Lsn::new(10))); - } - - #[test] - fn index_eq_same_value_conflict_aborts() { - // A write to the SAME read value after the read LSN must abort. - let mut index = WriteVersionIndex::new(); - index - .index_values - .record(db(), tenant(), "orders", "email", "a@b.c", Lsn::new(20)); - let key = ReadKeyIdent::IndexEq { - field: "email".to_string(), - value: "a@b.c".to_string(), - }; - assert!(!index.read_is_valid(db(), tenant(), "orders", &key, Lsn::new(10))); - } - - #[test] - fn index_range_in_range_added_value_conflict_aborts() { - // An added value INSIDE the read range after the read LSN must abort. - let mut index = WriteVersionIndex::new(); - index - .index_values - .record(db(), tenant(), "orders", "age", "15", Lsn::new(20)); - let key = ReadKeyIdent::IndexRange { - field: "age".to_string(), - lo: Some("10".to_string()), - hi: Some("20".to_string()), - }; - assert!(!index.read_is_valid(db(), tenant(), "orders", &key, Lsn::new(10))); - } - - #[test] - fn index_range_in_range_removed_value_conflict_aborts() { - // A delete of an in-range value also records that value's LSN, so a - // removal inside the read range must abort the read just like an insert. - let mut index = WriteVersionIndex::new(); - index - .index_values - .record(db(), tenant(), "orders", "age", "17", Lsn::new(20)); - let key = ReadKeyIdent::IndexRange { - field: "age".to_string(), - lo: Some("10".to_string()), - hi: Some("20".to_string()), - }; - assert!(!index.read_is_valid(db(), tenant(), "orders", &key, Lsn::new(10))); - } - - #[test] - fn index_range_phantom_insert_aborts() { - // Phantom protection: the range is current while it holds no in-range - // value (tracked → `Some(ZERO)`), then a NEW in-range value recorded - // after the read LSN invalidates it — proving the range captures the - // predicate, not just values extant at read time. - let mut index = WriteVersionIndex::new(); - // Track the dimension with an OUT-of-range value so the read starts - // current (tracked, no in-range write). - index - .index_values - .record(db(), tenant(), "orders", "age", "99", Lsn::new(5)); - let key = ReadKeyIdent::IndexRange { - field: "age".to_string(), - lo: Some("10".to_string()), - hi: Some("20".to_string()), - }; - assert!( - index.read_is_valid(db(), tenant(), "orders", &key, Lsn::new(10)), - "range with no in-range write is current" - ); - - // Phantom insert inside the range after the read LSN. - index - .index_values - .record(db(), tenant(), "orders", "age", "15", Lsn::new(20)); - assert!( - !index.read_is_valid(db(), tenant(), "orders", &key, Lsn::new(10)), - "phantom in-range insert must abort" - ); - } - - #[test] - fn tracked_index_eq_missing_value_is_current() { - // A tracked dimension (some other value recorded) queried for a value - // with no entry returns `Some(ZERO)` → current. - let mut index = WriteVersionIndex::new(); - index - .index_values - .record(db(), tenant(), "orders", "email", "other@x.y", Lsn::new(20)); - let key = ReadKeyIdent::IndexEq { - field: "email".to_string(), - value: "a@b.c".to_string(), - }; - assert!(index.read_is_valid(db(), tenant(), "orders", &key, Lsn::new(10))); - } - - #[test] - fn tracked_index_range_empty_range_is_current() { - // A tracked dimension queried for a range with no in-range entry returns - // `Some(ZERO)` → current. - let mut index = WriteVersionIndex::new(); - index - .index_values - .record(db(), tenant(), "orders", "age", "99", Lsn::new(20)); - let key = ReadKeyIdent::IndexRange { - field: "age".to_string(), - lo: Some("10".to_string()), - hi: Some("20".to_string()), - }; - assert!(index.read_is_valid(db(), tenant(), "orders", &key, Lsn::new(10))); - } - - // ── CoreLoop-level fixtures shared with the tests below and with other - // executor test modules (`make_core_with_dir` / `make_default_task` are - // `pub` for exactly that reason). ──────────────────────────────────────── - - fn make_core() -> ( - CoreLoop, - nodedb_bridge::buffer::Producer, - nodedb_bridge::buffer::Consumer, - tempfile::TempDir, - ) { - let dir = tempfile::tempdir().unwrap(); - let (req_tx, req_rx) = RingBuffer::channel::(64); - let (resp_tx, resp_rx) = RingBuffer::channel::(64); - let core = CoreLoop::open( - 0, - req_rx, - resp_tx, - dir.path(), - std::sync::Arc::new(nodedb_types::OrdinalClock::new()), - crate::data::executor::core_loop::test_governor(), - ) - .unwrap(); - (core, req_tx, resp_rx, dir) - } - - pub fn make_core_with_dir( - dir: &std::path::Path, - ) -> ( - CoreLoop, - nodedb_bridge::buffer::Producer, - nodedb_bridge::buffer::Consumer, - ) { - let (req_tx, req_rx) = RingBuffer::channel::(64); - let (resp_tx, resp_rx) = RingBuffer::channel::(64); - let core = CoreLoop::open( - 0, - req_rx, - resp_tx, - dir, - std::sync::Arc::new(nodedb_types::OrdinalClock::new()), - crate::data::executor::core_loop::test_governor(), - ) - .unwrap(); - (core, req_tx, resp_rx) - } - - fn make_request(plan: PhysicalPlan) -> Request { - Request { - request_id: RequestId::new(1), - tenant_id: TenantId::new(1), - database_id: DatabaseId::DEFAULT, - vshard_id: VShardId::new(0), - plan, - deadline: Instant::now() + Duration::from_secs(5), - priority: Priority::Normal, - trace_id: TraceId::ZERO, - consistency: ReadConsistency::Strong, - idempotency_key: None, - event_source: crate::event::EventSource::User, - user_roles: Vec::new(), - user_id: None, - statement_digest: None, - txn_id: None, - wal_lsn: None, - resolved_now_ms: None, - commit_hlc: None, - admission: crate::bridge::envelope::Admission::Admitted, - } - } - - /// A minimal `ExecutionTask` (DEFAULT database/tenant, vShard 0, no WAL LSN) - /// for handler unit tests that only read `request.database_id`. The carried - /// plan is inert — edge/point handlers take their parameters directly. - pub fn make_default_task() -> crate::data::executor::task::ExecutionTask { - crate::data::executor::task::ExecutionTask::new(make_request(PhysicalPlan::Document( - DocumentOp::PointGet { - collection: QualifiedCollection::new(DatabaseId::DEFAULT, "x"), - document_id: "y".into(), - surrogate: None, - pk_bytes: Vec::new(), - rls_filters: Vec::new(), - system_time: nodedb_types::SystemTimeScope::Current, - valid_at_ms: None, - }, - ))) - } - - /// A `{k: v}` document body: a standard MessagePack map. - fn doc_value(k: &str, v: &str) -> Vec { - let mut obj = std::collections::HashMap::new(); - obj.insert(k.to_string(), nodedb_types::Value::String(v.into())); - nodedb_types::value_to_msgpack(&nodedb_types::Value::Object(obj)).unwrap() - } - - /// An `ExecutionTask` carrying a known WAL LSN, tenant 1 / database DEFAULT. - fn wal_task(lsn: u64) -> ExecutionTask { - let plan = PhysicalPlan::Document(DocumentOp::PointGet { - collection: QualifiedCollection::new(DatabaseId::DEFAULT, "x"), - document_id: "y".into(), - surrogate: None, - pk_bytes: Vec::new(), - rls_filters: Vec::new(), - system_time: nodedb_types::SystemTimeScope::Current, - valid_at_ms: None, - }); - ExecutionTask::with_wal_lsn(make_request(plan), Some(Lsn::new(lsn))) - } - - fn surrogate_key(collection: &str, surrogate: u32) -> WriteKey { - WriteKey { - db: DatabaseId::DEFAULT, - tenant: TenantId::new(1), - collection: Box::from(collection), - key: KeyRepr::Surrogate(surrogate), - } - } - - fn coll_key(collection: &str) -> CollKey { - CollKey { - db: DatabaseId::DEFAULT, - tenant: TenantId::new(1), - collection: Box::from(collection), - } - } - - #[test] - fn point_put_records_write_version_and_advances_watermark() { - let (mut core, _, _, _dir) = make_core(); - - let task = wal_task(10); - let resp = core.execute_point_put( - &task, - PointPutExec { - tid: 1, - collection: "orders", - document_id: "o1", - surrogate: Surrogate::new(7), - value: &doc_value("a", "1"), - returning: None, - rls_filters: &[], - resolved_sum_targets: &[], - }, - ); - assert_eq!(resp.status, Status::Ok); - - assert_eq!( - core.write_index.key_write_lsn(&surrogate_key("orders", 7)), - Some(Lsn::new(10)) - ); - assert_eq!( - core.write_index.collection_write_lsn(&coll_key("orders")), - Some(Lsn::new(10)) - ); - assert_eq!(core.watermark, Lsn::new(10)); - - // Second write to the same key with a larger LSN overwrites monotonically. - let task = wal_task(20); - core.execute_point_put( - &task, - PointPutExec { - tid: 1, - collection: "orders", - document_id: "o1", - surrogate: Surrogate::new(7), - value: &doc_value("a", "2"), - returning: None, - rls_filters: &[], - resolved_sum_targets: &[], - }, - ); - assert_eq!( - core.write_index.key_write_lsn(&surrogate_key("orders", 7)), - Some(Lsn::new(20)) - ); - assert_eq!( - core.write_index.collection_write_lsn(&coll_key("orders")), - Some(Lsn::new(20)) - ); - assert_eq!(core.watermark, Lsn::new(20)); - - // A lower LSN never regresses an existing entry or the watermark. - let task = wal_task(15); - core.execute_point_put( - &task, - PointPutExec { - tid: 1, - collection: "orders", - document_id: "o1", - surrogate: Surrogate::new(7), - value: &doc_value("a", "3"), - returning: None, - rls_filters: &[], - resolved_sum_targets: &[], - }, - ); - assert_eq!( - core.write_index.key_write_lsn(&surrogate_key("orders", 7)), - Some(Lsn::new(20)) - ); - assert_eq!(core.watermark, Lsn::new(20)); - - // A second collection tracks its own max independently. - let task = wal_task(30); - core.execute_point_put( - &task, - PointPutExec { - tid: 1, - collection: "items", - document_id: "i1", - surrogate: Surrogate::new(9), - value: &doc_value("a", "4"), - returning: None, - rls_filters: &[], - resolved_sum_targets: &[], - }, - ); - assert_eq!( - core.write_index.collection_write_lsn(&coll_key("items")), - Some(Lsn::new(30)) - ); - assert_eq!( - core.write_index.collection_write_lsn(&coll_key("orders")), - Some(Lsn::new(20)) - ); - assert_eq!(core.watermark, Lsn::new(30)); - } - - #[test] - fn kv_put_records_kvkey_version() { - let (mut core, _, _, _dir) = make_core(); - let task = wal_task(42); - let resp = core.execute_kv_put( - &task, - KvWriteParams { - did: DatabaseId::DEFAULT.as_u64(), - tid: 1, - collection: "kv", - key: b"k1".as_slice(), - value: b"v1".as_slice(), - ttl_ms: 0, - surrogate: Surrogate::new(3), - returning: None, - rls_filters: &[], - }, - ); - assert_eq!(resp.status, Status::Ok); - - let wk = WriteKey { - db: DatabaseId::DEFAULT, - tenant: TenantId::new(1), - collection: Box::from("kv"), - key: KeyRepr::KvKey(Box::from(b"k1".as_slice())), - }; - assert_eq!(core.write_index.key_write_lsn(&wk), Some(Lsn::new(42))); - assert_eq!( - core.write_index.collection_write_lsn(&coll_key("kv")), - Some(Lsn::new(42)) - ); - assert_eq!(core.watermark, Lsn::new(42)); - } - - #[test] - fn an_autocommit_apply_is_named_by_the_next_replay_stamp() { - let (mut core, _, _, _dir) = make_core(); - core.floors - .applied_prefix - .observe_outcome_floor(Lsn::new(40)); - let resp = core.execute_kv_put( - &wal_task(42), - KvWriteParams { - did: DatabaseId::DEFAULT.as_u64(), - tid: 1, - collection: "kv", - key: b"k1".as_slice(), - value: b"v1".as_slice(), - ttl_ms: 0, - surrogate: Surrogate::new(3), - returning: None, - rls_filters: &[], - }, - ); - assert_eq!(resp.status, Status::Ok); - - let stamp = core.floors.applied_prefix.stamp().expect("exact stamp"); - assert!( - stamp.skips(42), - "the applied write is in every later artifact" - ); - assert!( - !stamp.skips(41), - "a record between the floor and the applied write still replays" - ); - } - - #[test] - fn edge_put_records_edge_version() { - let (mut core, _, _, _dir) = make_core(); - let task = wal_task(50); - let resp = core.execute_edge_put( - &task, - EdgePutParams { - tid: 1, - collection: "graph", - src_id: "a", - label: "KNOWS", - dst_id: "b", - properties: &[], - src_surrogate: Surrogate::new(1), - dst_surrogate: Surrogate::new(2), - }, - ); - assert_eq!(resp.status, Status::Ok); - - let wk = WriteKey { - db: DatabaseId::DEFAULT, - tenant: TenantId::new(1), - collection: Box::from("graph"), - key: KeyRepr::Edge { - src: Box::from("a"), - label: Box::from("KNOWS"), - dst: Box::from("b"), - }, - }; - assert_eq!(core.write_index.key_write_lsn(&wk), Some(Lsn::new(50))); - assert_eq!(core.watermark, Lsn::new(50)); - } - - #[test] - fn a_committed_transaction_records_its_write_versions() { - let (mut core, _, _, _dir) = make_core(); - let task = wal_task(60); - let plans = vec![PhysicalPlan::Document(DocumentOp::PointPut { - collection: QualifiedCollection::new(DatabaseId::DEFAULT, "batch"), - document_id: "d1".into(), - value: doc_value("a", "1"), - surrogate: Surrogate::new(11), - pk_bytes: Vec::new(), - returning: None, - rls_filters: Vec::new(), - resolved_sum_targets: Vec::new(), - })]; - let resp = core.commit_plans_for_test(&task, 1, &plans, 60); - assert_eq!(resp.status, Status::Ok, "{:?}", resp.error_code); - - assert_eq!( - core.write_index.key_write_lsn(&surrogate_key("batch", 11)), - Some(Lsn::new(60)) - ); - assert_eq!( - core.write_index.collection_write_lsn(&coll_key("batch")), - Some(Lsn::new(60)) - ); - assert_eq!(core.watermark, Lsn::new(60)); - } - - #[test] - fn no_wal_lsn_records_nothing() { - let (mut core, _, _, _dir) = make_core(); - // Task without a WAL LSN — the version index is skipped, not advanced. - let task = ExecutionTask::new(make_request(PhysicalPlan::Document(DocumentOp::PointGet { - collection: QualifiedCollection::new(DatabaseId::DEFAULT, "x"), - document_id: "y".into(), - surrogate: None, - pk_bytes: Vec::new(), - rls_filters: Vec::new(), - system_time: nodedb_types::SystemTimeScope::Current, - valid_at_ms: None, - }))); - core.execute_point_put( - &task, - PointPutExec { - tid: 1, - collection: "orders", - document_id: "o1", - surrogate: Surrogate::new(7), - value: &doc_value("a", "1"), - returning: None, - rls_filters: &[], - resolved_sum_targets: &[], - }, - ); - assert_eq!( - core.write_index.key_write_lsn(&surrogate_key("orders", 7)), - None - ); - assert_eq!(core.watermark, Lsn::ZERO); - } - - #[test] - fn horizon_gc_evicts_stale_keys_keeps_recent_and_collection() { - let (mut core, _, _, _dir) = make_core(); - let db = DatabaseId::DEFAULT; - let tenant = TenantId::new(1); - - // A stale per-key entry, then a recent write that drives the watermark far - // past the retain window. - core.note_write_lsn(db, tenant, "c", Some(KeyRepr::Surrogate(1)), Lsn::new(10)); - core.note_write_lsn( - db, - tenant, - "c", - Some(KeyRepr::Surrogate(2)), - Lsn::new(1_000_000), - ); - assert_eq!(core.watermark, Lsn::new(1_000_000)); - - core.gc_write_index(); - - // Stale key evicted; recent key retained; collection floor survives GC. - assert_eq!(core.write_index.key_write_lsn(&surrogate_key("c", 1)), None); - assert_eq!( - core.write_index.key_write_lsn(&surrogate_key("c", 2)), - Some(Lsn::new(1_000_000)) - ); - assert_eq!( - core.write_index.collection_write_lsn(&coll_key("c")), - Some(Lsn::new(1_000_000)) - ); - } - - // ── Read-set validation against the write-version index (integration, - // through `CoreLoop::read_set_still_current`) ────────────────────────── - - use nodedb_types::calvin::VersionedReadEntry; - - /// An `ExecutionTask` homing to `vshard_id`, carrying no WAL LSN. - fn task_with_vshard(vshard_id: VShardId) -> ExecutionTask { - ExecutionTask::new(Request { - vshard_id, - ..make_request(PhysicalPlan::Document(DocumentOp::PointGet { - collection: QualifiedCollection::new(DatabaseId::DEFAULT, "x"), - document_id: "y".into(), - surrogate: None, - pk_bytes: Vec::new(), - rls_filters: Vec::new(), - system_time: nodedb_types::SystemTimeScope::Current, - valid_at_ms: None, - })) - }) - } - - /// An `ExecutionTask` carrying WAL LSN `lsn` and homing to `vshard_id`. - fn wal_task_with_vshard(lsn: u64, vshard_id: VShardId) -> ExecutionTask { - ExecutionTask::with_wal_lsn( - Request { - vshard_id, - ..make_request(PhysicalPlan::Document(DocumentOp::PointGet { - collection: QualifiedCollection::new(DatabaseId::DEFAULT, "x"), - document_id: "y".into(), - surrogate: None, - pk_bytes: Vec::new(), - rls_filters: Vec::new(), - system_time: nodedb_types::SystemTimeScope::Current, - valid_at_ms: None, - })) - }, - Some(Lsn::new(lsn)), - ) - } - - /// The vShard `collection` homes to in the default database — mirrors the - /// homing `read_set_still_current` filters entries by. - fn local_vshard(collection: &str) -> VShardId { - nodedb_types::CollectionKey::from_bare(DatabaseId::DEFAULT, collection).vshard() - } - - /// Some vShard other than `than`, for exercising the cross-shard filter. - fn other_vshard(than: VShardId) -> VShardId { - VShardId::new((than.as_u32() + 1) % VShardId::COUNT) - } - - fn point_entry(collection: &str, surrogate: u32, read_lsn: u64) -> VersionedReadEntry { - VersionedReadEntry { - engine: EngineTag::Document, - collection: collection.to_string(), - key: ReadKeyIdent::Point(KeyRepr::Surrogate(surrogate)), - read_lsn: Lsn::new(read_lsn), - home_vshard: None, - served_by: 0, - } - } - - fn predicate_entry(collection: &str, read_lsn: u64) -> VersionedReadEntry { - VersionedReadEntry { - engine: EngineTag::Document, - collection: collection.to_string(), - key: ReadKeyIdent::Predicate, - read_lsn: Lsn::new(read_lsn), - home_vshard: None, - served_by: 0, - } - } - - #[test] - fn stale_point_read_is_detected_as_not_current() { - let (mut core, _, _, _dir) = make_core(); - core.note_write_lsn( - DatabaseId::DEFAULT, - TenantId::new(1), - "orders", - Some(KeyRepr::Surrogate(7)), - Lsn::new(20), - ); - - let task = task_with_vshard(local_vshard("orders")); - let reads = vec![point_entry("orders", 7, 10)]; - assert!(!core.read_set_still_current(&task, 1, &reads)); - } - - #[test] - fn fresh_point_read_is_still_current() { - let (mut core, _, _, _dir) = make_core(); - core.note_write_lsn( - DatabaseId::DEFAULT, - TenantId::new(1), - "orders", - Some(KeyRepr::Surrogate(7)), - Lsn::new(20), - ); - - let task = task_with_vshard(local_vshard("orders")); - assert!(core.read_set_still_current(&task, 1, &[point_entry("orders", 7, 20)])); - assert!(core.read_set_still_current(&task, 1, &[point_entry("orders", 7, 30)])); - } - - #[test] - fn read_entry_homing_to_a_different_vshard_is_filtered_out() { - let (mut core, _, _, _dir) = make_core(); - core.note_write_lsn( - DatabaseId::DEFAULT, - TenantId::new(1), - "orders", - Some(KeyRepr::Surrogate(7)), - Lsn::new(20), - ); - - let local = local_vshard("orders"); - let remote_task = task_with_vshard(other_vshard(local)); - // Would conflict (read_lsn 10 < write_lsn 20) if this shard owned the - // entry's collection; it homes elsewhere, so it is filtered out of this - // shard's slice and the vacuous (empty-after-filter) result is `true`. - let reads = vec![point_entry("orders", 7, 10)]; - assert!(core.read_set_still_current(&remote_task, 1, &reads)); - } - - #[test] - fn stale_predicate_read_is_detected_as_not_current() { - let (mut core, _, _, _dir) = make_core(); - core.note_write_lsn( - DatabaseId::DEFAULT, - TenantId::new(1), - "orders", - None, - Lsn::new(20), - ); - - let task = task_with_vshard(local_vshard("orders")); - let reads = vec![predicate_entry("orders", 10)]; - assert!(!core.read_set_still_current(&task, 1, &reads)); - } - - fn homed_entry(collection: &str, home: VShardId, read_lsn: u64) -> VersionedReadEntry { - VersionedReadEntry { - home_vshard: Some(home.as_u32()), - served_by: 0, - ..predicate_entry(collection, read_lsn) - } - } - - #[test] - fn a_homed_read_validates_on_its_home_not_its_collection_vshard() { - let (mut core, _, _, _dir) = make_core(); - core.note_write_lsn( - DatabaseId::DEFAULT, - TenantId::new(1), - "edges", - None, - Lsn::new(20), - ); - let home = other_vshard(local_vshard("edges")); - let stale = vec![homed_entry("edges", home, 10)]; - assert!(!core.read_set_still_current(&task_with_vshard(home), 1, &stale)); - assert!( - core.read_set_still_current(&task_with_vshard(local_vshard("edges")), 1, &stale), - "the collection's own vShard does not hold a homed read" - ); - let fresh = vec![homed_entry("edges", home, 20)]; - assert!(core.read_set_still_current(&task_with_vshard(home), 1, &fresh)); - } - - #[test] - fn a_homed_read_of_every_collection_validates_against_the_core_watermark() { - let (mut core, _, _, _dir) = make_core(); - let home = local_vshard("edges"); - core.note_write_lsn( - DatabaseId::DEFAULT, - TenantId::new(1), - "unrelated", - None, - Lsn::new(20), - ); - let task = task_with_vshard(home); - assert!(!core.read_set_still_current(&task, 1, &[homed_entry("", home, 10)])); - assert!(core.read_set_still_current(&task, 1, &[homed_entry("", home, 20)])); - } - - #[test] - fn fresh_predicate_read_is_still_current() { - let (mut core, _, _, _dir) = make_core(); - core.note_write_lsn( - DatabaseId::DEFAULT, - TenantId::new(1), - "orders", - None, - Lsn::new(20), - ); - - let task = task_with_vshard(local_vshard("orders")); - let reads = vec![predicate_entry("orders", 20)]; - assert!(core.read_set_still_current(&task, 1, &reads)); - } - - #[test] - fn empty_read_set_is_vacuously_current() { - let (core, _, _, _dir) = make_core(); - let task = task_with_vshard(VShardId::new(0)); - assert!(core.read_set_still_current(&task, 1, &[])); - } - - #[test] - fn a_read_that_predates_a_committed_write_is_no_longer_current() { - let (mut core, _, _, _dir) = make_core(); - let vshard = local_vshard("orders"); - - // A committed write to key 7 in "orders" records its version at LSN 10. - let write_task = wal_task_with_vshard(10, vshard); - let write_plans = vec![PhysicalPlan::Document(DocumentOp::PointPut { - collection: QualifiedCollection::new(DatabaseId::DEFAULT, "orders"), - document_id: "o7".into(), - value: doc_value("a", "1"), - surrogate: Surrogate::new(7), - pk_bytes: Vec::new(), - returning: None, - rls_filters: Vec::new(), - resolved_sum_targets: Vec::new(), - })]; - let write_resp = core.commit_plans_for_test(&write_task, 1, &write_plans, 10); - assert_eq!(write_resp.status, Status::Ok, "{:?}", write_resp.error_code); - - // A read of key 7 observed at LSN 5 predates the write; one observed - // at LSN 10 does not. - let task = task_with_vshard(vshard); - assert!(!core.read_set_still_current(&task, 1, &[point_entry("orders", 7, 5)])); - assert!(core.read_set_still_current(&task, 1, &[point_entry("orders", 7, 10)])); - } -} diff --git a/nodedb/src/data/executor/core_loop/write_index/index.rs b/nodedb/src/data/executor/core_loop/write_index/index.rs new file mode 100644 index 000000000..7fd5d7136 --- /dev/null +++ b/nodedb/src/data/executor/core_loop/write_index/index.rs @@ -0,0 +1,544 @@ +// SPDX-License-Identifier: BUSL-1.1 + +//! The per-core write-version index and its read validation. + +use std::collections::HashMap; + +use nodedb_types::calvin::ReadKeyIdent; +use nodedb_types::{DatabaseId, TenantId, WriteVersion}; + +use crate::types::VShardId; + +use super::super::index_value_versions::{IndexDimRef, IndexValueVersionIndex, IndexVersion}; +use super::keys::{CollKey, KeyRepr, WriteKey}; +use super::record_stamps::RecordStamps; + +/// Horizon retain window for per-key entries. Horizon GC evicts a +/// `last_write` entry more than this many log entries (or WAL LSNs, for +/// writes that apply no entry) below its vShard's latest version. Sized in +/// the same order of magnitude as the idempotency-cache cap (16,384 +/// entries): a bounded recent-write history, enough to validate in-flight +/// transactions, not an unbounded write log. +pub const RETAIN_WINDOW: u64 = 16_384; + +/// Hard upper bound on the `last_write` entry count. When horizon GC leaves +/// more entries than this, the lowest versions are dropped until the map is +/// back within bound. +const MAX_KEY_ENTRIES: usize = 65_536; + +/// One vShard's version bounds on this core. +#[derive(Debug, Clone, Copy, Default, PartialEq, Eq)] +struct VShardBounds { + /// The highest version recorded for the vShard. + latest: WriteVersion, + /// The version the vShard's state holds without per-row versions: a + /// snapshot install's cut, or the newest WAL record restart replay read. + installed: WriteVersion, + /// The highest per-key or per-value version horizon GC evicted. + evicted: WriteVersion, +} + +/// Per-core write-version index. +/// +/// Every version is a position in one vShard's history, so every map is +/// keyed by vShard and versions of two vShards are never compared. +#[derive(Default)] +pub struct WriteVersionIndex { + /// Last committed-write version per written key. + last_write: HashMap, + /// Last committed-write version per written collection (the phantom-safe + /// floor: a predicate reader validates against it). Never GC'd: bounded + /// by the live collections of the core's vShards. + coll_write: HashMap, + /// Per-secondary-index-dimension write-VALUE versions, the finer-grained + /// sibling of `coll_write` an index-range read validates against. + pub(in crate::data::executor) index_values: IndexValueVersionIndex, + /// Per-vShard bounds. + vshards: HashMap, + /// The stamps of the WAL records the replay arms apply: every record + /// during restart replay, the installed record during a committed redo + /// install. + pub(in crate::data::executor) record_stamps: RecordStamps, +} + +impl WriteVersionIndex { + pub fn new() -> Self { + Self::default() + } + + /// Record a committed write at `version` on `vshard`. + /// + /// Advances the collection's version and, when `key` is `Some`, the + /// key's version, each to the max of its current value and `version`. + /// Raises the vShard's latest version the same way. + pub fn note_write( + &mut self, + db: DatabaseId, + tenant: TenantId, + vshard: VShardId, + collection: &str, + key: Option, + version: WriteVersion, + ) { + let coll_key = CollKey { + vshard, + db, + tenant, + collection: Box::from(collection), + }; + let slot = self.coll_write.entry(coll_key).or_default(); + *slot = (*slot).max(version); + + if let Some(key) = key { + let write_key = WriteKey { + vshard, + db, + tenant, + collection: Box::from(collection), + key, + }; + let slot = self.last_write.entry(write_key).or_default(); + *slot = (*slot).max(version); + } + self.raise_latest(vshard, version); + } + + /// Raise `vshard`'s latest version to `version` when it is higher. + pub(in crate::data::executor) fn raise_latest( + &mut self, + vshard: VShardId, + version: WriteVersion, + ) { + let bounds = self.vshards.entry(vshard).or_default(); + bounds.latest = bounds.latest.max(version); + } + + /// The vShard's state now holds every write through `version`, with no + /// per-row versions: a snapshot install landed it. Every recorded version + /// of the vShard is dropped, because the install replaced the rows they + /// describe. A read older than `version` no longer validates. + pub fn install(&mut self, vshard: VShardId, version: WriteVersion) { + self.last_write.retain(|key, _| key.vshard != vshard); + self.coll_write.retain(|key, _| key.vshard != vshard); + self.index_values.drop_vshard(vshard); + let bounds = self.vshards.entry(vshard).or_default(); + bounds.installed = bounds.installed.max(version); + bounds.latest = bounds.latest.max(version); + } + + /// The vShard's state holds writes through `version` that no per-row + /// version describes: restart replay read records it did not apply. A + /// read older than `version` no longer validates against a row with no + /// recorded version. + pub fn raise_installed(&mut self, vshard: VShardId, version: WriteVersion) { + let bounds = self.vshards.entry(vshard).or_default(); + bounds.installed = bounds.installed.max(version); + bounds.latest = bounds.latest.max(version); + } + + /// Whether this core recorded any version of `vshard`. + pub(in crate::data::executor) fn holds(&self, vshard: VShardId) -> bool { + self.vshards.contains_key(&vshard) + } + + /// The highest version recorded for `vshard`, `ZERO` when none is. + pub(in crate::data::executor) fn latest(&self, vshard: VShardId) -> WriteVersion { + self.vshards + .get(&vshard) + .map(|bounds| bounds.latest) + .unwrap_or_default() + } + + /// Every vShard this core recorded a version of, with its latest one. + pub(in crate::data::executor) fn latest_by_vshard( + &self, + ) -> impl Iterator + '_ { + self.vshards + .iter() + .map(|(vshard, bounds)| (*vshard, bounds.latest)) + } + + /// Recorded per-key version, if any. + #[cfg(test)] + pub(crate) fn key_version(&self, key: &WriteKey) -> Option { + self.last_write.get(key).copied() + } + + /// Recorded per-collection version, if any. + #[cfg(test)] + pub(crate) fn collection_version(&self, key: &CollKey) -> Option { + self.coll_write.get(key).copied() + } + + /// Every recorded per-key version. + #[cfg(test)] + pub(crate) fn recorded_keys(&self) -> impl Iterator { + self.last_write.iter().map(|(key, version)| (key, *version)) + } + + /// Every recorded per-collection version. + #[cfg(test)] + pub(crate) fn recorded_collections(&self) -> impl Iterator { + self.coll_write.iter().map(|(key, version)| (key, *version)) + } + + /// The version of `collection` on `vshard` a read compares against: the + /// recorded one, else the version the vShard's state was installed at. + pub(in crate::data::executor) fn collection_current( + &self, + db: DatabaseId, + tenant: TenantId, + vshard: VShardId, + collection: &str, + ) -> WriteVersion { + let key = CollKey { + vshard, + db, + tenant, + collection: Box::from(collection), + }; + self.coll_write + .get(&key) + .copied() + .unwrap_or_else(|| self.bounds(vshard).installed) + } + + fn bounds(&self, vshard: VShardId) -> VShardBounds { + self.vshards.get(&vshard).copied().unwrap_or_default() + } + + /// Whether a read of `collection` on `vshard`, observed at `read`, is + /// still current. + /// + /// A `Point` read is current iff the key's version is at or below the + /// read's. A `Predicate` read compares the collection's version. An + /// index read compares the versions of the values it observed, and falls + /// back to the collection's version for a dimension no write recorded. + /// + /// A key or value with no recorded version was not written since the + /// vShard was installed, or GC evicted it. Its version is then at most + /// the installed or evicted bound, and never above the collection's. + pub(crate) fn read_is_valid( + &self, + db: DatabaseId, + tenant: TenantId, + vshard: VShardId, + collection: &str, + key: &ReadKeyIdent, + read: WriteVersion, + ) -> bool { + let collection_current = self.collection_current(db, tenant, vshard, collection); + let bounds = self.bounds(vshard); + let unrecorded = collection_current.min(bounds.installed.max(bounds.evicted)); + match key { + ReadKeyIdent::Point(repr) => { + let write_key = WriteKey { + vshard, + db, + tenant, + collection: Box::from(collection), + key: repr.clone(), + }; + self.last_write + .get(&write_key) + .copied() + .unwrap_or(unrecorded) + <= read + } + ReadKeyIdent::Predicate => collection_current <= read, + ReadKeyIdent::IndexEq { field, value } => { + let dim = IndexDimRef { + vshard, + db, + tenant, + collection, + field, + }; + match self.index_values.eq_version(dim, value) { + IndexVersion::Untracked => collection_current <= read, + IndexVersion::Tracked(recorded) => recorded.unwrap_or(unrecorded) <= read, + } + } + ReadKeyIdent::IndexRange { field, lo, hi } => { + let dim = IndexDimRef { + vshard, + db, + tenant, + collection, + field, + }; + match self + .index_values + .range_version(dim, lo.as_deref(), hi.as_deref()) + { + IndexVersion::Untracked => collection_current <= read, + // A value GC evicted from the range is unknown, so the + // range takes the unrecorded bound as well. + IndexVersion::Tracked(recorded) => { + recorded.unwrap_or_default().max(unrecorded) <= read + } + } + } + } + } + + /// Horizon garbage-collect the per-key and per-value maps. + /// + /// Evicts every entry more than [`RETAIN_WINDOW`] below its vShard's + /// latest version in the same epoch, then, if more than + /// [`MAX_KEY_ENTRIES`] per-key entries remain, drops the lowest versions + /// until back within bound. Each vShard's evicted bound rises to the + /// highest version it lost. The per-collection map is left untouched. + pub fn gc(&mut self) { + let latest: HashMap = self.latest_by_vshard().collect(); + let mut evicted: HashMap = HashMap::new(); + self.last_write.retain(|key, version| { + let keep = latest + .get(&key.vshard) + .and_then(|latest| latest.distance_above(*version)) + .is_none_or(|distance| distance <= RETAIN_WINDOW); + if !keep { + let slot = evicted.entry(key.vshard).or_default(); + *slot = (*slot).max(*version); + } + keep + }); + + if self.last_write.len() > MAX_KEY_ENTRIES { + let overflow = self.last_write.len() - MAX_KEY_ENTRIES; + let mut by_version: Vec<(WriteVersion, WriteKey)> = self + .last_write + .iter() + .map(|(key, version)| (*version, key.clone())) + .collect(); + // TOTAL order so tied-version eviction is replica-identical: a + // plain sort by version over a `HashMap`-collected Vec would let + // the dropped set depend on hash-iteration layout. `DatabaseId` / + // `TenantId` lack `Ord` (compare via `as_u64()`); `KeyRepr` is + // `Ord`. + by_version.sort_by(|a, b| { + a.0.cmp(&b.0) + .then_with(|| a.1.vshard.cmp(&b.1.vshard)) + .then_with(|| a.1.db.as_u64().cmp(&b.1.db.as_u64())) + .then_with(|| a.1.tenant.as_u64().cmp(&b.1.tenant.as_u64())) + .then_with(|| a.1.collection.cmp(&b.1.collection)) + .then_with(|| a.1.key.cmp(&b.1.key)) + }); + for (version, key) in by_version.into_iter().take(overflow) { + let slot = evicted.entry(key.vshard).or_default(); + *slot = (*slot).max(version); + self.last_write.remove(&key); + } + } + + for (vshard, version) in self.index_values.gc(&latest) { + let slot = evicted.entry(vshard).or_default(); + *slot = (*slot).max(version); + } + for (vshard, version) in evicted { + let bounds = self.vshards.entry(vshard).or_default(); + bounds.evicted = bounds.evicted.max(version); + } + } +} + +#[cfg(test)] +mod tests { + use super::*; + + fn db() -> DatabaseId { + DatabaseId::DEFAULT + } + + fn tenant() -> TenantId { + TenantId::new(1) + } + + fn vshard() -> VShardId { + VShardId::new(5) + } + + fn at(index: u64) -> WriteVersion { + WriteVersion::logged(0, index) + } + + fn point(surrogate: u32) -> ReadKeyIdent { + ReadKeyIdent::Point(KeyRepr::Surrogate(surrogate)) + } + + fn valid(index: &WriteVersionIndex, key: &ReadKeyIdent, read: u64) -> bool { + index.read_is_valid(db(), tenant(), vshard(), "orders", key, at(read)) + } + + fn write(index: &mut WriteVersionIndex, key: Option, version: u64) { + index.note_write(db(), tenant(), vshard(), "orders", key, at(version)); + } + + fn record_value(index: &mut WriteVersionIndex, field: &str, value: &str, version: u64) { + let dim = IndexDimRef { + vshard: vshard(), + db: db(), + tenant: tenant(), + collection: "orders", + field, + }; + index.index_values.record(dim, value, at(version)); + } + + #[test] + fn point_read_is_valid_when_key_never_written() { + let index = WriteVersionIndex::new(); + assert!(valid(&index, &point(7), 10)); + } + + #[test] + fn point_read_is_valid_when_write_at_or_before_read() { + let mut index = WriteVersionIndex::new(); + write(&mut index, Some(KeyRepr::Surrogate(7)), 10); + assert!(valid(&index, &point(7), 10)); + assert!(valid(&index, &point(7), 20)); + } + + #[test] + fn point_read_is_invalid_when_write_after_read() { + let mut index = WriteVersionIndex::new(); + write(&mut index, Some(KeyRepr::Surrogate(7)), 20); + assert!(!valid(&index, &point(7), 10)); + } + + #[test] + fn predicate_read_follows_the_collection_version() { + let mut index = WriteVersionIndex::new(); + assert!(valid(&index, &ReadKeyIdent::Predicate, 10)); + write(&mut index, None, 10); + assert!(valid(&index, &ReadKeyIdent::Predicate, 10)); + assert!(valid(&index, &ReadKeyIdent::Predicate, 20)); + write(&mut index, None, 20); + assert!(!valid(&index, &ReadKeyIdent::Predicate, 10)); + } + + #[test] + fn a_write_on_another_vshard_never_moves_a_read() { + let mut index = WriteVersionIndex::new(); + index.note_write( + db(), + tenant(), + VShardId::new(6), + "orders", + Some(KeyRepr::Surrogate(7)), + at(900), + ); + assert!(valid(&index, &point(7), 10)); + assert!(valid(&index, &ReadKeyIdent::Predicate, 10)); + } + + #[test] + fn untracked_index_dimension_falls_back_to_collection_version() { + let index_eq = ReadKeyIdent::IndexEq { + field: "email".to_string(), + value: "a@b.c".to_string(), + }; + let index_range = ReadKeyIdent::IndexRange { + field: "age".to_string(), + lo: Some("18".to_string()), + hi: None, + }; + for (floor, read) in [(5u64, 10u64), (10, 10), (20, 10)] { + let mut index = WriteVersionIndex::new(); + write(&mut index, None, floor); + let want = valid(&index, &ReadKeyIdent::Predicate, read); + assert_eq!(valid(&index, &index_eq, read), want, "eq {floor} {read}"); + assert_eq!( + valid(&index, &index_range, read), + want, + "range {floor} {read}" + ); + } + } + + #[test] + fn a_disjoint_index_value_write_does_not_abort() { + let mut index = WriteVersionIndex::new(); + write(&mut index, None, 20); + record_value(&mut index, "email", "z@z.z", 20); + record_value(&mut index, "age", "50", 20); + let eq = ReadKeyIdent::IndexEq { + field: "email".to_string(), + value: "a@b.c".to_string(), + }; + let range = ReadKeyIdent::IndexRange { + field: "age".to_string(), + lo: Some("10".to_string()), + hi: Some("20".to_string()), + }; + assert!(valid(&index, &eq, 10)); + assert!(valid(&index, &range, 10)); + } + + #[test] + fn an_index_value_written_after_the_read_aborts() { + let mut index = WriteVersionIndex::new(); + record_value(&mut index, "email", "a@b.c", 20); + record_value(&mut index, "age", "15", 20); + let eq = ReadKeyIdent::IndexEq { + field: "email".to_string(), + value: "a@b.c".to_string(), + }; + let range = ReadKeyIdent::IndexRange { + field: "age".to_string(), + lo: Some("10".to_string()), + hi: Some("20".to_string()), + }; + assert!(!valid(&index, &eq, 10)); + assert!(!valid(&index, &range, 10)); + } + + #[test] + fn a_phantom_insert_into_a_read_range_aborts() { + let mut index = WriteVersionIndex::new(); + record_value(&mut index, "age", "99", 5); + let range = ReadKeyIdent::IndexRange { + field: "age".to_string(), + lo: Some("10".to_string()), + hi: Some("20".to_string()), + }; + assert!(valid(&index, &range, 10), "no in-range write"); + record_value(&mut index, "age", "15", 20); + assert!(!valid(&index, &range, 10), "phantom in-range insert"); + } + + #[test] + fn horizon_gc_evicts_stale_keys_and_an_evicted_key_bounds_old_reads() { + let mut index = WriteVersionIndex::new(); + write(&mut index, Some(KeyRepr::Surrogate(1)), 10); + write(&mut index, Some(KeyRepr::Surrogate(2)), 1_000_000); + index.gc(); + + let key = |surrogate| WriteKey { + vshard: vshard(), + db: db(), + tenant: tenant(), + collection: Box::from("orders"), + key: KeyRepr::Surrogate(surrogate), + }; + assert_eq!(index.key_version(&key(1)), None); + assert_eq!(index.key_version(&key(2)), Some(at(1_000_000))); + // The evicted key's version is unknown, so a read older than the + // eviction bound no longer validates; a newer one does. + assert!(!valid(&index, &point(1), 5)); + assert!(valid(&index, &point(1), 10)); + } + + #[test] + fn an_install_drops_the_vshards_versions_and_bounds_every_read() { + let mut index = WriteVersionIndex::new(); + write(&mut index, Some(KeyRepr::Surrogate(7)), 10); + index.install(vshard(), at(50)); + assert_eq!(index.latest(vshard()), at(50)); + // The install replaced key 7: its version is unknown up to the cut. + assert!(!valid(&index, &point(7), 20)); + assert!(!valid(&index, &ReadKeyIdent::Predicate, 20)); + assert!(valid(&index, &point(7), 50)); + assert!(valid(&index, &ReadKeyIdent::Predicate, 50)); + } +} diff --git a/nodedb/src/data/executor/core_loop/write_index/keys.rs b/nodedb/src/data/executor/core_loop/write_index/keys.rs new file mode 100644 index 000000000..c0bfcbbfc --- /dev/null +++ b/nodedb/src/data/executor/core_loop/write_index/keys.rs @@ -0,0 +1,48 @@ +// SPDX-License-Identifier: BUSL-1.1 + +//! The keys the write-version index records versions under, and the stamp a +//! committed write records them with. + +use nodedb_types::{DatabaseId, TenantId, WriteVersion}; + +use crate::types::{Lsn, VShardId}; + +/// Row identity type, re-exported from its plane-neutral home +/// ([`crate::types::KeyRepr`]) so Data-Plane call sites can keep referring to +/// it through this module. Read keys and write keys share this one namespace. +pub use crate::types::KeyRepr; + +/// Fully-qualified per-key version-index key. Scoped by `(database, tenant)` +/// exactly like the write path, so two tenants (or databases) never alias. +/// A version positions a write within one vShard, so the vShard is part of +/// the key: an edge written on two endpoint vShards holds one version on +/// each. +#[derive(Debug, Clone, PartialEq, Eq, Hash)] +pub struct WriteKey { + pub vshard: VShardId, + pub db: DatabaseId, + pub tenant: TenantId, + pub collection: Box, + pub key: KeyRepr, +} + +/// Fully-qualified per-collection version-index key, on one vShard. +#[derive(Debug, Clone, PartialEq, Eq, Hash)] +pub struct CollKey { + pub vshard: VShardId, + pub db: DatabaseId, + pub tenant: TenantId, + pub collection: Box, +} + +/// What the version index records one committed write under. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub struct WriteStamp { + /// The vShard whose history the write extends. + pub vshard: VShardId, + /// The LSN of the write's WAL record on this node. + pub lsn: Lsn, + /// The log position of the data-group entry the write applies. `None` + /// for a write that applies no entry. + pub entry: Option, +} diff --git a/nodedb/src/data/executor/core_loop/write_index/mod.rs b/nodedb/src/data/executor/core_loop/write_index/mod.rs new file mode 100644 index 000000000..e6756e9c0 --- /dev/null +++ b/nodedb/src/data/executor/core_loop/write_index/mod.rs @@ -0,0 +1,35 @@ +// SPDX-License-Identifier: BUSL-1.1 + +//! Per-core write-version index. +//! +//! Records, for every committed write applied on this Data-Plane core, the +//! version of the write against the written key and against the written +//! collection, on the write's vShard. A write that applies a data-group Raft +//! entry takes the entry's log position as its version, so every replica of +//! the vShard records the same version for it. A write that applies no entry +//! takes its WAL LSN on top of the vShard's latest version. +//! +//! This is the shard-local write-version substrate the optimistic-concurrency +//! commit path validates a transaction's read-set against (see +//! `CoreLoop::read_set_still_current`). Because the index lives on the +//! `!Send` core it is a plain `HashMap`: no atomics, no locks, no cross-core +//! sharing. +//! +//! The per-key map is bounded: horizon GC (run from the periodic maintenance +//! hook) evicts entries far below their vShard's latest version and enforces +//! a hard entry-count backstop. The per-collection map is bounded by the +//! number of live collections and is never GC'd. + +pub mod index; +pub mod keys; +mod record; +pub mod record_stamps; +#[cfg(test)] +pub mod tests; +mod validate; + +pub use index::{RETAIN_WINDOW, WriteVersionIndex}; +#[cfg(test)] +pub use keys::{CollKey, WriteKey}; +pub use keys::{KeyRepr, WriteStamp}; +pub use record_stamps::RecordStamps; diff --git a/nodedb/src/data/executor/core_loop/write_index/record.rs b/nodedb/src/data/executor/core_loop/write_index/record.rs new file mode 100644 index 000000000..6b54575a7 --- /dev/null +++ b/nodedb/src/data/executor/core_loop/write_index/record.rs @@ -0,0 +1,548 @@ +// SPDX-License-Identifier: BUSL-1.1 + +//! The apply chokepoints that record committed writes into the per-core +//! write-version index. + +use nodedb_types::{DatabaseId, ShardVersion, TenantId, WriteVersion}; + +use crate::data::executor::task::ExecutionTask; +use crate::types::{Lsn, VShardId}; + +use super::super::CoreLoop; +use super::keys::{KeyRepr, WriteStamp}; + +impl CoreLoop { + /// The stamp of the write `task` applies: its vShard, its WAL record and + /// the log position of the data-group entry it applies. `None` for a task + /// that carries no WAL record. A record replay or a committed redo install + /// applies takes the stamp that record names. + pub(in crate::data::executor) fn task_write_stamp( + &self, + task: &ExecutionTask, + ) -> Option { + task.wal_lsn() + .map(|lsn| self.task_write_stamp_at(task, lsn)) + } + + /// The stamp of the write `task` applies from the WAL record at `lsn`, + /// for a handler that reads the record's LSN off its plan. + pub(in crate::data::executor) fn task_write_stamp_at( + &self, + task: &ExecutionTask, + lsn: Lsn, + ) -> WriteStamp { + self.write_index.record_stamps.resolve(WriteStamp { + vshard: task.request.vshard_id, + lsn, + entry: task.request.entry_version, + }) + } + + /// The version a write with `stamp` records: the log position of the + /// entry it applies, or, for a write that applies no entry, its WAL LSN on + /// top of the latest version its vShard holds here. + pub(in crate::data::executor) fn write_version_of(&self, stamp: WriteStamp) -> WriteVersion { + stamp.entry.unwrap_or_else(|| { + WriteVersion::local_after(self.write_index.latest(stamp.vshard), stamp.lsn.as_u64()) + }) + } + + /// Record a committed write into the per-core version index and advance + /// the core watermark monotonically. + /// + /// Called once per written key at every Data-Plane apply chokepoint. `key` + /// is `None` for engines whose per-key identity is internal (columnar / + /// timeseries / array / spatial / FTS): those record only the collection + /// version. + /// + /// In the install pass of a committed-redo apply the version waits in the + /// apply scope. The apply publishes it once the record settled, so a + /// rolled-back install leaves no version and no watermark behind. + pub(in crate::data::executor) fn note_write( + &mut self, + db: DatabaseId, + tenant: TenantId, + collection: &str, + key: Option, + stamp: WriteStamp, + ) { + if let Some(scope) = self.redo_apply.scope.as_mut() { + scope.defer_write_version(db, tenant, collection, key, stamp); + return; + } + self.publish_write_version(db, tenant, collection, key, stamp); + } + + /// Record a write version and advance the core watermark monotonically. + /// + /// Every applied write reaches here, so this is also where the core notes + /// the write's LSN as applied: a checkpoint's replay stamp names it. + pub(in crate::data::executor) fn publish_write_version( + &mut self, + db: DatabaseId, + tenant: TenantId, + collection: &str, + key: Option, + stamp: WriteStamp, + ) { + self.floors.applied_prefix.note_applied(stamp.lsn); + let version = self.write_version_of(stamp); + self.write_index + .note_write(db, tenant, stamp.vshard, collection, key, version); + if stamp.lsn > self.watermark { + self.watermark = stamp.lsn; + } + } + + /// Record a committed write's collection version only (no per-key + /// entry), if `task` carries a WAL record. Shared by the columnar-family + /// write handlers (columnar / timeseries / array / spatial / FTS) whose + /// per-key identity is internal: a predicate reader validates against + /// the collection version when it owns no per-key version. + pub(in crate::data::executor) fn note_collection_write( + &mut self, + task: &ExecutionTask, + collection: &str, + ) { + if let Some(stamp) = self.task_write_stamp(task) { + self.note_write( + task.request.database_id, + task.request.tenant_id, + collection, + None, + stamp, + ); + } + } + + /// Record a committed document/vector write's version, keyed by the + /// written row's cross-engine surrogate, if `task` carries a WAL record. + /// Shared by every per-surrogate write chokepoint (point put, point + /// insert, point delete, bulk update, bulk delete). + pub(in crate::data::executor) fn note_surrogate_write( + &mut self, + task: &ExecutionTask, + tid: u64, + collection: &str, + surrogate: u32, + ) { + if let Some(stamp) = self.task_write_stamp(task) { + self.note_write( + task.request.database_id, + TenantId::new(tid), + collection, + Some(KeyRepr::Surrogate(surrogate)), + stamp, + ); + } + } + + /// Record the version of a write a replay arm applied from the WAL record + /// at `record_lsn`. A no-op when `record_lsn == 0` (no durable record). + /// `key` is `None` for collection-only entries (e.g. truncate) and `Some` + /// for per-key/per-surrogate entries, exactly like [`Self::note_write`]. + /// + /// The replay arms run for restart replay and for a committed redo + /// install. Both register the stamp of the record they apply, so the + /// write takes that record's vShard and entry position. A record nothing + /// registered applies on its collection's home vShard, the vShard a live + /// write of the collection routes to, and names no entry. + pub(in crate::data::executor) fn note_replay_write( + &mut self, + database_id: u64, + tenant_id: u64, + collection: &str, + key: Option, + record_lsn: u64, + ) { + if record_lsn == 0 { + return; + } + let db = DatabaseId::new(database_id); + let stamp = match self.write_index.record_stamps.get(record_lsn) { + Some(stamp) => stamp, + None => WriteStamp { + // A name without its database's qualifier is a bare name. + vshard: nodedb_types::CollectionKey::from_qualified_str(db, collection) + .unwrap_or_else(|_| nodedb_types::CollectionKey::from_bare(db, collection)) + .vshard(), + lsn: Lsn::new(record_lsn), + entry: None, + }, + }; + self.note_write(db, TenantId::new(tenant_id), collection, key, stamp); + } + + /// Run horizon GC on the per-core version index. Invoked from the periodic + /// maintenance hook: no dedicated timer. + pub(in crate::data::executor) fn gc_write_index(&mut self) { + self.write_index.gc(); + } + + /// Land a snapshot install's cut on each of `floor`'s vShards: the + /// installed rows hold every write through the cut and carry no per-row + /// versions. + pub(in crate::data::executor) fn install_version_floor(&mut self, floor: &[ShardVersion]) { + for shard in floor { + if shard.vshard < VShardId::COUNT { + self.write_index + .install(VShardId::new(shard.vshard), shard.version); + } + } + } +} + +#[cfg(test)] +mod tests { + use nodedb_physical::physical_plan::DocumentOp; + use nodedb_types::{QualifiedCollection, Surrogate}; + + use super::super::keys::{CollKey, WriteKey}; + use super::super::tests::{ + doc_value, entry_task, local, make_core, make_request, point_put, wal_task, + }; + use super::*; + use crate::bridge::envelope::{PhysicalPlan, Status}; + use crate::data::executor::handlers::graph::EdgePutParams; + use crate::data::executor::handlers::kv::crud::KvWriteParams; + + fn vshard0() -> VShardId { + VShardId::new(0) + } + + fn surrogate_key(collection: &str, surrogate: u32) -> WriteKey { + WriteKey { + vshard: vshard0(), + db: DatabaseId::DEFAULT, + tenant: TenantId::new(1), + collection: Box::from(collection), + key: KeyRepr::Surrogate(surrogate), + } + } + + fn coll_key(collection: &str) -> CollKey { + CollKey { + vshard: vshard0(), + db: DatabaseId::DEFAULT, + tenant: TenantId::new(1), + collection: Box::from(collection), + } + } + + #[test] + fn a_write_with_no_entry_records_a_local_version_and_advances_the_watermark() { + let (mut core, _, _, _dir) = make_core(); + + let resp = point_put(&mut core, &wal_task(10), "orders", "o1", 7, "1"); + assert_eq!(resp.status, Status::Ok); + assert_eq!( + core.write_index.key_version(&surrogate_key("orders", 7)), + Some(local(10)) + ); + assert_eq!( + core.write_index.collection_version(&coll_key("orders")), + Some(local(10)) + ); + assert_eq!(core.watermark, Lsn::new(10)); + + point_put(&mut core, &wal_task(20), "orders", "o1", 7, "2"); + assert_eq!( + core.write_index.key_version(&surrogate_key("orders", 7)), + Some(local(20)) + ); + assert_eq!(core.watermark, Lsn::new(20)); + + // A lower LSN never regresses an existing entry or the watermark. + point_put(&mut core, &wal_task(15), "orders", "o1", 7, "3"); + assert_eq!( + core.write_index.key_version(&surrogate_key("orders", 7)), + Some(local(20)) + ); + assert_eq!(core.watermark, Lsn::new(20)); + + // A second collection tracks its own version independently. + point_put(&mut core, &wal_task(30), "items", "i1", 9, "4"); + assert_eq!( + core.write_index.collection_version(&coll_key("items")), + Some(local(30)) + ); + assert_eq!( + core.write_index.collection_version(&coll_key("orders")), + Some(local(20)) + ); + assert_eq!(core.watermark, Lsn::new(30)); + } + + #[test] + fn the_version_recorded_is_the_applying_entrys_log_position() { + let (mut core, _, _, _dir) = make_core(); + let entry = WriteVersion::logged(4, 812); + + // The WAL LSN is this node's own. The version is the entry's position. + let resp = point_put(&mut core, &entry_task(57, entry), "orders", "o1", 7, "1"); + assert_eq!(resp.status, Status::Ok); + assert_eq!( + core.write_index.key_version(&surrogate_key("orders", 7)), + Some(entry) + ); + assert_eq!( + core.write_index.collection_version(&coll_key("orders")), + Some(entry) + ); + assert_eq!(core.write_index.latest(vshard0()), entry); + assert_eq!(core.watermark, Lsn::new(57)); + + // A replica that applies the same entry at another WAL LSN records + // the same version. + let (mut replica, _, _, _replica_dir) = make_core(); + point_put( + &mut replica, + &entry_task(9_001, entry), + "orders", + "o1", + 7, + "1", + ); + assert_eq!( + replica.write_index.key_version(&surrogate_key("orders", 7)), + Some(entry) + ); + } + + #[test] + fn a_point_update_records_the_rows_version() { + let (mut core, _, _, _dir) = make_core(); + point_put( + &mut core, + &entry_task(40, WriteVersion::logged(1, 5)), + "orders", + "o1", + 7, + "1", + ); + let raised = nodedb_types::json_to_msgpack(&serde_json::json!("2")).expect("encode"); + let updates = vec![( + "a".to_string(), + nodedb_physical::physical_plan::UpdateValue::Literal(raised), + )]; + let updated = WriteVersion::logged(1, 9); + let resp = core.execute_point_update( + &entry_task(41, updated), + crate::data::executor::handlers::point::update::PointUpdateParams { + tid: 1, + collection: "orders", + document_id: "o1", + surrogate: Some(Surrogate::new(7)), + updates: &updates, + returning: None, + rls_filters: &[], + rls_write_check: &nodedb_types::RlsWriteCheck::NoPolicyApplies, + resolved_sum_targets: &[], + declared_primary_key: None, + }, + ); + assert_eq!(resp.status, Status::Ok, "{:?}", resp.error_code); + assert_eq!( + core.write_index.key_version(&surrogate_key("orders", 7)), + Some(updated), + "a read of the row from before the update must fail validation" + ); + assert_eq!( + core.write_index.collection_version(&coll_key("orders")), + Some(updated) + ); + } + + #[test] + fn a_local_write_sorts_between_the_entries_around_it() { + let (mut core, _, _, _dir) = make_core(); + point_put( + &mut core, + &entry_task(40, WriteVersion::logged(0, 5)), + "orders", + "o1", + 7, + "1", + ); + point_put(&mut core, &wal_task(41), "orders", "o2", 8, "1"); + let local_write = core + .write_index + .key_version(&surrogate_key("orders", 8)) + .expect("local write recorded"); + assert_eq!( + local_write, + WriteVersion::local_after(WriteVersion::logged(0, 5), 41) + ); + assert!(local_write > WriteVersion::logged(0, 5)); + assert!(local_write < WriteVersion::logged(0, 6)); + } + + #[test] + fn kv_put_records_kvkey_version() { + let (mut core, _, _, _dir) = make_core(); + let resp = core.execute_kv_put( + &wal_task(42), + KvWriteParams { + did: DatabaseId::DEFAULT.as_u64(), + tid: 1, + collection: "kv", + key: b"k1".as_slice(), + value: b"v1".as_slice(), + ttl_ms: 0, + surrogate: Surrogate::new(3), + returning: None, + rls_filters: &[], + }, + ); + assert_eq!(resp.status, Status::Ok); + + let wk = WriteKey { + vshard: vshard0(), + db: DatabaseId::DEFAULT, + tenant: TenantId::new(1), + collection: Box::from("kv"), + key: KeyRepr::KvKey(Box::from(b"k1".as_slice())), + }; + assert_eq!(core.write_index.key_version(&wk), Some(local(42))); + assert_eq!( + core.write_index.collection_version(&coll_key("kv")), + Some(local(42)) + ); + assert_eq!(core.watermark, Lsn::new(42)); + } + + #[test] + fn an_autocommit_apply_is_named_by_the_next_replay_stamp() { + let (mut core, _, _, _dir) = make_core(); + core.floors + .applied_prefix + .observe_outcome_floor(Lsn::new(40)); + let resp = core.execute_kv_put( + &wal_task(42), + KvWriteParams { + did: DatabaseId::DEFAULT.as_u64(), + tid: 1, + collection: "kv", + key: b"k1".as_slice(), + value: b"v1".as_slice(), + ttl_ms: 0, + surrogate: Surrogate::new(3), + returning: None, + rls_filters: &[], + }, + ); + assert_eq!(resp.status, Status::Ok); + + let stamp = core.floors.applied_prefix.stamp().expect("exact stamp"); + assert!( + stamp.skips(42), + "the applied write is in every later artifact" + ); + assert!( + !stamp.skips(41), + "a record between the floor and the applied write still replays" + ); + } + + #[test] + fn edge_put_records_edge_version() { + let (mut core, _, _, _dir) = make_core(); + let resp = core.execute_edge_put( + &wal_task(50), + EdgePutParams { + tid: 1, + collection: "graph", + src_id: "a", + label: "KNOWS", + dst_id: "b", + properties: &[], + src_surrogate: Surrogate::new(1), + dst_surrogate: Surrogate::new(2), + }, + ); + assert_eq!(resp.status, Status::Ok); + + let wk = WriteKey { + vshard: vshard0(), + db: DatabaseId::DEFAULT, + tenant: TenantId::new(1), + collection: Box::from("graph"), + key: KeyRepr::Edge { + src: Box::from("a"), + label: Box::from("KNOWS"), + dst: Box::from("b"), + }, + }; + assert_eq!(core.write_index.key_version(&wk), Some(local(50))); + assert_eq!(core.watermark, Lsn::new(50)); + } + + #[test] + fn a_committed_transaction_records_its_write_versions() { + let (mut core, _, _, _dir) = make_core(); + let task = wal_task(60); + let plans = vec![PhysicalPlan::Document(DocumentOp::PointPut { + collection: QualifiedCollection::new(DatabaseId::DEFAULT, "batch"), + document_id: "d1".into(), + value: doc_value("a", "1"), + surrogate: Surrogate::new(11), + pk_bytes: Vec::new(), + returning: None, + rls_filters: Vec::new(), + resolved_sum_targets: Vec::new(), + })]; + let resp = core.commit_plans_for_test(&task, 1, &plans, 60); + assert_eq!(resp.status, Status::Ok, "{:?}", resp.error_code); + + assert_eq!( + core.write_index.key_version(&surrogate_key("batch", 11)), + Some(local(60)) + ); + assert_eq!( + core.write_index.collection_version(&coll_key("batch")), + Some(local(60)) + ); + assert_eq!(core.watermark, Lsn::new(60)); + } + + #[test] + fn no_wal_lsn_records_nothing() { + let (mut core, _, _, _dir) = make_core(); + let task = ExecutionTask::new(make_request(PhysicalPlan::Document(DocumentOp::PointGet { + collection: QualifiedCollection::new(DatabaseId::DEFAULT, "x"), + document_id: "y".into(), + surrogate: None, + pk_bytes: Vec::new(), + rls_filters: Vec::new(), + system_time: nodedb_types::SystemTimeScope::Current, + valid_at_ms: None, + }))); + point_put(&mut core, &task, "orders", "o1", 7, "1"); + assert_eq!( + core.write_index.key_version(&surrogate_key("orders", 7)), + None + ); + assert_eq!(core.watermark, Lsn::ZERO); + } + + #[test] + fn an_install_floor_bounds_the_installed_vshard_only() { + let (mut core, _, _, _dir) = make_core(); + point_put(&mut core, &wal_task(10), "orders", "o1", 7, "1"); + let cut = WriteVersion::logged(2, 300); + core.install_version_floor(&[ShardVersion { + vshard: 0, + version: cut, + }]); + assert_eq!(core.write_index.latest(vshard0()), cut); + assert_eq!( + core.write_index.key_version(&surrogate_key("orders", 7)), + None, + "the install replaced the row its version described" + ); + assert_eq!( + core.write_index.latest(VShardId::new(1)), + WriteVersion::ZERO + ); + } +} diff --git a/nodedb/src/data/executor/core_loop/write_index/record_stamps.rs b/nodedb/src/data/executor/core_loop/write_index/record_stamps.rs new file mode 100644 index 000000000..2977b25b3 --- /dev/null +++ b/nodedb/src/data/executor/core_loop/write_index/record_stamps.rs @@ -0,0 +1,202 @@ +// SPDX-License-Identifier: BUSL-1.1 + +//! The stamps of the WAL records the replay arms apply. +//! +//! A replay arm sees a record's payload and LSN, not its header. Restart +//! replay registers every record of the WAL here before the arms run, and a +//! committed redo install registers the one record it applies. A write an +//! arm applies then takes its record's vShard and the log position of the +//! data-group entry the record applies. +//! +//! A replicated write's records follow a `ChangePosition` marker. The marker +//! names the entry's position and the proposal key every record of the entry +//! carries in its header. Restart replay recovers each record's position by +//! that key, so a recovered write records the version its live apply did. + +use std::collections::HashMap; + +use nodedb_types::WriteVersion; +use nodedb_wal::WalRecord; +use nodedb_wal::record::RecordType; + +use crate::event::cdc::position::ChangePositionMarker; +use crate::types::{Lsn, VShardId}; + +use super::keys::WriteStamp; + +/// WAL record LSN to the stamp the record's writes take. +#[derive(Debug, Default)] +pub struct RecordStamps { + by_lsn: HashMap, + /// The highest version each vShard's registered records hold. + latest: HashMap, +} + +impl RecordStamps { + /// The stamps of `records`, a node's WAL. A record that follows the + /// `ChangePosition` marker of its proposal key takes the marker's entry + /// position. Every other record names no entry. + pub fn from_wal(records: &[WalRecord]) -> Self { + let mut ordered: Vec<&WalRecord> = records.iter().collect(); + ordered.sort_by_key(|record| record.header.lsn); + + let mut stamps = Self::default(); + let mut entry_of_key: HashMap = HashMap::new(); + for record in ordered { + if RecordType::from_raw(record.logical_record_type()) + == Some(RecordType::ChangePosition) + { + match ChangePositionMarker::from_bytes(&record.payload) { + Ok(marker) if marker.apply_key != 0 => { + entry_of_key.insert( + marker.apply_key, + WriteVersion::logged(marker.position.epoch, marker.position.log_index), + ); + } + Ok(_) => {} + Err(error) => tracing::warn!( + lsn = record.header.lsn, + %error, + "skipping an undecodable ChangePosition WAL marker" + ), + } + continue; + } + if record.header.vshard_id >= VShardId::COUNT { + continue; + } + let vshard = VShardId::new(record.header.vshard_id); + let key = record.apply_key(); + let entry = (key != 0) + .then(|| entry_of_key.get(&key).copied()) + .flatten(); + let latest = stamps.latest.entry(vshard).or_default(); + *latest = (*latest).max( + entry.unwrap_or_else(|| WriteVersion::local_after(*latest, record.header.lsn)), + ); + stamps.by_lsn.insert( + record.header.lsn, + WriteStamp { + vshard, + lsn: Lsn::new(record.header.lsn), + entry, + }, + ); + } + stamps + } + + /// Register the record a committed redo install applies. + pub fn insert(&mut self, stamp: WriteStamp) { + self.by_lsn.insert(stamp.lsn.as_u64(), stamp); + } + + /// Forget the record at `lsn`. + pub fn remove(&mut self, lsn: Lsn) { + self.by_lsn.remove(&lsn.as_u64()); + } + + /// Forget every record: restart replay finished. + pub fn clear(&mut self) { + self.by_lsn.clear(); + self.latest.clear(); + } + + /// The stamp of the record at `lsn`, if registered. + pub fn get(&self, lsn: u64) -> Option { + self.by_lsn.get(&lsn).copied() + } + + /// `stamp`, or the stamp of its record when the record is registered. + pub fn resolve(&self, stamp: WriteStamp) -> WriteStamp { + self.get(stamp.lsn.as_u64()).unwrap_or(stamp) + } + + /// The highest version each vShard's registered WAL records hold. + pub fn latest_by_vshard(&self) -> impl Iterator + '_ { + self.latest + .iter() + .map(|(vshard, version)| (*vshard, *version)) + } +} + +#[cfg(test)] +mod tests { + use nodedb_wal::record::WalRecordArgs; + + use super::*; + use crate::event::cdc::position::ReplicatedPosition; + + fn record(record_type: RecordType, lsn: u64, vshard: u32, apply_key: u64) -> WalRecord { + let payload = match record_type { + RecordType::ChangePosition => ChangePositionMarker { + apply_key, + position: ReplicatedPosition { + epoch: 3, + group_id: 7, + log_index: 100 + lsn, + }, + } + .to_bytes() + .to_vec(), + _ => vec![1], + }; + let header_key = match record_type { + RecordType::ChangePosition => 0, + _ => apply_key, + }; + WalRecord::new_stamped( + WalRecordArgs { + record_type: record_type as u32, + lsn, + tenant_id: 1, + vshard_id: vshard, + database_id: 0, + payload, + encryption_key: None, + preamble_bytes: None, + }, + nodedb_wal::record::RecordStamp { + apply_key: header_key, + event_source: nodedb_wal::NO_EVENT_SOURCE, + commit_hlc: 0, + }, + ) + .expect("build record") + } + + #[test] + fn a_record_after_its_marker_takes_the_entry_position() { + let records = vec![ + record(RecordType::ChangePosition, 10, 4, 0xAB), + record(RecordType::Put, 11, 4, 0xAB), + record(RecordType::Put, 12, 4, 0), + ]; + let stamps = RecordStamps::from_wal(&records); + + let replicated = stamps.get(11).expect("registered"); + assert_eq!(replicated.vshard, VShardId::new(4)); + assert_eq!(replicated.entry, Some(WriteVersion::logged(3, 110))); + + let local = stamps.get(12).expect("registered"); + assert_eq!(local.entry, None); + assert_eq!(stamps.get(10), None, "a marker writes no row"); + + let latest: HashMap = stamps.latest_by_vshard().collect(); + assert_eq!( + latest.get(&VShardId::new(4)), + Some(&WriteVersion::local_after(WriteVersion::logged(3, 110), 12)) + ); + } + + #[test] + fn resolve_keeps_an_unregistered_stamp() { + let stamps = RecordStamps::default(); + let stamp = WriteStamp { + vshard: VShardId::new(2), + lsn: Lsn::new(5), + entry: None, + }; + assert_eq!(stamps.resolve(stamp), stamp); + } +} diff --git a/nodedb/src/data/executor/core_loop/write_index/tests.rs b/nodedb/src/data/executor/core_loop/write_index/tests.rs new file mode 100644 index 000000000..ee7c8f336 --- /dev/null +++ b/nodedb/src/data/executor/core_loop/write_index/tests.rs @@ -0,0 +1,165 @@ +// SPDX-License-Identifier: BUSL-1.1 + +//! `CoreLoop` fixtures shared by the write-version-index tests and by other +//! executor test modules (`make_core_with_dir` / `make_default_task` are +//! `pub` for exactly that reason). + +use std::time::{Duration, Instant}; + +use nodedb_bridge::buffer::RingBuffer; +use nodedb_physical::physical_plan::DocumentOp; +use nodedb_types::{DatabaseId, QualifiedCollection, Surrogate, TenantId, WriteVersion}; + +use crate::bridge::dispatch::{BridgeRequest, BridgeResponse}; +use crate::bridge::envelope::{PhysicalPlan, Priority, Request, Response}; +use crate::data::executor::core_loop::CoreLoop; +use crate::data::executor::handlers::point::put::PointPutExec; +use crate::data::executor::task::ExecutionTask; +use crate::types::{Lsn, ReadConsistency, RequestId, TraceId, VShardId}; + +pub(crate) fn make_core() -> ( + CoreLoop, + nodedb_bridge::buffer::Producer, + nodedb_bridge::buffer::Consumer, + tempfile::TempDir, +) { + let dir = tempfile::tempdir().unwrap(); + let (core, req_tx, resp_rx) = make_core_with_dir(dir.path()); + (core, req_tx, resp_rx, dir) +} + +pub fn make_core_with_dir( + dir: &std::path::Path, +) -> ( + CoreLoop, + nodedb_bridge::buffer::Producer, + nodedb_bridge::buffer::Consumer, +) { + let (req_tx, req_rx) = RingBuffer::channel::(64); + let (resp_tx, resp_rx) = RingBuffer::channel::(64); + let core = CoreLoop::open( + 0, + req_rx, + resp_tx, + dir, + std::sync::Arc::new(nodedb_types::OrdinalClock::new()), + crate::data::executor::core_loop::test_governor(), + ) + .unwrap(); + (core, req_tx, resp_rx) +} + +fn point_get() -> PhysicalPlan { + PhysicalPlan::Document(DocumentOp::PointGet { + collection: QualifiedCollection::new(DatabaseId::DEFAULT, "x"), + document_id: "y".into(), + surrogate: None, + pk_bytes: Vec::new(), + rls_filters: Vec::new(), + system_time: nodedb_types::SystemTimeScope::Current, + valid_at_ms: None, + }) +} + +pub(crate) fn make_request(plan: PhysicalPlan) -> Request { + Request { + request_id: RequestId::new(1), + tenant_id: TenantId::new(1), + database_id: DatabaseId::DEFAULT, + vshard_id: VShardId::new(0), + plan, + deadline: Instant::now() + Duration::from_secs(5), + priority: Priority::Normal, + trace_id: TraceId::ZERO, + consistency: ReadConsistency::Strong, + idempotency_key: None, + event_source: crate::event::EventSource::User, + user_roles: Vec::new(), + user_id: None, + statement_digest: None, + txn_id: None, + wal_lsn: None, + resolved_now_ms: None, + commit_hlc: None, + entry_version: None, + admission: crate::bridge::envelope::Admission::Admitted, + } +} + +/// A minimal `ExecutionTask` (DEFAULT database/tenant, vShard 0, no WAL LSN) +/// for handler unit tests that only read `request.database_id`. The carried +/// plan is inert: edge/point handlers take their parameters directly. +pub fn make_default_task() -> ExecutionTask { + ExecutionTask::new(make_request(point_get())) +} + +/// A `{k: v}` document body: a standard MessagePack map. +pub(crate) fn doc_value(k: &str, v: &str) -> Vec { + let mut obj = std::collections::HashMap::new(); + obj.insert(k.to_string(), nodedb_types::Value::String(v.into())); + nodedb_types::value_to_msgpack(&nodedb_types::Value::Object(obj)).unwrap() +} + +/// An `ExecutionTask` carrying WAL LSN `lsn` and applying no data-group +/// entry: tenant 1, database DEFAULT, vShard 0. +pub(crate) fn wal_task(lsn: u64) -> ExecutionTask { + ExecutionTask::with_wal_lsn(make_request(point_get()), Some(Lsn::new(lsn))) +} + +/// An `ExecutionTask` carrying WAL LSN `lsn` that applies the data-group +/// entry at `entry`. +pub(crate) fn entry_task(lsn: u64, entry: WriteVersion) -> ExecutionTask { + ExecutionTask::with_wal_lsn( + Request { + entry_version: Some(entry), + ..make_request(point_get()) + }, + Some(Lsn::new(lsn)), + ) +} + +/// An `ExecutionTask` homing to `vshard_id`, carrying no WAL LSN. +pub(crate) fn task_with_vshard(vshard_id: VShardId) -> ExecutionTask { + ExecutionTask::new(Request { + vshard_id, + ..make_request(point_get()) + }) +} + +/// The version a write at `lsn` records on a vShard no entry ever wrote: +/// a single-node write. +pub(crate) fn local(lsn: u64) -> WriteVersion { + WriteVersion::local_after(WriteVersion::ZERO, lsn) +} + +/// The vShard a replayed write of `collection` in `db` records under when no +/// registered record stamp names one: the collection's home vShard. +pub(crate) fn replay_home(db: DatabaseId, collection: &str) -> VShardId { + nodedb_types::CollectionKey::from_qualified_str(db, collection) + .unwrap_or_else(|_| nodedb_types::CollectionKey::from_bare(db, collection)) + .vshard() +} + +/// Put `{a: value}` as document `document_id` of `collection` under `task`. +pub(crate) fn point_put( + core: &mut CoreLoop, + task: &ExecutionTask, + collection: &str, + document_id: &str, + surrogate: u32, + value: &str, +) -> Response { + core.execute_point_put( + task, + PointPutExec { + tid: 1, + collection, + document_id, + surrogate: Surrogate::new(surrogate), + value: &doc_value("a", value), + returning: None, + rls_filters: &[], + resolved_sum_targets: &[], + }, + ) +} diff --git a/nodedb/src/data/executor/core_loop/write_index/validate.rs b/nodedb/src/data/executor/core_loop/write_index/validate.rs new file mode 100644 index 000000000..51991a501 --- /dev/null +++ b/nodedb/src/data/executor/core_loop/write_index/validate.rs @@ -0,0 +1,389 @@ +// SPDX-License-Identifier: BUSL-1.1 + +//! Calvin read-set validation against the per-core write versions. + +use nodedb_types::calvin::VersionedReadEntry; + +use crate::data::executor::task::ExecutionTask; +use crate::types::{TenantId, VShardId}; + +use super::super::CoreLoop; + +impl CoreLoop { + /// Whether this shard's slice of a transaction's versioned read-set is + /// still current against the local write versions. + /// + /// Filters the read-set to the entries homed on this request's vShard, + /// the only reads this core holds versions for, then checks each against + /// the write-version index. Short-circuits on the first entry that is no + /// longer current. An empty or fully-remote slice is vacuously current. + /// The `(database, tenant)` scope mirrors the write-version recorder so a + /// read validates against the same key space it was recorded in. + /// + /// Read versions are data-group log positions, so the verdict is the same + /// on every replica of the vShard, whichever node served the read. + /// + /// An entry's home is its `home_vshard` when set (a cross-shard graph read + /// names the key vShard it read edges on), else its collection's vShard, + /// by the same collection-in-database function the scheduler routes plans + /// with. A homed entry with no collection observed every collection on its + /// vShard, so it is current only while the vShard's latest version has + /// not passed its read version. + pub(in crate::data::executor) fn read_set_still_current( + &self, + task: &ExecutionTask, + tid: u64, + versioned_reads: &[VersionedReadEntry], + ) -> bool { + let db = task.request.database_id; + let tenant = TenantId::new(tid); + let local_vshard = task.request.vshard_id; + versioned_reads.iter().all(|entry| { + let home = match entry.home_vshard { + Some(home) if home < VShardId::COUNT => VShardId::new(home), + // A home past the vShard space names no shard: fail closed. + Some(_) => return false, + // An entry carries the plan's database-qualified name. One + // that does not de-qualify cannot be homed or validated, so + // the read set fails closed. + None => { + match nodedb_types::CollectionKey::from_qualified_str(db, &entry.collection) { + Ok(key) => key.vshard(), + Err(_) => return false, + } + } + }; + if home != local_vshard { + return true; + } + if entry.collection.is_empty() { + return self.write_index.latest(home) <= entry.read_version; + } + self.write_index.read_is_valid( + db, + tenant, + home, + &entry.collection, + &entry.key, + entry.read_version, + ) + }) + } +} + +#[cfg(test)] +mod tests { + use nodedb_physical::physical_plan::DocumentOp; + use nodedb_types::calvin::{EngineTag, ReadKeyIdent}; + use nodedb_types::{DatabaseId, QualifiedCollection, Surrogate, WriteVersion}; + + use super::super::keys::{KeyRepr, WriteStamp}; + use super::super::tests::{doc_value, make_core, make_request, task_with_vshard}; + use super::*; + use crate::bridge::envelope::{PhysicalPlan, Status}; + use crate::data::executor::task::ExecutionTask; + use crate::types::Lsn; + + /// The vShard `collection` homes to in the default database: mirrors the + /// homing `read_set_still_current` filters entries by. + fn local_vshard(collection: &str) -> VShardId { + nodedb_types::CollectionKey::from_bare(DatabaseId::DEFAULT, collection).vshard() + } + + /// Some vShard other than `than`, for exercising the cross-shard filter. + fn other_vshard(than: VShardId) -> VShardId { + VShardId::new((than.as_u32() + 1) % VShardId::COUNT) + } + + fn at(index: u64) -> WriteVersion { + WriteVersion::logged(0, index) + } + + /// Record a write applying the entry at `index` on `vshard`. + fn write( + core: &mut CoreLoop, + vshard: VShardId, + collection: &str, + key: Option, + index: u64, + ) { + core.note_write( + DatabaseId::DEFAULT, + TenantId::new(1), + collection, + key, + WriteStamp { + vshard, + lsn: Lsn::new(index), + entry: Some(at(index)), + }, + ); + } + + fn point_entry(collection: &str, surrogate: u32, read: u64) -> VersionedReadEntry { + VersionedReadEntry { + engine: EngineTag::Document, + collection: collection.to_string(), + key: ReadKeyIdent::Point(KeyRepr::Surrogate(surrogate)), + read_version: at(read), + home_vshard: None, + } + } + + fn predicate_entry(collection: &str, read: u64) -> VersionedReadEntry { + VersionedReadEntry { + key: ReadKeyIdent::Predicate, + ..point_entry(collection, 0, read) + } + } + + fn homed_entry(collection: &str, home: VShardId, read: u64) -> VersionedReadEntry { + VersionedReadEntry { + home_vshard: Some(home.as_u32()), + ..predicate_entry(collection, read) + } + } + + #[test] + fn a_stale_point_read_is_not_current_and_a_fresh_one_is() { + let (mut core, _, _, _dir) = make_core(); + let vshard = local_vshard("orders"); + write(&mut core, vshard, "orders", Some(KeyRepr::Surrogate(7)), 20); + let task = task_with_vshard(vshard); + assert!(!core.read_set_still_current(&task, 1, &[point_entry("orders", 7, 10)])); + assert!(core.read_set_still_current(&task, 1, &[point_entry("orders", 7, 20)])); + assert!(core.read_set_still_current(&task, 1, &[point_entry("orders", 7, 30)])); + } + + /// The read was served on another replica of the vShard. Its version is + /// the log position that replica had applied, which this replica records + /// for the same writes, so the read validates here by version alone. + #[test] + fn a_read_served_on_another_replica_validates_by_its_log_position() { + let vshard = local_vshard("orders"); + let (mut serving, _, _, _serving_dir) = make_core(); + let (mut validating, _, _, _validating_dir) = make_core(); + // Both replicas apply the entry at index 20, at different WAL LSNs. + for (core, lsn) in [(&mut serving, 7_000u64), (&mut validating, 31u64)] { + core.note_write( + DatabaseId::DEFAULT, + TenantId::new(1), + "orders", + Some(KeyRepr::Surrogate(7)), + WriteStamp { + vshard, + lsn: Lsn::new(lsn), + entry: Some(at(20)), + }, + ); + } + let read_version = serving + .write_index + .key_version(&super::super::keys::WriteKey { + vshard, + db: DatabaseId::DEFAULT, + tenant: TenantId::new(1), + collection: Box::from("orders"), + key: KeyRepr::Surrogate(7), + }) + .expect("serving replica recorded the write"); + let read = VersionedReadEntry { + read_version, + ..point_entry("orders", 7, 0) + }; + let task = task_with_vshard(vshard); + assert!(validating.read_set_still_current(&task, 1, std::slice::from_ref(&read))); + + // A conflicting write the validating replica applies after the read + // makes the same read stale. + write( + &mut validating, + vshard, + "orders", + Some(KeyRepr::Surrogate(7)), + 21, + ); + assert!(!validating.read_set_still_current(&task, 1, &[read])); + } + + /// A read that found no row answers `NotFound` with the version it + /// observed. That version, carried to the coordinator on the wire, keeps + /// the miss valid until a write of the missed key lands after it. + #[test] + fn a_missed_read_carries_its_version_and_stays_valid_until_the_key_is_written() { + let (mut core, _, _, _dir) = make_core(); + let vshard = local_vshard("orders"); + write(&mut core, vshard, "orders", Some(KeyRepr::Surrogate(7)), 20); + let read = ExecutionTask::new(make_request(PhysicalPlan::Document(DocumentOp::PointGet { + collection: QualifiedCollection::new(DatabaseId::DEFAULT, "orders"), + document_id: "missing".into(), + surrogate: Some(Surrogate::new(8)), + pk_bytes: Vec::new(), + rls_filters: Vec::new(), + system_time: nodedb_types::SystemTimeScope::Current, + valid_at_ms: None, + }))); + let miss = core.response_error(&read, crate::bridge::envelope::ErrorCode::NotFound); + assert_eq!(miss.status, Status::Error); + let observed = miss + .read_versions + .of(vshard) + .expect("a miss reports the version it observed"); + assert_eq!(observed, at(20)); + + let entry = VersionedReadEntry { + read_version: observed, + ..point_entry("orders", 8, 0) + }; + let task = task_with_vshard(vshard); + assert!(core.read_set_still_current(&task, 1, std::slice::from_ref(&entry))); + write(&mut core, vshard, "orders", Some(KeyRepr::Surrogate(8)), 21); + assert!(!core.read_set_still_current(&task, 1, &[entry])); + } + + fn array_slice(vshard: VShardId) -> ExecutionTask { + ExecutionTask::new(crate::bridge::envelope::Request { + vshard_id: vshard, + ..make_request(PhysicalPlan::Array( + nodedb_physical::physical_plan::ArrayOp::Slice { + array_id: nodedb_array::types::ArrayId::in_database( + TenantId::new(1), + DatabaseId::DEFAULT, + "grid", + ), + slice_msgpack: Vec::new(), + attr_projection: Vec::new(), + limit: 0, + cell_filter: None, + hilbert_range: None, + system_time: nodedb_types::SystemTimeScope::Current, + valid_at_ms: None, + }, + )) + }) + } + + /// An array read reports the vShard it covers even when no write of it + /// reached this core. A first write of that vShard then makes the read + /// stale: no tile can appear after the read unseen. + #[test] + fn an_array_read_covers_a_vshard_no_write_reached_and_a_first_write_makes_it_stale() { + let (mut core, _, _, _dir) = make_core(); + let vshard = local_vshard("grid"); + let observed = core + .read_versions(&array_slice(vshard)) + .of(vshard) + .expect("an array read reports its vShard with no write recorded"); + assert_eq!(observed, WriteVersion::ZERO); + + let entry = VersionedReadEntry { + engine: EngineTag::Array, + read_version: observed, + ..predicate_entry("grid", 0) + }; + let task = task_with_vshard(vshard); + assert!(core.read_set_still_current(&task, 1, std::slice::from_ref(&entry))); + write(&mut core, vshard, "grid", None, 5); + assert!(!core.read_set_still_current(&task, 1, &[entry])); + } + + /// An installed vShard with no recorded array write reports its install + /// bound, so the read stays current until a write lands above it. + #[test] + fn an_array_read_of_an_installed_vshard_reports_the_install_bound() { + let (mut core, _, _, _dir) = make_core(); + let vshard = local_vshard("grid"); + core.install_version_floor(&[nodedb_types::ShardVersion { + vshard: vshard.as_u32(), + version: at(9), + }]); + let observed = core + .read_versions(&array_slice(vshard)) + .of(vshard) + .expect("an array read reports its vShard"); + assert_eq!(observed, at(9)); + } + + #[test] + fn read_entry_homing_to_a_different_vshard_is_filtered_out() { + let (mut core, _, _, _dir) = make_core(); + let local = local_vshard("orders"); + write(&mut core, local, "orders", Some(KeyRepr::Surrogate(7)), 20); + let remote_task = task_with_vshard(other_vshard(local)); + assert!(core.read_set_still_current(&remote_task, 1, &[point_entry("orders", 7, 10)])); + } + + #[test] + fn a_stale_predicate_read_is_not_current_and_a_fresh_one_is() { + let (mut core, _, _, _dir) = make_core(); + let vshard = local_vshard("orders"); + write(&mut core, vshard, "orders", None, 20); + let task = task_with_vshard(vshard); + assert!(!core.read_set_still_current(&task, 1, &[predicate_entry("orders", 10)])); + assert!(core.read_set_still_current(&task, 1, &[predicate_entry("orders", 20)])); + } + + #[test] + fn a_homed_read_validates_on_its_home_not_its_collection_vshard() { + let (mut core, _, _, _dir) = make_core(); + let home = other_vshard(local_vshard("edges")); + write(&mut core, home, "edges", None, 20); + let stale = vec![homed_entry("edges", home, 10)]; + assert!(!core.read_set_still_current(&task_with_vshard(home), 1, &stale)); + assert!( + core.read_set_still_current(&task_with_vshard(local_vshard("edges")), 1, &stale), + "the collection's own vShard does not hold a homed read" + ); + let fresh = vec![homed_entry("edges", home, 20)]; + assert!(core.read_set_still_current(&task_with_vshard(home), 1, &fresh)); + } + + #[test] + fn a_homed_read_of_every_collection_validates_against_the_vshard_latest_version() { + let (mut core, _, _, _dir) = make_core(); + let home = local_vshard("edges"); + write(&mut core, home, "unrelated", None, 20); + // A write to another vShard on the same core never moves the home. + write(&mut core, other_vshard(home), "unrelated", None, 900); + let task = task_with_vshard(home); + assert!(!core.read_set_still_current(&task, 1, &[homed_entry("", home, 10)])); + assert!(core.read_set_still_current(&task, 1, &[homed_entry("", home, 20)])); + } + + #[test] + fn empty_read_set_is_vacuously_current() { + let (core, _, _, _dir) = make_core(); + let task = task_with_vshard(VShardId::new(0)); + assert!(core.read_set_still_current(&task, 1, &[])); + } + + #[test] + fn a_read_that_predates_a_committed_write_is_no_longer_current() { + let (mut core, _, _, _dir) = make_core(); + let vshard = local_vshard("orders"); + let entry = at(10); + let write_task = ExecutionTask::with_wal_lsn( + crate::bridge::envelope::Request { + entry_version: Some(entry), + ..task_with_vshard(vshard).request + }, + Some(Lsn::new(77)), + ); + let write_plans = vec![PhysicalPlan::Document(DocumentOp::PointPut { + collection: QualifiedCollection::new(DatabaseId::DEFAULT, "orders"), + document_id: "o7".into(), + value: doc_value("a", "1"), + surrogate: Surrogate::new(7), + pk_bytes: Vec::new(), + returning: None, + rls_filters: Vec::new(), + resolved_sum_targets: Vec::new(), + })]; + let write_resp = core.commit_plans_for_test(&write_task, 1, &write_plans, 77); + assert_eq!(write_resp.status, Status::Ok, "{:?}", write_resp.error_code); + + let task = task_with_vshard(vshard); + assert!(!core.read_set_still_current(&task, 1, &[point_entry("orders", 7, 5)])); + assert!(core.read_set_still_current(&task, 1, &[point_entry("orders", 7, 10)])); + } +} diff --git a/nodedb/src/data/executor/deadline.rs b/nodedb/src/data/executor/deadline.rs index 19475922c..1262cf82c 100644 --- a/nodedb/src/data/executor/deadline.rs +++ b/nodedb/src/data/executor/deadline.rs @@ -172,6 +172,7 @@ mod tests { wal_lsn: None, resolved_now_ms: None, commit_hlc: None, + entry_version: None, admission, }) } diff --git a/nodedb/src/data/executor/dispatch/array/aggregate.rs b/nodedb/src/data/executor/dispatch/array/aggregate.rs index 967ce26b6..bd0a03648 100644 --- a/nodedb/src/data/executor/dispatch/array/aggregate.rs +++ b/nodedb/src/data/executor/dispatch/array/aggregate.rs @@ -401,6 +401,7 @@ mod tests { wal_lsn: None, resolved_now_ms: None, commit_hlc: None, + entry_version: None, admission: crate::bridge::envelope::Admission::Admitted, } } diff --git a/nodedb/src/data/executor/dispatch/array/elementwise.rs b/nodedb/src/data/executor/dispatch/array/elementwise.rs index 1829389d1..844268f72 100644 --- a/nodedb/src/data/executor/dispatch/array/elementwise.rs +++ b/nodedb/src/data/executor/dispatch/array/elementwise.rs @@ -332,6 +332,7 @@ mod tests { wal_lsn: None, resolved_now_ms: None, commit_hlc: None, + entry_version: None, admission: crate::bridge::envelope::Admission::Admitted, } } diff --git a/nodedb/src/data/executor/dispatch/array/mutate.rs b/nodedb/src/data/executor/dispatch/array/mutate.rs index ce5be1428..e66ad9ef0 100644 --- a/nodedb/src/data/executor/dispatch/array/mutate.rs +++ b/nodedb/src/data/executor/dispatch/array/mutate.rs @@ -184,12 +184,13 @@ impl CoreLoop { ) -> crate::Result<()> { if wal_lsn > 0 { let lsn = crate::types::Lsn::new(wal_lsn); - self.note_write_lsn( + let stamp = self.task_write_stamp_at(task, lsn); + self.note_write( task.request.database_id, task.request.tenant_id, &array_id.name, None, - lsn, + stamp, ); self.floors.applied_prefix.note_applied(lsn); } @@ -330,6 +331,7 @@ mod tests { wal_lsn: None, resolved_now_ms: None, commit_hlc: None, + entry_version: None, admission: crate::bridge::envelope::Admission::Admitted, } } diff --git a/nodedb/src/data/executor/dispatch/array/read.rs b/nodedb/src/data/executor/dispatch/array/read.rs index adb5746c9..387469017 100644 --- a/nodedb/src/data/executor/dispatch/array/read.rs +++ b/nodedb/src/data/executor/dispatch/array/read.rs @@ -533,6 +533,7 @@ mod tests { wal_lsn: None, resolved_now_ms: None, commit_hlc: None, + entry_version: None, admission: crate::bridge::envelope::Admission::Admitted, } } diff --git a/nodedb/src/data/executor/dispatch/array/surrogate_scan.rs b/nodedb/src/data/executor/dispatch/array/surrogate_scan.rs index 6f0128a90..afc225c9a 100644 --- a/nodedb/src/data/executor/dispatch/array/surrogate_scan.rs +++ b/nodedb/src/data/executor/dispatch/array/surrogate_scan.rs @@ -173,6 +173,7 @@ mod tests { wal_lsn: None, resolved_now_ms: None, commit_hlc: None, + entry_version: None, admission: crate::bridge::envelope::Admission::Admitted, } } diff --git a/nodedb/src/data/executor/dispatch/bitmap/hashjoin_inline.rs b/nodedb/src/data/executor/dispatch/bitmap/hashjoin_inline.rs index 33ff4b011..6c384b9e5 100644 --- a/nodedb/src/data/executor/dispatch/bitmap/hashjoin_inline.rs +++ b/nodedb/src/data/executor/dispatch/bitmap/hashjoin_inline.rs @@ -153,6 +153,7 @@ mod tests { wal_lsn: None, resolved_now_ms: None, commit_hlc: None, + entry_version: None, admission: Admission::Exempt(ExemptReason::Read), }) } diff --git a/nodedb/src/data/executor/dispatch/graph/node_labels.rs b/nodedb/src/data/executor/dispatch/graph/node_labels.rs index d4840843e..659bce946 100644 --- a/nodedb/src/data/executor/dispatch/graph/node_labels.rs +++ b/nodedb/src/data/executor/dispatch/graph/node_labels.rs @@ -163,6 +163,7 @@ mod tests { wal_lsn: Some(Lsn::new(lsn)), resolved_now_ms: None, commit_hlc: None, + entry_version: None, admission: Admission::Exempt(ExemptReason::Read), }) } diff --git a/nodedb/src/data/executor/dispatch/meta.rs b/nodedb/src/data/executor/dispatch/meta.rs index a0ed13128..3fb037953 100644 --- a/nodedb/src/data/executor/dispatch/meta.rs +++ b/nodedb/src/data/executor/dispatch/meta.rs @@ -23,7 +23,7 @@ impl CoreLoop { MetaOp::Cancel { target_request_id } => self.execute_cancel(task, *target_request_id), // A committed transaction installs only through its redo record - // (`ApplyTransactionRedo`, `CalvinFlush`). A plan batch carries no + // (`ApplyTransactionRedo`). A plan batch carries no // record restart replay reads, so an Origin core refuses it. MetaOp::TransactionBatch { .. } => self.response_error( task, @@ -75,14 +75,23 @@ impl CoreLoop { replace_mode, collections_to_clear, group_vshards, - } => self.execute_restore_tenant_snapshot( - task, - *tenant_id, - snapshot, - *replace_mode, - collections_to_clear, - group_vshards, - ), + version_floor, + } => { + let response = self.execute_restore_tenant_snapshot( + task, + *tenant_id, + snapshot, + *replace_mode, + collections_to_clear, + group_vshards, + ); + // The installed rows carry no versions: each vShard holds + // every write through the snapshot's cut. + if response.status == crate::bridge::envelope::Status::Ok { + self.install_version_floor(version_floor); + } + response + } MetaOp::ConvertCollection { collection, @@ -167,7 +176,6 @@ impl CoreLoop { tenant_id, plans, epoch_system_ms, - is_group_leader, versioned_reads, body_plans, } => self.execute_calvin_execute_static( @@ -176,7 +184,6 @@ impl CoreLoop { epoch: *epoch, position: *position, epoch_system_ms: *epoch_system_ms, - is_group_leader: *is_group_leader, }, tenant_id, plans, @@ -204,14 +211,12 @@ impl CoreLoop { plans, injected_reads, epoch_system_ms, - is_group_leader, } => self.execute_calvin_execute_active( task, CalvinExecCtx { epoch: *epoch, position: *position, epoch_system_ms: *epoch_system_ms, - is_group_leader: *is_group_leader, }, tenant_id, plans, @@ -236,33 +241,6 @@ impl CoreLoop { self.execute_delete_synonym_group(task, *tenant_id, name) } - MetaOp::RecordCalvinWriteVersions { tenant_id, plans } => { - // The Calvin apply already committed; this records the write - // version of every key it wrote at the applied WAL LSN the - // scheduler threaded onto the request envelope. A no-op when - // the envelope carries no LSN. The install recorded the - // index-value versions of its document rows itself. - self.record_batch_write_versions(task, tenant_id.as_u64(), plans); - self.response_ok(task) - } - - MetaOp::CalvinFlush { - epoch, - position, - redo, - collections, - sum_targets, - } => self.execute_calvin_flush( - task, - crate::data::executor::handlers::control::calvin::CalvinFlushRedo { - epoch: *epoch, - position: *position, - redo, - collections, - sum_targets, - }, - ), - MetaOp::CalvinDrop { epoch, position } => { self.execute_calvin_drop(task, *epoch, *position) } @@ -282,22 +260,28 @@ impl CoreLoop { } // Install a committed transaction's redo record through the WAL - // replay arms, on every replica, in Raft log order. + // replay arms, on every replica, in Raft log order. A committed + // Calvin slice also ends its staged state and answers its reply. MetaOp::ApplyTransactionRedo { redo, collections, sum_targets, origin, - } => self.install_redo( - task, - tid, - crate::data::executor::handlers::transaction::redo_apply::CommittedRedo { - redo, - collections, - sum_targets, - }, - *origin, - ), + calvin, + } => { + let committed = + crate::data::executor::handlers::transaction::redo_apply::CommittedRedo { + redo, + collections, + sum_targets, + }; + match calvin { + Some(install) => { + self.install_calvin_redo(task, tid, committed, *origin, install) + } + None => self.install_redo(task, tid, committed, *origin), + } + } // A RESTORE batch installs only inside its Calvin transaction: // its resolve appends the batch to the transaction's redo record. @@ -420,6 +404,7 @@ mod txn_created_columnar_engine_tests { wal_lsn: None, resolved_now_ms: None, commit_hlc: None, + entry_version: None, admission: crate::bridge::envelope::Admission::Exempt( crate::bridge::envelope::ExemptReason::Read, ), diff --git a/nodedb/src/data/executor/dispatch/timeseries.rs b/nodedb/src/data/executor/dispatch/timeseries.rs index e0f305f09..56b7aaf8d 100644 --- a/nodedb/src/data/executor/dispatch/timeseries.rs +++ b/nodedb/src/data/executor/dispatch/timeseries.rs @@ -244,6 +244,7 @@ mod tests { wal_lsn: envelope_lsn.map(Lsn::new), resolved_now_ms: None, commit_hlc: None, + entry_version: None, admission: Admission::Admitted, }) } diff --git a/nodedb/src/data/executor/enforcement/materialized_sum/apply.rs b/nodedb/src/data/executor/enforcement/materialized_sum/apply.rs index 699ed597d..d36f30657 100644 --- a/nodedb/src/data/executor/enforcement/materialized_sum/apply.rs +++ b/nodedb/src/data/executor/enforcement/materialized_sum/apply.rs @@ -43,11 +43,12 @@ //! ([`sum_target_is_co_resident`](crate::query::sum_target_is_co_resident)) //! that exists so the two answers cannot differ. //! -//! A cross-shard target the plan DOES resolve is still applied here: the -//! Control-Plane orchestrators (`MERGE`, `UPDATE ... FROM`, `INSERT ... SELECT`, -//! and the staged-transaction expanders) resolve their rows and dispatch their -//! own concrete work without appending a sibling balance task, so for them the -//! resolution still means "this transaction owns it". +//! No write path hands this core a cross-shard target it must apply. Plain +//! writes and the in-transaction expanders take the same Control-Plane pass, +//! which ships every cross-shard balance on its own task. A `MERGE`, +//! `UPDATE ... FROM` or `INSERT ... SELECT` into a source that drives a +//! cross-shard target never reaches the autocommit orchestrators: every +//! protocol runs it through the expanders in an implicit transaction. //! //! # Identity comes from the plan, never from a store probe //! @@ -174,11 +175,7 @@ impl CoreLoop { // collection consults. // // The absence of the resolution IS the instruction; nothing is - // re-derived. A cross-shard target the plan DID resolve was - // resolved by a Control-Plane orchestrator that ships no - // sibling task — `MERGE`, `UPDATE ... FROM`, `INSERT ... SELECT` - // and the staged-transaction expanders all dispatch their own - // concrete work — so it is still this transaction's to apply. + // re-derived. if !co_resident && resolved_target(ctx, &binding.target_collection, &join_value).is_none() { diff --git a/nodedb/src/data/executor/enforcement/materialized_sum/divergence.rs b/nodedb/src/data/executor/enforcement/materialized_sum/divergence.rs index f19fca434..35bbf3fa6 100644 --- a/nodedb/src/data/executor/enforcement/materialized_sum/divergence.rs +++ b/nodedb/src/data/executor/enforcement/materialized_sum/divergence.rs @@ -71,16 +71,13 @@ impl CoreLoop { /// require. /// /// `rows` are already-decoded row images. A collection that declares no - /// materialized-sum binding, and every replica that is not the group leader, - /// diverge vacuously — they return `false` without reading anything. + /// materialized-sum binding diverges vacuously: it returns `false` without + /// reading anything. pub(in crate::data::executor) fn sum_targets_diverged( &self, check: &SumTargetCheck<'_>, rows: &[serde_json::Value], ) -> bool { - if !self.calvin.ollp_is_group_leader { - return false; - } let key = ( DatabaseId::new(check.database_id), TenantId::new(check.tid), @@ -152,7 +149,7 @@ impl CoreLoop { check: &SumTargetCheck<'_>, doc_ids: &[nodedb_types::StorageKey], ) -> bool { - if !self.calvin.ollp_is_group_leader || !self.declares_materialized_sums(check) { + if !self.declares_materialized_sums(check) { return false; } let mut rows: Vec = Vec::with_capacity(doc_ids.len()); diff --git a/nodedb/src/data/executor/graph_label_checkpoint/mod.rs b/nodedb/src/data/executor/graph_label_checkpoint/mod.rs index 8be4f5996..7c0052790 100644 --- a/nodedb/src/data/executor/graph_label_checkpoint/mod.rs +++ b/nodedb/src/data/executor/graph_label_checkpoint/mod.rs @@ -17,7 +17,7 @@ //! durable trace a label ever existed, which is exactly why //! `wal_replay_graph_labels.rs` is a standalone replay pass. Yet a label write //! advances the core watermark (`GraphOp::SetNodeLabels` goes through -//! `note_write_lsn` like any other write), so the periodic checkpoint reported +//! `note_write` like any other write), so the periodic checkpoint reported //! it as durable and the manager truncated the segments holding the only copy. //! The label silently vanished on the next restart, while the edges around it //! came back intact — the node was still there, `MATCH (a:Person)` just stopped diff --git a/nodedb/src/data/executor/handlers/aggregate/exec.rs b/nodedb/src/data/executor/handlers/aggregate/exec.rs index cb05daed7..a6174469d 100644 --- a/nodedb/src/data/executor/handlers/aggregate/exec.rs +++ b/nodedb/src/data/executor/handlers/aggregate/exec.rs @@ -456,6 +456,7 @@ mod tests { wal_lsn: None, resolved_now_ms: None, commit_hlc: None, + entry_version: None, admission: Admission::Exempt(ExemptReason::Read), }) } diff --git a/nodedb/src/data/executor/handlers/bulk_dml/admission.rs b/nodedb/src/data/executor/handlers/bulk_dml/admission.rs index 9e5a36920..eca591263 100644 --- a/nodedb/src/data/executor/handlers/bulk_dml/admission.rs +++ b/nodedb/src/data/executor/handlers/bulk_dml/admission.rs @@ -44,11 +44,9 @@ impl CoreLoop { /// WITHOUT writing. /// /// The apply set is the carried surrogate prediction when there is one, NOT - /// the local scan: that is the determinism anchor for multi-replica OLLP — - /// every replica, leader and follower, mutates exactly the leader's verified - /// set, so all replicas mutate identical state. With no prediction (the - /// single-shard / non-OLLP path) the local scan stands as the apply set, - /// unchanged. + /// the local scan. The prediction is checked against the scan first. With + /// no prediction (the single-shard / non-OLLP path) the local scan stands + /// as the apply set, unchanged. pub(in crate::data::executor) fn admit_bulk_predicate_write( &self, database_id: u64, @@ -59,9 +57,7 @@ impl CoreLoop { let apply_ids: Vec = match admission.predicted_surrogates { Some(predicted) => { // The set comparison is deterministic: both sides are sorted. - if self.calvin.ollp_is_group_leader - && !super::scan::ollp_surrogates_match(&matching_ids, predicted) - { + if !super::scan::ollp_surrogates_match(&matching_ids, predicted) { return Err(ErrorCode::OllpRetryRequired); } super::scan::ollp_predicted_doc_ids(predicted) @@ -77,9 +73,7 @@ impl CoreLoop { // surrogate-set check above cannot see this: the surrogate set is // unchanged. This runs BEFORE any write, so `sparse.get` still returns // pre-mutation content. - if let Some(predicted) = admission.predicted_edges - && self.calvin.ollp_is_group_leader - { + if let Some(predicted) = admission.predicted_edges { let actual = self.ollp_actual_edges(database_id, tid, admission.collection, &apply_ids); if !super::scan::ollp_edges_match(actual, predicted) { return Err(ErrorCode::OllpRetryRequired); diff --git a/nodedb/src/data/executor/handlers/bulk_dml/delete_cascade.rs b/nodedb/src/data/executor/handlers/bulk_dml/delete_cascade.rs index 0b9f625bc..fd94e30ff 100644 --- a/nodedb/src/data/executor/handlers/bulk_dml/delete_cascade.rs +++ b/nodedb/src/data/executor/handlers/bulk_dml/delete_cascade.rs @@ -184,18 +184,18 @@ impl CoreLoop { ); // Record the committed delete's write version against its // surrogate + collection. - self.note_surrogate_write_lsn(task, tid, collection, row_surrogate.as_u32()); + self.note_surrogate_write(task, tid, collection, row_surrogate.as_u32()); // Record the removed secondary-index tuples into the // per-index write-value substrate, recomputed from the // pre-delete document (see `index_paths` comment above). - if let (Some(lsn), Some(doc)) = (task.wal_lsn(), pre_delete_doc.as_ref()) { + if let (Some(stamp), Some(doc)) = (self.task_write_stamp(task), pre_delete_doc.as_ref()) { let tuples = self.index_tuples_for_doc(doc, index_paths); self.note_index_write_values( task.request.database_id, crate::types::TenantId::new(tid), collection, &tuples, - lsn, + stamp, ); } // The removal, journalled after apply: the plan carries no diff --git a/nodedb/src/data/executor/handlers/bulk_dml/update.rs b/nodedb/src/data/executor/handlers/bulk_dml/update.rs index a2b5a9b20..52cf20d79 100644 --- a/nodedb/src/data/executor/handlers/bulk_dml/update.rs +++ b/nodedb/src/data/executor/handlers/bulk_dml/update.rs @@ -324,13 +324,13 @@ impl CoreLoop { // Published only after the commit succeeded — the same // ordering the reindex helper used when it owned the // transaction. - if let Some(lsn) = task.wal_lsn() { + if let Some(stamp) = self.task_write_stamp(task) { self.note_index_write_values( task.request.database_id, crate::types::TenantId::new(tid), collection, &touched, - lsn, + stamp, ); } self.doc_cache.put( @@ -343,7 +343,7 @@ impl CoreLoop { // Record the committed row's write version against its // surrogate + collection. let surrogate = storage_key.surrogate(); - self.note_surrogate_write_lsn(task, tid, collection, surrogate.as_u32()); + self.note_surrogate_write(task, tid, collection, surrogate.as_u32()); // Re-index the row's vectors from the new body (soft-delete the // old HNSW node + insert the new one, keyed by the stable // surrogate). No-op unless the collection has a vector field diff --git a/nodedb/src/data/executor/handlers/columnar_mutation.rs b/nodedb/src/data/executor/handlers/columnar_mutation.rs index 72f240929..7f4a214a3 100644 --- a/nodedb/src/data/executor/handlers/columnar_mutation.rs +++ b/nodedb/src/data/executor/handlers/columnar_mutation.rs @@ -143,7 +143,7 @@ impl CoreLoop { // re-executing it against the identical restored state matches nothing // again. if affected > 0 { - self.note_collection_write_lsn(task, collection); + self.note_collection_write(task, collection); } debug!(core = self.core_id, %collection, affected, "columnar update complete"); @@ -250,7 +250,7 @@ impl CoreLoop { // it, needlessly holding WAL segments and denying a concurrent // transaction the conflict it should see against the rows this removed. if affected > 0 { - self.note_collection_write_lsn(task, collection); + self.note_collection_write(task, collection); } debug!(core = self.core_id, %collection, affected, "columnar delete complete"); diff --git a/nodedb/src/data/executor/handlers/columnar_read/materialize_scan.rs b/nodedb/src/data/executor/handlers/columnar_read/materialize_scan.rs index 84f1552e0..c6f3bddef 100644 --- a/nodedb/src/data/executor/handlers/columnar_read/materialize_scan.rs +++ b/nodedb/src/data/executor/handlers/columnar_read/materialize_scan.rs @@ -444,6 +444,7 @@ mod tests { wal_lsn: None, resolved_now_ms: None, commit_hlc: None, + entry_version: None, admission: crate::bridge::envelope::Admission::Exempt( crate::bridge::envelope::ExemptReason::Read, ), diff --git a/nodedb/src/data/executor/handlers/columnar_read/scan/execute.rs b/nodedb/src/data/executor/handlers/columnar_read/scan/execute.rs index 88a7c64e9..17dff10bb 100644 --- a/nodedb/src/data/executor/handlers/columnar_read/scan/execute.rs +++ b/nodedb/src/data/executor/handlers/columnar_read/scan/execute.rs @@ -468,6 +468,7 @@ mod tests { wal_lsn: None, resolved_now_ms: None, commit_hlc: None, + entry_version: None, admission: crate::bridge::envelope::Admission::Exempt( crate::bridge::envelope::ExemptReason::Read, ), diff --git a/nodedb/src/data/executor/handlers/columnar_resolved_mutation.rs b/nodedb/src/data/executor/handlers/columnar_resolved_mutation.rs index aa0d280b1..f7ff1b524 100644 --- a/nodedb/src/data/executor/handlers/columnar_resolved_mutation.rs +++ b/nodedb/src/data/executor/handlers/columnar_resolved_mutation.rs @@ -145,7 +145,7 @@ impl CoreLoop { // a checkpoint stamp that already contains it, and replay would // duplicate the row. if affected > 0 { - self.note_collection_write_lsn(task, collection); + self.note_collection_write(task, collection); } debug!(core = self.core_id, %collection, affected, "columnar resolved update complete"); @@ -224,7 +224,7 @@ impl CoreLoop { } if affected > 0 { - self.note_collection_write_lsn(task, collection); + self.note_collection_write(task, collection); } debug!(core = self.core_id, %collection, affected, "columnar resolved delete complete"); diff --git a/nodedb/src/data/executor/handlers/columnar_truncate.rs b/nodedb/src/data/executor/handlers/columnar_truncate.rs index cdf166124..51c4bc833 100644 --- a/nodedb/src/data/executor/handlers/columnar_truncate.rs +++ b/nodedb/src/data/executor/handlers/columnar_truncate.rs @@ -78,7 +78,7 @@ impl CoreLoop { self.checkpoint_coordinator .mark_dirty("columnar", truncated); - self.note_collection_write_lsn(task, collection); + self.note_collection_write(task, collection); self.invalidate_aggregate_cache_for_collection(db.as_u64(), tid.as_u64(), collection); debug!(core = self.core_id, %collection, truncated, "columnar truncate complete"); diff --git a/nodedb/src/data/executor/handlers/columnar_write/insert.rs b/nodedb/src/data/executor/handlers/columnar_write/insert.rs index dcd51cb13..b992b8539 100644 --- a/nodedb/src/data/executor/handlers/columnar_write/insert.rs +++ b/nodedb/src/data/executor/handlers/columnar_write/insert.rs @@ -201,7 +201,7 @@ impl CoreLoop { // Advance the collection floor for this committed columnar write. if accepted > 0 { - self.note_collection_write_lsn(task, collection); + self.note_collection_write(task, collection); } // On the sync path, advance the HWM and return SyncAckResult payload. @@ -243,8 +243,8 @@ impl CoreLoop { payload: Payload::from_vec(json), watermark_lsn: self.watermark, error_code: None, - read_set_valid: None, - read_version_lsn: crate::types::Lsn::ZERO, + stage_vote: None, + read_versions: crate::types::ReadVersions::new(), write_set: Vec::new(), } } diff --git a/nodedb/src/data/executor/handlers/control/calvin/active_passive.rs b/nodedb/src/data/executor/handlers/control/calvin/active_passive.rs index 3f07e7dc4..ea73dc536 100644 --- a/nodedb/src/data/executor/handlers/control/calvin/active_passive.rs +++ b/nodedb/src/data/executor/handlers/control/calvin/active_passive.rs @@ -10,11 +10,10 @@ use tracing::{debug, info_span}; use nodedb_cluster::calvin::types::PassiveReadKey; use nodedb_types::Value; -use crate::bridge::envelope::{ErrorCode, Payload, Response, Status}; +use crate::bridge::envelope::{ErrorCode, Payload, Response, StageVote, Status}; use crate::data::executor::core_loop::CoreLoop; use crate::data::executor::core_loop::commit_pending::PendingCommit; use crate::data::executor::handlers::control::calvin_reply::{CalvinReply, CalvinStaging}; -use crate::data::executor::response_codec; use crate::data::executor::task::ExecutionTask; use crate::types::TenantId; use nodedb_physical::physical_plan::PhysicalPlan; @@ -28,14 +27,13 @@ use super::shared::CalvinExecCtx; impl CoreLoop { /// Execute a passive-participant dependent-read Calvin txn. /// - /// Reads each key from the local engine state and returns a - /// msgpack-encoded `Vec<(PassiveReadKeyId, Value)>` as the response - /// payload. The Control Plane scheduler collects these values and - /// proposes a `ReplicatedWrite::CalvinReadResult` entry to the - /// per-vshard Raft group so all replicas see the same read results. - /// - /// `Instant::now()` is intentionally absent here — this is a - /// synchronous Data Plane read with no timer interaction. + /// Reads each key from base storage and returns a zerompk-encoded + /// `Vec<(PassiveReadKeyId, Value)>` as the response payload. The + /// scheduler holds the transaction's locks on these keys, so no other + /// write changes them before the transaction's verdict. A successful + /// read votes commit: a passive participant writes nothing. The scheduler + /// proposes the payload as a `ReplicatedWrite::CalvinReadResult` entry + /// to the data group of every active vShard. pub(in crate::data::executor) fn execute_calvin_execute_passive( &mut self, task: &ExecutionTask, @@ -53,18 +51,21 @@ impl CoreLoop { "calvin execute passive: reading keys" ); + let database_id = task.request.database_id.as_u64(); let mut results: Vec<(PassiveReadKeyId, Value)> = Vec::with_capacity(keys_to_read.len()); - for passive_key in keys_to_read { - // Build a PassiveReadKeyId for each surrogate in the engine key set. - // For this v1 handler the engine key set carries single surrogates per - // key (as specified in the design); we iterate all surrogates to be safe. - let values = self.read_passive_key(tenant_id, &passive_key.engine_key); - results.extend(values); + match self.read_passive_key(database_id, tenant_id, &passive_key.engine_key) { + Ok(values) => results.extend(values), + Err(code) => return self.response_error(task, code), + } } - match response_codec::encode_serde(&results) { - Ok(payload) => self.response_with_payload(task, payload), + match zerompk::to_msgpack_vec(&results) { + Ok(payload) => { + let mut response = self.response_with_payload(task, payload); + response.stage_vote = Some(StageVote::Commit); + response + } Err(e) => self.response_error( task, ErrorCode::Internal { @@ -80,21 +81,20 @@ impl CoreLoop { /// mutation and fires NO side effects. It stages each write plan into /// `txn_overlays` under the synthetic `TxnId` and buffers the plans in /// `commit_pending`, so a subsequent `CalvinResolve` builds one redo record - /// and [`CoreLoop::execute_calvin_flush`] installs it. + /// and [`CoreLoop::install_calvin_redo`] installs it. /// /// The one divergence from the static path: OLLP predicate verification /// (leader-only) runs HERE, before staging, via /// [`CoreLoop::verify_calvin_active_ollp`]. The dependent-read path has no - /// LSN-versioned read-set to vote on; its conflict detector is the OLLP + /// write-versioned read-set to vote on; its conflict detector is the OLLP /// `actual != predicted` re-check. A mismatch returns `OllpRetryRequired` - /// and stages nothing, so no redo record is appended for it. The Control + /// and stages nothing, so no redo entry exists for it. The Control /// Plane scheduler releases locks and re-recons on `OllpRetryRequired`. /// - /// `injected_reads` is retained on the wire for future plan variants that - /// reference resolved read values by `PassiveReadKeyId`. The coordinator - /// bakes the read values into concrete point ops and the predicted - /// surrogate set at recon, so the plans are self-contained and stage - /// byte-identically to the static path. + /// The coordinator bakes the values it read into concrete point ops, so + /// the plans are self-contained and stage byte-identically to the static + /// path. The scheduler compared `injected_reads` with those values before + /// it dispatched this stage, and a mismatch never reaches the core. pub(in crate::data::executor) fn execute_calvin_execute_active( &mut self, task: &ExecutionTask, @@ -107,7 +107,6 @@ impl CoreLoop { epoch, position, epoch_system_ms, - is_group_leader, } = ctx; let vshard_id = task.request.vshard_id.as_u32(); debug!( @@ -116,7 +115,6 @@ impl CoreLoop { position, epoch_system_ms, vshard_id, - is_group_leader, plan_count = plans.len(), injected_count = injected_reads.len(), "calvin execute active" @@ -133,17 +131,12 @@ impl CoreLoop { // OLLP verification runs HERE, before staging, so a predicate-drift // mismatch surfaces on THIS stage response (where the scheduler releases - // locks and re-recons) and nothing is staged, resolved, or WAL-appended. - // Scoped to this replica's staged leadership for the check, then the - // resting (authoritative) state is restored. - let prev_group_leader = self.calvin.ollp_is_group_leader; - self.calvin.ollp_is_group_leader = is_group_leader; + // locks and re-recons) and nothing is staged or resolved. let verified = self.verify_calvin_active_ollp(task, tenant_id.as_u64(), plans); - self.calvin.ollp_is_group_leader = prev_group_leader; match verified { Ok(true) => {} - Ok(false) => return self.response_error(task, ErrorCode::OllpRetryRequired), - Err(e) => return self.response_error(task, e), + Ok(false) => return self.calvin_stage_refusal(task, ErrorCode::OllpRetryRequired), + Err(e) => return self.calvin_stage_refusal(task, e.into()), } let synthetic_txn_id = match calvin_synthetic_txn_id(epoch, position, vshard_id) { @@ -203,10 +196,10 @@ impl CoreLoop { payload: Payload::empty(), watermark_lsn: self.watermark, error_code: None, - // The dependent-read path carries no versioned read-set; `None` maps - // to "commit" in `resolve_staged_commit` (`read_set_valid != Some(false)`). - read_set_valid: None, - read_version_lsn: crate::types::Lsn::ZERO, + // The dependent-read path carries no versioned read-set. Its OLLP + // check passed above, so the slice votes commit. + stage_vote: Some(StageVote::Commit), + read_versions: crate::types::ReadVersions::new(), write_set: Vec::new(), } } @@ -226,10 +219,132 @@ mod tests { bulk_delete_plan, doc_value, make_task, point_insert_plan, seed_row, }; + fn kv_keys(collection: &str, keys: &[&[u8]]) -> PassiveReadKey { + PassiveReadKey { + engine_key: nodedb_cluster::calvin::types::EngineKeySet::Kv { + collection: collection.to_owned(), + keys: nodedb_cluster::calvin::types::SortedVec::new( + keys.iter().map(|key| key.to_vec()).collect(), + ), + }, + } + } + + fn decode_reads(response: &Response) -> Vec<(PassiveReadKeyId, Value)> { + zerompk::from_msgpack(response.payload.as_ref()).expect("passive payload decodes") + } + + /// A passive read returns each stored key-value row as its stored bytes + /// and an absent row as `Null`, and votes commit. + #[test] + fn calvin_execute_passive_reads_stored_kv_bytes_and_votes_commit() { + let dir = tempfile::tempdir().unwrap(); + let (mut core, _tx, _rx) = make_core_with_dir(dir.path()); + core.kv_engine + .put(crate::engine::kv::KvPutParams { + database_id: DatabaseId::DEFAULT.as_u64(), + tenant_id: 1, + collection: "items", + key: b"alice:sword", + value: b"stored-bytes", + ttl_ms: 0, + now_ms: crate::engine::kv::current_ms(), + surrogate: Surrogate::new(5), + }) + .expect("seed kv row"); + + let task = make_task(); + let resp = core.execute_calvin_execute_passive( + &task, + 1, + 0, + &TenantId::new(1), + &[kv_keys("items", &[b"alice:sword", b"bob:sword"])], + ); + + assert_eq!(resp.status, Status::Ok); + assert_eq!(resp.stage_vote, Some(StageVote::Commit)); + let collection = nodedb_types::QualifiedCollection::from_stored("items".to_owned()); + let reads: BTreeMap = decode_reads(&resp).into_iter().collect(); + assert_eq!( + reads.get(&PassiveReadKeyId::kv( + collection.clone(), + b"alice:sword".to_vec() + )), + Some(&Value::Bytes(b"stored-bytes".to_vec())) + ); + assert_eq!( + reads.get(&PassiveReadKeyId::kv(collection, b"bob:sword".to_vec())), + Some(&Value::Null) + ); + } + + /// A passive read returns a stored document row as its stored bytes. + #[test] + fn calvin_execute_passive_reads_stored_document_bytes() { + let dir = tempfile::tempdir().unwrap(); + let (mut core, _tx, _rx) = make_core_with_dir(dir.path()); + seed_row(&mut core, "orders", 11); + let stored = core + .sparse + .get( + DatabaseId::DEFAULT.as_u64(), + 1, + "orders", + &nodedb_types::StorageKey::for_surrogate(Surrogate::new(11)), + ) + .expect("read seeded row") + .expect("seeded row present"); + + let task = make_task(); + let keys = [PassiveReadKey { + engine_key: nodedb_cluster::calvin::types::EngineKeySet::Document { + collection: "orders".to_owned(), + surrogates: nodedb_cluster::calvin::types::SortedVec::new(vec![11]), + }, + }]; + let resp = core.execute_calvin_execute_passive(&task, 1, 0, &TenantId::new(1), &keys); + + assert_eq!(resp.status, Status::Ok); + assert_eq!( + decode_reads(&resp), + vec![( + PassiveReadKeyId::surrogate( + nodedb_types::QualifiedCollection::from_stored("orders".to_owned()), + 11 + ), + Value::Bytes(stored) + )] + ); + } + + /// A key set that names no readable row refuses the read, so the + /// passive participant votes abort instead of reporting a value. + #[test] + fn calvin_execute_passive_refuses_a_lock_only_key_set() { + let dir = tempfile::tempdir().unwrap(); + let (mut core, _tx, _rx) = make_core_with_dir(dir.path()); + let task = make_task(); + let keys = [PassiveReadKey { + engine_key: nodedb_cluster::calvin::types::EngineKeySet::Collection { + collection: "orders".to_owned(), + vshards: nodedb_cluster::calvin::types::SortedVec::new(Vec::new()), + }, + }]; + let resp = core.execute_calvin_execute_passive(&task, 1, 0, &TenantId::new(1), &keys); + + assert_eq!(resp.status, Status::Error); + assert_eq!(resp.stage_vote, None); + assert!(matches!( + resp.error_code.as_deref(), + Some(ErrorCode::Unsupported { .. }) + )); + } + /// The dependent-read ACTIVE path STAGES its writes (into `commit_pending` + /// the synthetic overlay) instead of applying them to base directly, so a /// Calvin-committed dependent-read write reaches the WAL as a redo record. - /// Staging routes it through the same resolve → redo → flush the static + /// Staging routes it through the same resolve → redo → install the static /// path uses. #[test] fn calvin_execute_active_stages_point_insert_into_overlay() { @@ -243,25 +358,22 @@ mod tests { epoch: 1, position: 0, epoch_system_ms: 0, - is_group_leader: true, }; let injected = BTreeMap::new(); let resp = core.execute_calvin_execute_active(&task, ctx, &tenant_id, &plans, &injected); assert_eq!(resp.status, Status::Ok); - // The dependent-read path carries no versioned read-set; `None` maps to - // "commit" in `resolve_staged_commit`. - assert_eq!(resp.read_set_valid, None); + assert_eq!(resp.stage_vote, Some(StageVote::Commit)); let vshard_id = task.request.vshard_id.as_u32(); - // STAGED, not applied: the plans are buffered for the flush replay. + // STAGED, not applied: the plans are buffered for the resolve. assert!( core.calvin.commit_pending.contains_key(&(1, 0, vshard_id)), "active-path write must be STAGED into commit_pending, not applied directly" ); - // No base mutation at stage time — the row appears only after flush. + // No base mutation at stage time — the row appears only after install. let doc_id = nodedb_types::StorageKey::for_surrogate(Surrogate::new(7)); assert!( core.sparse @@ -289,7 +401,7 @@ mod tests { /// predicted set no longer matches live state returns `OllpRetryRequired` /// BEFORE staging anything — so no stale redo is WAL-appended and the /// coordinator can re-recon under a fresh attempt. Verifying at stage time - /// (not flush) is the one divergence from the static path. + /// (not install) is the one divergence from the static path. #[test] fn calvin_execute_active_ollp_drift_returns_retry_and_stages_nothing() { let dir = tempfile::tempdir().unwrap(); @@ -306,7 +418,6 @@ mod tests { epoch: 1, position: 0, epoch_system_ms: 0, - is_group_leader: true, }; let injected = BTreeMap::new(); @@ -316,6 +427,7 @@ mod tests { resp.error_code.as_deref(), Some(&ErrorCode::OllpRetryRequired) ); + assert_eq!(resp.stage_vote, Some(StageVote::PredictionDrift)); // Drift stages NOTHING — neither the raw buffer nor the overlay. let vshard_id = task.request.vshard_id.as_u32(); @@ -329,4 +441,31 @@ mod tests { "an OLLP-drift retry must not leave a staged overlay" ); } + + #[test] + fn calvin_execute_active_stage_error_votes_participant_error() { + let dir = tempfile::tempdir().unwrap(); + let (mut core, _tx, _rx) = make_core_with_dir(dir.path()); + let task = make_task(); + let tenant_id = TenantId::new(1); + let epoch_outside_synthetic_range = 1_u64 << 33; + let ctx = CalvinExecCtx { + epoch: epoch_outside_synthetic_range, + position: 0, + epoch_system_ms: 0, + }; + + let resp = core.execute_calvin_execute_active( + &task, + ctx, + &tenant_id, + &[point_insert_plan("orders", "o1", 7)], + &BTreeMap::new(), + ); + + assert_eq!(resp.status, Status::Error); + assert_eq!(resp.stage_vote, Some(StageVote::ParticipantError)); + assert!(core.calvin.commit_pending.is_empty()); + assert!(core.txn_overlays.is_empty()); + } } diff --git a/nodedb/src/data/executor/handlers/control/calvin/discard.rs b/nodedb/src/data/executor/handlers/control/calvin/discard.rs index 0457561eb..f478165bd 100644 --- a/nodedb/src/data/executor/handlers/control/calvin/discard.rs +++ b/nodedb/src/data/executor/handlers/control/calvin/discard.rs @@ -12,9 +12,8 @@ impl CoreLoop { /// Discard a staged Calvin transaction. /// /// Removes the plans staged under `(epoch, position, vshard)` from the - /// commit-pending buffer and fires nothing — no base mutation, no side - /// effects. An - /// absent key (already flushed or dropped) is an idempotent no-op. + /// commit-pending buffer and fires nothing: no base mutation, no side + /// effects. An absent key (already installed or dropped) is a no-op. pub(in crate::data::executor) fn execute_calvin_drop( &mut self, task: &ExecutionTask, @@ -30,10 +29,6 @@ impl CoreLoop { // Discard the synthetic overlay entry alongside the raw plan buffer; // idempotent no-op if it was never staged or already removed. self.drop_calvin_synthetic_overlay(epoch, position, vshard_id); - // Writes waiting on the rows this transaction owned run next. - self.calvin - .fence - .note_resolved((epoch, position, vshard_id), None); debug!( core = self.core_id, epoch, position, vshard_id, existed, "calvin drop: discarding staged commit" @@ -64,7 +59,6 @@ mod tests { epoch: 1, position: 0, epoch_system_ms: 0, - is_group_leader: true, }; let resp = core.execute_calvin_execute_static(&task, ctx, &tenant_id, &plans, &[], &[]); assert_eq!(resp.status, Status::Ok); diff --git a/nodedb/src/data/executor/handlers/control/calvin/flush.rs b/nodedb/src/data/executor/handlers/control/calvin/flush.rs deleted file mode 100644 index 37e71f690..000000000 --- a/nodedb/src/data/executor/handlers/control/calvin/flush.rs +++ /dev/null @@ -1,523 +0,0 @@ -// SPDX-License-Identifier: BUSL-1.1 - -//! Flush a staged Calvin transaction: install its committed redo record. - -use nodedb_physical::physical_plan::{MetaOp, PhysicalPlan, RedoOrigin, RedoSumTargets}; -use tracing::{debug, info_span}; - -use crate::bridge::envelope::{Payload, Response, Status}; -use crate::data::executor::core_loop::CoreLoop; -use crate::data::executor::handlers::transaction::redo_apply::CommittedRedo; -use crate::data::executor::task::ExecutionTask; - -/// The `MetaOp::CalvinFlush` fields the flush reads. -pub(in crate::data::executor) struct CalvinFlushRedo<'a> { - pub epoch: u64, - pub position: u32, - /// The encoded `RedoRecord` the scheduler appended at the request's LSN. - /// Empty when the transaction wrote nothing on this vShard. - pub redo: &'a [u8], - /// Every collection the local slice of the transaction writes. - pub collections: &'a [String], - /// The materialized-sum targets the local slice folds into. - pub sum_targets: &'a [RedoSumTargets], -} - -impl CoreLoop { - /// Flush the Calvin transaction staged under `(epoch, position)`. - /// - /// The flush installs the transaction's committed redo record through - /// [`CoreLoop::install_committed_redo_into`], the install every committed - /// transaction and restart replay run: validate, install with undo, then - /// settle and cover. An absent staged entry means a duplicate dispatch, - /// which answers `Ok` and installs nothing. - /// - /// The staged entry and its overlay stay until the install succeeds. A - /// refused install rolled every write back, so the scheduler can send the - /// same flush again. - /// - /// The reply reads at the epoch instant, so every replica sees the same - /// live rows. The response carries the reply the plans decided when they - /// staged. A document or CRDT `RETURNING` row is read from base after the - /// install, so it reports the row the install stored. A reply that fails - /// to render leaves the flush `Ok`, since the record installed. Its error - /// travels in `error_code` for the statement to report. - pub(in crate::data::executor) fn execute_calvin_flush( - &mut self, - task: &ExecutionTask, - flush: CalvinFlushRedo<'_>, - ) -> Response { - let CalvinFlushRedo { - epoch, - position, - redo, - collections, - sum_targets, - } = flush; - let vshard_id = task.request.vshard_id.as_u32(); - let key = (epoch, position, vshard_id); - let Some((tenant_id, epoch_system_ms, origin)) = - self.calvin.commit_pending.get(&key).map(|pending| { - ( - pending.tenant_id, - pending.epoch_system_ms, - redo_origin(&pending.plans), - ) - }) - else { - debug!( - core = self.core_id, - epoch, position, vshard_id, "calvin flush: no staged commit (already resolved)" - ); - return self.response_ok(task); - }; - let _apply_span = info_span!( - "executor_apply", - epoch, - position, - vshard = vshard_id, - tenant_id = tenant_id.as_u64(), - trace_id = ?task.request.trace_id, - ) - .entered(); - // The record's graph versions stamp at least the transaction's - // ordinal, so a later local write stamps above them. - let txn_ordinal = nodedb_types::calvin_txn_ordinal(epoch_system_ms, position); - self.hlc.update_from_remote(txn_ordinal); - - // What each resolved timeseries batch of the record stored: the - // reply answers the apply's counts and stored rows, not the resolve's. - let mut ts_installs = Vec::new(); - let mut response = if redo.is_empty() { - self.response_ok(task) - } else { - self.install_committed_redo_into( - task, - tenant_id.as_u64(), - CommittedRedo { - redo, - collections, - sum_targets, - }, - origin, - &mut ts_installs, - ) - }; - if response.status != Status::Ok { - return response; - } - // Restart replay folds the record's sum targets from its stamp, so the - // folded target rows stay out of the record's group. - response.write_set.clear(); - // The record installed: the staged entry and its overlay are spent. - let reply = self - .calvin - .commit_pending - .remove(&key) - .map(|pending| pending.reply) - .unwrap_or_default(); - self.drop_calvin_synthetic_overlay(epoch, position, vshard_id); - // Writes waiting on the rows this transaction owned run next. - self.calvin.fence.note_resolved(key, task.wal_lsn()); - let prev_epoch_ms = self.epoch_system_ms; - self.epoch_system_ms = Some(epoch_system_ms); - let rendered = self.calvin_reply_payload(task, tenant_id.as_u64(), reply, &ts_installs); - self.epoch_system_ms = prev_epoch_ms; - match rendered { - Ok(payload) => response.payload = Payload::from_vec(payload), - Err(error) => response.error_code = Some(Box::new(error)), - } - response - } -} - -/// The origin of a Calvin transaction's redo record: a RESTORE when its -/// local plans install a RESTORE batch, a commit otherwise. A RESTORE's rows -/// passed their collection's rules when they were first written, so the -/// install runs only the checks a RESTORE runs. -fn redo_origin(plans: &[PhysicalPlan]) -> RedoOrigin { - if plans - .iter() - .any(|plan| matches!(plan, PhysicalPlan::Meta(MetaOp::RestoreRedo(_)))) - { - RedoOrigin::Restore - } else { - RedoOrigin::Commit - } -} - -#[cfg(test)] -mod tests { - use super::*; - use crate::bridge::envelope::ErrorCode; - use crate::data::executor::core_loop::tests::make_core_with_dir; - use crate::data::executor::handlers::control::calvin_txn_id::calvin_synthetic_txn_id; - use crate::types::{DatabaseId, Lsn, TenantId}; - use nodedb_physical::physical_plan::{ - DocumentOp, PhysicalPlan, ReturningColumns, ReturningSpec, - }; - use nodedb_types::{QualifiedCollection, StorageKey, Surrogate}; - - use crate::engine::document::store::{CollectionConfig, IndexPath}; - - use super::super::shared::CalvinExecCtx; - use super::super::shared::test_support::{ - bulk_delete_plan, doc_value, make_task, point_insert_plan, seed_row, - }; - - const CTX: CalvinExecCtx = CalvinExecCtx { - epoch: 1, - position: 0, - epoch_system_ms: 0, - is_group_leader: true, - }; - - /// Stage `plans` at `(1, 0)`, resolve them, and return the redo bytes. - fn stage_and_resolve(core: &mut CoreLoop, plans: &[PhysicalPlan]) -> Vec { - let task = make_task(); - let staged = - core.execute_calvin_execute_static(&task, CTX, &TenantId::new(1), plans, &[], &[]); - assert_eq!(staged.status, Status::Ok, "{:?}", staged.error_code); - let resolved = core.execute_calvin_resolve(&task, 1, 0); - assert_eq!(resolved.status, Status::Ok, "{:?}", resolved.error_code); - resolved.payload.as_bytes().to_vec() - } - - fn flush_at(core: &mut CoreLoop, lsn: u64, redo: &[u8], collections: &[String]) -> Response { - let mut task = make_task(); - task.wal_lsn = Some(Lsn::new(lsn)); - core.execute_calvin_flush( - &task, - CalvinFlushRedo { - epoch: 1, - position: 0, - redo, - collections, - sum_targets: &[], - }, - ) - } - - fn base_row(core: &CoreLoop, collection: &str, surrogate: u32) -> Option> { - core.sparse - .get( - DatabaseId::DEFAULT.as_u64(), - 1, - collection, - &StorageKey::for_surrogate(Surrogate::new(surrogate)), - ) - .expect("read base row") - } - - #[test] - fn calvin_flush_drops_synthetic_overlay() { - let dir = tempfile::tempdir().unwrap(); - let (mut core, _tx, _rx) = make_core_with_dir(dir.path()); - let redo = stage_and_resolve(&mut core, &[point_insert_plan("orders", "o1", 7)]); - - let vshard_id = make_task().request.vshard_id.as_u32(); - let synthetic = calvin_synthetic_txn_id(1, 0, vshard_id).unwrap(); - assert!(core.txn_overlays.contains_key(&synthetic)); - - let flushed = flush_at(&mut core, 40, &redo, &["orders".to_string()]); - assert_eq!(flushed.status, Status::Ok, "{:?}", flushed.error_code); - - assert!( - !core.txn_overlays.contains_key(&synthetic), - "flush must drop the synthetic overlay entry alongside commit_pending" - ); - assert!( - base_row(&core, "orders", 7).is_some(), - "the flush installs the staged row from the redo record" - ); - } - - /// The flush installs the redo record, not the staged plans. A row that - /// joins the bulk delete's predicate after staging stays, and the - /// predicted row goes. - #[test] - fn a_flush_installs_the_resolved_record_not_a_replay_of_the_plans() { - let dir = tempfile::tempdir().unwrap(); - let (mut core, _tx, _rx) = make_core_with_dir(dir.path()); - seed_row(&mut core, "orders", 1); - let redo = stage_and_resolve(&mut core, &[bulk_delete_plan("orders", Some(vec![1]))]); - - seed_row(&mut core, "orders", 2); - let flushed = flush_at(&mut core, 41, &redo, &["orders".to_string()]); - - assert_eq!(flushed.status, Status::Ok, "{:?}", flushed.error_code); - assert!(base_row(&core, "orders", 1).is_none()); - assert!(base_row(&core, "orders", 2).is_some()); - } - - /// The flush answers with the reply the staged statement computed: - /// RETURNING rows for a point insert that asked for them. - #[test] - fn a_calvin_flush_answers_the_returning_rows_its_statement_staged() { - let dir = tempfile::tempdir().unwrap(); - let (mut core, _tx, _rx) = make_core_with_dir(dir.path()); - let plan = PhysicalPlan::Document(DocumentOp::PointInsert { - collection: QualifiedCollection::new(DatabaseId::DEFAULT, "orders"), - document_id: "o9".to_string(), - value: doc_value("a", "returned"), - if_absent: false, - surrogate: Surrogate::new(9), - returning: Some(ReturningSpec { - columns: ReturningColumns::Star, - }), - rls_filters: Vec::new(), - resolved_sum_targets: Vec::new(), - deferred_sum_targets: Vec::new(), - }); - let redo = stage_and_resolve(&mut core, &[plan]); - - let flushed = flush_at(&mut core, 42, &redo, &["orders".to_string()]); - - assert_eq!(flushed.status, Status::Ok, "{:?}", flushed.error_code); - let payload = flushed.payload.as_bytes(); - assert!( - payload.windows(b"returned".len()).any(|w| w == b"returned"), - "the flush reply carries the inserted row" - ); - assert!(base_row(&core, "orders", 9).is_some()); - } - - /// The install notes the index values of every document row at the - /// flush's LSN, so no flush-time staging carries them to a later op. - #[test] - fn a_calvin_flush_notes_index_values_at_the_redo_lsn() { - let dir = tempfile::tempdir().unwrap(); - let (mut core, _tx, _rx) = make_core_with_dir(dir.path()); - let mut config = CollectionConfig::new("orders"); - config.index_paths.push(IndexPath::new("a")); - core.doc_configs.insert( - (DatabaseId::DEFAULT, TenantId::new(1), "orders".to_string()), - config, - ); - let redo = stage_and_resolve(&mut core, &[point_insert_plan("orders", "o1", 7)]); - - let flushed = flush_at(&mut core, 43, &redo, &["orders".to_string()]); - - assert_eq!(flushed.status, Status::Ok, "{:?}", flushed.error_code); - assert_eq!( - core.write_index.index_values.value_lsn( - DatabaseId::DEFAULT, - TenantId::new(1), - "orders", - "a", - "1" - ), - Some(Lsn::new(43)), - "the install records the row's index value at the redo LSN" - ); - } - - /// A flush whose record fails to install answers the install's error and - /// leaves nothing of the record in base. - #[test] - fn a_calvin_flush_whose_record_cannot_install_answers_an_error() { - let dir = tempfile::tempdir().unwrap(); - let (mut core, _tx, _rx) = make_core_with_dir(dir.path()); - stage_and_resolve(&mut core, &[point_insert_plan("orders", "o1", 7)]); - - let flushed = flush_at(&mut core, 44, &[0xff, 0x00], &["orders".to_string()]); - - assert_eq!(flushed.status, Status::Error); - assert!(!matches!( - flushed.error_code.as_deref(), - Some(ErrorCode::OllpRetryRequired) - )); - assert!(base_row(&core, "orders", 7).is_none()); - } - - /// A refused install keeps the staged entry and its overlay, so the - /// scheduler's resent flush installs the record. - #[test] - fn a_refused_install_keeps_the_staged_commit_for_the_resent_flush() { - use crate::data::executor::handlers::transaction::redo_apply::test_commit::failing_install_sub_record; - use crate::wal::RedoRecord; - - let dir = tempfile::tempdir().unwrap(); - let (mut core, _tx, _rx) = make_core_with_dir(dir.path()); - let redo = stage_and_resolve(&mut core, &[point_insert_plan("orders", "o1", 7)]); - let mut failing = RedoRecord::from_bytes(&redo).expect("decode redo"); - failing.ops.push(failing_install_sub_record()); - let failing = failing.to_bytes().expect("encode redo"); - let vshard_id = make_task().request.vshard_id.as_u32(); - let synthetic = calvin_synthetic_txn_id(1, 0, vshard_id).unwrap(); - - let refused = flush_at(&mut core, 45, &failing, &["orders".to_string()]); - - assert!( - matches!( - refused.error_code.as_deref(), - Some(ErrorCode::RetryableRefusal { .. }) - ), - "{:?}", - refused.error_code - ); - assert!(core.calvin.commit_pending.contains_key(&(1, 0, vshard_id))); - assert!(core.txn_overlays.contains_key(&synthetic)); - assert!(base_row(&core, "orders", 7).is_none()); - - let flushed = flush_at(&mut core, 46, &redo, &["orders".to_string()]); - - assert_eq!(flushed.status, Status::Ok, "{:?}", flushed.error_code); - assert!(base_row(&core, "orders", 7).is_some()); - assert!(!core.calvin.commit_pending.contains_key(&(1, 0, vshard_id))); - assert!(!core.txn_overlays.contains_key(&synthetic)); - } - - /// A reply that fails to render after a successful install leaves the - /// flush `Ok` and carries the render error for the statement. - #[test] - fn a_reply_render_error_leaves_the_installed_flush_ok() { - use crate::data::executor::handlers::control::calvin_reply::CalvinReply; - - let dir = tempfile::tempdir().unwrap(); - let (mut core, _tx, _rx) = make_core_with_dir(dir.path()); - let redo = stage_and_resolve(&mut core, &[point_insert_plan("orders", "o1", 7)]); - let vshard_id = make_task().request.vshard_id.as_u32(); - core.calvin - .commit_pending - .get_mut(&(1, 0, vshard_id)) - .expect("staged commit") - .reply = CalvinReply::unrenderable_for_test("orders"); - - let flushed = flush_at(&mut core, 47, &redo, &["orders".to_string()]); - - assert_eq!(flushed.status, Status::Ok); - assert!( - matches!( - flushed.error_code.as_deref(), - Some(ErrorCode::Internal { .. }) - ), - "{:?}", - flushed.error_code - ); - assert!(base_row(&core, "orders", 7).is_some()); - assert!(!core.calvin.commit_pending.contains_key(&(1, 0, vshard_id))); - } - /// An autocommit put of row `surrogate` in "orders" at `wal_lsn`. - fn autocommit_put(surrogate: u32, value: &str, wal_lsn: u64) -> ExecutionTask { - let mut task = make_task(); - task.request.plan = PhysicalPlan::Document(DocumentOp::PointPut { - collection: QualifiedCollection::new(DatabaseId::DEFAULT, "orders"), - document_id: "o1".to_string(), - value: doc_value("a", value), - surrogate: Surrogate::new(surrogate), - pk_bytes: Vec::new(), - returning: None, - rls_filters: Vec::new(), - resolved_sum_targets: Vec::new(), - }); - task.wal_lsn = Some(Lsn::new(wal_lsn)); - task - } - - /// The flush of the transaction staged at `(1, 0)`, as a queued task. - fn flush_task(redo: Vec, lsn: u64) -> ExecutionTask { - let mut task = make_task(); - task.request.plan = - PhysicalPlan::Meta(nodedb_physical::physical_plan::MetaOp::CalvinFlush { - epoch: 1, - position: 0, - redo, - collections: vec!["orders".to_string()], - sum_targets: Vec::new(), - }); - task.wal_lsn = Some(Lsn::new(lsn)); - task - } - - fn holds(core: &CoreLoop, needle: &[u8]) -> bool { - base_row(core, "orders", 7) - .is_some_and(|row| row.windows(needle.len()).any(|w| w == needle)) - } - - /// An autocommit write to a row a staged Calvin transaction owns waits - /// for the flush, then applies on top of it. - #[test] - fn a_write_to_an_owned_row_applies_after_the_flush() { - let dir = tempfile::tempdir().unwrap(); - let (mut core, _tx, _rx) = make_core_with_dir(dir.path()); - let redo = stage_and_resolve(&mut core, &[point_insert_plan("orders", "o1", 7)]); - - core.task_queue.push(autocommit_put(7, "autocommit", 60)); - core.tick(); - assert_eq!(core.calvin.fence.len(), 1, "the write waits for the owner"); - assert!(base_row(&core, "orders", 7).is_none()); - - core.task_queue.push(flush_task(redo, 50)); - core.tick(); - - assert_eq!(core.calvin.fence.len(), 0); - assert!( - holds(&core, b"autocommit"), - "the waiting write applies after the flush and is not lost" - ); - } - - /// A waiting write whose record sits below the flush's redo record is - /// refused: restart replay would apply it before the flush. - #[test] - fn a_waiting_write_below_the_flush_lsn_is_refused() { - let dir = tempfile::tempdir().unwrap(); - let (mut core, _tx, _rx) = make_core_with_dir(dir.path()); - let redo = stage_and_resolve(&mut core, &[point_insert_plan("orders", "o1", 7)]); - - core.task_queue.push(autocommit_put(7, "autocommit", 40)); - core.task_queue.push(flush_task(redo, 50)); - core.tick(); - - assert_eq!(core.calvin.fence.len(), 0); - assert!(base_row(&core, "orders", 7).is_some()); - assert!( - !holds(&core, b"autocommit"), - "the refused write never applies" - ); - } - - /// A constraint-set install on a collection a staged transaction writes - /// runs at once. It writes no row the flush installs, and a parked - /// install holds its data group's apply loop behind the flush. - #[test] - fn a_constraint_install_on_an_owned_collection_does_not_wait() { - let dir = tempfile::tempdir().unwrap(); - let (mut core, _tx, _rx) = make_core_with_dir(dir.path()); - stage_and_resolve(&mut core, &[point_insert_plan("orders", "o1", 7)]); - - let mut install = make_task(); - install.request.plan = - PhysicalPlan::Crdt(nodedb_physical::physical_plan::CrdtOp::SetConstraints { - collection: QualifiedCollection::new(DatabaseId::DEFAULT, "orders"), - constraint_version: 1, - constraints: Vec::new(), - }); - install.request.admission = crate::bridge::envelope::Admission::Exempt( - crate::bridge::envelope::ExemptReason::AlreadyOrdered, - ); - core.task_queue.push(install); - core.tick(); - - assert_eq!( - core.calvin.fence.len(), - 0, - "a constraint install never waits for a Calvin flush" - ); - } - - /// A write to a row the staged transaction does not own runs at once. - #[test] - fn a_write_to_an_unowned_row_does_not_wait() { - let dir = tempfile::tempdir().unwrap(); - let (mut core, _tx, _rx) = make_core_with_dir(dir.path()); - stage_and_resolve(&mut core, &[point_insert_plan("orders", "o1", 7)]); - - core.task_queue.push(autocommit_put(8, "autocommit", 60)); - core.tick(); - - assert_eq!(core.calvin.fence.len(), 0); - assert!(base_row(&core, "orders", 8).is_some()); - } -} diff --git a/nodedb/src/data/executor/handlers/control/calvin/mod.rs b/nodedb/src/data/executor/handlers/control/calvin/mod.rs index 33fb2b19c..4f8437fd7 100644 --- a/nodedb/src/data/executor/handlers/control/calvin/mod.rs +++ b/nodedb/src/data/executor/handlers/control/calvin/mod.rs @@ -8,15 +8,15 @@ //! (the common case). It VALIDATES the read-set to compute the local commit //! vote and STAGES the transaction's plans into the synthetic overlay and //! the commit-pending buffer WITHOUT mutating base or firing side effects, -//! then returns the vote. `CoreLoop::execute_calvin_flush` later installs -//! the transaction's committed redo record, or -//! `CoreLoop::execute_calvin_drop` discards the staged state. +//! then returns the vote. The data group's apply of the slice's stamped +//! redo entry later installs it and consumes the staged state, or +//! `CoreLoop::execute_calvin_drop` discards it. //! //! - `CoreLoop::execute_calvin_execute_passive`: passive participant for a -//! dependent-read txn. Reads each declared key from the local engine and -//! returns a msgpack-encoded `Vec<(PassiveReadKeyId, Value)>` payload. The -//! Control Plane scheduler proposes a `CalvinReadResult` Raft entry after -//! receiving this response. +//! dependent-read txn. Reads each declared key from base storage and +//! returns a zerompk-encoded `Vec<(PassiveReadKeyId, Value)>` payload with +//! a commit vote. The Control Plane scheduler proposes the payload as a +//! `CalvinReadResult` entry to the data group of every active vShard. //! //! - `CoreLoop::execute_calvin_execute_active`: active participant for a //! dependent-read txn. Stages the physical plans, with the injected read @@ -26,17 +26,16 @@ //! returns `OllpRetryRequired` and stages nothing. The OLLP orchestrator on //! the Control Plane retries via `Inbox::submit`. //! -//! The scheduler appends the transaction's `TransactionRedo` record, or a -//! `CalvinApplied` marker when the transaction wrote nothing here, on the -//! Control Plane. The Data Plane writes no WAL record. +//! The data-group leader's scheduler proposes the slice's stamped redo entry +//! to the data group. The Data Plane writes no WAL record for a slice. mod active_passive; mod discard; -mod flush; mod shared; mod static_stage; #[cfg(test)] mod test_commit; -pub(in crate::data::executor) use flush::CalvinFlushRedo; pub(in crate::data::executor) use shared::CalvinExecCtx; +#[cfg(test)] +pub(in crate::data::executor) use shared::test_support; diff --git a/nodedb/src/data/executor/handlers/control/calvin/shared.rs b/nodedb/src/data/executor/handlers/control/calvin/shared.rs index bbabcade5..540f694e6 100644 --- a/nodedb/src/data/executor/handlers/control/calvin/shared.rs +++ b/nodedb/src/data/executor/handlers/control/calvin/shared.rs @@ -3,7 +3,7 @@ //! `CalvinExecCtx`, the helpers the static and active Calvin stage paths //! share, and the test fixtures every concern file's tests build on. -use crate::bridge::envelope::{ErrorCode, Response}; +use crate::bridge::envelope::{ErrorCode, Response, StageVote}; use crate::data::executor::core_loop::CoreLoop; use crate::data::executor::task::ExecutionTask; @@ -16,11 +16,21 @@ pub(in crate::data::executor) struct CalvinExecCtx { pub epoch: u64, pub position: u32, pub epoch_system_ms: i64, - pub is_group_leader: bool, } impl CoreLoop { - /// Clean failed static staging and return an explicit abort vote. + /// Refuse a Calvin stage that staged nothing, with the vote `error` names. + /// + /// `OllpRetryRequired` votes `PredictionDrift`. Every other error votes + /// `ParticipantError`. + pub(super) fn calvin_stage_refusal(&self, task: &ExecutionTask, error: ErrorCode) -> Response { + let vote = StageVote::of_stage_error(&error); + let mut response = self.response_error(task, error); + response.stage_vote = Some(vote); + response + } + + /// Clean failed staging and return its abort vote. /// Defensive removal clears document/KV and graph overlays plus their gauge. pub(super) fn calvin_stage_failure( &mut self, @@ -37,19 +47,14 @@ impl CoreLoop { .commit_pending .remove(&(epoch, position, vshard_id)); self.drop_calvin_synthetic_overlay(epoch, position, vshard_id); - self.calvin - .fence - .note_resolved((epoch, position, vshard_id), None); - let mut response = self.response_error(task, error.into()); - // Scheduler treats this as a durable local abort vote and still waits - // for the authoritative global verdict before issuing any drop. - response.read_set_valid = Some(false); - response + // The scheduler proposes this abort vote and waits for the global + // verdict before it drops anything. + self.calvin_stage_refusal(task, error.into()) } } #[cfg(test)] -pub(in crate::data::executor::handlers::control::calvin) mod test_support { +pub(in crate::data::executor) mod test_support { use std::time::{Duration, Instant}; use nodedb_physical::physical_plan::{DocumentOp, PhysicalPlan, TimeseriesOp}; @@ -93,6 +98,7 @@ pub(in crate::data::executor::handlers::control::calvin) mod test_support { wal_lsn: None, resolved_now_ms: None, commit_hlc: None, + entry_version: None, admission: Admission::Exempt(ExemptReason::Read), }; ExecutionTask::new(request) @@ -107,7 +113,7 @@ pub(in crate::data::executor::handlers::control::calvin) mod test_support { nodedb_types::value_to_msgpack(&Value::Object(obj)).unwrap() } - pub(in crate::data::executor::handlers::control::calvin) fn point_insert_plan( + pub(in crate::data::executor) fn point_insert_plan( collection: &str, document_id: &str, surrogate: u32, @@ -143,7 +149,7 @@ pub(in crate::data::executor::handlers::control::calvin) mod test_support { }) } - pub(in crate::data::executor::handlers::control::calvin) fn bulk_delete_plan( + pub(in crate::data::executor) fn bulk_delete_plan( collection: &str, predicted: Option>, ) -> PhysicalPlan { @@ -162,7 +168,7 @@ pub(in crate::data::executor::handlers::control::calvin) mod test_support { /// Seed a row directly into base storage (bypassing Calvin staging), the /// pre-existing state the active-path OLLP verifier scans against. - pub(in crate::data::executor::handlers::control::calvin) fn seed_row( + pub(in crate::data::executor) fn seed_row( core: &mut CoreLoop, collection: &str, surrogate: u32, @@ -174,3 +180,48 @@ pub(in crate::data::executor::handlers::control::calvin) mod test_support { .expect("seed row"); } } + +#[cfg(test)] +mod tests { + use super::test_support::make_task; + use crate::bridge::envelope::{ErrorCode, StageVote, Status}; + use crate::data::executor::core_loop::tests::make_core_with_dir; + use crate::data::executor::handlers::control::calvin_txn_id::calvin_synthetic_txn_id; + + #[test] + fn a_stage_drift_refusal_votes_prediction_drift_and_drops_staged_state() { + let dir = tempfile::tempdir().unwrap(); + let (mut core, _tx, _rx) = make_core_with_dir(dir.path()); + let task = make_task(); + let vshard = task.request.vshard_id.as_u32(); + let synthetic = calvin_synthetic_txn_id(4, 2, vshard).unwrap(); + core.graph_txn_overlay_mut(synthetic); + + let response = core.calvin_stage_failure(&task, 4, 2, vshard, ErrorCode::OllpRetryRequired); + + assert_eq!(response.status, Status::Error); + assert_eq!(response.stage_vote, Some(StageVote::PredictionDrift)); + assert_eq!( + response.error_code.as_deref(), + Some(&ErrorCode::OllpRetryRequired) + ); + assert!(!core.graph_txn_overlays.contains_key(&synthetic)); + } + + #[test] + fn any_other_stage_refusal_votes_participant_error() { + let dir = tempfile::tempdir().unwrap(); + let (core, _tx, _rx) = make_core_with_dir(dir.path()); + let task = make_task(); + + let response = core.calvin_stage_refusal( + &task, + ErrorCode::Internal { + detail: "stage".into(), + }, + ); + + assert_eq!(response.status, Status::Error); + assert_eq!(response.stage_vote, Some(StageVote::ParticipantError)); + } +} diff --git a/nodedb/src/data/executor/handlers/control/calvin/static_stage.rs b/nodedb/src/data/executor/handlers/control/calvin/static_stage.rs index 8fc7ba688..6738bea4a 100644 --- a/nodedb/src/data/executor/handlers/control/calvin/static_stage.rs +++ b/nodedb/src/data/executor/handlers/control/calvin/static_stage.rs @@ -8,7 +8,7 @@ use tracing::{debug, info_span}; use nodedb_types::calvin::VersionedReadEntry; -use crate::bridge::envelope::{ErrorCode, Payload, Response, Status}; +use crate::bridge::envelope::{ErrorCode, Payload, Response, StageVote, Status}; use crate::data::executor::core_loop::CoreLoop; use crate::data::executor::core_loop::commit_pending::PendingCommit; use crate::data::executor::handlers::control::calvin_reply::{CalvinReply, CalvinStaging}; @@ -25,14 +25,13 @@ impl CoreLoop { /// Validate a static-set Calvin transaction and stage it for commit. /// /// Computes the local commit vote by checking whether this participant's - /// slice of the transaction's LSN-versioned read-set is still current + /// slice of the transaction's versioned read-set is still current /// against the per-core write versions, then STAGES the write plans into /// the synthetic overlay and the commit-pending buffer keyed by /// `(epoch, position)`. It performs NO base mutation and fires NO side - /// effects — nothing is observable until a subsequent - /// [`CoreLoop::execute_calvin_flush`] installs the transaction's redo - /// record (or [`CoreLoop::execute_calvin_drop`] discards the staged - /// state). The response carries the vote on `read_set_valid`. Staging runs + /// effects. Nothing is observable until the apply of the slice's stamped + /// redo entry installs it, or [`CoreLoop::execute_calvin_drop`] discards + /// the staged state. The response carries the vote on `stage_vote`. Staging runs /// under the epoch's deterministic time anchor, which the pending entry /// keeps for resolve. `body_plans` indexes the plans a trigger body /// buffered: they stage under `Trigger`, so their rows fire no trigger. @@ -49,7 +48,6 @@ impl CoreLoop { epoch, position, epoch_system_ms, - is_group_leader, } = ctx; let vshard_id = task.request.vshard_id.as_u32(); debug!( @@ -58,7 +56,6 @@ impl CoreLoop { position, epoch_system_ms, vshard_id, - is_group_leader, plan_count = plans.len(), read_count = versioned_reads.len(), "calvin stage for commit" @@ -95,7 +92,7 @@ impl CoreLoop { self.begin_calvin_plan_staging(task, synthetic_txn_id, *tenant_id, plan, body); self.stage_calvin_plan(task, synthetic_txn_id, *tenant_id, plan, &mut staging)?; // Test-only fault boundary after a potentially-mutating stage. - crate::fail_point!("calvin_static::during_overlay_stage"); + crate::fail_point!(self.fail_scope, "calvin_static::during_overlay_stage"); } Ok::(staging.reply) })); @@ -125,7 +122,11 @@ impl CoreLoop { // current against the local write versions? Empty read-set is vacuously // current. Read-only — no base mutation here. A stale-read false vote // retains the fully staged state until the durable global verdict. - let vote = self.read_set_still_current(task, tenant_id.as_u64(), versioned_reads); + let vote = StageVote::of_read_set(self.read_set_still_current( + task, + tenant_id.as_u64(), + versioned_reads, + )); // Publish only a fully staged transaction. Resolve reads these plans // against the overlay; a global abort drops both. @@ -147,8 +148,8 @@ impl CoreLoop { payload: Payload::empty(), watermark_lsn: self.watermark, error_code: None, - read_set_valid: Some(vote), - read_version_lsn: crate::types::Lsn::ZERO, + stage_vote: Some(vote), + read_versions: crate::types::ReadVersions::new(), write_set: Vec::new(), } } @@ -211,11 +212,12 @@ mod tests { epoch: 1, position: 0, epoch_system_ms: 0, - is_group_leader: true, }; let resp = core.execute_calvin_execute_static(&task, ctx, &tenant_id, &plans, &[], &[]); assert_eq!(resp.status, Status::Ok); + // An empty read-set is current. + assert_eq!(resp.stage_vote, Some(StageVote::Commit)); let vshard_id = task.request.vshard_id.as_u32(); @@ -239,6 +241,89 @@ mod tests { ); } + /// A predicate read of `collection` on vShard 0, taken after the entry at + /// `index`. + fn homed_read(collection: &str, index: u64) -> VersionedReadEntry { + VersionedReadEntry { + engine: nodedb_types::calvin::EngineTag::Document, + collection: collection.to_string(), + key: nodedb_types::calvin::ReadKeyIdent::Predicate, + read_version: nodedb_types::WriteVersion::logged(0, index), + home_vshard: Some(0), + } + } + + /// Record a write of `collection` applying the entry at `index` on + /// vShard 0. + fn write_at(core: &mut CoreLoop, collection: &str, index: u64) { + core.note_write( + DatabaseId::DEFAULT, + TenantId::new(1), + collection, + None, + crate::data::executor::core_loop::write_index::WriteStamp { + vshard: crate::types::VShardId::new(0), + lsn: crate::types::Lsn::new(index), + entry: Some(nodedb_types::WriteVersion::logged(0, index)), + }, + ); + } + + #[test] + fn static_stage_with_a_superseded_read_votes_serialization_conflict() { + let dir = tempfile::tempdir().unwrap(); + let (mut core, _tx, _rx) = make_core_with_dir(dir.path()); + write_at(&mut core, "orders", 20); + let task = make_task(); + let tenant_id = TenantId::new(1); + let ctx = CalvinExecCtx { + epoch: 3, + position: 0, + epoch_system_ms: 0, + }; + + let resp = core.execute_calvin_execute_static( + &task, + ctx, + &tenant_id, + &[point_insert_plan("orders", "o1", 7)], + &[homed_read("orders", 10)], + &[], + ); + + assert_eq!(resp.status, Status::Ok); + assert_eq!(resp.stage_vote, Some(StageVote::SerializationConflict)); + // A stale slice stays staged until the global verdict drops it. + let vshard = task.request.vshard_id.as_u32(); + assert!(core.calvin.commit_pending.contains_key(&(3, 0, vshard))); + } + + #[test] + fn static_stage_with_a_current_read_votes_commit() { + let dir = tempfile::tempdir().unwrap(); + let (mut core, _tx, _rx) = make_core_with_dir(dir.path()); + write_at(&mut core, "orders", 20); + let task = make_task(); + let tenant_id = TenantId::new(1); + let ctx = CalvinExecCtx { + epoch: 3, + position: 1, + epoch_system_ms: 0, + }; + + let resp = core.execute_calvin_execute_static( + &task, + ctx, + &tenant_id, + &[point_insert_plan("orders", "o1", 7)], + &[homed_read("orders", 20)], + &[], + ); + + assert_eq!(resp.status, Status::Ok); + assert_eq!(resp.stage_vote, Some(StageVote::Commit)); + } + #[test] fn synthetic_id_failure_leaves_no_pending_or_overlay_state() { let dir = tempfile::tempdir().unwrap(); @@ -253,7 +338,6 @@ mod tests { epoch: epoch_outside_synthetic_range, position: 0, epoch_system_ms: 0, - is_group_leader: true, }, &tenant_id, &[point_insert_plan("orders", "o1", 7)], @@ -262,7 +346,7 @@ mod tests { ); assert_eq!(response.status, Status::Error); - assert_eq!(response.read_set_valid, Some(false)); + assert_eq!(response.stage_vote, Some(StageVote::ParticipantError)); assert!(core.calvin.commit_pending.is_empty()); assert!(core.txn_overlays.is_empty()); assert!(core.graph_txn_overlays.is_empty()); @@ -286,7 +370,6 @@ mod tests { epoch: 9, position: 3, epoch_system_ms: 0, - is_group_leader: true, }; let vshard = task.request.vshard_id.as_u32(); let synthetic = calvin_synthetic_txn_id(9, 3, vshard).unwrap(); @@ -306,7 +389,7 @@ mod tests { let response = core.execute_calvin_execute_static(&task, ctx, &tenant_id, &plans, &[], &[]); assert_eq!(response.status, Status::Error); - assert_eq!(response.read_set_valid, Some(false)); + assert_eq!(response.stage_vote, Some(StageVote::ParticipantError)); assert!(!core.calvin.commit_pending.contains_key(&(9, 3, vshard))); assert!(!core.txn_overlays.contains_key(&synthetic)); assert!(!core.graph_txn_overlays.contains_key(&synthetic)); @@ -335,7 +418,6 @@ mod tests { epoch: 9, position: 5, epoch_system_ms: 0, - is_group_leader: true, }, &tenant_id, &[bulk_delete_plan("orders", None)], @@ -343,7 +425,7 @@ mod tests { &[], ); - assert_eq!(response.read_set_valid, Some(false)); + assert_eq!(response.stage_vote, Some(StageVote::ParticipantError)); assert!(!core.graph_txn_overlays.contains_key(&synthetic)); } @@ -366,7 +448,6 @@ mod tests { epoch: 9, position: 4, epoch_system_ms: 0, - is_group_leader: true, }; let vshard = task.request.vshard_id.as_u32(); let synthetic = calvin_synthetic_txn_id(9, 4, vshard).unwrap(); @@ -391,7 +472,7 @@ mod tests { ); assert_eq!(response.status, Status::Error); - assert_eq!(response.read_set_valid, Some(false)); + assert_eq!(response.stage_vote, Some(StageVote::ParticipantError)); assert!(!core.calvin.commit_pending.contains_key(&(9, 4, vshard))); assert!(!core.txn_overlays.contains_key(&synthetic)); assert!(!core.graph_txn_overlays.contains_key(&synthetic)); @@ -428,14 +509,13 @@ mod tests { epoch: 21, position: 1, epoch_system_ms: 0, - is_group_leader: true, }, &tenant, &[malformed], &[], &[], ); - assert_eq!(failed.read_set_valid, Some(false)); + assert_eq!(failed.stage_vote, Some(StageVote::ParticipantError)); assert!(matches!( failed.error_code.as_deref(), Some(ErrorCode::RejectedPrevalidation { .. }) @@ -450,14 +530,13 @@ mod tests { epoch: 21, position: 1, epoch_system_ms: 0, - is_group_leader: true, }, &tenant, &[valid], &[], &[], ); - assert_eq!(staged.read_set_valid, Some(true)); + assert_eq!(staged.stage_vote, Some(StageVote::Commit)); assert_eq!( core.txn_overlays .get(&synthetic) @@ -472,7 +551,6 @@ mod tests { epoch: 21, position: 2, epoch_system_ms: 0, - is_group_leader: true, }, &tenant, &[mismatch], @@ -480,7 +558,7 @@ mod tests { &[], ); let mismatch_id = calvin_synthetic_txn_id(21, 2, vshard).expect("synthetic transaction id"); - assert_eq!(failed.read_set_valid, Some(false)); + assert_eq!(failed.stage_vote, Some(StageVote::ParticipantError)); assert!(!core.calvin.commit_pending.contains_key(&(21, 2, vshard))); assert!(!core.txn_overlays.contains_key(&mismatch_id)); @@ -496,7 +574,6 @@ mod tests { epoch: 21, position: 3, epoch_system_ms: 0, - is_group_leader: true, }, &tenant, &[overflow], @@ -505,14 +582,14 @@ mod tests { ); let overflow_id = calvin_synthetic_txn_id(21, 3, vshard).expect("synthetic transaction id"); assert_eq!(failed.status, Status::Error); - assert_eq!(failed.read_set_valid, Some(false)); + assert_eq!(failed.stage_vote, Some(StageVote::ParticipantError)); assert!(matches!( failed.error_code.as_deref(), Some(ErrorCode::RejectedPrevalidation { .. }) )); assert!( !core.calvin.commit_pending.contains_key(&(21, 3, vshard)), - "a rejected stage cannot reach the TransactionRedo-producing flush path" + "a rejected stage cannot reach the TransactionRedo-producing resolve path" ); assert!(!core.txn_overlays.contains_key(&overflow_id)); } diff --git a/nodedb/src/data/executor/handlers/control/calvin/test_commit.rs b/nodedb/src/data/executor/handlers/control/calvin/test_commit.rs index 89afdc475..bff0e14e4 100644 --- a/nodedb/src/data/executor/handlers/control/calvin/test_commit.rs +++ b/nodedb/src/data/executor/handlers/control/calvin/test_commit.rs @@ -1,9 +1,12 @@ // SPDX-License-Identifier: BUSL-1.1 //! Test driver for a committed Calvin transaction on one core: stage the -//! plans, resolve them into the redo record, and flush it. +//! plans, resolve them into the redo record, and install the record as the +//! slice's stamped `ApplyTransactionRedo`. -use nodedb_physical::physical_plan::PhysicalPlan; +use nodedb_physical::physical_plan::{ + CalvinInstall, CalvinResolved, MetaOp, PhysicalPlan, RedoOrigin, +}; use crate::bridge::envelope::{Response, Status}; use crate::control::wal_replication::transaction_redo::collections::written_collections; @@ -12,15 +15,14 @@ use crate::data::executor::core_loop::CoreLoop; use crate::data::executor::task::ExecutionTask; use crate::types::{Lsn, TenantId}; -use super::flush::CalvinFlushRedo; use super::shared::CalvinExecCtx; impl CoreLoop { - /// Commit `plans` as the Calvin transaction at `(epoch, 0)` and flush its - /// redo record at `lsn`, the path a committed Calvin transaction takes on - /// its participant core. + /// Commit `plans` as the Calvin transaction at `(epoch, 0)` and install + /// its redo record at `lsn`, the path a committed Calvin slice takes on + /// every replica of its vShard. /// - /// Returns the first refusal: the stage, the resolve, or the flush. + /// Returns the first refusal: the stage, the resolve, or the install. pub(in crate::data::executor) fn calvin_commit_for_test( &mut self, task: &ExecutionTask, @@ -47,7 +49,6 @@ impl CoreLoop { epoch, position: 0, epoch_system_ms, - is_group_leader: true, }; let staged = self.execute_calvin_execute_static(task, ctx, &TenantId::new(tid), plans, &[], &[]); @@ -58,21 +59,33 @@ impl CoreLoop { if resolved.status != Status::Ok { return resolved; } - let redo = resolved.payload.as_bytes().to_vec(); - let mut request = task.request.clone(); - request.wal_lsn = Some(Lsn::new(lsn)); - let flush_task = ExecutionTask::new(request); - let collections = written_collections(plans); - let sum_targets = redo_sum_targets(plans); - self.execute_calvin_flush( - &flush_task, - CalvinFlushRedo { + let answer = match zerompk::from_msgpack::(resolved.payload.as_bytes()) { + Ok(answer) => answer, + Err(error) => { + return self.response_error( + task, + crate::bridge::envelope::ErrorCode::Internal { + detail: format!("decode the resolved answer: {error}"), + }, + ); + } + }; + let install = PhysicalPlan::Meta(MetaOp::ApplyTransactionRedo { + redo: answer.redo, + collections: written_collections(plans), + sum_targets: redo_sum_targets(plans), + origin: RedoOrigin::Commit, + calvin: Some(CalvinInstall { epoch, position: 0, - redo: &redo, - collections: &collections, - sum_targets: &sum_targets, - }, - ) + epoch_system_ms, + reply: answer.reply, + user_write: true, + }), + }); + let mut request = task.request.clone(); + request.wal_lsn = Some(Lsn::new(lsn)); + let install_task = ExecutionTask::new(request); + self.execute_plan(&install_task, &install) } } diff --git a/nodedb/src/data/executor/handlers/control/calvin_active_verify.rs b/nodedb/src/data/executor/handlers/control/calvin_active_verify.rs index 189c9e2af..0309aa5d6 100644 --- a/nodedb/src/data/executor/handlers/control/calvin_active_verify.rs +++ b/nodedb/src/data/executor/handlers/control/calvin_active_verify.rs @@ -2,7 +2,7 @@ //! Stage-time OLLP predicate verification for the dependent-read ACTIVE Calvin //! path, which carries no versioned read-set. The leader-only `actual != -//! predicted` re-check must run HERE, before staging — at flush time a +//! predicted` re-check must run HERE, before staging — at install time a //! mismatch is swallowed as a degraded shard instead. Mirrors the //! `BulkDelete`/`BulkUpdate` arms of [`CoreLoop::stage_calvin_overlay`]. @@ -16,8 +16,8 @@ use crate::data::executor::handlers::bulk_dml::scan::{ use crate::data::executor::task::ExecutionTask; impl CoreLoop { - /// Leader-only OLLP verification of every predicate-DML plan in a - /// dependent-read ACTIVE Calvin txn, before staging. `Ok(false)` means a + /// OLLP verification of every predicate-DML plan in a dependent-read + /// ACTIVE Calvin txn, before staging. Only the data-group leader stages. `Ok(false)` means a /// prediction drifted; caller returns `OllpRetryRequired`, stages nothing. pub(in crate::data::executor) fn verify_calvin_active_ollp( &self, @@ -25,9 +25,6 @@ impl CoreLoop { tid: u64, plans: &[PhysicalPlan], ) -> crate::Result { - if !self.calvin.ollp_is_group_leader { - return Ok(true); - } let database_id = task.request.database_id.as_u64(); for plan in plans { // Only the document engine carries an OLLP prediction; exhaustive diff --git a/nodedb/src/data/executor/handlers/control/calvin_overlay_stage.rs b/nodedb/src/data/executor/handlers/control/calvin_overlay_stage.rs index 7a7dcd93a..7b343c716 100644 --- a/nodedb/src/data/executor/handlers/control/calvin_overlay_stage.rs +++ b/nodedb/src/data/executor/handlers/control/calvin_overlay_stage.rs @@ -6,8 +6,8 @@ //! //! Staging is the producer side for `CalvinResolve`, which reads the overlay //! the same way `MetaOp::ResolveTxn` does for session transactions -//! (`resolve/entry.rs`). The redo record it builds is what the flush -//! installs. +//! (`resolve/entry.rs`). The redo record it builds is what the stamped redo +//! entry installs. use nodedb_physical::physical_plan::{ ArrayOp, ColumnarOp, CrdtOp, DocumentOp, GraphOp, PhysicalPlan, TimeseriesOp, VectorOp, @@ -288,7 +288,7 @@ impl CoreLoop { } // Every columnar write stages its post-image the way a session // write does, so the transaction's redo carries the rows the - // flush installs: an ON CONFLICT merge and a predicate DML's + // install writes: an ON CONFLICT merge and a predicate DML's // matched rows are resolved once, here, against the state this // position observes. PhysicalPlan::Columnar( @@ -395,7 +395,7 @@ impl CoreLoop { /// own parallel overlay, same as a session transaction's /// `execute_stage_graph`), and `array_txn_overlays` (ARRAY cell ops, /// same as `execute_stage_array`). Called from both - /// [`CoreLoop::execute_calvin_flush`] and [`CoreLoop::execute_calvin_drop`] + /// [`CoreLoop::install_calvin_redo`] and [`CoreLoop::execute_calvin_drop`] /// so neither overlay outlives the `commit_pending` entry it shadows. /// Idempotent: a missing key (already removed, or the id derivation /// itself failing) is a silent no-op — the same shape as the @@ -411,7 +411,7 @@ impl CoreLoop { ) { if let Ok(synthetic_txn_id) = calvin_synthetic_txn_id(epoch, position, vshard) { // The shared teardown also drops a columnar engine staging - // auto-created that the flush left empty. + // auto-created that the install left empty. self.drop_overlay_entry(synthetic_txn_id); } } diff --git a/nodedb/src/data/executor/handlers/control/calvin_overlay_stage_bulk.rs b/nodedb/src/data/executor/handlers/control/calvin_overlay_stage_bulk.rs index 8a05ca4cc..1bd0d93ff 100644 --- a/nodedb/src/data/executor/handlers/control/calvin_overlay_stage_bulk.rs +++ b/nodedb/src/data/executor/handlers/control/calvin_overlay_stage_bulk.rs @@ -11,8 +11,8 @@ //! live predicate rescan is NOT used as the row set, because a follower's //! local snapshot can legitimately lag the leader's verified prediction //! window. Re-deriving the row set locally would diverge across replicas. -//! The flush installs the redo record `CalvinResolve` builds from this -//! staging, so the staged rows are the rows the flush writes. The +//! The install applies the redo record `CalvinResolve` builds from this +//! staging, so the staged rows are the rows the install writes. The //! `stage_bulk_delete` / `stage_bulk_update` session-transaction handlers do //! a live rescan and are NOT reused here for this reason. //! @@ -92,7 +92,7 @@ pub(in crate::data::executor) struct CalvinBulkUpdateStage<'a> { impl CoreLoop { /// Stage a Calvin `BulkDelete` into the overlay: one tombstone per /// predicted surrogate, resolved to its doc-id via - /// `ollp_predicted_doc_ids` — the identical primitive the flush apply + /// `ollp_predicted_doc_ids` — the identical primitive the install /// uses to derive `apply_ids`. NOT a live predicate rescan. Returns the /// number of predicted rows that exist. pub(in crate::data::executor) fn stage_calvin_bulk_delete( @@ -181,7 +181,7 @@ impl CoreLoop { /// via the exact same per-row transform `execute_bulk_update` / /// `stage_point_update` use (`CoreLoop::stage_apply_update`), and stage /// the post-image as a `Put`. Row set = predicted surrogates, matching - /// the flush apply set exactly (and correctly excluding rows a + /// the install set exactly (and correctly excluding rows a /// same-transaction `INSERT` created after the predicted set was /// computed at recon). /// @@ -241,7 +241,7 @@ impl CoreLoop { declared_primary_key, )?; // Decide the staged post-image against the write policy: this is - // the row the Calvin flush will install. + // the row the stamped redo install writes. let identity = stored_row_identity( &new_body, strict_schema.as_ref(), diff --git a/nodedb/src/data/executor/handlers/control/calvin_passive_read.rs b/nodedb/src/data/executor/handlers/control/calvin_passive_read.rs index 1e6f76929..831bd828d 100644 --- a/nodedb/src/data/executor/handlers/control/calvin_passive_read.rs +++ b/nodedb/src/data/executor/handlers/control/calvin_passive_read.rs @@ -3,169 +3,106 @@ //! Passive-read helpers for the Calvin dependent-read path. //! //! [`CoreLoop::execute_calvin_execute_passive`] reads each declared key from -//! the local engine to build the `Vec<(PassiveReadKeyId, Value)>` payload the -//! Control Plane scheduler proposes as a `CalvinReadResult` Raft entry. The -//! per-engine lookups and the deterministic key-hashing they rely on live -//! here, split out of `calvin.rs` to keep that file within the size limit. +//! base storage to build the `Vec<(PassiveReadKeyId, Value)>` payload the +//! Control Plane scheduler proposes as a `CalvinReadResult` Raft entry. //! -//! The KV/edge surrogate hashes MUST be deterministic across replicas: the -//! same byte key or `(src, dst)` edge must produce the same `u32` on every -//! node, so `DefaultHasher` (RandomState) is explicitly NOT used. +//! A row's value is its stored bytes as `Value::Bytes`, `Value::Null` when +//! the row is absent. Stored bytes compare exactly, so an active participant +//! matches them against the bytes its coordinator read. //! -//! `EngineKeySet.collection` arrives already database-qualified: the Calvin -//! write-key extraction reads it off the physical plan's own qualified -//! collection field. So every `PassiveReadKeyId` below rebuilds it via -//! `from_stored` rather than re-qualifying. +//! `EngineKeySet.collection` arrives database-qualified: the Calvin key +//! extraction reads it off the physical plan's qualified collection field. +//! The engines key their tables by that same string. use nodedb_physical::physical_plan::meta::PassiveReadKeyId; -use nodedb_types::Value; +use nodedb_types::{QualifiedCollection, StorageKey, Surrogate, Value}; +use crate::bridge::envelope::ErrorCode; use crate::data::executor::core_loop::CoreLoop; +use crate::engine::kv::current_ms; use crate::types::TenantId; impl CoreLoop { - /// Read a single `EngineKeySet` from the local engine state, returning - /// `(PassiveReadKeyId, Value)` pairs. + /// Read every row of `engine_key` from base storage. /// - /// For Document/Vector/Edge engine sets: looks up each surrogate in the - /// sparse engine (document store) and returns the stored value or `Null`. - /// For KV engine sets: looks up each byte key in the KV engine. + /// A passive participant reads document rows by surrogate and key-value + /// rows by key. Every other key set names no row a passive read returns, + /// so it is refused with `Unsupported`. pub(super) fn read_passive_key( &self, + database_id: u64, tenant_id: &TenantId, engine_key: &nodedb_cluster::calvin::types::EngineKeySet, - ) -> Vec<(PassiveReadKeyId, Value)> { + ) -> Result, ErrorCode> { use nodedb_cluster::calvin::types::EngineKeySet; + let tid = tenant_id.as_u64(); match engine_key { EngineKeySet::Document { collection, surrogates, - } - | EngineKeySet::Vector { - collection, - surrogates, } => surrogates .iter() .map(|&surrogate| { - let value = self - .read_surrogate_value(tenant_id, collection, surrogate) - .unwrap_or(Value::Null); - ( - PassiveReadKeyId { - collection: nodedb_types::QualifiedCollection::from_stored( - collection.clone(), + let stored = self + .sparse + .get( + database_id, + tid, + collection, + &StorageKey::for_surrogate(Surrogate::new(surrogate)), + ) + .map_err(|e| ErrorCode::Internal { + detail: format!( + "calvin passive read of {collection} row {surrogate}: {e}" ), + })?; + Ok(( + PassiveReadKeyId::surrogate( + QualifiedCollection::from_stored(collection.clone()), surrogate, - }, - value, - ) + ), + stored_value(stored), + )) }) .collect(), - - EngineKeySet::Kv { collection, keys } => keys - .iter() - .map(|k| { - let value = self - .read_kv_value(tenant_id, collection, k) - .unwrap_or(Value::Null); - // For KV, use key bytes as surrogate placeholder (0 sentinel). - // KV keys don't have surrogates; the PassiveReadKeyId identifies - // the collection and a stable u32 hash of the key. - let key_hash = stable_kv_hash(k); - ( - PassiveReadKeyId { - collection: nodedb_types::QualifiedCollection::from_stored( - collection.clone(), + EngineKeySet::Kv { collection, keys } => { + let now_ms = current_ms(); + Ok(keys + .iter() + .map(|key| { + let stored = self + .kv_engine + .get(database_id, tid, collection, key, now_ms); + ( + PassiveReadKeyId::kv( + QualifiedCollection::from_stored(collection.clone()), + key.clone(), ), - surrogate: key_hash, - }, - value, - ) - }) - .collect(), - - EngineKeySet::Edge { - collection, edges, .. - } => edges - .iter() - .map(|&(src, dst)| { - // Edge reads: use a stable hash of (src, dst) as surrogate. - let edge_hash = stable_edge_hash(src, dst); - ( - PassiveReadKeyId { - collection: nodedb_types::QualifiedCollection::from_stored( - collection.clone(), - ), - surrogate: edge_hash, - }, - Value::Null, // Edge existence read: Null = absent, non-Null = present. - ) - }) - .collect(), - - // An array write is a static write: no dependent transaction - // predicts its cells, so no passive participant reads them. - EngineKeySet::Array { .. } => Vec::new(), + stored_value(stored), + ) + }) + .collect()) + } + EngineKeySet::Vector { collection, .. } + | EngineKeySet::Edge { collection, .. } + | EngineKeySet::Array { collection, .. } + | EngineKeySet::Collection { collection, .. } + | EngineKeySet::Unique { collection, .. } => Err(ErrorCode::Unsupported { + detail: format!( + "a passive Calvin read names rows of {collection} by a key set that is \ + neither document surrogates nor key-value keys" + ), + }), } } - - /// Read a single document surrogate from the sparse engine. - /// - /// Returns `None` if the surrogate is not present in this core's partition. - pub(super) fn read_surrogate_value( - &self, - tenant_id: &TenantId, - collection: &str, - surrogate: u32, - ) -> Option { - // In v1 this is a thin stub: the full implementation requires a - // synchronous lookup through the sparse engine's redb B-Tree. - // The engine lookup path is available via `self.engine_state` once - // the Data Plane engine access APIs are wired. For now, return None - // (caller maps None → Null). - let _ = (tenant_id, collection, surrogate); - None - } - - /// Read a single KV entry from the KV engine. - /// - /// Returns `None` if the key is not present. - pub(super) fn read_kv_value( - &self, - tenant_id: &TenantId, - collection: &str, - key: &[u8], - ) -> Option { - let _ = (tenant_id, collection, key); - None - } } -/// Stable, deterministic hash of a KV byte key into a u32 surrogate -/// placeholder for use in `PassiveReadKeyId`. -/// -/// Uses xxhash with a fixed seed to satisfy the determinism contract: -/// the same byte key must produce the same hash on every replica. -/// `DefaultHasher` (RandomState) is explicitly NOT used here. -fn stable_kv_hash(key: &[u8]) -> u32 { - // FNV-1a 32-bit with fixed offset basis — no external dependency needed. - // This is a placeholder; a production implementation would use xxhash-rust - // with a fixed seed once the crate is available in this crate's deps. - const FNV_OFFSET: u32 = 2_166_136_261; - const FNV_PRIME: u32 = 16_777_619; - let mut hash = FNV_OFFSET; - for &byte in key { - hash ^= u32::from(byte); - hash = hash.wrapping_mul(FNV_PRIME); +/// The value a passive read reports for a row: its stored bytes, or `Null` +/// for an absent row. +fn stored_value(stored: Option>) -> Value { + match stored { + Some(bytes) => Value::Bytes(bytes), + None => Value::Null, } - hash -} - -/// Stable, deterministic hash of an edge `(src, dst)` into a u32 surrogate -/// placeholder for `PassiveReadKeyId`. -fn stable_edge_hash(src: u32, dst: u32) -> u32 { - // Combine src and dst with a deterministic mix. - let combined: u64 = (u64::from(src) << 32) | u64::from(dst); - stable_kv_hash(&combined.to_le_bytes()) } diff --git a/nodedb/src/data/executor/handlers/control/calvin_reply/mod.rs b/nodedb/src/data/executor/handlers/control/calvin_reply/mod.rs index f570f830b..ce9c7742b 100644 --- a/nodedb/src/data/executor/handlers/control/calvin_reply/mod.rs +++ b/nodedb/src/data/executor/handlers/control/calvin_reply/mod.rs @@ -1,8 +1,9 @@ // SPDX-License-Identifier: BUSL-1.1 -mod flush_read; mod images; +mod payload; mod reply; +mod spec; mod stage; mod target; diff --git a/nodedb/src/data/executor/handlers/control/calvin_reply/flush_read.rs b/nodedb/src/data/executor/handlers/control/calvin_reply/payload.rs similarity index 94% rename from nodedb/src/data/executor/handlers/control/calvin_reply/flush_read.rs rename to nodedb/src/data/executor/handlers/control/calvin_reply/payload.rs index ddb6a763d..9f0f408a3 100644 --- a/nodedb/src/data/executor/handlers/control/calvin_reply/flush_read.rs +++ b/nodedb/src/data/executor/handlers/control/calvin_reply/payload.rs @@ -1,6 +1,6 @@ // SPDX-License-Identifier: BUSL-1.1 -//! The payload a flushed Calvin transaction answers with. +//! The payload an installed Calvin slice answers with. use super::images::{RowLocation, StoredRow}; use super::reply::{CalvinReply, InstalledTimeseries}; @@ -14,8 +14,8 @@ use crate::engine::timeseries::install_counts::TsInstallCounts; use crate::util::rmpv_value::rmpv_to_value; impl CoreLoop { - /// The payload `reply` answers with once the flush has installed the - /// transaction. Post-images are read from base after the install, so each row is what + /// The payload `reply` answers with once the redo install has written + /// the slice. Post-images are read from base after the install, so each row is what /// the install stored. /// /// A row the plan wrote that base does not hold after the install is an @@ -93,7 +93,7 @@ fn installed_timeseries_rows( return Err(ErrorCode::Internal { detail: format!( "calvin RETURNING: the install of timeseries ingest {} into '{}' is absent \ - from the flushed record", + from the installed record", installed.ordinal, installed.collection ), }); diff --git a/nodedb/src/data/executor/handlers/control/calvin_reply/reply.rs b/nodedb/src/data/executor/handlers/control/calvin_reply/reply.rs index 443ce0d98..015c01769 100644 --- a/nodedb/src/data/executor/handlers/control/calvin_reply/reply.rs +++ b/nodedb/src/data/executor/handlers/control/calvin_reply/reply.rs @@ -1,14 +1,14 @@ // SPDX-License-Identifier: BUSL-1.1 -//! The reply a staged Calvin transaction answers with when its flush -//! installs it. +//! The reply a staged Calvin transaction answers with when its redo record +//! installs. use nodedb_physical::physical_plan::ReturningSpec; use nodedb_types::{RowIdentity, Surrogate}; use super::target::RowEngine; -/// The reply a Calvin transaction's flush answers with. +/// The reply a Calvin transaction's install answers with. /// /// The transaction answers with its last `RETURNING` plan's rows, or with /// its last plan's affected count when no plan carries `RETURNING`. A plan @@ -20,7 +20,7 @@ pub(in crate::data::executor) enum CalvinReply { Count(Vec), /// `RETURNING` rows, decided when the plan staged. Rows(Vec), - /// `RETURNING` rows the flush reads from base after the install, so each + /// `RETURNING` rows the install reads from base after it writes, so each /// row is exactly what a `SELECT` reads. PostImages(PostImages), /// `RETURNING` rows of a resolved timeseries ingest: the rows its install @@ -36,7 +36,7 @@ impl Default for CalvinReply { } impl CalvinReply { - /// A reply the flush cannot render: post-images of a columnar row, which + /// A reply the install cannot render: post-images of a columnar row, which /// base keys by no surrogate. #[cfg(test)] pub(in crate::data::executor) fn unrenderable_for_test(collection: &str) -> Self { @@ -85,7 +85,7 @@ pub(in crate::data::executor) struct CalvinStaging { pub(super) owned_edges: u64, } -/// The rows of a `RETURNING` plan the flush reads after the install. +/// The rows of a `RETURNING` plan the install reads after it writes. #[derive(Debug)] pub(in crate::data::executor) struct PostImages { pub(super) spec: ReturningSpec, diff --git a/nodedb/src/data/executor/handlers/control/calvin_reply/spec.rs b/nodedb/src/data/executor/handlers/control/calvin_reply/spec.rs new file mode 100644 index 000000000..7fcff0930 --- /dev/null +++ b/nodedb/src/data/executor/handlers/control/calvin_reply/spec.rs @@ -0,0 +1,151 @@ +// SPDX-License-Identifier: BUSL-1.1 + +//! A staged reply as the redo entry carries it, and back. +//! +//! The leader's resolve turns the reply its stage decided into a +//! [`CalvinReplySpec`]. Every replica's install turns that spec back into a +//! [`CalvinReply`] and renders it. + +use nodedb_physical::physical_plan::{ + CalvinInstalledTimeseriesSpec, CalvinPostImagesSpec, CalvinReplyRow, CalvinReplySpec, + CalvinRowEngine, +}; +use nodedb_types::RowIdentity; + +use super::reply::{CalvinReply, InstalledTimeseries, PostImages}; +use super::target::RowEngine; + +impl From for CalvinRowEngine { + fn from(engine: RowEngine) -> Self { + match engine { + RowEngine::Document => Self::Document, + RowEngine::Crdt => Self::Crdt, + RowEngine::Kv => Self::Kv, + RowEngine::Vector => Self::Vector, + RowEngine::Columnar => Self::Columnar, + RowEngine::Timeseries => Self::Timeseries, + } + } +} + +impl From for RowEngine { + fn from(engine: CalvinRowEngine) -> Self { + match engine { + CalvinRowEngine::Document => Self::Document, + CalvinRowEngine::Crdt => Self::Crdt, + CalvinRowEngine::Kv => Self::Kv, + CalvinRowEngine::Vector => Self::Vector, + CalvinRowEngine::Columnar => Self::Columnar, + CalvinRowEngine::Timeseries => Self::Timeseries, + } + } +} + +impl From<&CalvinReply> for CalvinReplySpec { + fn from(reply: &CalvinReply) -> Self { + match reply { + CalvinReply::Count(payload) => Self::Count(payload.clone()), + CalvinReply::Rows(payload) => Self::Rows(payload.clone()), + CalvinReply::PostImages(images) => Self::PostImages(CalvinPostImagesSpec { + spec: images.spec.clone(), + rls_filters: images.rls_filters.clone(), + collection: images.collection.clone(), + engine: images.engine.into(), + rows: images + .rows + .iter() + .map(|(identity, surrogate)| CalvinReplyRow { + identity: identity.as_str().to_string(), + surrogate: *surrogate, + }) + .collect(), + }), + CalvinReply::InstalledTimeseries(installed) => { + Self::InstalledTimeseries(CalvinInstalledTimeseriesSpec { + spec: installed.spec.clone(), + rls_filters: installed.rls_filters.clone(), + collection: installed.collection.clone(), + ordinal: installed.ordinal as u64, + }) + } + } + } +} + +impl From for CalvinReply { + fn from(spec: CalvinReplySpec) -> Self { + match spec { + CalvinReplySpec::Count(payload) => Self::Count(payload), + CalvinReplySpec::Rows(payload) => Self::Rows(payload), + CalvinReplySpec::PostImages(images) => Self::PostImages(PostImages { + spec: images.spec, + rls_filters: images.rls_filters, + collection: images.collection, + engine: images.engine.into(), + rows: images + .rows + .into_iter() + .map(|row| (RowIdentity::from_user_key(row.identity), row.surrogate)) + .collect(), + }), + // An ordinal past the address space names no install, so the + // render reports the install absent. + CalvinReplySpec::InstalledTimeseries(installed) => { + Self::InstalledTimeseries(InstalledTimeseries { + spec: installed.spec, + rls_filters: installed.rls_filters, + collection: installed.collection, + ordinal: usize::try_from(installed.ordinal).unwrap_or(usize::MAX), + }) + } + } + } +} + +#[cfg(test)] +mod tests { + use super::*; + use nodedb_physical::physical_plan::{ReturningColumns, ReturningSpec}; + use nodedb_types::Surrogate; + + fn returning() -> ReturningSpec { + ReturningSpec { + columns: ReturningColumns::Star, + } + } + + fn round_trip(spec: CalvinReplySpec) { + let reply = CalvinReply::from(spec.clone()); + assert_eq!(CalvinReplySpec::from(&reply), spec); + } + + #[test] + fn every_reply_shape_round_trips_through_its_spec() { + round_trip(CalvinReplySpec::Count(vec![1, 2])); + round_trip(CalvinReplySpec::Rows(vec![3])); + round_trip(CalvinReplySpec::PostImages(CalvinPostImagesSpec { + spec: returning(), + rls_filters: vec![4], + collection: "orders".into(), + engine: CalvinRowEngine::Vector, + rows: vec![ + CalvinReplyRow { + identity: "o1".into(), + surrogate: Surrogate::new(1), + }, + CalvinReplyRow { + identity: "o2".into(), + surrogate: Surrogate::new(2), + }, + ], + })); + round_trip(CalvinReplySpec::InstalledTimeseries( + CalvinInstalledTimeseriesSpec { + spec: returning(), + rls_filters: Vec::new(), + collection: "metrics".into(), + ordinal: 3, + }, + )); + } +} diff --git a/nodedb/src/data/executor/handlers/control/calvin_reply/stage.rs b/nodedb/src/data/executor/handlers/control/calvin_reply/stage.rs index b823adaaf..70a2c45f4 100644 --- a/nodedb/src/data/executor/handlers/control/calvin_reply/stage.rs +++ b/nodedb/src/data/executor/handlers/control/calvin_reply/stage.rs @@ -2,8 +2,8 @@ //! Stage one Calvin plan and fold its reply into the transaction's reply. //! -//! A Calvin flush installs the transaction's redo record, and a redo record -//! carries no reply. Each plan therefore decides its reply when it stages. +//! A Calvin slice installs from its redo record, and a redo record carries no +//! reply. Each plan therefore decides its reply when it stages. //! //! - A plan without `RETURNING` answers its staging handler's reply: the //! affected count against BASE ∪ OVERLAY. @@ -12,8 +12,7 @@ //! taken from BASE ∪ OVERLAY, so a row an earlier plan of the same //! transaction wrote is reported as that plan left it. //! - A delete reports each removed row's image from before the plan. -//! - A document or CRDT write reports each row as the flush's install -//! stored it, read back from base after the install. The install can +//! - A document or CRDT write reports each row as the install stored it, read back from base after the install. The install can //! rewrite a body, as hash chaining does. //! - A KV, vector-primary or columnar write reports the rows it staged, //! which are the exact bytes the install stores. @@ -58,7 +57,7 @@ impl CoreLoop { ) -> Result<(), ErrorCode> { let tid = tenant_id.as_u64(); // Every timeseries ingest becomes one redo sub-record, in plan order, - // so its position among them names its install at the flush. + // so its position among them names its install. let ts_ordinal = staging.ts_ingests; if matches!(plan, PhysicalPlan::Timeseries(TimeseriesOp::Ingest { .. })) { staging.ts_ingests += 1; @@ -75,7 +74,7 @@ impl CoreLoop { let reply = &mut staging.reply; match returning_target(plan) { // A resolved ingest's install decides which rows land and how - // they read, so the flush renders them from its own install. + // they read, so the reply renders them from that install. Some(target) if is_resolved_ingest(plan) => { *reply = CalvinReply::InstalledTimeseries(InstalledTimeseries { spec: target.spec.clone(), @@ -491,7 +490,7 @@ mod tests { } /// Commit `plans` as one Calvin transaction on a fresh core prepared by - /// `setup`, and return the flush's reply. + /// `setup`, and return the install's reply. fn commit(plans: &[PhysicalPlan], setup: impl FnOnce(&mut CoreLoop)) -> Response { let dir = tempfile::tempdir().expect("tempdir"); let (mut core, _tx, _rx) = make_core_with_dir(dir.path()); diff --git a/nodedb/src/data/executor/handlers/control/calvin_resolve.rs b/nodedb/src/data/executor/handlers/control/calvin_resolve.rs index 8f36fac1f..52c16f3f2 100644 --- a/nodedb/src/data/executor/handlers/control/calvin_resolve.rs +++ b/nodedb/src/data/executor/handlers/control/calvin_resolve.rs @@ -13,8 +13,13 @@ //! [`CoreLoop::stage_calvin_overlay`][super::calvin_overlay_stage]). Reusing //! `execute_resolve_txn` directly means the redo serialization logic itself is //! never duplicated. +//! +//! The answer is a [`CalvinResolved`]: the redo record and the reply the +//! staged plans decided, as a [`nodedb_physical::physical_plan::CalvinReplySpec`]. + +use nodedb_physical::physical_plan::{CalvinReplySpec, CalvinResolved}; -use crate::bridge::envelope::Response; +use crate::bridge::envelope::{ErrorCode, Payload, Response, Status}; use crate::data::executor::core_loop::CoreLoop; use crate::data::executor::task::ExecutionTask; @@ -22,13 +27,14 @@ use super::calvin_txn_id::calvin_synthetic_txn_id; impl CoreLoop { /// Resolve the Calvin transaction staged under `(epoch, position)` on - /// this vshard into a [`RedoRecord`][crate::wal::RedoRecord] and return - /// its encoded bytes, without touching any base engine. + /// this vshard into a [`RedoRecord`][crate::wal::RedoRecord], without + /// touching any base engine. Answers a [`CalvinResolved`] with the + /// record's encoded bytes and the staged reply. /// /// Errors (rather than silently dropping data or producing an empty /// record) when no `commit_pending` entry exists for /// `(epoch, position, vshard)` — the transaction was never staged (or was - /// already flushed/dropped), and there is nothing to resolve. + /// already installed/dropped), and there is nothing to resolve. /// /// `DocumentOp::BulkUpdate` / `BulkDelete` plans are staged into the /// overlay by `stage_calvin_overlay` (via the predicted-surrogate-set @@ -48,11 +54,11 @@ impl CoreLoop { Err(e) => return self.response_error(task, e), }; - // Clone the staged plans (and the deterministic epoch anchor) out of - // `commit_pending` so the `&mut self` resolve call — which assigns - // bitemporal stamps into the overlay — does not overlap the immutable - // borrow of the pending buffer. - let (tid, plans, epoch_system_ms) = match self + // Clone the staged plans, the deterministic epoch anchor, and the + // staged reply out of `commit_pending` so the `&mut self` resolve + // call — which assigns bitemporal stamps into the overlay — does not + // overlap the immutable borrow of the pending buffer. + let (tid, plans, epoch_system_ms, reply) = match self .calvin .commit_pending .get(&(epoch, position, vshard_id)) @@ -61,6 +67,7 @@ impl CoreLoop { pending.tenant_id.as_u64(), pending.plans.clone(), pending.epoch_system_ms, + CalvinReplySpec::from(&pending.reply), ), None => { return self.response_error( @@ -89,10 +96,31 @@ impl CoreLoop { let prev_epoch_ms = self.epoch_system_ms; self.epoch_system_ms = Some(epoch_system_ms); self.apply_scope.calvin_txn_ordinal = Some(txn_ordinal); - let resp = self.execute_resolve_txn(task, tid, synthetic_txn_id, &plans); + let mut resp = self.execute_resolve_txn(task, tid, synthetic_txn_id, &plans); self.apply_scope.calvin_txn_ordinal = None; self.epoch_system_ms = prev_epoch_ms; - resp + if resp.status != Status::Ok { + return resp; + } + let resolved = CalvinResolved { + redo: resp.payload.as_bytes().to_vec(), + reply, + }; + match zerompk::to_msgpack_vec(&resolved) { + Ok(bytes) => { + resp.payload = Payload::from_vec(bytes); + resp + } + Err(error) => self.response_error( + task, + ErrorCode::Internal { + detail: format!( + "calvin resolve: encode the resolved answer of epoch={epoch} \ + position={position} vshard={vshard_id}: {error}" + ), + }, + ), + } } } @@ -115,7 +143,7 @@ mod tests { /// A minimal `ExecutionTask` homing to vShard 0, tenant 1, database /// DEFAULT, matching what the Calvin scheduler dispatches with (see /// `dispatch.rs`'s `DatabaseId::DEFAULT` for `CalvinExecuteStatic` / - /// `CalvinFlush` / `CalvinDrop`; `CalvinResolve` must match). + /// `CalvinDrop`; `CalvinResolve` must match). fn make_task() -> ExecutionTask { let plan = PhysicalPlan::Document(DocumentOp::PointGet { collection: QualifiedCollection::new(DatabaseId::DEFAULT, "x"), @@ -145,6 +173,7 @@ mod tests { wal_lsn: None, resolved_now_ms: None, commit_hlc: None, + entry_version: None, admission: Admission::Exempt(ExemptReason::Read), }; ExecutionTask::new(request) @@ -219,6 +248,45 @@ mod tests { .expect("seed row"); } + /// The redo record a resolve answered with. + fn resolved_record(resp: &Response) -> RedoRecord { + let resolved: CalvinResolved = + zerompk::from_msgpack(resp.payload.as_bytes()).expect("decode resolved answer"); + RedoRecord::from_bytes(&resolved.redo).expect("decode redo record") + } + + /// The resolve answers the reply the stage decided, beside the record. + #[test] + fn a_resolve_answers_the_staged_reply() { + let dir = tempfile::tempdir().unwrap(); + let (mut core, _tx, _rx) = make_core_with_dir(dir.path()); + let task = make_task(); + stage( + &mut core, + &task, + 2, + 0, + &[point_insert_plan("orders", "o1", 7)], + ); + let vshard_id = task.request.vshard_id.as_u32(); + let staged = CalvinReplySpec::from( + &core + .calvin + .commit_pending + .get(&(2, 0, vshard_id)) + .expect("staged commit") + .reply, + ); + + let resp = core.execute_calvin_resolve(&task, 2, 0); + + assert_eq!(resp.status, Status::Ok, "resolve must succeed: {resp:?}"); + let resolved: CalvinResolved = + zerompk::from_msgpack(resp.payload.as_bytes()).expect("decode resolved answer"); + assert_eq!(resolved.reply, staged); + assert_eq!(resolved_record(&resp).ops.len(), 1); + } + fn stage( core: &mut CoreLoop, task: &ExecutionTask, @@ -230,7 +298,6 @@ mod tests { epoch, position, epoch_system_ms: 0, - is_group_leader: true, }; let resp = core.execute_calvin_execute_static(task, ctx, &TenantId::new(1), plans, &[], &[]); @@ -254,7 +321,7 @@ mod tests { let resp = core.execute_calvin_resolve(&task, 1, 0); assert_eq!(resp.status, Status::Ok, "resolve must succeed: {resp:?}"); - let record = RedoRecord::from_bytes(resp.payload.as_bytes()).expect("decode redo record"); + let record = resolved_record(&resp); assert!( record.calvin_stamp.is_none(), "calvin_stamp is filled in by a later unit, not resolve itself" @@ -320,7 +387,7 @@ mod tests { let resp = core.execute_calvin_resolve(&task, 3, 0); assert_eq!(resp.status, Status::Ok, "resolve must succeed: {resp:?}"); - let record = RedoRecord::from_bytes(resp.payload.as_bytes()).expect("decode redo record"); + let record = resolved_record(&resp); assert_eq!( record.ops.len(), 2, @@ -340,7 +407,7 @@ mod tests { // them -- if staging re-derived the row set via a live scan instead // of trusting the predicted set, the redo would carry 3 deletes, not // 2. This is the determinism proof: staging must key off the - // predicted set the flush applies, not a fresh predicate scan. + // predicted set the install applies, not a fresh predicate scan. seed_row(&mut core, "orders", 20, "a", "1"); seed_row(&mut core, "orders", 21, "a", "1"); seed_row(&mut core, "orders", 22, "a", "1"); @@ -355,7 +422,7 @@ mod tests { let resp = core.execute_calvin_resolve(&task, 4, 0); assert_eq!(resp.status, Status::Ok, "resolve must succeed: {resp:?}"); - let record = RedoRecord::from_bytes(resp.payload.as_bytes()).expect("decode redo record"); + let record = resolved_record(&resp); assert_eq!( record.ops.len(), 2, @@ -387,7 +454,7 @@ mod tests { let resp = core.execute_calvin_resolve(&task, 5, 0); assert_eq!(resp.status, Status::Ok, "resolve must succeed: {resp:?}"); - let record = RedoRecord::from_bytes(resp.payload.as_bytes()).expect("decode redo record"); + let record = resolved_record(&resp); assert_eq!(record.ops.len(), 2, "one Put sub-record per predicted row"); // The expected post-image is the exact same decode -> apply -> encode @@ -422,7 +489,6 @@ mod tests { epoch: 6, position: 0, epoch_system_ms: 0, - is_group_leader: true, }; let resp = core.execute_calvin_execute_static( &task, @@ -507,7 +573,7 @@ mod tests { let resp = core.execute_calvin_resolve(&task, 6, 0); assert_eq!(resp.status, Status::Ok, "resolve must succeed: {resp:?}"); - let record = RedoRecord::from_bytes(resp.payload.as_bytes()).expect("decode redo"); + let record = resolved_record(&resp); let image = record .ops .iter() @@ -564,7 +630,6 @@ mod tests { epoch: 8, position: 3, epoch_system_ms: EPOCH_MS, - is_group_leader: true, }; let staged = core.execute_calvin_execute_static( &task, @@ -581,7 +646,7 @@ mod tests { core.apply_scope.calvin_txn_ordinal, None, "the resolve clears the transaction ordinal" ); - let record = RedoRecord::from_bytes(resp.payload.as_bytes()).expect("decode redo"); + let record = resolved_record(&resp); let put = record .ops .iter() diff --git a/nodedb/src/data/executor/handlers/control/crdt.rs b/nodedb/src/data/executor/handlers/control/crdt.rs index 7a05e657e..e7a0beabf 100644 --- a/nodedb/src/data/executor/handlers/control/crdt.rs +++ b/nodedb/src/data/executor/handlers/control/crdt.rs @@ -183,7 +183,7 @@ impl CoreLoop { ) { crate::engine::crdt::tenant_state::ValidatedApplyOutcome::Clean { .. } => { self.checkpoint_coordinator.mark_dirty("crdt", 1); - self.note_collection_write_lsn(task, collection); + self.note_collection_write(task, collection); self.response_ok(task) } crate::engine::crdt::tenant_state::ValidatedApplyOutcome::Rejected(reason) => { diff --git a/nodedb/src/data/executor/handlers/control/crdt_apply/gated.rs b/nodedb/src/data/executor/handlers/control/crdt_apply/gated.rs index e7113d0ea..c9f86ef4e 100644 --- a/nodedb/src/data/executor/handlers/control/crdt_apply/gated.rs +++ b/nodedb/src/data/executor/handlers/control/crdt_apply/gated.rs @@ -407,7 +407,7 @@ impl CoreLoop { ); } if imported_authoritative { - self.note_collection_write_lsn(task, collection); + self.note_collection_write(task, collection); } // The high-water-mark and the client-visible frame are decided diff --git a/nodedb/src/data/executor/handlers/control/crdt_apply/local.rs b/nodedb/src/data/executor/handlers/control/crdt_apply/local.rs index 68eaab503..2e3c15ea0 100644 --- a/nodedb/src/data/executor/handlers/control/crdt_apply/local.rs +++ b/nodedb/src/data/executor/handlers/control/crdt_apply/local.rs @@ -162,13 +162,13 @@ impl CoreLoop { &bytes, ); if imported_authoritative { - self.note_collection_write_lsn(task, collection); + self.note_collection_write(task, collection); } } Ok(None) if imported_authoritative => { // A headless import has no sparse projection, but still // changed authoritative Loro state. - self.note_collection_write_lsn(task, collection); + self.note_collection_write(task, collection); } Ok(None) => {} Err(refusal) => { diff --git a/nodedb/src/data/executor/handlers/control/crdt_preview.rs b/nodedb/src/data/executor/handlers/control/crdt_preview.rs index 12568e53b..b02e8d6ef 100644 --- a/nodedb/src/data/executor/handlers/control/crdt_preview.rs +++ b/nodedb/src/data/executor/handlers/control/crdt_preview.rs @@ -208,7 +208,13 @@ mod tests { ); let import = core.execute_crdt_import_snapshot(&task, 1, "docs", &snapshot); assert_eq!(import.status, Status::Ok); - assert_eq!(import.read_version_lsn, crate::types::Lsn::new(73)); + assert_eq!( + import.read_versions.of(task.request.vshard_id), + Some(nodedb_types::WriteVersion::local_after( + nodedb_types::WriteVersion::ZERO, + 73 + )) + ); assert_eq!(import.watermark_lsn, crate::types::Lsn::new(73)); let before = core .execute_crdt_read(&task, "docs", "one") @@ -455,7 +461,13 @@ mod tests { }, ); assert_eq!(applied.status, Status::Ok); - assert_eq!(applied.read_version_lsn, crate::types::Lsn::new(91)); + assert_eq!( + applied.read_versions.of(task.request.vshard_id), + Some(nodedb_types::WriteVersion::local_after( + nodedb_types::WriteVersion::ZERO, + 91 + )) + ); assert_eq!(applied.watermark_lsn, crate::types::Lsn::new(91)); assert!( core.crdt_engines diff --git a/nodedb/src/data/executor/handlers/control/mod.rs b/nodedb/src/data/executor/handlers/control/mod.rs index a645588d5..5a2284b64 100644 --- a/nodedb/src/data/executor/handlers/control/mod.rs +++ b/nodedb/src/data/executor/handlers/control/mod.rs @@ -26,4 +26,5 @@ pub mod reindex; pub mod snapshot; pub mod synonym_group; +#[cfg(test)] pub(in crate::data::executor) use calvin_txn_id::calvin_synthetic_txn_id; diff --git a/nodedb/src/data/executor/handlers/control/reindex/csr.rs b/nodedb/src/data/executor/handlers/control/reindex/csr.rs index 800fbea57..370258059 100644 --- a/nodedb/src/data/executor/handlers/control/reindex/csr.rs +++ b/nodedb/src/data/executor/handlers/control/reindex/csr.rs @@ -38,11 +38,12 @@ impl CoreLoop { }; let token = seed.token(); let memory = self.graph_memory(target); + let fail_scope = self.fail_scope; let (tx, rx) = mpsc::sync_channel::>(1); let spawned = std::thread::Builder::new() .name(format!("reindex-csr-{}", self.core_id)) .spawn(move || { - hold_build(CSR_BUILD_HOLD); + hold_build(fail_scope, CSR_BUILD_HOLD); // The receiver is gone only when the core shut down. let _ = tx.send(seed.build(memory)); }); diff --git a/nodedb/src/data/executor/handlers/control/reindex/fts.rs b/nodedb/src/data/executor/handlers/control/reindex/fts.rs index 4d7e39a1a..3f24276c5 100644 --- a/nodedb/src/data/executor/handlers/control/reindex/fts.rs +++ b/nodedb/src/data/executor/handlers/control/reindex/fts.rs @@ -33,11 +33,12 @@ impl CoreLoop { return Ok(()); }; let token = ticket.token(); + let fail_scope = self.fail_scope; let (tx, rx) = mpsc::sync_channel::>(1); let spawned = std::thread::Builder::new() .name(format!("reindex-fts-{}", self.core_id)) .spawn(move || { - hold_build(FTS_BUILD_HOLD); + hold_build(fail_scope, FTS_BUILD_HOLD); // The receiver is gone only when the core shut down. let _ = tx.send(ticket.read().map(FtsSnapshot::compact)); }); diff --git a/nodedb/src/data/executor/handlers/control/reindex/hold.rs b/nodedb/src/data/executor/handlers/control/reindex/hold.rs index 4d11035e9..0333b0f8e 100644 --- a/nodedb/src/data/executor/handlers/control/reindex/hold.rs +++ b/nodedb/src/data/executor/handlers/control/reindex/hold.rs @@ -6,25 +6,28 @@ //! The hold runs on the rebuild's own OS thread, never on a core. Without //! the `failpoints` feature it compiles to nothing. +use crate::fail_point::FailScope; + /// Holds a full-text rebuild thread before it reads its snapshot. pub const FTS_BUILD_HOLD: &str = "reindex::fts_build_hold"; /// Holds a CSR rebuild thread before it compacts its snapshot. pub const CSR_BUILD_HOLD: &str = "reindex::csr_build_hold"; -/// Hold the calling rebuild thread at failpoint `name`. +/// Hold the calling rebuild thread at failpoint `name`, evaluated in +/// `scope`, the scope of the core that started the rebuild. /// /// `WaitForFile(path)` parks the thread until `path` exists, for at most /// two minutes. Every other action runs as at any failpoint. -pub(super) fn hold_build(name: &str) { +pub(super) fn hold_build(scope: FailScope, name: &str) { #[cfg(feature = "failpoints")] - hold_armed(name); + hold_armed(scope, name); #[cfg(not(feature = "failpoints"))] - let _ = name; + let _ = (scope, name); } #[cfg(feature = "failpoints")] -fn hold_armed(name: &str) { +fn hold_armed(scope: FailScope, name: &str) { use std::time::{Duration, Instant}; use crate::fail_point::{FailAction, eval, lookup}; @@ -32,14 +35,14 @@ fn hold_armed(name: &str) { const HOLD_LIMIT: Duration = Duration::from_secs(120); const POLL: Duration = Duration::from_millis(5); - match lookup(name) { + match lookup(scope, name) { Some(FailAction::WaitForFile(path)) => { let deadline = Instant::now() + HOLD_LIMIT; while !path.exists() && Instant::now() < deadline { std::thread::sleep(POLL); } } - Some(_) => eval(name), + Some(_) => eval(scope, name), None => {} } } diff --git a/nodedb/src/data/executor/handlers/control/reindex/waiter.rs b/nodedb/src/data/executor/handlers/control/reindex/waiter.rs index 0e43d6aae..83006404a 100644 --- a/nodedb/src/data/executor/handlers/control/reindex/waiter.rs +++ b/nodedb/src/data/executor/handlers/control/reindex/waiter.rs @@ -4,10 +4,9 @@ //! //! The rebuilds run exactly as for REINDEX CONCURRENTLY: off the core, with //! the core serving every other request meanwhile. Only the answer waits. -//! The core holds the request's task here and answers it from the tick, -//! the way it holds a write parked behind a staged Calvin transaction -//! (`core_loop::calvin_fence`): the held task builds the response, the -//! response ring carries it, and the task's deadline bounds the wait. +//! The core holds the request's task here and answers it from the tick: +//! the held task builds the response, the response ring carries it, and the +//! task's deadline bounds the wait. //! //! - Every rebuild cut over: `Ok`. //! - A rebuild was discarded, or an HNSW segment rebuild failed: the first diff --git a/nodedb/src/data/executor/handlers/document/apply_balance_delta.rs b/nodedb/src/data/executor/handlers/document/apply_balance_delta.rs index e35c43b4f..7922753c7 100644 --- a/nodedb/src/data/executor/handlers/document/apply_balance_delta.rs +++ b/nodedb/src/data/executor/handlers/document/apply_balance_delta.rs @@ -143,7 +143,7 @@ impl CoreLoop { ); } - self.note_surrogate_write_lsn(task, tid, collection, surrogate.as_u32()); + self.note_surrogate_write(task, tid, collection, surrogate.as_u32()); self.checkpoint_coordinator.mark_dirty("sparse", 1); let mut response = self.response_affected(task, 1); diff --git a/nodedb/src/data/executor/handlers/document/resolve/apply_row.rs b/nodedb/src/data/executor/handlers/document/resolve/apply_row.rs index 102f17d4d..a00e1785f 100644 --- a/nodedb/src/data/executor/handlers/document/resolve/apply_row.rs +++ b/nodedb/src/data/executor/handlers/document/resolve/apply_row.rs @@ -205,7 +205,7 @@ impl CoreLoop { self.checkpoint_coordinator.mark_dirty("sparse", 1); // Same post-commit index bookkeeping `execute_point_update` runs. - if let Some(lsn) = task.wal_lsn() { + if let Some(stamp) = self.task_write_stamp(task) { let mut tuples = std::mem::take(&mut outcome.secondary_index_added); tuples.append(&mut outcome.secondary_index_removed); tuples.append(&mut outcome.bitemporal_index_tuples); @@ -214,7 +214,7 @@ impl CoreLoop { crate::types::TenantId::new(tid), collection, &tuples, - lsn, + stamp, ); } @@ -227,7 +227,7 @@ impl CoreLoop { &stored_bytes, precondition, ); - self.note_surrogate_write_lsn(task, tid, collection, surrogate.as_u32()); + self.note_surrogate_write(task, tid, collection, surrogate.as_u32()); // The row, journalled after apply from the body `apply_point_put` // took. A resolved write's mutations can name several collections, @@ -347,8 +347,8 @@ impl CoreLoop { self.checkpoint_coordinator.mark_dirty("sparse", 1); if let Some(prior_bytes) = outcome.prior_value.as_deref() { - self.note_surrogate_write_lsn(task, tid, collection, surrogate.as_u32()); - if let Some(lsn) = task.wal_lsn() { + self.note_surrogate_write(task, tid, collection, surrogate.as_u32()); + if let Some(stamp) = self.task_write_stamp(task) { let mut tuples = outcome.secondary_index_tuples; tuples.extend(outcome.bitemporal_index_tuples); self.note_index_write_values( @@ -356,7 +356,7 @@ impl CoreLoop { crate::types::TenantId::new(tid), collection, &tuples, - lsn, + stamp, ); } self.emit_document_delete_event( diff --git a/nodedb/src/data/executor/handlers/document/write/batch_insert.rs b/nodedb/src/data/executor/handlers/document/write/batch_insert.rs index 873e56b93..04aa5dd0d 100644 --- a/nodedb/src/data/executor/handlers/document/write/batch_insert.rs +++ b/nodedb/src/data/executor/handlers/document/write/batch_insert.rs @@ -358,17 +358,23 @@ impl CoreLoop { return self.response_error(task, e); } + // Record each committed row's version against its surrogate and + // collection. + for (_, key) in &applied { + self.note_surrogate_write(task, tid, collection, key.surrogate().as_u32()); + } + // Record each committed row's touched secondary-index values into the // per-index write-value substrate. This runs only after the batch has // durably committed. - if let Some(lsn) = task.wal_lsn() { + if let Some(stamp) = self.task_write_stamp(task) { for tuples in &row_index_tuples { self.note_index_write_values( task.request.database_id, crate::types::TenantId::new(tid), collection, tuples, - lsn, + stamp, ); } } @@ -501,6 +507,7 @@ mod tests { wal_lsn: None, resolved_now_ms: None, commit_hlc: None, + entry_version: None, admission: crate::bridge::envelope::Admission::Admitted, }) } diff --git a/nodedb/src/data/executor/handlers/fts_sync.rs b/nodedb/src/data/executor/handlers/fts_sync.rs index 927cb84b7..a693eb1f1 100644 --- a/nodedb/src/data/executor/handlers/fts_sync.rs +++ b/nodedb/src/data/executor/handlers/fts_sync.rs @@ -69,7 +69,7 @@ impl CoreLoop { { Ok(()) => { // Advance the collection floor for this committed FTS write. - self.note_collection_write_lsn(task, collection); + self.note_collection_write(task, collection); if let Some(prov) = provenance { self.sync_commit(prov); return self.sync_ack_response(task, AckStatus::Applied, prov.seq); @@ -133,7 +133,7 @@ impl CoreLoop { { Ok(()) => { // Advance the collection floor for this committed FTS delete. - self.note_collection_write_lsn(task, collection); + self.note_collection_write(task, collection); if let Some(prov) = provenance { self.sync_commit(prov); return self.sync_ack_response(task, AckStatus::Applied, prov.seq); diff --git a/nodedb/src/data/executor/handlers/graph_edge_write/delete.rs b/nodedb/src/data/executor/handlers/graph_edge_write/delete.rs index 36330f675..337316276 100644 --- a/nodedb/src/data/executor/handlers/graph_edge_write/delete.rs +++ b/nodedb/src/data/executor/handlers/graph_edge_write/delete.rs @@ -145,8 +145,8 @@ impl CoreLoop { undo.push(UndoEntry::EdgeWrite(Box::new(edge_undo))); } self.checkpoint_coordinator.mark_dirty("sparse", 1); - self.note_edge_write_lsn(task, tid, collection, src_id, label, dst_id); - // CDC: emit after `note_edge_write_lsn` so the event LSN matches + self.note_edge_write(task, tid, collection, src_id, label, dst_id); + // CDC: emit after `note_edge_write` so the event LSN matches // this edge's WAL LSN (the WAL-replay reconstruction key). self.emit_graph_edge_event( task, diff --git a/nodedb/src/data/executor/handlers/graph_edge_write/delete_batch.rs b/nodedb/src/data/executor/handlers/graph_edge_write/delete_batch.rs index 3a18c67ae..b533a3101 100644 --- a/nodedb/src/data/executor/handlers/graph_edge_write/delete_batch.rs +++ b/nodedb/src/data/executor/handlers/graph_edge_write/delete_batch.rs @@ -144,7 +144,7 @@ impl CoreLoop { .mark_dirty("sparse", edges.len()); } for edge in edges { - self.note_edge_write_lsn( + self.note_edge_write( task, tid, edge.collection.as_str(), diff --git a/nodedb/src/data/executor/handlers/graph_edge_write/put.rs b/nodedb/src/data/executor/handlers/graph_edge_write/put.rs index 9fadb68bc..d4ba40f4c 100644 --- a/nodedb/src/data/executor/handlers/graph_edge_write/put.rs +++ b/nodedb/src/data/executor/handlers/graph_edge_write/put.rs @@ -152,8 +152,8 @@ impl CoreLoop { partition.set_node_surrogate(src_id, src_surrogate); partition.set_node_surrogate(dst_id, dst_surrogate); self.checkpoint_coordinator.mark_dirty("sparse", 1); - self.note_edge_write_lsn(task, tid, collection, src_id, label, dst_id); - // CDC: emit after `note_edge_write_lsn` so the core + self.note_edge_write(task, tid, collection, src_id, label, dst_id); + // CDC: emit after `note_edge_write` so the core // watermark (the event's LSN) already reflects this // edge's WAL LSN, matching the WAL-replay reconstruction. self.emit_graph_edge_event( diff --git a/nodedb/src/data/executor/handlers/graph_edge_write/put_batch.rs b/nodedb/src/data/executor/handlers/graph_edge_write/put_batch.rs index 3e1fae0f4..157fb5f7f 100644 --- a/nodedb/src/data/executor/handlers/graph_edge_write/put_batch.rs +++ b/nodedb/src/data/executor/handlers/graph_edge_write/put_batch.rs @@ -142,7 +142,7 @@ impl CoreLoop { .mark_dirty("sparse", edges.len()); } for edge in edges { - self.note_edge_write_lsn( + self.note_edge_write( task, tid, edge.collection.as_str(), diff --git a/nodedb/src/data/executor/handlers/graph_edge_write/shared.rs b/nodedb/src/data/executor/handlers/graph_edge_write/shared.rs index de22a0ca1..34a8e7b4e 100644 --- a/nodedb/src/data/executor/handlers/graph_edge_write/shared.rs +++ b/nodedb/src/data/executor/handlers/graph_edge_write/shared.rs @@ -114,7 +114,7 @@ impl CoreLoop { /// Record a committed edge write's version, keyed by the edge's /// `(src, label, dst)` identity, if a WAL LSN was threaded onto the task. - pub(in crate::data::executor) fn note_edge_write_lsn( + pub(in crate::data::executor) fn note_edge_write( &mut self, task: &ExecutionTask, tid: u64, @@ -123,10 +123,10 @@ impl CoreLoop { label: &str, dst_id: &str, ) { - let Some(lsn) = task.wal_lsn() else { + let Some(stamp) = self.task_write_stamp(task) else { return; }; - self.note_write_lsn( + self.note_write( task.request.database_id, TenantId::new(tid), collection, @@ -137,7 +137,7 @@ impl CoreLoop { dst: Box::from(dst_id), }, ), - lsn, + stamp, ); } } @@ -210,6 +210,7 @@ pub(super) mod test_support { wal_lsn: Some(Lsn::new(lsn)), resolved_now_ms: None, commit_hlc: None, + entry_version: None, admission: Admission::Exempt(ExemptReason::Read), }) } diff --git a/nodedb/src/data/executor/handlers/graph_node_guard.rs b/nodedb/src/data/executor/handlers/graph_node_guard.rs index d942ce35b..d10cdbaa7 100644 --- a/nodedb/src/data/executor/handlers/graph_node_guard.rs +++ b/nodedb/src/data/executor/handlers/graph_node_guard.rs @@ -293,6 +293,7 @@ mod tests { wal_lsn: None, resolved_now_ms: None, commit_hlc: None, + entry_version: None, admission: Admission::Exempt(ExemptReason::Read), }) } diff --git a/nodedb/src/data/executor/handlers/graph_wcc.rs b/nodedb/src/data/executor/handlers/graph_wcc.rs index 78d285619..48284b3f3 100644 --- a/nodedb/src/data/executor/handlers/graph_wcc.rs +++ b/nodedb/src/data/executor/handlers/graph_wcc.rs @@ -144,9 +144,10 @@ impl CoreLoop { // Fails this one core's round, so a test can check that the all-core // gather fails the query instead of merging the other cores. #[cfg(feature = "failpoints")] - if let Some(detail) = - crate::fail_point::eval_fail(&format!("graph::wcc_superstep::core{}", self.core_id)) - { + if let Some(detail) = crate::fail_point::eval_fail( + self.fail_scope, + &format!("graph::wcc_superstep::core{}", self.core_id), + ) { return self.response_error(task, ErrorCode::Internal { detail }); } diff --git a/nodedb/src/data/executor/handlers/home_versions.rs b/nodedb/src/data/executor/handlers/home_versions.rs index ee2cb32d9..ec48fea78 100644 --- a/nodedb/src/data/executor/handlers/home_versions.rs +++ b/nodedb/src/data/executor/handlers/home_versions.rs @@ -3,17 +3,17 @@ //! `MetaOp::HomeVersions`: this core's current version of each probed home. //! //! The Control Plane hands a core only the probes whose vShard the core owns. -//! A probe with a collection answers the collection's write floor on this -//! core. A probe without one answers the core watermark. Nothing is read -//! beyond those two values, and nothing is written. +//! A probe with a collection answers the collection's version on the probe's +//! vShard. A probe without one answers the vShard's latest version. Nothing +//! is read beyond those two values, and nothing is written. use nodedb_physical::physical_plan::{HomeAnswer, HomeVersion, HomeVersionProbe}; +use nodedb_types::WriteVersion; use crate::bridge::envelope::{ErrorCode, Response}; use crate::data::executor::core_loop::CoreLoop; -use crate::data::executor::core_loop::write_index::CollKey; use crate::data::executor::task::ExecutionTask; -use crate::types::Lsn; +use crate::types::VShardId; impl CoreLoop { pub(in crate::data::executor) fn execute_home_versions( @@ -21,13 +21,24 @@ impl CoreLoop { task: &ExecutionTask, probes: &[HomeVersionProbe], ) -> Response { - let answers: Vec = probes - .iter() - .map(|probe| HomeVersion { + let mut answers: Vec = Vec::with_capacity(probes.len()); + for probe in probes { + let Some(version) = self.home_version(task, probe) else { + return self.response_error( + task, + ErrorCode::Internal { + detail: format!( + "home versions: probe names vShard {}, past the vShard space", + probe.vshard + ), + }, + ); + }; + answers.push(HomeVersion { probe: probe.clone(), - answer: HomeAnswer::Version(self.home_version(task, probe).as_u64()), - }) - .collect(); + answer: HomeAnswer::Version(version), + }); + } match zerompk::to_msgpack_vec(&answers) { Ok(payload) => self.response_with_payload(task, payload), Err(e) => self.response_error( @@ -39,17 +50,21 @@ impl CoreLoop { } } - fn home_version(&self, task: &ExecutionTask, probe: &HomeVersionProbe) -> Lsn { - match probe.collection.as_deref() { - Some(collection) => self - .write_index - .collection_write_lsn(&CollKey { - db: task.request.database_id, - tenant: task.request.tenant_id, - collection: Box::from(collection), - }) - .unwrap_or(Lsn::ZERO), - None => self.watermark, + /// The probe's current version, `None` for a vShard id past the vShard + /// space. + fn home_version(&self, task: &ExecutionTask, probe: &HomeVersionProbe) -> Option { + if probe.vshard >= VShardId::COUNT { + return None; } + let vshard = VShardId::new(probe.vshard); + Some(match probe.collection.as_deref() { + Some(collection) => self.write_index.collection_current( + task.request.database_id, + task.request.tenant_id, + vshard, + collection, + ), + None => self.write_index.latest(vshard), + }) } } diff --git a/nodedb/src/data/executor/handlers/join/params.rs b/nodedb/src/data/executor/handlers/join/params.rs index a3d08eaa5..7c9b7d7cb 100644 --- a/nodedb/src/data/executor/handlers/join/params.rs +++ b/nodedb/src/data/executor/handlers/join/params.rs @@ -137,6 +137,7 @@ fn make_dummy_task() -> ExecutionTask { wal_lsn: None, resolved_now_ms: None, commit_hlc: None, + entry_version: None, admission: crate::bridge::envelope::Admission::Exempt( crate::bridge::envelope::ExemptReason::Read, ), diff --git a/nodedb/src/data/executor/handlers/join/shuffle_join.rs b/nodedb/src/data/executor/handlers/join/shuffle_join.rs index 826b35b83..239a58ce8 100644 --- a/nodedb/src/data/executor/handlers/join/shuffle_join.rs +++ b/nodedb/src/data/executor/handlers/join/shuffle_join.rs @@ -216,6 +216,7 @@ mod tests { wal_lsn: None, resolved_now_ms: None, commit_hlc: None, + entry_version: None, admission: crate::bridge::envelope::Admission::Exempt( crate::bridge::envelope::ExemptReason::Read, ), diff --git a/nodedb/src/data/executor/handlers/kv/atomic.rs b/nodedb/src/data/executor/handlers/kv/atomic.rs index 40c32d3fc..5bf3d5884 100644 --- a/nodedb/src/data/executor/handlers/kv/atomic.rs +++ b/nodedb/src/data/executor/handlers/kv/atomic.rs @@ -127,7 +127,7 @@ impl CoreLoop { Some(written.as_slice()), None, ); - self.note_kv_write_lsn(task, did, tid, collection, key); + self.note_kv_write(task, did, tid, collection, key); match response_codec::encode_json_as_msgpack(&serde_json::json!({ "value": value })) { Ok(payload) => self.response_with_payload(task, payload), @@ -195,7 +195,7 @@ impl CoreLoop { Some(written.as_slice()), None, ); - self.note_kv_write_lsn(task, did, tid, collection, key); + self.note_kv_write(task, did, tid, collection, key); match response_codec::encode_json_as_msgpack(&incr_float_reply(value, &written)) { Ok(payload) => self.response_with_payload(task, payload), Err(e) => self.response_error(task, ErrorCode::from(e)), @@ -266,7 +266,7 @@ impl CoreLoop { Some(written.as_slice()), None, ); - self.note_kv_write_lsn(task, did, tid, collection, key); + self.note_kv_write(task, did, tid, collection, key); } let current_b64 = result @@ -345,7 +345,7 @@ impl CoreLoop { Some(written.as_slice()), old.as_deref(), ); - self.note_kv_write_lsn(task, did, tid, collection, key); + self.note_kv_write(task, did, tid, collection, key); // A row the read policy excludes is reported exactly as an absent row, // the same convention `execute_kv_get` uses — the caller cannot tell it diff --git a/nodedb/src/data/executor/handlers/kv/batch.rs b/nodedb/src/data/executor/handlers/kv/batch.rs index 18fd6fe2c..92394b662 100644 --- a/nodedb/src/data/executor/handlers/kv/batch.rs +++ b/nodedb/src/data/executor/handlers/kv/batch.rs @@ -143,7 +143,7 @@ impl CoreLoop { // version against that single LSN. if task.wal_lsn().is_some() { for (key, _) in entries { - self.note_kv_write_lsn(task, did, tid, collection, key); + self.note_kv_write(task, did, tid, collection, key); } } if let Some(spec) = returning { diff --git a/nodedb/src/data/executor/handlers/kv/crud/write_basic.rs b/nodedb/src/data/executor/handlers/kv/crud/write_basic.rs index 760e50a7e..36bc60aa0 100644 --- a/nodedb/src/data/executor/handlers/kv/crud/write_basic.rs +++ b/nodedb/src/data/executor/handlers/kv/crud/write_basic.rs @@ -68,7 +68,7 @@ impl CoreLoop { }; self.emit_kv_write_event(task, collection, op, key, Some(value), old_slice); - self.note_kv_write_lsn(task, did, tid, collection, key); + self.note_kv_write(task, did, tid, collection, key); if let Some(spec) = returning { // `value` IS the stored body on this path: the put writes the // caller's bytes verbatim, so projecting them is projecting the @@ -159,7 +159,7 @@ impl CoreLoop { None, ); - self.note_kv_write_lsn(task, did, tid, collection, key); + self.note_kv_write(task, did, tid, collection, key); if let Some(spec) = returning { return self.kv_stored_returning_response(task, spec, rls_filters, &[(key, value)]); } @@ -241,7 +241,7 @@ impl CoreLoop { None, ); - self.note_kv_write_lsn(task, did, tid, collection, key); + self.note_kv_write(task, did, tid, collection, key); if let Some(spec) = returning { return self.kv_stored_returning_response(task, spec, rls_filters, &[(key, value)]); } @@ -251,7 +251,7 @@ impl CoreLoop { /// Record a committed KV point write's version, keyed by the raw KV key /// bytes, if a WAL LSN was threaded onto the task. Shared by every KV write /// chokepoint (basic put/delete, atomic ops, batch, field, TTL). - pub(in crate::data::executor) fn note_kv_write_lsn( + pub(in crate::data::executor) fn note_kv_write( &mut self, task: &ExecutionTask, did: u64, @@ -259,15 +259,15 @@ impl CoreLoop { collection: &str, key: &[u8], ) { - let Some(lsn) = task.wal_lsn() else { + let Some(stamp) = self.task_write_stamp(task) else { return; }; - self.note_write_lsn( + self.note_write( crate::types::DatabaseId::new(did), crate::types::TenantId::new(tid), collection, Some(crate::data::executor::core_loop::write_index::KeyRepr::KvKey(Box::from(key))), - lsn, + stamp, ); } } diff --git a/nodedb/src/data/executor/handlers/kv/field.rs b/nodedb/src/data/executor/handlers/kv/field.rs index d5a68451e..4c3f57292 100644 --- a/nodedb/src/data/executor/handlers/kv/field.rs +++ b/nodedb/src/data/executor/handlers/kv/field.rs @@ -201,7 +201,7 @@ impl CoreLoop { Some(computed.new_value.as_slice()), current.as_deref(), ); - self.note_kv_write_lsn(task, did, tid, collection, key); + self.note_kv_write(task, did, tid, collection, key); if let Some(spec) = returning { // `computed.new_value` IS the stored body: the merge is persisted // verbatim, so projecting it is projecting the post-image. diff --git a/nodedb/src/data/executor/handlers/kv/predicate/apply.rs b/nodedb/src/data/executor/handlers/kv/predicate/apply.rs index 7ca5020ec..a623d6718 100644 --- a/nodedb/src/data/executor/handlers/kv/predicate/apply.rs +++ b/nodedb/src/data/executor/handlers/kv/predicate/apply.rs @@ -105,7 +105,7 @@ impl CoreLoop { Some(new_value), Some(old_body), ); - self.note_kv_write_lsn(task, did, tid, collection, key); + self.note_kv_write(task, did, tid, collection, key); } if let Some(spec) = returning { diff --git a/nodedb/src/data/executor/handlers/kv/resolve/apply.rs b/nodedb/src/data/executor/handlers/kv/resolve/apply.rs index 1e345a348..182673f8c 100644 --- a/nodedb/src/data/executor/handlers/kv/resolve/apply.rs +++ b/nodedb/src/data/executor/handlers/kv/resolve/apply.rs @@ -148,7 +148,7 @@ impl CoreLoop { Some(value), precondition.as_deref(), ); - self.note_kv_write_lsn(task, did, tid, collection.as_str(), key); + self.note_kv_write(task, did, tid, collection.as_str(), key); } // The drift check confirmed the row holds `precondition`, so it is // present and keeps its bound identity. @@ -183,7 +183,7 @@ impl CoreLoop { Some(value), Some(precondition.as_slice()), ); - self.note_kv_write_lsn(task, did, tid, collection.as_str(), key); + self.note_kv_write(task, did, tid, collection.as_str(), key); } KvResolvedMutation::Delete { collection, @@ -203,7 +203,7 @@ impl CoreLoop { None, precondition.as_deref(), ); - self.note_kv_write_lsn(task, did, tid, collection.as_str(), key); + self.note_kv_write(task, did, tid, collection.as_str(), key); } // No body change, so no event — matches the live handlers. KvResolvedMutation::Expire { @@ -220,7 +220,7 @@ impl CoreLoop { key, resolved_now_ms.saturating_add(*ttl_ms), ); - self.note_kv_write_lsn(task, did, tid, collection.as_str(), key); + self.note_kv_write(task, did, tid, collection.as_str(), key); } KvResolvedMutation::Persist { collection, @@ -228,7 +228,7 @@ impl CoreLoop { precondition: _, } => { self.kv_engine.persist(did, tid, collection.as_str(), key); - self.note_kv_write_lsn(task, did, tid, collection.as_str(), key); + self.note_kv_write(task, did, tid, collection.as_str(), key); } } Ok(()) diff --git a/nodedb/src/data/executor/handlers/kv/ttl.rs b/nodedb/src/data/executor/handlers/kv/ttl.rs index 7d71934ba..4d9e1bf8b 100644 --- a/nodedb/src/data/executor/handlers/kv/ttl.rs +++ b/nodedb/src/data/executor/handlers/kv/ttl.rs @@ -69,7 +69,7 @@ impl CoreLoop { .kv_engine .expire(did, tid, collection, key, ttl_ms, now_ms) { - self.note_kv_write_lsn(task, did, tid, collection, key); + self.note_kv_write(task, did, tid, collection, key); self.response_ok(task) } else { self.response_error(task, ErrorCode::NotFound) @@ -99,7 +99,7 @@ impl CoreLoop { } if self.kv_engine.persist(did, tid, collection, key) { - self.note_kv_write_lsn(task, did, tid, collection, key); + self.note_kv_write(task, did, tid, collection, key); self.response_ok(task) } else { self.response_error(task, ErrorCode::NotFound) diff --git a/nodedb/src/data/executor/handlers/point/apply_put/core.rs b/nodedb/src/data/executor/handlers/point/apply_put/core.rs index 8802fdb63..097803849 100644 --- a/nodedb/src/data/executor/handlers/point/apply_put/core.rs +++ b/nodedb/src/data/executor/handlers/point/apply_put/core.rs @@ -469,6 +469,7 @@ mod tests { wal_lsn: None, resolved_now_ms: None, commit_hlc: None, + entry_version: None, admission: crate::bridge::envelope::Admission::Admitted, }) } diff --git a/nodedb/src/data/executor/handlers/point/apply_put/index.rs b/nodedb/src/data/executor/handlers/point/apply_put/index.rs index 0521c89c7..cecd41aa3 100644 --- a/nodedb/src/data/executor/handlers/point/apply_put/index.rs +++ b/nodedb/src/data/executor/handlers/point/apply_put/index.rs @@ -267,6 +267,7 @@ mod tests { wal_lsn: None, resolved_now_ms: None, commit_hlc: None, + entry_version: None, admission: crate::bridge::envelope::Admission::Admitted, }) } diff --git a/nodedb/src/data/executor/handlers/point/apply_put/types.rs b/nodedb/src/data/executor/handlers/point/apply_put/types.rs index f9082bb4d..71ee70832 100644 --- a/nodedb/src/data/executor/handlers/point/apply_put/types.rs +++ b/nodedb/src/data/executor/handlers/point/apply_put/types.rs @@ -149,6 +149,7 @@ pub(in crate::data::executor) fn map_enforcement_error(e: ErrorCode) -> crate::E | ErrorCode::RejectedConstraint { .. } | ErrorCode::RejectedPrevalidation { .. } | ErrorCode::RetryableRefusal { .. } + | ErrorCode::CoreFailStopped { .. } | ErrorCode::SyncRejected { .. } | ErrorCode::SyncNotApplied { .. } | ErrorCode::NotFound diff --git a/nodedb/src/data/executor/handlers/point/delete.rs b/nodedb/src/data/executor/handlers/point/delete.rs index 9dbb4a651..1cc5cfc7c 100644 --- a/nodedb/src/data/executor/handlers/point/delete.rs +++ b/nodedb/src/data/executor/handlers/point/delete.rs @@ -181,11 +181,11 @@ impl CoreLoop { // collection, but only when a row was actually removed — a delete that // matched nothing changes no state and creates no OCC conflict. if prior.is_some() { - self.note_surrogate_write_lsn(task, tid, collection, surrogate.as_u32()); + self.note_surrogate_write(task, tid, collection, surrogate.as_u32()); // Record the removed secondary-index values into the per-index // write-value substrate (plain cascade ∪ bitemporal tombstones). - if let Some(lsn) = task.wal_lsn() { + if let Some(stamp) = self.task_write_stamp(task) { let mut tuples = outcome.secondary_index_tuples; tuples.extend(outcome.bitemporal_index_tuples); self.note_index_write_values( @@ -193,7 +193,7 @@ impl CoreLoop { crate::types::TenantId::new(tid), collection, &tuples, - lsn, + stamp, ); } } diff --git a/nodedb/src/data/executor/handlers/point/insert.rs b/nodedb/src/data/executor/handlers/point/insert.rs index 8ad6005e8..c1145eb45 100644 --- a/nodedb/src/data/executor/handlers/point/insert.rs +++ b/nodedb/src/data/executor/handlers/point/insert.rs @@ -298,7 +298,7 @@ impl CoreLoop { self.checkpoint_coordinator.mark_dirty("sparse", 1); - self.note_surrogate_write_lsn(task, tid, collection, surrogate.as_u32()); + self.note_surrogate_write(task, tid, collection, surrogate.as_u32()); // The exact bytes storage now holds, taken before the index tuples are // consumed below. A `RETURNING` projection reads these rather than @@ -308,7 +308,7 @@ impl CoreLoop { // Record the touched secondary-index values into the per-index // write-value substrate (added ∪ removed ∪ bitemporal tuples). - if let Some(lsn) = task.wal_lsn() { + if let Some(stamp) = self.task_write_stamp(task) { let mut tuples = outcome.secondary_index_added; tuples.extend(outcome.secondary_index_removed); tuples.extend(outcome.bitemporal_index_tuples); @@ -317,7 +317,7 @@ impl CoreLoop { crate::types::TenantId::new(tid), collection, &tuples, - lsn, + stamp, ); } diff --git a/nodedb/src/data/executor/handlers/point/put.rs b/nodedb/src/data/executor/handlers/point/put.rs index ff7b64753..255d0337d 100644 --- a/nodedb/src/data/executor/handlers/point/put.rs +++ b/nodedb/src/data/executor/handlers/point/put.rs @@ -200,11 +200,11 @@ impl CoreLoop { } // Record the committed write's version against its surrogate + collection. - self.note_surrogate_write_lsn(task, tid, collection, surrogate.as_u32()); + self.note_surrogate_write(task, tid, collection, surrogate.as_u32()); // Record the touched secondary-index values into the per-index // write-value substrate (added ∪ removed ∪ bitemporal tuples). - if let Some(lsn) = task.wal_lsn() { + if let Some(stamp) = self.task_write_stamp(task) { let mut tuples = std::mem::take(&mut prior.secondary_index_added); tuples.append(&mut prior.secondary_index_removed); tuples.append(&mut prior.bitemporal_index_tuples); @@ -213,7 +213,7 @@ impl CoreLoop { crate::types::TenantId::new(tid), collection, &tuples, - lsn, + stamp, ); } diff --git a/nodedb/src/data/executor/handlers/point/update/exec.rs b/nodedb/src/data/executor/handlers/point/update/exec.rs index 5e396b166..818a2f040 100644 --- a/nodedb/src/data/executor/handlers/point/update/exec.rs +++ b/nodedb/src/data/executor/handlers/point/update/exec.rs @@ -242,10 +242,15 @@ impl CoreLoop { bitemporal, sys_from_ms: sys_from_for_encode, wal_lsn: task.wal_lsn(), + write_stamp: self.task_write_stamp(task), resolved_sum_targets, }); match write_result { Ok(target_write_set) => { + // Record the committed row's write version against + // its surrogate and collection. A read of the row + // from before this update then fails validation. + self.note_surrogate_write(task, tid, collection, surrogate.as_u32()); self.doc_cache.put( task.request.database_id.as_u64(), tid, diff --git a/nodedb/src/data/executor/handlers/point/update/persist.rs b/nodedb/src/data/executor/handlers/point/update/persist.rs index 8ed06e1eb..a4021563d 100644 --- a/nodedb/src/data/executor/handlers/point/update/persist.rs +++ b/nodedb/src/data/executor/handlers/point/update/persist.rs @@ -40,6 +40,10 @@ pub(in crate::data::executor) struct PointUpdatePersist<'a> { pub(in crate::data::executor) bitemporal: bool, pub(in crate::data::executor) sys_from_ms: i64, pub(in crate::data::executor) wal_lsn: Option, + /// The stamp the update's index values record under, `None` without a + /// WAL record. + pub(in crate::data::executor) write_stamp: + Option, /// `(target collection, join-key value)` → target row surrogate for every /// materialized-sum target this update may touch — BOTH sides when the /// update moves a row between targets by changing its join key. Resolved on @@ -70,6 +74,7 @@ impl CoreLoop { bitemporal, sys_from_ms, wal_lsn, + write_stamp, resolved_sum_targets, } = params; @@ -169,7 +174,7 @@ impl CoreLoop { // Index write-versions are published only once the write they describe // is durable. - if let Some(lsn) = wal_lsn + if let Some(stamp) = write_stamp && !touched.is_empty() { self.note_index_write_values( @@ -177,7 +182,7 @@ impl CoreLoop { TenantId::new(tid), collection, &touched, - lsn, + stamp, ); } diff --git a/nodedb/src/data/executor/handlers/query_collection_size.rs b/nodedb/src/data/executor/handlers/query_collection_size.rs index 5f4d1d76c..ffb2fb116 100644 --- a/nodedb/src/data/executor/handlers/query_collection_size.rs +++ b/nodedb/src/data/executor/handlers/query_collection_size.rs @@ -79,8 +79,8 @@ impl CoreLoop { payload: Payload::from(payload), watermark_lsn: self.watermark, error_code: None, - read_set_valid: None, - read_version_lsn: crate::types::Lsn::ZERO, + stage_vote: None, + read_versions: crate::types::ReadVersions::new(), write_set: Vec::new(), } } diff --git a/nodedb/src/data/executor/handlers/snapshot/capture_arrays.rs b/nodedb/src/data/executor/handlers/snapshot/capture_arrays.rs index 74cd92e39..ab14cad02 100644 --- a/nodedb/src/data/executor/handlers/snapshot/capture_arrays.rs +++ b/nodedb/src/data/executor/handlers/snapshot/capture_arrays.rs @@ -3,15 +3,15 @@ //! Tenant snapshot capture of the array engine. //! //! Every cell version of each of the tenant's arrays in the snapshot's -//! database is exported, grouped by the vShard its Hilbert prefix routes to. +//! database is exported, grouped by the vShard its tile's key routes to. //! A backup keeps each group on the source node of that vShard, so every //! version is captured once. use std::collections::BTreeMap; use std::sync::Arc; -use nodedb_array::coord::encode::encode_hilbert_prefix; use nodedb_array::schema::ArraySchema; +use nodedb_array::tile::cell_tile_prefix; use nodedb_cluster::distributed_array::array_vshard_for_tile; use crate::data::executor::core_loop::CoreLoop; @@ -81,7 +81,7 @@ fn by_vshard( ) -> crate::Result>> { let mut grouped: BTreeMap> = BTreeMap::new(); for version in versions { - let prefix = match encode_hilbert_prefix(schema, &version.coord) { + let prefix = match cell_tile_prefix(schema, &version.coord) { Ok(prefix) => prefix, Err(_) if version.payload.is_none() => continue, Err(e) => { @@ -152,7 +152,7 @@ mod tests { ); for (vshard, cells) in &grouped { for cell in cells { - let prefix = encode_hilbert_prefix(&s, &cell.coord).expect("prefix"); + let prefix = cell_tile_prefix(&s, &cell.coord).expect("prefix"); assert_eq!(array_vshard_for_tile(prefix, 8).expect("route"), *vshard); } } diff --git a/nodedb/src/data/executor/handlers/snapshot/install_arrays.rs b/nodedb/src/data/executor/handlers/snapshot/install_arrays.rs index 42421eaf6..63b16c4e0 100644 --- a/nodedb/src/data/executor/handlers/snapshot/install_arrays.rs +++ b/nodedb/src/data/executor/handlers/snapshot/install_arrays.rs @@ -10,8 +10,8 @@ use std::collections::{HashMap, HashSet}; use std::sync::Arc; -use nodedb_array::coord::encode::encode_hilbert_prefix; use nodedb_array::schema::ArraySchema; +use nodedb_array::tile::cell_tile_prefix; use nodedb_array::types::coord::value::CoordValue; use nodedb_cluster::distributed_array::array_vshard_for_tile; @@ -93,7 +93,7 @@ impl CoreLoop { let routes = Arc::clone(&schema); let prefix_bits = entry.prefix_bits; let replaced = |coord: &[CoordValue]| { - encode_hilbert_prefix(&routes, coord) + cell_tile_prefix(&routes, coord) .ok() .and_then(|prefix| array_vshard_for_tile(prefix, prefix_bits).ok()) .is_some_and(|vshard| group.contains(&vshard)) diff --git a/nodedb/src/data/executor/handlers/snapshot/restore/tenant_snapshot.rs b/nodedb/src/data/executor/handlers/snapshot/restore/tenant_snapshot.rs index 60306fbdb..de2cc38cb 100644 --- a/nodedb/src/data/executor/handlers/snapshot/restore/tenant_snapshot.rs +++ b/nodedb/src/data/executor/handlers/snapshot/restore/tenant_snapshot.rs @@ -496,6 +496,7 @@ mod tests { PhysicalPlan::Meta(MetaOp::WalAppend { payload: Vec::new(), }), + None, ); let response = core.execute_restore_tenant_snapshot( @@ -568,6 +569,7 @@ mod tests { PhysicalPlan::Meta(MetaOp::WalAppend { payload: Vec::new(), }), + None, ); (bytes, task) } @@ -653,6 +655,7 @@ mod tests { PhysicalPlan::Meta(MetaOp::WalAppend { payload: Vec::new(), }), + None, ); let response = core.execute_restore_tenant_snapshot(&task, 0, &bytes, true, &[], &[]); assert_eq!(response.status, Status::Ok, "restore must succeed"); diff --git a/nodedb/src/data/executor/handlers/spatial/rtree_scan.rs b/nodedb/src/data/executor/handlers/spatial/rtree_scan.rs index 5c77fd621..9acf744fa 100644 --- a/nodedb/src/data/executor/handlers/spatial/rtree_scan.rs +++ b/nodedb/src/data/executor/handlers/spatial/rtree_scan.rs @@ -388,6 +388,7 @@ mod tests { wal_lsn: None, resolved_now_ms: None, commit_hlc: None, + entry_version: None, admission: crate::bridge::envelope::Admission::Exempt( crate::bridge::envelope::ExemptReason::Read, ), diff --git a/nodedb/src/data/executor/handlers/spatial_sync.rs b/nodedb/src/data/executor/handlers/spatial_sync.rs index c6d05df41..fcbe5e9e0 100644 --- a/nodedb/src/data/executor/handlers/spatial_sync.rs +++ b/nodedb/src/data/executor/handlers/spatial_sync.rs @@ -199,7 +199,7 @@ impl CoreLoop { self.spatial_doc_map.insert(doc_map_key, doc_id); // Advance the collection floor for this committed spatial write. - self.note_collection_write_lsn(task, collection); + self.note_collection_write(task, collection); // Advance HWM and return gate result payload when provenance is present. if let Some(prov) = provenance { @@ -297,7 +297,7 @@ impl CoreLoop { self.spatial_doc_map.remove(&doc_map_key); // Advance the collection floor for this committed spatial delete. - self.note_collection_write_lsn(task, collection); + self.note_collection_write(task, collection); if let Some(prov) = provenance { self.sync_commit(prov); diff --git a/nodedb/src/data/executor/handlers/timeseries/ingest.rs b/nodedb/src/data/executor/handlers/timeseries/ingest.rs index 9771b5586..e4d96e440 100644 --- a/nodedb/src/data/executor/handlers/timeseries/ingest.rs +++ b/nodedb/src/data/executor/handlers/timeseries/ingest.rs @@ -431,8 +431,8 @@ impl CoreLoop { payload: Payload::from_vec(json), watermark_lsn: self.watermark, error_code: None, - read_set_valid: None, - read_version_lsn: crate::types::Lsn::ZERO, + stage_vote: None, + read_versions: crate::types::ReadVersions::new(), write_set: Vec::new(), } } diff --git a/nodedb/src/data/executor/handlers/timeseries/ingest_dispatch.rs b/nodedb/src/data/executor/handlers/timeseries/ingest_dispatch.rs index 46888bca3..63c5585f0 100644 --- a/nodedb/src/data/executor/handlers/timeseries/ingest_dispatch.rs +++ b/nodedb/src/data/executor/handlers/timeseries/ingest_dispatch.rs @@ -173,8 +173,8 @@ impl CoreLoop { payload: Payload::from_vec(json), watermark_lsn: self.watermark, error_code: None, - read_set_valid: None, - read_version_lsn: crate::types::Lsn::ZERO, + stage_vote: None, + read_versions: crate::types::ReadVersions::new(), write_set: Vec::new(), }; } @@ -265,7 +265,7 @@ impl CoreLoop { return self.sync_ack_response(task, AckStatus::Applied, applied_seq); } if ingest_response.status == Status::Ok { - self.note_collection_write_lsn(task, collection); + self.note_collection_write(task, collection); } ingest_response } diff --git a/nodedb/src/data/executor/handlers/timeseries/ingest_resolved.rs b/nodedb/src/data/executor/handlers/timeseries/ingest_resolved.rs index f2c387510..e516dc301 100644 --- a/nodedb/src/data/executor/handlers/timeseries/ingest_resolved.rs +++ b/nodedb/src/data/executor/handlers/timeseries/ingest_resolved.rs @@ -382,8 +382,8 @@ impl CoreLoop { payload: Payload::from_vec(json), watermark_lsn: self.watermark, error_code: None, - read_set_valid: None, - read_version_lsn: crate::types::Lsn::ZERO, + stage_vote: None, + read_versions: crate::types::ReadVersions::new(), write_set: Vec::new(), }, Err(e) => self.response_error(task, ErrorCode::from(e)), diff --git a/nodedb/src/data/executor/handlers/timeseries/ingest_resolved_outcome.rs b/nodedb/src/data/executor/handlers/timeseries/ingest_resolved_outcome.rs index 1e0c78413..e4d3d7c39 100644 --- a/nodedb/src/data/executor/handlers/timeseries/ingest_resolved_outcome.rs +++ b/nodedb/src/data/executor/handlers/timeseries/ingest_resolved_outcome.rs @@ -10,7 +10,7 @@ //! its durable outcome (`install_outcome`) before the first row lands, so //! WAL catch-up rebuilds exactly the events the install emits. //! - A committed redo install records its counts and landed images in the -//! open apply scope, so the apply answers the counts and a Calvin flush +//! open apply scope, so the apply answers the counts and a Calvin install //! renders its `RETURNING` rows from the stored rows. use crate::data::executor::core_loop::CoreLoop; diff --git a/nodedb/src/data/executor/handlers/timeseries/scan.rs b/nodedb/src/data/executor/handlers/timeseries/scan.rs index f986f9b46..399723f5b 100644 --- a/nodedb/src/data/executor/handlers/timeseries/scan.rs +++ b/nodedb/src/data/executor/handlers/timeseries/scan.rs @@ -274,8 +274,8 @@ impl CoreLoop { payload: Payload::from_vec(buf), watermark_lsn: self.watermark, error_code: None, - read_set_valid: None, - read_version_lsn: crate::types::Lsn::ZERO, + stage_vote: None, + read_versions: crate::types::ReadVersions::new(), write_set: Vec::new(), } } diff --git a/nodedb/src/data/executor/handlers/timeseries/truncate.rs b/nodedb/src/data/executor/handlers/timeseries/truncate.rs index b1a7f8fbd..05aa0eb8f 100644 --- a/nodedb/src/data/executor/handlers/timeseries/truncate.rs +++ b/nodedb/src/data/executor/handlers/timeseries/truncate.rs @@ -165,7 +165,7 @@ impl CoreLoop { self.checkpoint_coordinator .mark_dirty("timeseries", truncated as usize); - self.note_collection_write_lsn(task, collection); + self.note_collection_write(task, collection); self.invalidate_aggregate_cache_for_collection(db.as_u64(), tid.as_u64(), collection); debug!(core = self.core_id, %collection, truncated, "timeseries truncate complete"); diff --git a/nodedb/src/data/executor/handlers/timeseries_wal.rs b/nodedb/src/data/executor/handlers/timeseries_wal.rs index bb888678a..6e5ee75ca 100644 --- a/nodedb/src/data/executor/handlers/timeseries_wal.rs +++ b/nodedb/src/data/executor/handlers/timeseries_wal.rs @@ -306,7 +306,23 @@ impl CoreLoop { mod tests { use crate::data::executor::core_loop::CoreLoop; use crate::data::executor::core_loop::write_index::CollKey; - use crate::types::{DatabaseId, Lsn, TenantId}; + use crate::data::executor::core_loop::write_index::tests::local; + use crate::types::{DatabaseId, TenantId}; + + /// The collection key a replayed write of `collection` in database 0 + /// records under: the collection's home vShard. + fn replayed_key(collection: &str) -> CollKey { + let vshard = + nodedb_types::CollectionKey::from_qualified_str(DatabaseId::new(0), collection) + .expect("collection key") + .vshard(); + CollKey { + vshard, + db: DatabaseId::new(0), + tenant: TenantId::new(7), + collection: Box::from(collection), + } + } use crate::wal::{DecodedBatchRecord, decode_batch_record}; use nodedb_types::Surrogate; use nodedb_types::columnar::ColumnarWalRecord; @@ -557,14 +573,14 @@ mod tests { } /// WAL replay threads the record LSN into `replay_task` so - /// `execute_columnar_insert`'s `note_collection_write_lsn(task, ..)` call + /// `execute_columnar_insert`'s `note_collection_write(task, ..)` call /// (gated on `task.wal_lsn().is_some()`) fires during WAL replay too, not - /// just on live writes. This proves the collection floor in + /// just on live writes. This proves the collection version in /// `WriteVersionIndex` is populated end-to-end through /// `replay_timeseries_wal` -> `replay_columnar_payload` -> /// `execute_columnar_insert`. #[test] - fn columnar_insert_replay_populates_collection_write_lsn_floor() { + fn columnar_insert_replay_populates_the_collection_write_version() { let mut h = make_core(); let record = columnar_wal_record("events_wv", 123, 7); @@ -574,16 +590,13 @@ mod tests { &nodedb_wal::TombstoneSet::new(), ); - let coll_key = CollKey { - db: DatabaseId::new(0), - tenant: TenantId::new(7), - collection: Box::from("events_wv"), - }; assert_eq!( - h.core.write_index.collection_write_lsn(&coll_key), - Some(Lsn::new(123)), - "columnar insert replay must record the record LSN as the \ - collection write-version floor" + h.core + .write_index + .collection_version(&replayed_key("events_wv")), + Some(local(123)), + "columnar insert replay must record the record's version as the \ + collection write version" ); } @@ -630,14 +643,11 @@ mod tests { &nodedb_wal::TombstoneSet::new(), ); - let coll_key = CollKey { - db: DatabaseId::new(0), - tenant: TenantId::new(7), - collection: Box::from("events_ungated"), - }; assert_eq!( - h.core.write_index.collection_write_lsn(&coll_key), - Some(Lsn::new(150)), + h.core + .write_index + .collection_version(&replayed_key("events_ungated")), + Some(local(150)), "a record above the floor must be applied" ); } diff --git a/nodedb/src/data/executor/handlers/transaction/mod.rs b/nodedb/src/data/executor/handlers/transaction/mod.rs index 5d6e8c40d..b99ec2f07 100644 --- a/nodedb/src/data/executor/handlers/transaction/mod.rs +++ b/nodedb/src/data/executor/handlers/transaction/mod.rs @@ -8,5 +8,9 @@ pub(in crate::data::executor) mod resolve; pub(in crate::data::executor) mod savepoint_marks; pub(in crate::data::executor) mod stage_write; pub(in crate::data::executor) mod undo; +#[cfg(test)] mod write_version; +#[cfg(test)] mod write_version_kv; +#[cfg(test)] +mod write_version_parity_tests; diff --git a/nodedb/src/data/executor/handlers/transaction/redo_apply/calvin_fold_tests.rs b/nodedb/src/data/executor/handlers/transaction/redo_apply/calvin_fold_tests.rs index abbb33c39..eaca0fb7d 100644 --- a/nodedb/src/data/executor/handlers/transaction/redo_apply/calvin_fold_tests.rs +++ b/nodedb/src/data/executor/handlers/transaction/redo_apply/calvin_fold_tests.rs @@ -1,7 +1,8 @@ // SPDX-License-Identifier: BUSL-1.1 -//! A Calvin redo record folds its materialized sums from its stamp, the same -//! way live and in restart replay. +//! A Calvin redo record folds its materialized sums at its live install. The +//! fold rows travel in the install's write set, so the funnel journals them +//! as parts of the record's group, and restart replay applies those parts. use nodedb_physical::physical_plan::{MaterializedSumBinding, RedoSumTargets, ResolvedSumTarget}; use nodedb_types::{DatabaseId, Surrogate, TenantId}; @@ -11,27 +12,47 @@ use nodedb_wal::{TombstoneSet, WalRecord}; use super::entry::CommittedRedo; use super::test_commit::doc_put_sub_record; use crate::bridge::envelope::Status; +use crate::control::server::wal_dispatch::{GroupOrigin, WriteSetTarget, plan_group_parts}; use crate::data::executor::core_loop::CoreLoop; use crate::data::executor::core_loop::tests::{make_core_with_dir, make_default_task}; use crate::data::executor::doc_format; use crate::engine::document::store::CollectionConfig; -use crate::types::Lsn; -use crate::wal::{CalvinStamp, RedoRecord}; +use crate::types::{Lsn, VShardId}; +use crate::wal::{CalvinStamp, RedoRecord, WriteGroup, WriteGroupRecord}; const DB: u64 = 0; -const TID: u64 = 1; +pub(in crate::data::executor) const TID: u64 = 1; /// Source and target share a vShard, so the inline fold writes the target /// on this core. -const SOURCE: &str = "local_charges"; -const TARGET: &str = "local_balances"; +pub(in crate::data::executor) const SOURCE: &str = "local_charges"; +pub(in crate::data::executor) const TARGET: &str = "local_balances"; const ACCOUNT: &str = "a1"; -const TARGET_SURROGATE: Surrogate = Surrogate(4242); +pub(super) const TARGET_SURROGATE: Surrogate = Surrogate(4242); const LSN: u64 = 70; /// A core whose source collection folds `amount` into the target's /// `balance`, with the target row seeded at 100. -fn core_with_sum(dir: &std::path::Path) -> (CoreLoop, Box) { +pub(in crate::data::executor) fn core_with_sum( + dir: &std::path::Path, +) -> (CoreLoop, Box) { let (mut core, req, resp) = make_core_with_dir(dir); + configure_sum(&mut core); + let seed = serde_json::json!({"id": ACCOUNT, "balance": "100"}); + core.sparse + .put( + DB, + TID, + TARGET, + &nodedb_types::StorageKey::for_surrogate(TARGET_SURROGATE), + &doc_format::encode_to_msgpack(&seed), + ) + .expect("seed target row"); + (core, Box::new((req, resp))) +} + +/// Declare on `core` that the source collection folds `amount` into the +/// target's `balance`. The target row is left as the stores hold it. +pub(in crate::data::executor) fn configure_sum(core: &mut CoreLoop) { core.doc_configs.insert( (DatabaseId::DEFAULT, TenantId::new(TID), TARGET.to_string()), CollectionConfig::new(TARGET), @@ -48,29 +69,17 @@ fn core_with_sum(dir: &std::path::Path) -> (CoreLoop, Box) { (DatabaseId::DEFAULT, TenantId::new(TID), SOURCE.to_string()), source, ); - let seed = serde_json::json!({"id": ACCOUNT, "balance": "100"}); - core.sparse - .put( - DB, - TID, - TARGET, - &nodedb_types::StorageKey::for_surrogate(TARGET_SURROGATE), - &doc_format::encode_to_msgpack(&seed), - ) - .expect("seed target row"); - (core, Box::new((req, resp))) } -fn sum_targets() -> Vec { +pub(in crate::data::executor) fn sum_targets() -> Vec { vec![RedoSumTargets { collection: SOURCE.to_string(), resolved: vec![ResolvedSumTarget::new(TARGET, ACCOUNT, TARGET_SURROGATE)], - deferred: Vec::new(), }] } /// A Calvin redo record writing one source row of `amount` 25. -fn calvin_redo() -> Vec { +pub(in crate::data::executor) fn calvin_redo() -> Vec { let body = doc_format::encode_to_msgpack(&serde_json::json!({ "account_id": ACCOUNT, "amount": 25, @@ -82,8 +91,6 @@ fn calvin_redo() -> Vec { epoch: 3, position: 0, vshard_id: 0, - collections: vec![SOURCE.to_string()], - sum_targets: sum_targets(), }), cross_shard_applied: None, row_sources: Vec::new(), @@ -94,7 +101,7 @@ fn calvin_redo() -> Vec { .expect("encode redo") } -fn balance(core: &CoreLoop) -> Option { +pub(in crate::data::executor) fn balance(core: &CoreLoop) -> Option { let stored = core .sparse .get( @@ -111,8 +118,27 @@ fn balance(core: &CoreLoop) -> Option { .map(str::to_string) } +/// The WAL record at `lsn` of `record_type` carrying `payload`. +fn wal_record(record_type: RecordType, lsn: u64, payload: Vec) -> WalRecord { + WalRecord::new(WalRecordArgs { + record_type: record_type as u32, + lsn, + tenant_id: TID, + vshard_id: 0, + database_id: DB, + payload, + encryption_key: None, + preamble_bytes: None, + }) + .expect("wal record") +} + +/// The live install folds the target row and reports it in its write set. +/// Restart replay of the record and the parts journalled from that write +/// set reaches the live total. A replay of the record alone folds nothing: +/// the stamp carries no fold. #[test] -fn restart_replay_folds_a_calvin_record_to_the_live_total() { +fn restart_replay_applies_the_journalled_fold_rows_not_the_stamp() { let redo = calvin_redo(); let live_dir = tempfile::tempdir().expect("tempdir"); @@ -130,35 +156,60 @@ fn restart_replay_folds_a_calvin_record_to_the_live_total() { ); assert_eq!(response.status, Status::Ok, "{:?}", response.error_code); assert_eq!(balance(&live).as_deref(), Some("125")); + assert!( + !response.write_set.is_empty(), + "the fold target row travels in the write set" + ); + + let parts = plan_group_parts( + &WriteSetTarget { + tenant_id: TenantId::new(TID), + vshard_id: VShardId::new(0), + database_id: DatabaseId::DEFAULT, + collection: SOURCE, + origin: GroupOrigin { lsn: Lsn::new(LSN) }, + }, + &response.write_set, + 1 << 20, + ) + .expect("plan parts"); + let count = parts.count().expect("part count"); + let mut records = vec![wal_record(RecordType::TransactionRedo, LSN, redo.clone())]; + for (part, (_, ops)) in (1..=count).zip(parts.runs) { + let group = WriteGroupRecord { + group: WriteGroup::part_of(LSN, part, count), + ops, + redo: None, + }; + records.push(wal_record( + RecordType::WriteGroup, + LSN + u64::from(part), + group.to_bytes().expect("encode part"), + )); + } let replay_dir = tempfile::tempdir().expect("tempdir"); let (mut replayed, _replay_ends) = core_with_sum(replay_dir.path()); - let record = WalRecord::new(WalRecordArgs { - record_type: RecordType::TransactionRedo as u32, - lsn: LSN, - tenant_id: TID, - vshard_id: 0, - database_id: DB, - payload: redo, - encryption_key: None, - preamble_bytes: None, - }) - .expect("wal record"); replayed - .replay_transaction_redo_wal(std::slice::from_ref(&record), 1, &TombstoneSet::new()) + .replay_transaction_redo_wal(&records, 1, &TombstoneSet::new()) .expect("replay"); assert_eq!( balance(&replayed), balance(&live), - "restart replay folds to the total the live install produced" + "restart replay of the record and its parts reaches the live total" ); - replayed - .replay_transaction_redo_wal(std::slice::from_ref(&record), 1, &TombstoneSet::new()) - .expect("replay again"); + let bare_dir = tempfile::tempdir().expect("tempdir"); + let (mut bare, _bare_ends) = core_with_sum(bare_dir.path()); + bare.replay_transaction_redo_wal( + &[wal_record(RecordType::TransactionRedo, LSN, redo)], + 1, + &TombstoneSet::new(), + ) + .expect("replay record alone"); assert_eq!( - balance(&replayed).as_deref(), - Some("125"), - "a second replay over the installed row folds nothing" + balance(&bare).as_deref(), + Some("100"), + "the record alone folds nothing" ); } diff --git a/nodedb/src/data/executor/handlers/transaction/redo_apply/calvin_install.rs b/nodedb/src/data/executor/handlers/transaction/redo_apply/calvin_install.rs new file mode 100644 index 000000000..3139d745d --- /dev/null +++ b/nodedb/src/data/executor/handlers/transaction/redo_apply/calvin_install.rs @@ -0,0 +1,95 @@ +// SPDX-License-Identifier: BUSL-1.1 + +//! The install of a committed Calvin slice from its stamped redo entry. +//! +//! Every replica runs it from the vShard's data-group log, in log order. The +//! install is the committed-redo install every transaction takes. Around it +//! the slice adds: +//! +//! 1. the slice's ordinal on the core clock, so a later local write stamps +//! above the record's graph versions; +//! 2. the end of the slice's staged state on a core that staged it: the +//! staged entry and its synthetic overlay go; +//! 3. the reply the leader's stage decided, rendered at the epoch instant +//! from the entry's [`CalvinReplySpec`], so every replica answers the same +//! bytes. +//! +//! The fold target rows stay in `Response::write_set`, so the funnel +//! journals them with the record. + +use nodedb_physical::physical_plan::{CalvinInstall, RedoOrigin}; +use tracing::info_span; + +use crate::bridge::envelope::{Payload, Response, Status}; +use crate::data::executor::core_loop::CoreLoop; +use crate::data::executor::handlers::control::calvin_reply::CalvinReply; +use crate::data::executor::task::ExecutionTask; + +use super::entry::CommittedRedo; + +impl CoreLoop { + /// Install the committed Calvin slice `install` names from its redo + /// record `committed`, and answer the slice's reply. + /// + /// A refused install rolled every write back and leaves any staged entry + /// in place. A reply that fails to render leaves the install `Ok`, since + /// the record installed. Its error travels in `error_code`. + pub(in crate::data::executor) fn install_calvin_redo( + &mut self, + task: &ExecutionTask, + tid: u64, + committed: CommittedRedo<'_>, + origin: RedoOrigin, + install: &CalvinInstall, + ) -> Response { + let CalvinInstall { + epoch, + position, + epoch_system_ms, + reply, + // The Control Plane raises the write mark; the install writes rows. + user_write: _, + } = install; + let (epoch, position, epoch_system_ms) = (*epoch, *position, *epoch_system_ms); + let vshard_id = task.request.vshard_id.as_u32(); + let _apply_span = info_span!( + "executor_apply", + epoch, + position, + vshard = vshard_id, + tenant_id = tid, + trace_id = ?task.request.trace_id, + ) + .entered(); + // The record's graph versions stamp at least the slice's ordinal, so + // a later local write stamps above them. + let txn_ordinal = nodedb_types::calvin_txn_ordinal(epoch_system_ms, position); + self.hlc.update_from_remote(txn_ordinal); + + // What each resolved timeseries batch of the record stored: the + // reply answers the install's counts and stored rows. + let mut ts_installs = Vec::new(); + let mut response = + self.install_committed_redo_into(task, tid, committed, origin, &mut ts_installs); + if response.status != Status::Ok { + return response; + } + // The record installed: a staged entry of the slice and its overlay + // are spent. + self.calvin + .commit_pending + .remove(&(epoch, position, vshard_id)); + self.drop_calvin_synthetic_overlay(epoch, position, vshard_id); + + let prev_epoch_ms = self.epoch_system_ms; + self.epoch_system_ms = Some(epoch_system_ms); + let rendered = + self.calvin_reply_payload(task, tid, CalvinReply::from(reply.clone()), &ts_installs); + self.epoch_system_ms = prev_epoch_ms; + match rendered { + Ok(payload) => response.payload = Payload::from_vec(payload), + Err(error) => response.error_code = Some(Box::new(error)), + } + response + } +} diff --git a/nodedb/src/data/executor/handlers/transaction/redo_apply/calvin_install_tests.rs b/nodedb/src/data/executor/handlers/transaction/redo_apply/calvin_install_tests.rs new file mode 100644 index 000000000..c6ed0201f --- /dev/null +++ b/nodedb/src/data/executor/handlers/transaction/redo_apply/calvin_install_tests.rs @@ -0,0 +1,389 @@ +// SPDX-License-Identifier: BUSL-1.1 + +//! A committed Calvin slice installs from its stamped redo entry: the +//! `ApplyTransactionRedo` plan with a `CalvinInstall`. + +use nodedb_physical::physical_plan::{ + CalvinInstall, CalvinReplySpec, CalvinResolved, DocumentOp, MetaOp, PhysicalPlan, RedoOrigin, + RedoSumTargets, ReturningColumns, ReturningSpec, +}; +use nodedb_types::{QualifiedCollection, StorageKey, Surrogate}; + +use super::calvin_fold_tests::{TARGET, TID, balance, calvin_redo, core_with_sum, sum_targets}; +use super::test_commit::failing_install_sub_record; +use crate::bridge::envelope::{ErrorCode, Response, Status}; +use crate::data::executor::core_loop::CoreLoop; +use crate::data::executor::core_loop::tests::{make_core_with_dir, make_default_task}; +use crate::data::executor::handlers::control::calvin::CalvinExecCtx; +use crate::data::executor::handlers::control::calvin::test_support::{ + bulk_delete_plan, point_insert_plan, seed_row, +}; +use crate::data::executor::handlers::control::calvin_reply::CalvinReply; +use crate::data::executor::handlers::control::calvin_synthetic_txn_id; +use crate::data::executor::task::ExecutionTask; +use crate::engine::document::store::{CollectionConfig, IndexPath}; +use crate::types::{DatabaseId, Lsn, TenantId}; +use crate::wal::RedoRecord; + +const ORDERS: &str = "orders"; +/// The vShard `make_default_task` homes to. +const VSHARD: u32 = 0; + +fn doc_body(field: &str, value: &str) -> Vec { + let mut obj = std::collections::HashMap::new(); + obj.insert( + field.to_string(), + nodedb_types::Value::String(value.to_string()), + ); + nodedb_types::value_to_msgpack(&nodedb_types::Value::Object(obj)).expect("encode body") +} + +/// An insert of row `surrogate` that asks for its row back. +fn returning_insert(document_id: &str, surrogate: u32) -> PhysicalPlan { + PhysicalPlan::Document(DocumentOp::PointInsert { + collection: QualifiedCollection::new(DatabaseId::DEFAULT, ORDERS), + document_id: document_id.to_string(), + value: doc_body("a", "returned"), + if_absent: false, + surrogate: Surrogate::new(surrogate), + returning: Some(ReturningSpec { + columns: ReturningColumns::Star, + }), + rls_filters: Vec::new(), + resolved_sum_targets: Vec::new(), + deferred_sum_targets: Vec::new(), + }) +} + +/// Stage `plans` as the leader does at `(1, 0)` and resolve them. +fn stage_and_resolve(core: &mut CoreLoop, plans: &[PhysicalPlan]) -> CalvinResolved { + let task = make_default_task(); + let ctx = CalvinExecCtx { + epoch: 1, + position: 0, + epoch_system_ms: 0, + }; + let staged = + core.execute_calvin_execute_static(&task, ctx, &TenantId::new(TID), plans, &[], &[]); + assert_eq!(staged.status, Status::Ok, "{:?}", staged.error_code); + let resolved = core.execute_calvin_resolve(&task, 1, 0); + assert_eq!(resolved.status, Status::Ok, "{:?}", resolved.error_code); + zerompk::from_msgpack(resolved.payload.as_bytes()).expect("decode resolved answer") +} + +/// The stamped install of `redo` as the slice at `(epoch, 0)`. +fn install_plan( + redo: Vec, + collections: Vec, + sum_targets: Vec, + epoch: u64, + reply: CalvinReplySpec, +) -> PhysicalPlan { + PhysicalPlan::Meta(MetaOp::ApplyTransactionRedo { + redo, + collections, + sum_targets, + origin: RedoOrigin::Commit, + calvin: Some(CalvinInstall { + epoch, + position: 0, + epoch_system_ms: 0, + reply, + user_write: true, + }), + }) +} + +/// Run `plan` on `core` with its record at `lsn`, through the plan +/// dispatcher. +fn run_at(core: &mut CoreLoop, plan: &PhysicalPlan, lsn: u64) -> Response { + let mut task: ExecutionTask = make_default_task(); + task.wal_lsn = Some(Lsn::new(lsn)); + core.execute_plan(&task, plan) +} + +fn base_row(core: &CoreLoop, surrogate: u32) -> Option> { + core.sparse + .get( + DatabaseId::DEFAULT.as_u64(), + TID, + ORDERS, + &StorageKey::for_surrogate(Surrogate::new(surrogate)), + ) + .expect("read base row") +} + +fn holds(payload: &[u8], needle: &[u8]) -> bool { + payload.windows(needle.len()).any(|w| w == needle) +} + +#[test] +fn a_calvin_install_consumes_its_staged_entry_and_renders_the_reply() { + let dir = tempfile::tempdir().expect("tempdir"); + let (mut core, _tx, _rx) = make_core_with_dir(dir.path()); + let resolved = stage_and_resolve(&mut core, &[returning_insert("o9", 9)]); + let synthetic = calvin_synthetic_txn_id(1, 0, VSHARD).expect("synthetic id"); + assert!(core.calvin.commit_pending.contains_key(&(1, 0, VSHARD))); + assert!(core.txn_overlays.contains_key(&synthetic)); + + let plan = install_plan( + resolved.redo, + vec![ORDERS.to_string()], + Vec::new(), + 1, + resolved.reply, + ); + let installed = run_at(&mut core, &plan, 42); + + assert_eq!(installed.status, Status::Ok, "{:?}", installed.error_code); + assert!(installed.error_code.is_none(), "{:?}", installed.error_code); + assert!( + !core.calvin.commit_pending.contains_key(&(1, 0, VSHARD)), + "the install consumes the staged entry" + ); + assert!( + !core.txn_overlays.contains_key(&synthetic), + "the install drops the synthetic overlay" + ); + assert!(base_row(&core, 9).is_some(), "the install writes the row"); + assert!( + holds(installed.payload.as_bytes(), b"returned"), + "the reply carries the RETURNING row" + ); +} + +#[test] +fn a_calvin_install_without_a_staged_entry_renders_the_same_reply() { + let leader_dir = tempfile::tempdir().expect("tempdir"); + let (mut leader, _ltx, _lrx) = make_core_with_dir(leader_dir.path()); + let resolved = stage_and_resolve(&mut leader, &[returning_insert("o9", 9)]); + let plan = install_plan( + resolved.redo, + vec![ORDERS.to_string()], + Vec::new(), + 1, + resolved.reply, + ); + let on_leader = run_at(&mut leader, &plan, 42); + assert_eq!(on_leader.status, Status::Ok, "{:?}", on_leader.error_code); + + // A replica that staged nothing installs the same entry. + let replica_dir = tempfile::tempdir().expect("tempdir"); + let (mut replica, _rtx, _rrx) = make_core_with_dir(replica_dir.path()); + assert!(replica.calvin.commit_pending.is_empty()); + let on_replica = run_at(&mut replica, &plan, 42); + + assert_eq!(on_replica.status, Status::Ok, "{:?}", on_replica.error_code); + assert!( + on_replica.error_code.is_none(), + "{:?}", + on_replica.error_code + ); + assert!( + base_row(&replica, 9).is_some(), + "the replica writes the row" + ); + assert!(holds(on_replica.payload.as_bytes(), b"returned")); + assert_eq!( + on_replica.payload.as_bytes(), + on_leader.payload.as_bytes(), + "every replica answers the same reply" + ); +} + +#[test] +fn a_calvin_install_keeps_fold_rows_in_the_write_set() { + let dir = tempfile::tempdir().expect("tempdir"); + let (mut core, _ends) = core_with_sum(dir.path()); + let plan = install_plan( + calvin_redo(), + vec![super::calvin_fold_tests::SOURCE.to_string()], + sum_targets(), + 3, + CalvinReplySpec::Count(Vec::new()), + ); + + let installed = run_at(&mut core, &plan, 70); + + assert_eq!(installed.status, Status::Ok, "{:?}", installed.error_code); + assert_eq!(balance(&core).as_deref(), Some("125")); + assert!( + installed + .write_set + .iter() + .any(|entry| entry.collection.as_deref() == Some(TARGET)), + "the folded target row stays in the write set for the funnel to journal: {:?}", + installed.write_set + ); +} + +/// The install applies the resolved record, not a replay of the staged +/// plans. A row that joins the bulk delete's predicate after the stage +/// stays, and the predicted row goes. +#[test] +fn a_calvin_install_applies_the_resolved_record_not_a_replay_of_the_plans() { + let dir = tempfile::tempdir().expect("tempdir"); + let (mut core, _tx, _rx) = make_core_with_dir(dir.path()); + seed_row(&mut core, ORDERS, 1); + let resolved = stage_and_resolve(&mut core, &[bulk_delete_plan(ORDERS, Some(vec![1]))]); + seed_row(&mut core, ORDERS, 2); + + let plan = install_plan( + resolved.redo, + vec![ORDERS.to_string()], + Vec::new(), + 1, + resolved.reply, + ); + let installed = run_at(&mut core, &plan, 41); + + assert_eq!(installed.status, Status::Ok, "{:?}", installed.error_code); + assert!(base_row(&core, 1).is_none()); + assert!(base_row(&core, 2).is_some()); +} + +/// The install notes the index values of every document row at the +/// record's LSN. +#[test] +fn a_calvin_install_notes_index_values_at_the_redo_lsn() { + let dir = tempfile::tempdir().expect("tempdir"); + let (mut core, _tx, _rx) = make_core_with_dir(dir.path()); + let mut config = CollectionConfig::new(ORDERS); + config.index_paths.push(IndexPath::new("a")); + core.doc_configs.insert( + (DatabaseId::DEFAULT, TenantId::new(TID), ORDERS.to_string()), + config, + ); + let resolved = stage_and_resolve(&mut core, &[point_insert_plan(ORDERS, "o1", 7)]); + + let plan = install_plan( + resolved.redo, + vec![ORDERS.to_string()], + Vec::new(), + 1, + resolved.reply, + ); + let installed = run_at(&mut core, &plan, 43); + + assert_eq!(installed.status, Status::Ok, "{:?}", installed.error_code); + assert_eq!( + core.write_index.index_values.value_version( + crate::data::executor::core_loop::index_value_versions::IndexDimRef { + vshard: crate::types::VShardId::new(0), + db: DatabaseId::DEFAULT, + tenant: TenantId::new(TID), + collection: ORDERS, + field: "a", + }, + "1" + ), + Some(crate::data::executor::core_loop::write_index::tests::local( + 43 + )), + "the install records the row's index value at the redo's version" + ); +} + +/// An install whose record cannot decode answers an error and leaves +/// nothing of the record in base. +#[test] +fn a_calvin_install_whose_record_cannot_decode_answers_an_error() { + let dir = tempfile::tempdir().expect("tempdir"); + let (mut core, _tx, _rx) = make_core_with_dir(dir.path()); + let resolved = stage_and_resolve(&mut core, &[point_insert_plan(ORDERS, "o1", 7)]); + + let plan = install_plan( + vec![0xff, 0x00], + vec![ORDERS.to_string()], + Vec::new(), + 1, + resolved.reply, + ); + let installed = run_at(&mut core, &plan, 44); + + assert_eq!(installed.status, Status::Error); + assert!(!matches!( + installed.error_code.as_deref(), + Some(ErrorCode::OllpRetryRequired) + )); + assert!(base_row(&core, 7).is_none()); +} + +/// A refused install keeps the staged entry and its overlay, so a later +/// install of the record consumes them. +#[test] +fn a_refused_calvin_install_keeps_the_staged_entry() { + let dir = tempfile::tempdir().expect("tempdir"); + let (mut core, _tx, _rx) = make_core_with_dir(dir.path()); + let resolved = stage_and_resolve(&mut core, &[point_insert_plan(ORDERS, "o1", 7)]); + let mut failing = RedoRecord::from_bytes(&resolved.redo).expect("decode redo"); + failing.ops.push(failing_install_sub_record()); + let failing = failing.to_bytes().expect("encode redo"); + let synthetic = calvin_synthetic_txn_id(1, 0, VSHARD).expect("synthetic id"); + + let refused_plan = install_plan( + failing, + vec![ORDERS.to_string()], + Vec::new(), + 1, + resolved.reply.clone(), + ); + let refused = run_at(&mut core, &refused_plan, 45); + + assert!( + matches!( + refused.error_code.as_deref(), + Some(ErrorCode::RetryableRefusal { .. }) + ), + "{:?}", + refused.error_code + ); + assert!(core.calvin.commit_pending.contains_key(&(1, 0, VSHARD))); + assert!(core.txn_overlays.contains_key(&synthetic)); + assert!(base_row(&core, 7).is_none()); + + let plan = install_plan( + resolved.redo, + vec![ORDERS.to_string()], + Vec::new(), + 1, + resolved.reply, + ); + let installed = run_at(&mut core, &plan, 46); + + assert_eq!(installed.status, Status::Ok, "{:?}", installed.error_code); + assert!(base_row(&core, 7).is_some()); + assert!(!core.calvin.commit_pending.contains_key(&(1, 0, VSHARD))); + assert!(!core.txn_overlays.contains_key(&synthetic)); +} + +/// A reply that fails to render after a successful install leaves the +/// install `Ok` and carries the render error for the statement. +#[test] +fn a_reply_render_error_leaves_the_calvin_install_ok() { + let dir = tempfile::tempdir().expect("tempdir"); + let (mut core, _tx, _rx) = make_core_with_dir(dir.path()); + let resolved = stage_and_resolve(&mut core, &[point_insert_plan(ORDERS, "o1", 7)]); + let unrenderable = CalvinReplySpec::from(&CalvinReply::unrenderable_for_test(ORDERS)); + + let plan = install_plan( + resolved.redo, + vec![ORDERS.to_string()], + Vec::new(), + 1, + unrenderable, + ); + let installed = run_at(&mut core, &plan, 47); + + assert_eq!(installed.status, Status::Ok); + assert!( + matches!( + installed.error_code.as_deref(), + Some(ErrorCode::Internal { .. }) + ), + "{:?}", + installed.error_code + ); + assert!(base_row(&core, 7).is_some()); + assert!(!core.calvin.commit_pending.contains_key(&(1, 0, VSHARD))); +} diff --git a/nodedb/src/data/executor/handlers/transaction/redo_apply/cross_shard_fold_tests.rs b/nodedb/src/data/executor/handlers/transaction/redo_apply/cross_shard_fold_tests.rs new file mode 100644 index 000000000..975112b54 --- /dev/null +++ b/nodedb/src/data/executor/handlers/transaction/redo_apply/cross_shard_fold_tests.rs @@ -0,0 +1,160 @@ +// SPDX-License-Identifier: BUSL-1.1 + +//! A committed record never folds a cross-shard sum target on the source's +//! core. +//! +//! A cross-shard balance travels on an `ApplyBalanceDelta` task homed on the +//! target's vShard. An INSERT says so by its deferral list. A DELETE says so +//! by leaving its join value unresolved. The install folds every row of one +//! collection against one merged table. So an INSERT and a DELETE of one +//! account in one transaction must leave that account out of the table. +//! Otherwise the DELETE row folds the account here as well, and the shipped +//! task moves it a second time. + +use nodedb_physical::physical_plan::{ + DocumentOp, MaterializedSumBinding, PhysicalPlan, ResolvedSumTarget, +}; +use nodedb_types::{ + CollectionKey, DatabaseId, QualifiedCollection, RlsWriteCheck, StorageKey, Surrogate, TenantId, +}; + +use crate::bridge::envelope::Status; +use crate::data::executor::core_loop::CoreLoop; +use crate::data::executor::core_loop::tests::{make_core_with_dir, make_default_task}; +use crate::data::executor::doc_format; +use crate::engine::document::store::CollectionConfig; + +const DB: u64 = 0; +const TID: u64 = 1; +/// The sum's source collection. +const SOURCE: &str = "local_charges"; +/// A target on another vShard than [`SOURCE`]. +const TARGET: &str = "remote_balances"; +const ACCOUNT: &str = "a1"; +const TARGET_SURROGATE: Surrogate = Surrogate(4242); +/// The stored row the transaction deletes. +const DELETED: Surrogate = Surrogate(1); +/// The row the transaction inserts. +const INSERTED: Surrogate = Surrogate(2); + +/// A core where [`SOURCE`] sums `amount` into [`TARGET`]'s `balance`, with +/// the target row at 10 and one source row of 10 for the account. +fn seeded_core(dir: &std::path::Path) -> (CoreLoop, Box) { + let (mut core, req, resp) = make_core_with_dir(dir); + core.doc_configs.insert( + (DatabaseId::DEFAULT, TenantId::new(TID), TARGET.to_string()), + CollectionConfig::new(TARGET), + ); + let mut source = CollectionConfig::new(SOURCE); + source.enforcement.materialized_sum_sources = vec![MaterializedSumBinding { + target_collection: TARGET.to_string(), + target_column: "balance".to_string(), + join_column: "account_id".to_string(), + value_expr: nodedb_query::expr::SqlExpr::Column("amount".to_string()), + declared_primary_key: None, + }]; + core.doc_configs.insert( + (DatabaseId::DEFAULT, TenantId::new(TID), SOURCE.to_string()), + source, + ); + // The target row lives in this core's store, so a fold on this core is + // visible as a moved balance rather than a refusal. + put_row( + &mut core, + TARGET, + TARGET_SURROGATE, + &serde_json::json!({"id": ACCOUNT, "balance": "10"}), + ); + put_row(&mut core, SOURCE, DELETED, &entry(10)); + (core, Box::new((req, resp))) +} + +fn entry(amount: i64) -> serde_json::Value { + serde_json::json!({"account_id": ACCOUNT, "amount": amount}) +} + +fn put_row(core: &mut CoreLoop, collection: &str, surrogate: Surrogate, doc: &serde_json::Value) { + core.sparse + .put( + DB, + TID, + collection, + &StorageKey::for_surrogate(surrogate), + &doc_format::encode_to_msgpack(doc), + ) + .expect("seed row"); +} + +fn row(core: &CoreLoop, collection: &str, surrogate: Surrogate) -> Option { + let stored = core + .sparse + .get(DB, TID, collection, &StorageKey::for_surrogate(surrogate)) + .expect("read row")?; + doc_format::decode_document(&stored).ok() +} + +/// The source slice of `BEGIN; INSERT ...; DELETE ...; COMMIT` as the +/// Control Plane plans it. The INSERT keeps its resolution and defers the +/// target. The DELETE's balance was settled from its stored row, so its join +/// value is gone from its resolution. +fn source_slice() -> Vec { + let collection = QualifiedCollection::new(DatabaseId::DEFAULT, SOURCE); + vec![ + PhysicalPlan::Document(DocumentOp::PointInsert { + collection: collection.clone(), + document_id: "e1".to_string(), + value: doc_format::encode_to_msgpack(&entry(4)), + if_absent: false, + surrogate: INSERTED, + returning: None, + rls_filters: Vec::new(), + resolved_sum_targets: vec![ResolvedSumTarget::new(TARGET, ACCOUNT, TARGET_SURROGATE)], + deferred_sum_targets: vec![TARGET.to_string()], + }), + PhysicalPlan::Document(DocumentOp::PointDelete { + collection, + document_id: "e0".to_string(), + surrogate: Some(DELETED), + pk_bytes: Vec::new(), + returning: None, + rls_filters: Vec::new(), + rls_write_check: RlsWriteCheck::NoPolicyApplies, + resolved_sum_targets: Vec::new(), + }), + ] +} + +/// The premise: the target is cross-shard, so its balance moves only on its +/// own task. +#[test] +fn the_fixture_target_is_cross_shard() { + assert!( + !crate::query::sum_target_is_co_resident( + CollectionKey::from_bare(DatabaseId::DEFAULT, SOURCE), + TARGET, + ), + "'{TARGET}' must not share '{SOURCE}''s vShard" + ); +} + +/// The source slice of an INSERT and a DELETE on one account installs both +/// rows and leaves the cross-shard balance to the shipped tasks. +#[test] +fn a_delete_next_to_an_insert_of_its_account_folds_no_cross_shard_balance() { + let dir = tempfile::tempdir().expect("tempdir"); + let (mut core, _ends) = seeded_core(dir.path()); + + let response = core.commit_plans_for_test(&make_default_task(), TID, &source_slice(), 30); + + assert_eq!(response.status, Status::Ok, "{:?}", response.error_code); + assert!(row(&core, SOURCE, DELETED).is_none(), "e0 is deleted"); + assert!(row(&core, SOURCE, INSERTED).is_some(), "e1 is inserted"); + assert_eq!( + row(&core, TARGET, TARGET_SURROGATE) + .as_ref() + .and_then(|doc| doc.get("balance")) + .and_then(|balance| balance.as_str()), + Some("10"), + "the source install moves no cross-shard balance; the shipped tasks move it once" + ); +} diff --git a/nodedb/src/data/executor/handlers/transaction/redo_apply/document.rs b/nodedb/src/data/executor/handlers/transaction/redo_apply/document.rs index 6ae8bb6f0..861a91ac5 100644 --- a/nodedb/src/data/executor/handlers/transaction/redo_apply/document.rs +++ b/nodedb/src/data/executor/handlers/transaction/redo_apply/document.rs @@ -92,7 +92,7 @@ impl CoreLoop { row: &CommittedDocWrite<'_>, value: &[u8], ) -> crate::Result<()> { - let (resolved, deferred) = self.committed_sum_targets(row.collection, row.record_lsn); + let resolved = self.committed_sum_targets(row.collection); let surrogate = Surrogate::new(row.surrogate); let storage_key = StorageKey::for_surrogate(surrogate); let wal_lsn = (row.record_lsn != 0).then(|| Lsn::new(row.record_lsn)); @@ -101,7 +101,8 @@ impl CoreLoop { tid: row.tenant_id, collection: row.collection, resolved_targets: &resolved, - deferred_sum_targets: &deferred, + // A committed record defers a target by omission from `resolved`. + deferred_sum_targets: &[], wal_lsn, }; @@ -238,7 +239,7 @@ impl CoreLoop { } fn committed_document_delete(&mut self, row: &CommittedDocWrite<'_>) -> crate::Result { - let (resolved, _) = self.committed_sum_targets(row.collection, row.record_lsn); + let resolved = self.committed_sum_targets(row.collection); let surrogate = Surrogate::new(row.surrogate); let storage_key = StorageKey::for_surrogate(surrogate); let hook_ctx = HookCtx { @@ -246,7 +247,7 @@ impl CoreLoop { tid: row.tenant_id, collection: row.collection, resolved_targets: &resolved, - // A delete is deferred by omission from `resolved`, never by list. + // A committed record defers a target by omission from `resolved`. deferred_sum_targets: &[], wal_lsn: (row.record_lsn != 0).then(|| Lsn::new(row.record_lsn)), }; @@ -333,24 +334,17 @@ impl CoreLoop { Ok(true) } - /// The sum targets a write to `collection` folds into: the open scope's - /// under a committed-redo apply, else the restart-replay folds of the - /// record at `record_lsn`. + /// The sum targets a write to `collection` folds into: the open + /// committed-redo scope's. With no scope open nothing folds. fn committed_sum_targets( &self, collection: &str, - record_lsn: u64, - ) -> ( - Vec, - Vec, - ) { - match self.redo_apply.scope.as_ref() { - Some(scope) => scope.sum_targets_for(collection), - None => self - .redo_apply - .replay_folds_for(record_lsn, collection) - .unwrap_or_default(), - } + ) -> Vec { + self.redo_apply + .scope + .as_ref() + .map(|scope| scope.sum_targets_for(collection)) + .unwrap_or_default() } fn record_committed_doc_write(&mut self, write: AppliedDocWrite) { diff --git a/nodedb/src/data/executor/handlers/transaction/redo_apply/entry.rs b/nodedb/src/data/executor/handlers/transaction/redo_apply/entry.rs index 4a0ac94de..edd5d42f2 100644 --- a/nodedb/src/data/executor/handlers/transaction/redo_apply/entry.rs +++ b/nodedb/src/data/executor/handlers/transaction/redo_apply/entry.rs @@ -163,7 +163,12 @@ impl CoreLoop { if let Err(refusal) = self.validate_redo_pass(&target, committed.sum_targets) { return self.response_error(task, refusal.into_code()); } + // The replay arms install the record. Each write they apply takes the + // record's stamp: its vShard and the entry it applies. + let stamp = self.task_write_stamp_at(task, lsn); + self.write_index.record_stamps.insert(stamp); let installed = self.install_redo_pass(&target, committed.sum_targets); + self.write_index.record_stamps.remove(lsn); let mut scope = match installed { Ok(scope) => scope, Err(refusal) => return self.response_error(task, refusal.into_code()), @@ -198,7 +203,7 @@ impl CoreLoop { version.tenant, &version.collection, version.key, - version.lsn, + version.stamp, ); } // Events leave only once the record is settled. Every @@ -232,7 +237,7 @@ impl CoreLoop { let tenant = TenantId::new(tid); for collection in committed.collections { - self.note_write_lsn(task.request.database_id, tenant, collection, None, lsn); + self.note_write(task.request.database_id, tenant, collection, None, stamp); } for write in &scope.doc_writes { self.note_index_write_values( @@ -240,7 +245,7 @@ impl CoreLoop { tenant, &write.collection, &write.index_tuples, - lsn, + stamp, ); } let write_set = target_write_set(&scope.target_writes); diff --git a/nodedb/src/data/executor/handlers/transaction/redo_apply/install_refusal_tests.rs b/nodedb/src/data/executor/handlers/transaction/redo_apply/install_refusal_tests.rs index cacb3654d..4559bffd4 100644 --- a/nodedb/src/data/executor/handlers/transaction/redo_apply/install_refusal_tests.rs +++ b/nodedb/src/data/executor/handlers/transaction/redo_apply/install_refusal_tests.rs @@ -521,12 +521,17 @@ fn a_refused_install_publishes_no_write_version_and_no_watermark() { let point = nodedb_types::calvin::ReadKeyIdent::Point(crate::types::KeyRepr::Surrogate(41)); let committed = run(&[document_insert()], false, |_| {}); assert_eq!(committed.core.watermark, Lsn::new(200)); + // The install runs on the default task's vShard 0 and applies no + // data-group entry, so its version is its WAL LSN there. + let vshard = crate::types::VShardId::new(0); + let before_install = crate::data::executor::core_loop::write_index::tests::local(199); assert!(!committed.core.write_index.read_is_valid( DatabaseId::DEFAULT, TenantId::new(TID), + vshard, "notes", &point, - Lsn::new(199), + before_install, )); let refused = run(&[document_insert()], true, |_| {}); @@ -538,9 +543,10 @@ fn a_refused_install_publishes_no_write_version_and_no_watermark() { refused.core.write_index.read_is_valid( DatabaseId::DEFAULT, TenantId::new(TID), + vshard, "notes", &point, - Lsn::new(199), + before_install, ), "the rolled-back install publishes no write version" ); diff --git a/nodedb/src/data/executor/handlers/transaction/redo_apply/mod.rs b/nodedb/src/data/executor/handlers/transaction/redo_apply/mod.rs index 63ffb21aa..8c5accbb2 100644 --- a/nodedb/src/data/executor/handlers/transaction/redo_apply/mod.rs +++ b/nodedb/src/data/executor/handlers/transaction/redo_apply/mod.rs @@ -4,6 +4,7 @@ //! the core that owns its vShard, through the WAL replay arms. //! //! - [`entry`]: the `MetaOp::ApplyTransactionRedo` handler. +//! - [`calvin_install`]: the install of a committed Calvin slice. //! - [`validate`]: commit-boundary checks that run before any write. //! - [`passes`]: the validate and install passes over the replay arms. //! - [`settle`]: the work an install defers until every sub-record landed. @@ -14,12 +15,21 @@ //! - [`test_commit`]: a test driver for a session commit on one core. //! - [`install_refusal_tests`]: one committed and one refused install per //! engine kind. -//! - [`calvin_fold_tests`]: a Calvin record folds the same live and in -//! restart replay. +//! - [`calvin_fold_tests`]: a Calvin record's fold rows journal as parts, +//! and restart replay applies them. +//! - [`calvin_install_tests`]: a Calvin slice installs from its stamped +//! redo entry. +//! - [`cross_shard_fold_tests`]: a record never folds a cross-shard sum +//! target on the source's core. //! - [`unique_handover_tests`]: UNIQUE judged on a record's post-state. #[cfg(test)] -mod calvin_fold_tests; +pub(in crate::data::executor) mod calvin_fold_tests; +mod calvin_install; +#[cfg(test)] +mod calvin_install_tests; +#[cfg(test)] +mod cross_shard_fold_tests; mod document; mod entry; mod events; diff --git a/nodedb/src/data/executor/handlers/transaction/redo_apply/state.rs b/nodedb/src/data/executor/handlers/transaction/redo_apply/state.rs index ef0395978..6de4a417d 100644 --- a/nodedb/src/data/executor/handlers/transaction/redo_apply/state.rs +++ b/nodedb/src/data/executor/handlers/transaction/redo_apply/state.rs @@ -19,11 +19,10 @@ use nodedb_physical::physical_plan::{RedoSumTargets, ResolvedSumTarget}; use nodedb_types::RowIdentity; use crate::bridge::envelope::ErrorCode; -use crate::data::executor::core_loop::write_index::KeyRepr; +use crate::data::executor::core_loop::write_index::{KeyRepr, WriteStamp}; use crate::data::executor::enforcement::materialized_sum::apply::TargetWrite; use crate::data::executor::handlers::transaction::undo::UndoEntry; use crate::event::WriteOp; -use crate::types::Lsn; /// Committed-redo apply state owned by one core. pub(in crate::data::executor) struct RedoApplyState { @@ -32,57 +31,19 @@ pub(in crate::data::executor) struct RedoApplyState { pub(in crate::data::executor) num_cores: usize, /// `Some` only while one committed redo record applies on this core. pub(in crate::data::executor) scope: Option, - /// During restart replay: the materialized-sum targets each Calvin redo - /// record's stamp carries, keyed by the record's LSN. The document redo - /// arm folds a row at such an LSN into its targets, as the live install - /// did. Empty outside restart replay. - pub(in crate::data::executor) replay_folds: HashMap>, } impl RedoApplyState { - /// The sum targets restart replay folds a write to `collection` at - /// `record_lsn` into, when the record carries any. - pub(in crate::data::executor) fn replay_folds_for( - &self, - record_lsn: u64, - collection: &str, - ) -> Option<(Vec, Vec)> { - self.replay_folds - .get(&record_lsn)? - .iter() - .find(|targets| targets.collection == collection) - .map(|targets| (targets.resolved.clone(), targets.deferred.clone())) - } - /// A single-core default. Every multi-core runtime sets the real count /// through `CoreLoop::set_num_cores` before the core serves requests. pub(in crate::data::executor) fn new() -> Self { Self { num_cores: 1, scope: None, - replay_folds: HashMap::new(), } } } -impl crate::data::executor::core_loop::CoreLoop { - /// Arm restart replay with the sum targets each Calvin record carries. - /// Returns whether this is restart replay: a committed-redo apply folds - /// from its open scope instead, and leaves them unused. - pub(crate) fn begin_replay_folds(&mut self, folds: HashMap>) -> bool { - let restart = self.redo_apply.scope.is_none(); - if restart { - self.redo_apply.replay_folds = folds; - } - restart - } - - /// Drop the restart-replay sum targets once the document redo arm ran. - pub(crate) fn end_replay_folds(&mut self) { - self.redo_apply.replay_folds.clear(); - } -} - /// One document row the committed-redo apply wrote. pub(in crate::data::executor) struct AppliedDocWrite { pub collection: String, @@ -161,7 +122,7 @@ pub(in crate::data::executor) struct DeferredWriteVersion { pub tenant: crate::types::TenantId, pub collection: String, pub key: Option, - pub lsn: Lsn, + pub stamp: WriteStamp, } /// `(database, tenant, collection)`. @@ -211,7 +172,7 @@ impl RedoApplyScope { tenant: crate::types::TenantId, collection: &str, key: Option, - lsn: Lsn, + stamp: WriteStamp, ) { if self.pass == RedoApplyPass::Install { self.write_versions.push(DeferredWriteVersion { @@ -219,20 +180,21 @@ impl RedoApplyScope { tenant, collection: collection.to_string(), key, - lsn, + stamp, }); } } - /// The resolved and deferred sum targets for writes to `collection`. + /// The sum targets writes to `collection` fold on this core. A + /// cross-shard target with no entry travels on its own task. pub(in crate::data::executor) fn sum_targets_for( &self, collection: &str, - ) -> (Vec, Vec) { - match self.sum_targets.get(collection) { - Some(targets) => (targets.resolved.clone(), targets.deferred.clone()), - None => (Vec::new(), Vec::new()), - } + ) -> Vec { + self.sum_targets + .get(collection) + .map(|targets| targets.resolved.clone()) + .unwrap_or_default() } /// Keep the first error; a later one is a consequence of it. diff --git a/nodedb/src/data/executor/handlers/transaction/redo_apply/validate.rs b/nodedb/src/data/executor/handlers/transaction/redo_apply/validate.rs index 56be3e760..c4489648e 100644 --- a/nodedb/src/data/executor/handlers/transaction/redo_apply/validate.rs +++ b/nodedb/src/data/executor/handlers/transaction/redo_apply/validate.rs @@ -68,7 +68,7 @@ impl CoreLoop { continue; }; let bitemporal = self.is_bitemporal(database_id, tid, collection); - let (resolved, _) = apply_scope.sum_targets_for(collection); + let resolved = apply_scope.sum_targets_for(collection); let scope = CollectionScope { database_id, tid, diff --git a/nodedb/src/data/executor/handlers/transaction/resolve/entry.rs b/nodedb/src/data/executor/handlers/transaction/resolve/entry.rs index 241f5a608..aea64aad8 100644 --- a/nodedb/src/data/executor/handlers/transaction/resolve/entry.rs +++ b/nodedb/src/data/executor/handlers/transaction/resolve/entry.rs @@ -548,6 +548,7 @@ mod tests { wal_lsn: None, resolved_now_ms: None, commit_hlc: None, + entry_version: None, admission: crate::bridge::envelope::Admission::Exempt( crate::bridge::envelope::ExemptReason::Read, ), diff --git a/nodedb/src/data/executor/handlers/transaction/stage_write/stage_graph.rs b/nodedb/src/data/executor/handlers/transaction/stage_write/stage_graph.rs index 7fe8e9778..b6c39cebe 100644 --- a/nodedb/src/data/executor/handlers/transaction/stage_write/stage_graph.rs +++ b/nodedb/src/data/executor/handlers/transaction/stage_write/stage_graph.rs @@ -447,6 +447,7 @@ mod tests { wal_lsn: None, resolved_now_ms: None, commit_hlc: None, + entry_version: None, admission: Admission::Exempt(ExemptReason::AlreadyOrdered), }, state: TaskState::Running, diff --git a/nodedb/src/data/executor/handlers/transaction/stage_write/stage_kv.rs b/nodedb/src/data/executor/handlers/transaction/stage_write/stage_kv.rs index 0a880f5bc..c707132d3 100644 --- a/nodedb/src/data/executor/handlers/transaction/stage_write/stage_kv.rs +++ b/nodedb/src/data/executor/handlers/transaction/stage_write/stage_kv.rs @@ -426,6 +426,7 @@ mod tests { wal_lsn: None, resolved_now_ms: None, commit_hlc: None, + entry_version: None, admission: Admission::Exempt(ExemptReason::Read), }; ExecutionTask::new(request) diff --git a/nodedb/src/data/executor/handlers/transaction/stage_write/stage_kv_predicate.rs b/nodedb/src/data/executor/handlers/transaction/stage_write/stage_kv_predicate.rs index c8eb39688..c8c0ffa58 100644 --- a/nodedb/src/data/executor/handlers/transaction/stage_write/stage_kv_predicate.rs +++ b/nodedb/src/data/executor/handlers/transaction/stage_write/stage_kv_predicate.rs @@ -295,6 +295,7 @@ mod tests { wal_lsn: None, resolved_now_ms: None, commit_hlc: None, + entry_version: None, admission: Admission::Exempt(ExemptReason::Read), }) } diff --git a/nodedb/src/data/executor/handlers/transaction/stage_write/stage_timeseries.rs b/nodedb/src/data/executor/handlers/transaction/stage_write/stage_timeseries.rs index 6f1cac54a..2377c1f6d 100644 --- a/nodedb/src/data/executor/handlers/transaction/stage_write/stage_timeseries.rs +++ b/nodedb/src/data/executor/handlers/transaction/stage_write/stage_timeseries.rs @@ -553,6 +553,7 @@ mod tests { wal_lsn: None, resolved_now_ms: None, commit_hlc: None, + entry_version: None, admission: Admission::Exempt(ExemptReason::AlreadyOrdered), }, state: TaskState::Running, diff --git a/nodedb/src/data/executor/handlers/transaction/stage_write/stage_vector.rs b/nodedb/src/data/executor/handlers/transaction/stage_write/stage_vector.rs index 808645e84..3777b1588 100644 --- a/nodedb/src/data/executor/handlers/transaction/stage_write/stage_vector.rs +++ b/nodedb/src/data/executor/handlers/transaction/stage_write/stage_vector.rs @@ -515,6 +515,7 @@ mod tests { wal_lsn: None, resolved_now_ms: None, commit_hlc: None, + entry_version: None, admission: Admission::Exempt(ExemptReason::Read), }) } diff --git a/nodedb/src/data/executor/handlers/transaction/undo/apply.rs b/nodedb/src/data/executor/handlers/transaction/undo/apply.rs index 45e145c2b..d6dac454a 100644 --- a/nodedb/src/data/executor/handlers/transaction/undo/apply.rs +++ b/nodedb/src/data/executor/handlers/transaction/undo/apply.rs @@ -602,6 +602,7 @@ mod tests { wal_lsn: Some(Lsn::new(lsn)), resolved_now_ms: None, commit_hlc: None, + entry_version: None, admission: Admission::Exempt(ExemptReason::AlreadyOrdered), }, Some(Lsn::new(lsn)), diff --git a/nodedb/src/data/executor/handlers/transaction/write_version.rs b/nodedb/src/data/executor/handlers/transaction/write_version.rs index 34f6bc683..b991ce8b1 100644 --- a/nodedb/src/data/executor/handlers/transaction/write_version.rs +++ b/nodedb/src/data/executor/handlers/transaction/write_version.rs @@ -1,15 +1,16 @@ // SPDX-License-Identifier: BUSL-1.1 -//! Records the per-core last-write-LSN version for every key a committed -//! transaction batch wrote. Runs once, after commit, over the buffered -//! sub-plans, covering both the fast-path commit and every Calvin apply. -//! One WAL LSN applies to every key in the batch. +//! The per-key write versions a committed batch of plans implies, read off +//! the plans. The redo install records versions through its replay arms. +//! The parity tests check the install against this oracle, engine by engine. +//! One write stamp applies to every key in the batch. use crate::bridge::envelope::PhysicalPlan; use crate::data::executor::core_loop::CoreLoop; use crate::data::executor::core_loop::write_index::KeyRepr; +use crate::data::executor::core_loop::write_index::WriteStamp; use crate::data::executor::task::ExecutionTask; -use crate::types::{Lsn, TenantId}; +use crate::types::TenantId; use nodedb_physical::physical_plan::{ ColumnarOp, CrdtOp, DocumentOp, GraphOp, SpatialOp, TextOp, TimeseriesOp, VectorOp, VectorWriteTargets, @@ -19,19 +20,19 @@ impl CoreLoop { /// Record the version of every key written by a committed transaction /// batch. No-op with no WAL LSN. Per-key engines record `KeyRepr`; /// engines with internal per-key identity record only the collection floor. - pub(in crate::data::executor) fn record_batch_write_versions( + pub(super) fn record_batch_write_versions( &mut self, task: &ExecutionTask, tid: u64, plans: &[PhysicalPlan], ) { - let Some(lsn) = task.wal_lsn() else { + let Some(stamp) = self.task_write_stamp(task) else { return; }; let db = task.request.database_id; let tenant = TenantId::new(tid); for plan in plans { - self.record_plan_write_version(db, tenant, plan, lsn); + self.record_plan_write_version(db, tenant, plan, stamp); } } @@ -40,13 +41,13 @@ impl CoreLoop { db: crate::types::DatabaseId, tenant: TenantId, plan: &PhysicalPlan, - lsn: Lsn, + stamp: WriteStamp, ) { match plan { - PhysicalPlan::Document(op) => self.record_document_version(db, tenant, op, lsn), - PhysicalPlan::Vector(op) => self.record_vector_version(db, tenant, op, lsn), - PhysicalPlan::Graph(op) => self.record_graph_version(db, tenant, op, lsn), - PhysicalPlan::Kv(op) => self.record_kv_version(db, tenant, op, lsn), + PhysicalPlan::Document(op) => self.record_document_version(db, tenant, op, stamp), + PhysicalPlan::Vector(op) => self.record_vector_version(db, tenant, op, stamp), + PhysicalPlan::Graph(op) => self.record_graph_version(db, tenant, op, stamp), + PhysicalPlan::Kv(op) => self.record_kv_version(db, tenant, op, stamp), // Collection-floor engines: per-key identity is engine-internal. PhysicalPlan::Columnar(op) => { let coll = match op { @@ -63,13 +64,13 @@ impl CoreLoop { ColumnarOp::ResolveDml { .. } => None, }; if let Some(c) = coll { - self.note_write_lsn(db, tenant, c, None, lsn); + self.note_write(db, tenant, c, None, stamp); } } PhysicalPlan::Timeseries( TimeseriesOp::Ingest { collection, .. } | TimeseriesOp::Truncate { collection, .. }, ) => { - self.note_write_lsn(db, tenant, collection.as_str(), None, lsn); + self.note_write(db, tenant, collection.as_str(), None, stamp); } PhysicalPlan::Spatial(op) => { let coll = match op { @@ -80,7 +81,7 @@ impl CoreLoop { SpatialOp::Scan { .. } => None, }; if let Some(c) = coll { - self.note_write_lsn(db, tenant, c, None, lsn); + self.note_write(db, tenant, c, None, stamp); } } PhysicalPlan::Text(op) => { @@ -97,13 +98,13 @@ impl CoreLoop { TextOp::SetTextConfig { .. } => None, }; if let Some(c) = coll { - self.note_write_lsn(db, tenant, c, None, lsn); + self.note_write(db, tenant, c, None, stamp); } } PhysicalPlan::Crdt(CrdtOp::Apply { collection, .. }) | PhysicalPlan::Crdt(CrdtOp::DocUpsert { collection, .. }) | PhysicalPlan::Crdt(CrdtOp::DocDelete { collection, .. }) => { - self.note_write_lsn(db, tenant, collection.as_str(), None, lsn); + self.note_write(db, tenant, collection.as_str(), None, stamp); } // No per-key/collection version recorded for reads, control ops, or // engines not (yet) part of the funnel (array is keyed by tile). @@ -122,7 +123,7 @@ impl CoreLoop { db: crate::types::DatabaseId, tenant: TenantId, op: &DocumentOp, - lsn: Lsn, + stamp: WriteStamp, ) { let (collection, surrogate) = match op { DocumentOp::PointPut { @@ -147,7 +148,7 @@ impl CoreLoop { // Whole-collection mutation: key set is every row, so only the // collection floor applies. DocumentOp::Truncate { collection, .. } => { - self.note_write_lsn(db, tenant, collection.as_str(), None, lsn); + self.note_write(db, tenant, collection.as_str(), None, stamp); return; } // No per-key version recorded: reads, DDL/index ops, and @@ -174,12 +175,12 @@ impl CoreLoop { | DocumentOp::ResolveWrite(_) | DocumentOp::ResolvedWrite { .. } => return, }; - self.note_write_lsn( + self.note_write( db, tenant, collection, Some(KeyRepr::Surrogate(surrogate.as_u32())), - lsn, + stamp, ); } @@ -193,7 +194,7 @@ impl CoreLoop { db: crate::types::DatabaseId, tenant: TenantId, op: &VectorOp, - lsn: Lsn, + stamp: WriteStamp, ) { match op { VectorOp::Insert { @@ -216,12 +217,12 @@ impl CoreLoop { surrogate, .. } => { - self.note_write_lsn( + self.note_write( db, tenant, collection.as_str(), Some(KeyRepr::Surrogate(surrogate.as_u32())), - lsn, + stamp, ); } // An unbound delete names no row: it records the collection @@ -231,12 +232,12 @@ impl CoreLoop { surrogate, .. } => { - self.note_write_lsn( + self.note_write( db, tenant, collection.as_str(), surrogate.map(|s| KeyRepr::Surrogate(s.as_u32())), - lsn, + stamp, ); } VectorOp::MultiVectorInsert { @@ -249,12 +250,12 @@ impl CoreLoop { document_surrogate, .. } => { - self.note_write_lsn( + self.note_write( db, tenant, collection.as_str(), Some(KeyRepr::Surrogate(document_surrogate.as_u32())), - lsn, + stamp, ); } VectorOp::BatchInsert { @@ -265,21 +266,21 @@ impl CoreLoop { // Every vector of a batch carries its bound surrogate. An // empty batch records the collection floor only. for s in surrogates { - self.note_write_lsn( + self.note_write( db, tenant, collection.as_str(), Some(KeyRepr::Surrogate(s.as_u32())), - lsn, + stamp, ); } if surrogates.is_empty() { - self.note_write_lsn(db, tenant, collection.as_str(), None, lsn); + self.note_write(db, tenant, collection.as_str(), None, stamp); } } // Every row of the collection: the collection floor only. VectorOp::DirectTruncate { collection, .. } => { - self.note_write_lsn(db, tenant, collection.as_str(), None, lsn); + self.note_write(db, tenant, collection.as_str(), None, stamp); } // A point-targeted delete or update names its surrogates; a // predicate-targeted one resolves them on apply, so it records the @@ -296,17 +297,17 @@ impl CoreLoop { } => match targets { VectorWriteTargets::Surrogates(surrogates) if !surrogates.is_empty() => { for s in surrogates { - self.note_write_lsn( + self.note_write( db, tenant, collection.as_str(), Some(KeyRepr::Surrogate(s.as_u32())), - lsn, + stamp, ); } } VectorWriteTargets::Surrogates(_) | VectorWriteTargets::Predicate(_) => { - self.note_write_lsn(db, tenant, collection.as_str(), None, lsn); + self.note_write(db, tenant, collection.as_str(), None, stamp); } }, // Every resolved mutation names its surrogate. @@ -316,16 +317,16 @@ impl CoreLoop { .. } => { for mutation in mutations { - self.note_write_lsn( + self.note_write( db, tenant, collection.as_str(), Some(KeyRepr::Surrogate(mutation.surrogate().as_u32())), - lsn, + stamp, ); } if mutations.is_empty() { - self.note_write_lsn(db, tenant, collection.as_str(), None, lsn); + self.note_write(db, tenant, collection.as_str(), None, stamp); } } // Sparse (doc_id-keyed) and `Delete` (vector_id isn't the @@ -333,7 +334,7 @@ impl CoreLoop { VectorOp::SparseInsert { collection, .. } | VectorOp::SparseDelete { collection, .. } | VectorOp::Delete { collection, .. } => { - self.note_write_lsn(db, tenant, collection.as_str(), None, lsn); + self.note_write(db, tenant, collection.as_str(), None, stamp); } // Read / config / query ops: nothing written, no version to record. // The resolve pass reads what a governed write depends on. @@ -356,7 +357,7 @@ impl CoreLoop { db: crate::types::DatabaseId, tenant: TenantId, op: &GraphOp, - lsn: Lsn, + stamp: WriteStamp, ) { match op { GraphOp::EdgePut { @@ -373,7 +374,7 @@ impl CoreLoop { dst_id, .. } => { - self.note_write_lsn( + self.note_write( db, tenant, collection.as_str(), @@ -382,12 +383,12 @@ impl CoreLoop { label: Box::from(label.as_str()), dst: Box::from(dst_id.as_str()), }), - lsn, + stamp, ); } GraphOp::EdgePutBatch { edges } | GraphOp::EdgeDeleteBatch { edges } => { for edge in edges { - self.note_write_lsn( + self.note_write( db, tenant, edge.collection.as_str(), @@ -396,7 +397,7 @@ impl CoreLoop { label: Box::from(edge.label.as_str()), dst: Box::from(edge.dst_id.as_str()), }), - lsn, + stamp, ); } } diff --git a/nodedb/src/data/executor/handlers/transaction/write_version_kv.rs b/nodedb/src/data/executor/handlers/transaction/write_version_kv.rs index 5e8712292..d2d9ef4b8 100644 --- a/nodedb/src/data/executor/handlers/transaction/write_version_kv.rs +++ b/nodedb/src/data/executor/handlers/transaction/write_version_kv.rs @@ -1,12 +1,13 @@ // SPDX-License-Identifier: BUSL-1.1 -//! `KvOp` write-version recording, split out of `write_version.rs` to keep -//! that file under the size limit. Same contract: no-op with no WAL LSN, -//! one `note_write_lsn` per key a committed KV write touched. +//! `KvOp` write versions for the parity oracle in `write_version.rs`. Same +//! contract: no-op with no WAL LSN, one `note_write` per key a committed +//! KV write touched. use crate::data::executor::core_loop::CoreLoop; use crate::data::executor::core_loop::write_index::KeyRepr; -use crate::types::{Lsn, TenantId}; +use crate::data::executor::core_loop::write_index::WriteStamp; +use crate::types::TenantId; use nodedb_physical::physical_plan::KvOp; impl CoreLoop { @@ -15,7 +16,7 @@ impl CoreLoop { db: crate::types::DatabaseId, tenant: TenantId, op: &KvOp, - lsn: Lsn, + stamp: WriteStamp, ) { match op { KvOp::Put { @@ -51,24 +52,24 @@ impl CoreLoop { | KvOp::GetSet { collection, key, .. } => { - self.note_write_lsn( + self.note_write( db, tenant, collection.as_str(), Some(KeyRepr::KvKey(Box::from(key.as_slice()))), - lsn, + stamp, ); } KvOp::Delete { collection, keys, .. } => { for key in keys { - self.note_write_lsn( + self.note_write( db, tenant, collection.as_str(), Some(KeyRepr::KvKey(Box::from(key.as_slice()))), - lsn, + stamp, ); } } @@ -78,12 +79,12 @@ impl CoreLoop { .. } => { for (key, _value) in entries { - self.note_write_lsn( + self.note_write( db, tenant, collection.as_str(), Some(KeyRepr::KvKey(Box::from(key.as_slice()))), - lsn, + stamp, ); } } @@ -92,7 +93,7 @@ impl CoreLoop { KvOp::Truncate { collection, .. } | KvOp::PredicateUpdate { collection, .. } | KvOp::PredicateDelete { collection, .. } => { - self.note_write_lsn(db, tenant, collection.as_str(), None, lsn); + self.note_write(db, tenant, collection.as_str(), None, stamp); } // Read-only: nothing written, no version to record. KvOp::Get { .. } diff --git a/nodedb/src/data/executor/handlers/transaction/write_version_parity_tests.rs b/nodedb/src/data/executor/handlers/transaction/write_version_parity_tests.rs new file mode 100644 index 000000000..12245ffa1 --- /dev/null +++ b/nodedb/src/data/executor/handlers/transaction/write_version_parity_tests.rs @@ -0,0 +1,330 @@ +// SPDX-License-Identifier: BUSL-1.1 + +//! A committed Calvin slice installs through its stamped redo entry, and the +//! install records every write version the slice's plans imply. +//! +//! The oracle is `record_batch_write_versions` over the slice's plans. For +//! each engine, every per-key version and collection floor the oracle records +//! holds on the installed core at the install's LSN or above. + +use nodedb_physical::physical_plan::{ + CalvinInstall, CalvinReplySpec, CalvinResolved, ColumnarInsertIntent, ColumnarOp, CrdtOp, + CrdtWriteVerb, DocumentOp, GraphOp, KvOp, MetaOp, PhysicalPlan, RedoOrigin, SpatialOp, TextOp, + TimeseriesOp, VectorOp, +}; +use nodedb_types::geometry::Geometry; +use nodedb_types::{DatabaseId, QualifiedCollection, RlsWriteCheck, Surrogate, Value}; + +use crate::bridge::envelope::{Response, Status}; +use crate::control::wal_replication::transaction_redo::collections::written_collections; +use crate::control::wal_replication::transaction_redo::sum_targets::redo_sum_targets; +use crate::data::executor::core_loop::CoreLoop; +use crate::data::executor::core_loop::tests::{make_core_with_dir, make_default_task}; +use crate::data::executor::handlers::control::calvin::CalvinExecCtx; +use crate::data::executor::task::ExecutionTask; +use crate::types::Lsn; +use crate::wal::{RedoRecord, RedoSubRecord}; + +/// The LSN every install and every oracle record names. +const LSN: u64 = 300; + +/// A core with its bridge ends and data directory kept alive. +struct Core { + core: CoreLoop, + _req: Box, + _resp: Box, + _dir: tempfile::TempDir, +} + +fn fresh_core() -> Core { + let dir = tempfile::tempdir().expect("tempdir"); + let (core, req, resp) = make_core_with_dir(dir.path()); + Core { + core, + _req: Box::new(req), + _resp: Box::new(resp), + _dir: dir, + } +} + +fn tid() -> u64 { + make_default_task().request.tenant_id.as_u64() +} + +fn task_at(lsn: u64) -> ExecutionTask { + let mut task = make_default_task(); + task.wal_lsn = Some(Lsn::new(lsn)); + task +} + +fn object(fields: &[(&str, Value)]) -> Vec { + let map: std::collections::HashMap = fields + .iter() + .map(|(k, v)| ((*k).to_string(), v.clone())) + .collect(); + nodedb_types::value_to_msgpack(&Value::Object(map)).expect("encode object") +} + +/// Stage `plans` as the slice at `(1, 0)` and resolve its redo. +fn stage_and_resolve(core: &mut CoreLoop, plans: &[PhysicalPlan]) -> CalvinResolved { + let task = make_default_task(); + let ctx = CalvinExecCtx { + epoch: 1, + position: 0, + epoch_system_ms: 0, + }; + let tenant = task.request.tenant_id; + let staged = core.execute_calvin_execute_static(&task, ctx, &tenant, plans, &[], &[]); + assert_eq!(staged.status, Status::Ok, "{:?}", staged.error_code); + let resolved = core.execute_calvin_resolve(&task, 1, 0); + assert_eq!(resolved.status, Status::Ok, "{:?}", resolved.error_code); + zerompk::from_msgpack(resolved.payload.as_bytes()).expect("decode resolved answer") +} + +/// Install `redo` as the stamped slice at `(1, 0)` at [`LSN`]. +fn install( + core: &mut CoreLoop, + redo: Vec, + plans: &[PhysicalPlan], + reply: CalvinReplySpec, +) -> Response { + let plan = PhysicalPlan::Meta(MetaOp::ApplyTransactionRedo { + redo, + collections: written_collections(plans), + sum_targets: redo_sum_targets(plans), + origin: RedoOrigin::Commit, + calvin: Some(CalvinInstall { + epoch: 1, + position: 0, + epoch_system_ms: 0, + reply, + user_write: true, + }), + }); + let installed = core.execute_plan(&task_at(LSN), &plan); + assert_eq!(installed.status, Status::Ok, "{:?}", installed.error_code); + installed +} + +/// Check that `installed` holds every version the oracle records for +/// `plans`. The oracle must record at least one collection floor. +fn assert_parity(installed: &CoreLoop, plans: &[PhysicalPlan]) { + let mut oracle = fresh_core(); + oracle + .core + .record_batch_write_versions(&task_at(LSN), tid(), plans); + let floors: Vec<_> = oracle.core.write_index.recorded_collections().collect(); + assert!(!floors.is_empty(), "the oracle records the slice's writes"); + for (key, version) in floors { + let held = installed.write_index.collection_version(key); + assert!( + held.is_some_and(|held| held >= version), + "the install records the collection floor of {key:?}: {held:?}" + ); + } + for (key, version) in oracle.core.write_index.recorded_keys() { + let held = installed.write_index.key_version(key); + assert!( + held.is_some_and(|held| held >= version), + "the install records the version of {key:?}: {held:?}" + ); + } +} + +/// Stage, resolve and install `plans` as one Calvin slice on a fresh core, +/// then check parity. +fn staged_slice_parity(plans: &[PhysicalPlan]) { + let mut core = fresh_core(); + let resolved = stage_and_resolve(&mut core.core, plans); + install(&mut core.core, resolved.redo, plans, resolved.reply); + assert_parity(&core.core, plans); +} + +#[test] +fn a_document_slice_records_every_row_version() { + let plans = [ + PhysicalPlan::Document(DocumentOp::PointInsert { + collection: QualifiedCollection::new(DatabaseId::DEFAULT, "notes"), + document_id: "n1".to_string(), + value: object(&[("title", Value::String("kept".into()))]), + if_absent: false, + surrogate: Surrogate::new(41), + returning: None, + rls_filters: Vec::new(), + resolved_sum_targets: Vec::new(), + deferred_sum_targets: Vec::new(), + }), + PhysicalPlan::Document(DocumentOp::PointPut { + collection: QualifiedCollection::new(DatabaseId::DEFAULT, "notes"), + document_id: "n2".to_string(), + value: object(&[("title", Value::String("also".into()))]), + surrogate: Surrogate::new(42), + pk_bytes: b"n2".to_vec(), + returning: None, + rls_filters: Vec::new(), + resolved_sum_targets: Vec::new(), + }), + ]; + staged_slice_parity(&plans); +} + +#[test] +fn a_kv_slice_records_every_key_version() { + let put = |key: &[u8], surrogate: u32| { + PhysicalPlan::Kv(KvOp::Put { + collection: QualifiedCollection::new(DatabaseId::DEFAULT, "cache"), + key: key.to_vec(), + value: object(&[("n", Value::Integer(1))]), + ttl_ms: 0, + surrogate: Surrogate::new(surrogate), + returning: None, + rls_filters: Vec::new(), + provenance: None, + }) + }; + staged_slice_parity(&[put(b"k1", 51), put(b"k2", 52)]); +} + +#[test] +fn a_vector_slice_records_every_surrogate_version() { + let plan = PhysicalPlan::Vector(VectorOp::DirectInsert { + collection: QualifiedCollection::new(DatabaseId::DEFAULT, "vp"), + field: "vec".into(), + surrogate: Surrogate::new(61), + pk_bytes: b"r".to_vec(), + vector: vec![1.0, 0.0], + payload: zerompk::to_msgpack_vec(&std::collections::HashMap::from([( + "id".to_string(), + Value::String("r".into()), + )])) + .expect("encode payload"), + quantization: nodedb_types::VectorQuantization::None, + storage_dtype: nodedb_types::VectorStorageDtype::F32, + payload_indexes: Vec::new(), + returning: None, + rls_filters: Vec::new(), + }); + staged_slice_parity(&[plan]); +} + +#[test] +fn a_graph_slice_records_every_edge_version() { + let plan = PhysicalPlan::Graph(GraphOp::EdgePut { + collection: QualifiedCollection::new(DatabaseId::DEFAULT, "social"), + src_id: "a".into(), + label: "KNOWS".into(), + dst_id: "b".into(), + properties: Vec::new(), + src_surrogate: Surrogate::new(1), + dst_surrogate: Surrogate::new(2), + }); + staged_slice_parity(&[plan]); +} + +#[test] +fn a_columnar_slice_records_its_collection_floor() { + let rows = Value::Array(vec![Value::Object(std::collections::HashMap::from([ + ("id".to_string(), Value::Integer(1)), + ("v".to_string(), Value::Integer(7)), + ]))]); + let plan = PhysicalPlan::Columnar(ColumnarOp::Insert { + collection: QualifiedCollection::new(DatabaseId::DEFAULT, "metrics_c"), + payload: nodedb_types::value_to_msgpack(&rows).expect("encode rows"), + format: "msgpack".into(), + intent: ColumnarInsertIntent::Insert, + on_conflict_updates: Vec::new(), + surrogates: vec![Surrogate::new(71)], + schema_bytes: Vec::new(), + provenance: None, + wal_lsn: None, + rls_write_check: RlsWriteCheck::already_decided_elsewhere(), + returning: None, + rls_filters: Vec::new(), + }); + staged_slice_parity(&[plan]); +} + +#[test] +fn a_timeseries_slice_records_its_collection_floor() { + let plan = PhysicalPlan::Timeseries(TimeseriesOp::Ingest { + collection: QualifiedCollection::new(DatabaseId::DEFAULT, "metrics"), + payload: b"metrics,host=a value=1 1000000000".to_vec(), + format: "ilp".to_owned(), + wal_lsn: None, + surrogates: Vec::new(), + provenance: None, + rls_write_check: RlsWriteCheck::NoPolicyApplies, + returning: None, + rls_filters: Vec::new(), + }); + staged_slice_parity(&[plan]); +} + +#[test] +fn a_spatial_slice_records_its_collection_floor() { + let plan = PhysicalPlan::Spatial(SpatialOp::Insert { + collection: QualifiedCollection::new(DatabaseId::DEFAULT, "places"), + field: "geom".to_string(), + surrogate: Surrogate::new(81), + geometry: Geometry::Point { + coordinates: [1.0, 2.0], + }, + provenance: None, + }); + staged_slice_parity(&[plan]); +} + +#[test] +fn a_crdt_slice_records_its_collection_floor() { + let plan = PhysicalPlan::Crdt(CrdtOp::DocUpsert { + collection: QualifiedCollection::new(DatabaseId::DEFAULT, "tasks"), + document_id: "t1".to_string(), + fields_json: r#"{"title":"kept"}"#.to_string(), + surrogate: Surrogate::new(71), + partial: false, + verb: CrdtWriteVerb::Insert, + returning: None, + rls_filters: Vec::new(), + }); + staged_slice_parity(&[plan]); +} + +/// A text write stages no row. Its slice installs the sub-record its +/// resolve serializes from the plan. +#[test] +fn a_text_slice_records_its_collection_floor() { + let op = TextOp::FtsIndexDoc { + collection: QualifiedCollection::new(DatabaseId::DEFAULT, "docs"), + surrogate: Surrogate::new(91), + fields: vec![("body".to_string(), "hello world".to_string())], + provenance: None, + }; + let (record_type, payload) = crate::control::server::wal_dispatch::encode_text_op_record(&op) + .expect("encode text op") + .expect("an index op journals a record"); + let redo = RedoRecord { + version: 1, + ops: vec![RedoSubRecord { + record_type: record_type as u32, + payload, + }], + calvin_stamp: None, + cross_shard_applied: None, + row_sources: Vec::new(), + publishes: Vec::new(), + row_changes: Vec::new(), + } + .to_bytes() + .expect("encode redo"); + let plans = [PhysicalPlan::Text(op)]; + + let mut core = fresh_core(); + install( + &mut core.core, + redo, + &plans, + CalvinReplySpec::Count(Vec::new()), + ); + + assert_parity(&core.core, &plans); +} diff --git a/nodedb/src/data/executor/handlers/truncate.rs b/nodedb/src/data/executor/handlers/truncate.rs index 46d0ed35e..84d3afaf8 100644 --- a/nodedb/src/data/executor/handlers/truncate.rs +++ b/nodedb/src/data/executor/handlers/truncate.rs @@ -229,6 +229,9 @@ impl CoreLoop { declared_primary_key, *storage_key, ); + // Record the removal's version against the row's surrogate + // and collection, as a point delete does. + self.note_surrogate_write(task, tid, collection, surrogate.as_u32()); // The row's text leaves the inverted index with every other // row's, in one purge once the loop ends. removed.push(surrogate); diff --git a/nodedb/src/data/executor/handlers/update_from_join_write.rs b/nodedb/src/data/executor/handlers/update_from_join_write.rs index d5b7fa7e7..774557320 100644 --- a/nodedb/src/data/executor/handlers/update_from_join_write.rs +++ b/nodedb/src/data/executor/handlers/update_from_join_write.rs @@ -230,7 +230,7 @@ impl CoreLoop { } // Index write-versions are published only once the write they // describe is durable. - if let Some(lsn) = task.wal_lsn() + if let Some(stamp) = self.task_write_stamp(task) && !touched.is_empty() { self.note_index_write_values( @@ -238,7 +238,7 @@ impl CoreLoop { crate::types::TenantId::new(tid), target_collection, &touched, - lsn, + stamp, ); } // The identity is the one INSERT minted: the declared primary diff --git a/nodedb/src/data/executor/handlers/upsert/exec/insert.rs b/nodedb/src/data/executor/handlers/upsert/exec/insert.rs index e929130e3..86ea23cbe 100644 --- a/nodedb/src/data/executor/handlers/upsert/exec/insert.rs +++ b/nodedb/src/data/executor/handlers/upsert/exec/insert.rs @@ -200,6 +200,22 @@ impl CoreLoop { return self.response_error(task, e); } + // Record the committed row's version and its touched index values, + // as a point put does. + self.note_surrogate_write(task, tid, collection, surrogate.as_u32()); + if let Some(stamp) = self.task_write_stamp(task) { + let mut tuples = std::mem::take(&mut prior.secondary_index_added); + tuples.append(&mut prior.secondary_index_removed); + tuples.append(&mut prior.bitemporal_index_tuples); + self.note_index_write_values( + task.request.database_id, + crate::types::TenantId::new(tid), + collection, + &tuples, + stamp, + ); + } + self.emit_put_event( task, tid, diff --git a/nodedb/src/data/executor/handlers/upsert/exec/overwrite.rs b/nodedb/src/data/executor/handlers/upsert/exec/overwrite.rs index 671958506..e3ff78b06 100644 --- a/nodedb/src/data/executor/handlers/upsert/exec/overwrite.rs +++ b/nodedb/src/data/executor/handlers/upsert/exec/overwrite.rs @@ -289,6 +289,22 @@ impl CoreLoop { return self.response_error(task, e); } + // Record the committed row's version and its touched index values, + // as a point put does. + self.note_surrogate_write(task, tid, collection, surrogate.as_u32()); + if let Some(stamp) = self.task_write_stamp(task) { + let mut tuples = std::mem::take(&mut outcome.secondary_index_added); + tuples.append(&mut outcome.secondary_index_removed); + tuples.append(&mut outcome.bitemporal_index_tuples); + self.note_index_write_values( + task.request.database_id, + crate::types::TenantId::new(tid), + collection, + &tuples, + stamp, + ); + } + // `current_bytes` is the pre-merge stored row, already read // above — thread it to the Event Plane as `old_value` so the // emitted WriteOp resolves to Update. diff --git a/nodedb/src/data/executor/handlers/vector.rs b/nodedb/src/data/executor/handlers/vector.rs index 2b180e7ea..ecff3f9eb 100644 --- a/nodedb/src/data/executor/handlers/vector.rs +++ b/nodedb/src/data/executor/handlers/vector.rs @@ -234,7 +234,7 @@ impl CoreLoop { // Record this write's version so cross-shard OCC read-set // validation (predicate reads always record the collection // floor) sees this insert. - self.note_surrogate_write_lsn(task, tid, collection, surrogate.as_u32()); + self.note_surrogate_write(task, tid, collection, surrogate.as_u32()); self.response_ok(task) } Err(err) => self.response_error(task, err), @@ -364,7 +364,7 @@ impl CoreLoop { // surrogate (a superset of `execute_vector_delete`'s // node-id-scoped floor-only record below — correct and // more precise since the surrogate identity is known here). - self.note_surrogate_write_lsn(task, tid, collection, surrogate.as_u32()); + self.note_surrogate_write(task, tid, collection, surrogate.as_u32()); } response } @@ -382,6 +382,7 @@ mod tests { use super::*; use crate::bridge::envelope::{PhysicalPlan, Priority, Request}; use crate::data::executor::core_loop::write_index::WriteKey; + use crate::data::executor::core_loop::write_index::tests::local; use crate::types::{Lsn, RequestId, TraceId, VShardId}; use nodedb_bridge::buffer::RingBuffer; use nodedb_physical::physical_plan::VectorOp; @@ -417,7 +418,7 @@ mod tests { } } - /// A task carrying `wal_lsn` so the handler's `note_*_write_lsn` calls + /// A task carrying `wal_lsn` so the handler's `note_*_write` calls /// (gated on `task.wal_lsn().is_some()`) actually fire, mirroring a live /// write dispatched with an allocated WAL LSN. fn make_task_with_lsn(lsn: u64) -> ExecutionTask { @@ -453,6 +454,7 @@ mod tests { wal_lsn: Some(Lsn::new(lsn)), resolved_now_ms: None, commit_hlc: None, + entry_version: None, admission: crate::bridge::envelope::Admission::Exempt( crate::bridge::envelope::ExemptReason::Read, ), @@ -478,6 +480,7 @@ mod tests { assert_eq!(response.status, crate::bridge::envelope::Status::Ok); let key = WriteKey { + vshard: VShardId::new(0), db: DatabaseId::DEFAULT, tenant: TenantId::new(1), collection: Box::from("docs"), @@ -486,20 +489,21 @@ mod tests { ), }; assert_eq!( - h.core.write_index.key_write_lsn(&key), - Some(Lsn::new(11)), + h.core.write_index.key_version(&key), + Some(local(11)), "vector insert must populate the per-key (surrogate) write-version index" ); let coll_key = crate::data::executor::core_loop::write_index::CollKey { + vshard: VShardId::new(0), db: DatabaseId::DEFAULT, tenant: TenantId::new(1), collection: Box::from("docs"), }; assert_eq!( - h.core.write_index.collection_write_lsn(&coll_key), - Some(Lsn::new(11)), - "vector insert must advance the collection write-version floor \ + h.core.write_index.collection_version(&coll_key), + Some(local(11)), + "vector insert must advance the collection write version \ (predicate reads validate against the floor)" ); } diff --git a/nodedb/src/data/executor/handlers/vector_direct_resolve/apply.rs b/nodedb/src/data/executor/handlers/vector_direct_resolve/apply.rs index e85cbc0c0..0e72178bc 100644 --- a/nodedb/src/data/executor/handlers/vector_direct_resolve/apply.rs +++ b/nodedb/src/data/executor/handlers/vector_direct_resolve/apply.rs @@ -391,6 +391,7 @@ mod tests { wal_lsn: Some(Lsn::new(7)), resolved_now_ms: None, commit_hlc: None, + entry_version: None, admission: Admission::Exempt(ExemptReason::Read), }) } diff --git a/nodedb/src/data/executor/handlers/vector_direct_row.rs b/nodedb/src/data/executor/handlers/vector_direct_row.rs index 08428cb46..6e79cdd26 100644 --- a/nodedb/src/data/executor/handlers/vector_direct_row.rs +++ b/nodedb/src/data/executor/handlers/vector_direct_row.rs @@ -282,7 +282,7 @@ impl CoreLoop { } self.checkpoint_coordinator.mark_dirty("vector", 1); for surrogate in surrogates { - self.note_surrogate_write_lsn(task, tid, collection, surrogate.as_u32()); + self.note_surrogate_write(task, tid, collection, surrogate.as_u32()); } } } diff --git a/nodedb/src/data/executor/handlers/vector_multi.rs b/nodedb/src/data/executor/handlers/vector_multi.rs index e6ea15310..e9df42590 100644 --- a/nodedb/src/data/executor/handlers/vector_multi.rs +++ b/nodedb/src/data/executor/handlers/vector_multi.rs @@ -122,7 +122,7 @@ impl CoreLoop { // Record this write's version keyed by the shared document surrogate // so cross-shard OCC read-set validation (predicate reads always // record the collection floor) sees this insert. - self.note_surrogate_write_lsn(task, tid, collection, document_surrogate.as_u32()); + self.note_surrogate_write(task, tid, collection, document_surrogate.as_u32()); match super::super::response_codec::encode_count("inserted_vectors", ids.len()) { Ok(bytes) => self.response_with_payload(task, bytes), @@ -161,7 +161,7 @@ impl CoreLoop { // Record this write's version keyed by the shared document // surrogate so cross-shard OCC read-set validation sees this // delete. - self.note_surrogate_write_lsn(task, tid, collection, document_surrogate.as_u32()); + self.note_surrogate_write(task, tid, collection, document_surrogate.as_u32()); } self.response_ok(task) } @@ -282,6 +282,7 @@ mod tests { Admission, ExemptReason, PhysicalPlan, Priority, Request, Status, }; use crate::data::executor::core_loop::CoreLoop; + use crate::data::executor::core_loop::write_index::tests::local; use crate::data::executor::core_loop::write_index::{CollKey, KeyRepr, WriteKey}; use crate::types::{DatabaseId, Lsn, ReadConsistency, RequestId, TenantId, TraceId, VShardId}; use nodedb_bridge::buffer::RingBuffer; @@ -317,7 +318,7 @@ mod tests { } } - /// A task carrying `wal_lsn` so `note_surrogate_write_lsn` (gated on + /// A task carrying `wal_lsn` so `note_surrogate_write` (gated on /// `task.wal_lsn().is_some()`) actually fires, mirroring a live write /// dispatched with an allocated WAL LSN. fn make_task_with_lsn(lsn: u64) -> ExecutionTask { @@ -353,6 +354,7 @@ mod tests { wal_lsn: Some(Lsn::new(lsn)), resolved_now_ms: None, commit_hlc: None, + entry_version: None, admission: Admission::Exempt(ExemptReason::Read), }) } @@ -376,26 +378,28 @@ mod tests { assert_eq!(response.status, Status::Ok); let key = WriteKey { + vshard: VShardId::new(0), db: DatabaseId::DEFAULT, tenant: TenantId::new(1), collection: Box::from("chunks"), key: KeyRepr::Surrogate(document_surrogate.as_u32()), }; assert_eq!( - h.core.write_index.key_write_lsn(&key), - Some(Lsn::new(31)), + h.core.write_index.key_version(&key), + Some(local(31)), "multi-vector insert must populate the per-key (surrogate) write-version index" ); let coll_key = CollKey { + vshard: VShardId::new(0), db: DatabaseId::DEFAULT, tenant: TenantId::new(1), collection: Box::from("chunks"), }; assert_eq!( - h.core.write_index.collection_write_lsn(&coll_key), - Some(Lsn::new(31)), - "multi-vector insert must advance the collection write-version floor" + h.core.write_index.collection_version(&coll_key), + Some(local(31)), + "multi-vector insert must advance the collection write version" ); } diff --git a/nodedb/src/data/executor/handlers/vector_sparse.rs b/nodedb/src/data/executor/handlers/vector_sparse.rs index 6e965c897..972ad5dbc 100644 --- a/nodedb/src/data/executor/handlers/vector_sparse.rs +++ b/nodedb/src/data/executor/handlers/vector_sparse.rs @@ -86,7 +86,7 @@ impl CoreLoop { self.checkpoint_coordinator.mark_dirty("vector", 1); // Sparse vectors are keyed by `doc_id: String` — no cross-engine // surrogate — so only the collection floor is recorded. - self.note_collection_write_lsn(task, collection); + self.note_collection_write(task, collection); self.response_ok(task) } @@ -191,7 +191,7 @@ impl CoreLoop { self.checkpoint_coordinator.mark_dirty("vector", 1); // Sparse vectors are keyed by `doc_id: String` — no cross-engine // surrogate — so only the collection floor is recorded. - self.note_collection_write_lsn(task, collection); + self.note_collection_write(task, collection); self.response_ok(task) } else { self.response_error(task, ErrorCode::NotFound) @@ -206,6 +206,7 @@ mod tests { Admission, ExemptReason, PhysicalPlan, Priority, Request, Status, }; use crate::data::executor::core_loop::CoreLoop; + use crate::data::executor::core_loop::write_index::tests::local; use crate::data::executor::core_loop::write_index::{CollKey, KeyRepr, WriteKey}; use crate::types::{DatabaseId, Lsn, ReadConsistency, RequestId, TenantId, TraceId, VShardId}; use nodedb_bridge::buffer::RingBuffer; @@ -242,7 +243,7 @@ mod tests { } } - /// A task carrying `wal_lsn` so `note_collection_write_lsn` (gated on + /// A task carrying `wal_lsn` so `note_collection_write` (gated on /// `task.wal_lsn().is_some()`) actually fires, mirroring a live write /// dispatched with an allocated WAL LSN. fn make_task_with_lsn(lsn: u64) -> ExecutionTask { @@ -278,6 +279,7 @@ mod tests { wal_lsn: Some(Lsn::new(lsn)), resolved_now_ms: None, commit_hlc: None, + entry_version: None, admission: Admission::Exempt(ExemptReason::Read), }) } @@ -298,26 +300,28 @@ mod tests { assert_eq!(response.status, Status::Ok); let coll_key = CollKey { + vshard: VShardId::new(0), db: DatabaseId::DEFAULT, tenant: TenantId::new(1), collection: Box::from("sparse_docs"), }; assert_eq!( - h.core.write_index.collection_write_lsn(&coll_key), - Some(Lsn::new(41)), - "sparse insert must advance the collection write-version floor" + h.core.write_index.collection_version(&coll_key), + Some(local(41)), + "sparse insert must advance the collection write version" ); // Sparse vectors carry no cross-engine surrogate: no would-be // per-key `Surrogate` entry must exist for this write. let would_be_key = WriteKey { + vshard: VShardId::new(0), db: DatabaseId::DEFAULT, tenant: TenantId::new(1), collection: Box::from("sparse_docs"), key: KeyRepr::Surrogate(0), }; assert_eq!( - h.core.write_index.key_write_lsn(&would_be_key), + h.core.write_index.key_version(&would_be_key), None, "sparse insert must not record a per-key surrogate entry" ); diff --git a/nodedb/src/data/executor/handlers/vector_upsert.rs b/nodedb/src/data/executor/handlers/vector_upsert.rs index 222b4b72e..39ac9763e 100644 --- a/nodedb/src/data/executor/handlers/vector_upsert.rs +++ b/nodedb/src/data/executor/handlers/vector_upsert.rs @@ -337,6 +337,7 @@ mod tests { Admission, ExemptReason, PhysicalPlan, Priority, Request, Status, }; use crate::data::executor::core_loop::CoreLoop; + use crate::data::executor::core_loop::write_index::tests::local; use crate::data::executor::core_loop::write_index::{CollKey, KeyRepr, WriteKey}; use crate::types::{DatabaseId, Lsn, ReadConsistency, RequestId, TenantId, TraceId, VShardId}; use nodedb_bridge::buffer::RingBuffer; @@ -373,7 +374,7 @@ mod tests { } } - /// A task carrying `wal_lsn` so `note_surrogate_write_lsn` (gated on + /// A task carrying `wal_lsn` so `note_surrogate_write` (gated on /// `task.wal_lsn().is_some()`) actually fires, mirroring a live write /// dispatched with an allocated WAL LSN. fn make_task_with_lsn(lsn: u64) -> ExecutionTask { @@ -409,6 +410,7 @@ mod tests { wal_lsn: Some(Lsn::new(lsn)), resolved_now_ms: None, commit_hlc: None, + entry_version: None, admission: Admission::Exempt(ExemptReason::Read), }) } @@ -460,26 +462,28 @@ mod tests { assert_eq!(response.status, Status::Ok); let key = WriteKey { + vshard: VShardId::new(0), db: DatabaseId::DEFAULT, tenant: TenantId::new(1), collection: Box::from("primary_docs"), key: KeyRepr::Surrogate(surrogate.as_u32()), }; assert_eq!( - h.core.write_index.key_write_lsn(&key), - Some(Lsn::new(21)), + h.core.write_index.key_version(&key), + Some(local(21)), "direct upsert must populate the per-key (surrogate) write-version index" ); let coll_key = CollKey { + vshard: VShardId::new(0), db: DatabaseId::DEFAULT, tenant: TenantId::new(1), collection: Box::from("primary_docs"), }; assert_eq!( - h.core.write_index.collection_write_lsn(&coll_key), - Some(Lsn::new(21)), - "direct upsert must advance the collection write-version floor" + h.core.write_index.collection_version(&coll_key), + Some(local(21)), + "direct upsert must advance the collection write version" ); } diff --git a/nodedb/src/data/executor/handlers/vector_write.rs b/nodedb/src/data/executor/handlers/vector_write.rs index 0b961b654..8abe344b2 100644 --- a/nodedb/src/data/executor/handlers/vector_write.rs +++ b/nodedb/src/data/executor/handlers/vector_write.rs @@ -67,10 +67,10 @@ impl CoreLoop { // validation sees this batch: per surrogate, a superset of the // collection floor. An empty batch records the floor only. for s in surrogates { - self.note_surrogate_write_lsn(task, tid, collection, s.as_u32()); + self.note_surrogate_write(task, tid, collection, s.as_u32()); } if surrogates.is_empty() { - self.note_collection_write_lsn(task, collection); + self.note_collection_write(task, collection); } match super::super::response_codec::encode_count("inserted", vectors.len()) { Ok(bytes) => self.response_with_payload(task, bytes), @@ -154,7 +154,7 @@ impl CoreLoop { // surrogate — recording it as a `KeyRepr::Surrogate` would be a // wrong identity in a different key space. Floor-only: a // predicate reader validates against the collection floor. - self.note_collection_write_lsn(task, collection); + self.note_collection_write(task, collection); self.response_ok(task) } else { self.response_error(task, ErrorCode::NotFound) @@ -169,6 +169,7 @@ mod tests { Admission, ExemptReason, PhysicalPlan, Priority, Request, Status, }; use crate::data::executor::core_loop::CoreLoop; + use crate::data::executor::core_loop::write_index::tests::local; use crate::data::executor::core_loop::write_index::{CollKey, KeyRepr, WriteKey}; use crate::types::{Lsn, ReadConsistency, RequestId, TraceId, VShardId}; use nodedb_bridge::buffer::RingBuffer; @@ -205,7 +206,7 @@ mod tests { } } - /// A task carrying `wal_lsn` so `note_collection_write_lsn` (gated on + /// A task carrying `wal_lsn` so `note_collection_write` (gated on /// `task.wal_lsn().is_some()`) actually fires, mirroring a live write /// dispatched with an allocated WAL LSN. fn make_task_with_lsn(lsn: u64) -> ExecutionTask { @@ -241,6 +242,7 @@ mod tests { wal_lsn: Some(Lsn::new(lsn)), resolved_now_ms: None, commit_hlc: None, + entry_version: None, admission: Admission::Exempt(ExemptReason::Read), }) } @@ -317,27 +319,29 @@ mod tests { assert_eq!(response.status, Status::Ok); let coll_key = CollKey { + vshard: VShardId::new(0), db: DatabaseId::DEFAULT, tenant: TenantId::new(1), collection: Box::from("docs"), }; assert_eq!( - h.core.write_index.collection_write_lsn(&coll_key), - Some(Lsn::new(51)), - "vector delete must advance the collection write-version floor" + h.core.write_index.collection_version(&coll_key), + Some(local(51)), + "vector delete must advance the collection write version" ); // BRIGHT-LINE: `vector_id` (the internal HNSW node id) must never be // recorded as a `KeyRepr::Surrogate` — that is a different key space // from the cross-engine surrogate. let would_be_key = WriteKey { + vshard: VShardId::new(0), db: DatabaseId::DEFAULT, tenant: TenantId::new(1), collection: Box::from("docs"), key: KeyRepr::Surrogate(vector_id), }; assert_eq!( - h.core.write_index.key_write_lsn(&would_be_key), + h.core.write_index.key_version(&would_be_key), None, "vector delete must not record vector_id as a surrogate key" ); diff --git a/nodedb/src/data/executor/replay_task.rs b/nodedb/src/data/executor/replay_task.rs index 55ab326b5..a63b0b383 100644 --- a/nodedb/src/data/executor/replay_task.rs +++ b/nodedb/src/data/executor/replay_task.rs @@ -40,6 +40,7 @@ impl CoreLoop { wal_lsn, resolved_now_ms: None, commit_hlc: None, + entry_version: None, admission: crate::bridge::envelope::Admission::Exempt( crate::bridge::envelope::ExemptReason::AlreadyOrdered, ), diff --git a/nodedb/src/data/executor/task.rs b/nodedb/src/data/executor/task.rs index 88100dc68..4141eb8fc 100644 --- a/nodedb/src/data/executor/task.rs +++ b/nodedb/src/data/executor/task.rs @@ -17,7 +17,7 @@ pub struct ExecutionTask { /// WAL LSN the Control Plane allocated for this write at wal-dispatch time, /// carried alongside the request so the apply chokepoint can record the /// per-key / per-collection write version (see - /// [`CoreLoop::note_write_lsn`](crate::data::executor::core_loop::CoreLoop)). + /// [`CoreLoop::note_write`](crate::data::executor::core_loop::CoreLoop)). /// `None` for reads, control ops, and writes whose LSN is not (yet) threaded /// — the version index is skipped rather than advanced with a wrong value. /// Copied from [`Request::wal_lsn`](crate::bridge::envelope::Request) in @@ -140,6 +140,7 @@ mod tests { wal_lsn, resolved_now_ms: None, commit_hlc: None, + entry_version: None, admission: crate::bridge::envelope::Admission::Exempt( crate::bridge::envelope::ExemptReason::Read, ), diff --git a/nodedb/src/data/executor/timeseries_checkpoint/flush.rs b/nodedb/src/data/executor/timeseries_checkpoint/flush.rs index e285dc949..444a71e06 100644 --- a/nodedb/src/data/executor/timeseries_checkpoint/flush.rs +++ b/nodedb/src/data/executor/timeseries_checkpoint/flush.rs @@ -164,6 +164,7 @@ mod tests { wal_lsn: wal_lsn.map(crate::types::Lsn::new), resolved_now_ms: None, commit_hlc: None, + entry_version: None, admission: crate::bridge::envelope::Admission::Admitted, })) .expect("push request"); @@ -174,7 +175,7 @@ mod tests { /// Ingest one ILP line through the real dispatch path, exactly as the /// Control Plane does. `wal_lsn` is threaded BOTH on the op (the /// record the partition's replay stamp names) and on the request (what - /// `note_collection_write_lsn` raises the watermark from). + /// `note_collection_write` raises the watermark from). fn ingest(&mut self, host: &str, value: f64, ts_ms: i64, wal_lsn: u64) { let line = format!("{COLL},host={host} value={value} {}\n", ts_ms * 1_000_000); let r = self.send( diff --git a/nodedb/src/data/executor/timeseries_checkpoint/mod.rs b/nodedb/src/data/executor/timeseries_checkpoint/mod.rs index 299c7fdf0..091941263 100644 --- a/nodedb/src/data/executor/timeseries_checkpoint/mod.rs +++ b/nodedb/src/data/executor/timeseries_checkpoint/mod.rs @@ -6,7 +6,7 @@ //! //! `columnar_memtables` holds one `ColumnarMemtable` per timeseries collection, //! and every ILP / JSON / msgpack ingest lands there. Those rows advance the -//! core watermark (`execute_timeseries_ingest` calls `note_collection_write_lsn` +//! core watermark (`execute_timeseries_ingest` calls `note_collection_write` //! with the Control Plane's `wal_lsn`). The periodic checkpoint reports them as //! durable, and the manager then removes the `TimeseriesBatch` records below //! that LSN. The checkpoint must therefore flush every memtable first. The diff --git a/nodedb/src/data/executor/vector_checkpoint/write.rs b/nodedb/src/data/executor/vector_checkpoint/write.rs index c2b5d91b0..5324a51fe 100644 --- a/nodedb/src/data/executor/vector_checkpoint/write.rs +++ b/nodedb/src/data/executor/vector_checkpoint/write.rs @@ -330,6 +330,7 @@ mod tests { PhysicalPlan::Meta(MetaOp::WalAppend { payload: Vec::new(), }), + None, ); let response = restored.execute_multi_vector_delete(&task, 7, "docs", "emb", one_vector); assert_eq!(response.status, Status::Ok); diff --git a/nodedb/src/data/executor/wal_replay/crdt.rs b/nodedb/src/data/executor/wal_replay/crdt.rs index 17c75acbd..e072d9383 100644 --- a/nodedb/src/data/executor/wal_replay/crdt.rs +++ b/nodedb/src/data/executor/wal_replay/crdt.rs @@ -401,7 +401,7 @@ impl CoreLoop { // Every successfully imported payload changed authoritative Loro // state, including a snapshot import that projects no row. Record // its exact durable collection floor either way. - self.note_replay_write_lsn( + self.note_replay_write( record.header.database_id, tid.as_u64(), collection, @@ -1054,15 +1054,20 @@ mod crdt_replay_tests { "matching fenced retry must rebuild its sparse projection" ); assert_eq!( - h.core.write_index.collection_write_lsn( + h.core.write_index.collection_version( &crate::data::executor::core_loop::write_index::CollKey { + vshard: crate::data::executor::core_loop::write_index::tests::replay_home( + db, collection + ), db, tenant: tid, collection: Box::from(collection), } ), - Some(crate::types::Lsn::new(3)), - "only the correctly fenced retry advances the replay write floor" + Some(crate::data::executor::core_loop::write_index::tests::local( + 3 + )), + "only the correctly fenced retry advances the replay write version" ); } @@ -1233,14 +1238,23 @@ mod crdt_replay_tests { h } + /// The replayed version of `users`, as the WAL LSN it was written at. fn users_write_lsn(h: &CoreHarness, tid: TenantId) -> Option { - h.core.write_index.collection_write_lsn( + use crate::data::executor::core_loop::write_index::tests::{local, replay_home}; + let version = h.core.write_index.collection_version( &crate::data::executor::core_loop::write_index::CollKey { + vshard: replay_home(DatabaseId::DEFAULT, "users"), db: DatabaseId::DEFAULT, tenant: tid, collection: Box::from("users"), }, - ) + )?; + assert_eq!( + version, + local(version.local), + "a single-node replay records a local version" + ); + Some(crate::types::Lsn::new(version.local)) } /// A replayed rejection the full queue refuses stops replay at its diff --git a/nodedb/src/data/executor/wal_replay/kv.rs b/nodedb/src/data/executor/wal_replay/kv.rs index 4364848d4..d6bf3ab68 100644 --- a/nodedb/src/data/executor/wal_replay/kv.rs +++ b/nodedb/src/data/executor/wal_replay/kv.rs @@ -68,7 +68,7 @@ impl CoreLoop { let record_lsn = record.header.lsn; let tombstones = &tombstones.for_database(database_id); - crate::fail_point!("replay::kv_mid_pass"); + crate::fail_point!(self.fail_scope, "replay::kv_mid_pass"); if is_put { // Absolute-overwrite puts — see `kv_put.rs` for the record @@ -303,7 +303,7 @@ impl CoreLoop { self.kv_engine .delete(database_id, tenant_id, &collection, &keys, now_ms); for deleted_key in &keys { - self.note_replay_write_lsn( + self.note_replay_write( database_id, tenant_id, &collection, @@ -336,13 +336,7 @@ impl CoreLoop { }]); } self.kv_engine.truncate(database_id, tenant_id, &collection); - self.note_replay_write_lsn( - database_id, - tenant_id, - &collection, - None, - record_lsn, - ); + self.note_replay_write(database_id, tenant_id, &collection, None, record_lsn); deletes += 1; continue; } diff --git a/nodedb/src/data/executor/wal_replay/kv_put.rs b/nodedb/src/data/executor/wal_replay/kv_put.rs index ef1ed814f..4b4c97c80 100644 --- a/nodedb/src/data/executor/wal_replay/kv_put.rs +++ b/nodedb/src/data/executor/wal_replay/kv_put.rs @@ -119,7 +119,7 @@ impl CoreLoop { self.replay_record_unapplied("kv", "put_identity", record_lsn, &e.to_string()); return Some(0); } - self.note_replay_write_lsn( + self.note_replay_write( database_id, tenant_id, &collection, @@ -176,7 +176,7 @@ impl CoreLoop { }, expire_at_ms, ); - self.note_replay_write_lsn( + self.note_replay_write( database_id, tenant_id, &collection, @@ -268,7 +268,7 @@ impl CoreLoop { return Some(0); } for (entry_key, _entry_value) in &entries { - self.note_replay_write_lsn( + self.note_replay_write( database_id, tenant_id, &collection, diff --git a/nodedb/src/data/executor/wal_replay_all.rs b/nodedb/src/data/executor/wal_replay_all.rs index 258edc698..ac725bf95 100644 --- a/nodedb/src/data/executor/wal_replay_all.rs +++ b/nodedb/src/data/executor/wal_replay_all.rs @@ -29,6 +29,26 @@ use super::core_loop::CoreLoop; use super::core_loop::fail_stop::FailStopCause; impl CoreLoop { + /// Bound the versions restart replay could not rebuild, then forget the + /// record stamps. + /// + /// A record an engine checkpoint already held applies no write, so its + /// keys keep no version. Each vShard this core owns therefore holds every + /// write through the newest of its WAL records, and a read older than + /// that no longer validates against a row with no recorded version. + fn settle_replayed_versions(&mut self, num_cores: usize) { + let owned: Vec<_> = self + .write_index + .record_stamps + .latest_by_vshard() + .filter(|(vshard, _)| crate::types::core_for_vshard(*vshard, num_cores) == self.core_id) + .collect(); + for (vshard, version) in owned { + self.write_index.raise_installed(vshard, version); + } + self.write_index.record_stamps.clear(); + } + /// Replay every WAL record class into this core's engines, in the exact /// order restart correctness requires. No-op when `records` is empty. /// @@ -59,7 +79,13 @@ impl CoreLoop { .seed_replayed_through(crate::types::Lsn::new(wal_end)); } - crate::fail_point!("replay::before_engine_passes"); + // Every write a replay arm applies takes its record's vShard and the + // log position of the data-group entry the record applies, so a + // recovered write records the version its live apply did. + self.write_index.record_stamps = + super::core_loop::write_index::RecordStamps::from_wal(records); + + crate::fail_point!(self.fail_scope, "replay::before_engine_passes"); // Every engine-bearing record class — standalone (autocommit) records // and the sub-records of committed `TransactionRedo` groups alike — @@ -71,7 +97,9 @@ impl CoreLoop { // Fatal on error: a redo group that cannot be reconstituted is a // committed transaction that cannot be applied, and continuing would // open the database with a hole in the replayed suffix. - if let Err(e) = self.replay_engines_in_lsn_order(records, num_cores, tombstones) { + let replayed = self.replay_engines_in_lsn_order(records, num_cores, tombstones); + self.settle_replayed_versions(num_cores); + if let Err(e) = replayed { error!( core_id, error = %e, @@ -86,7 +114,7 @@ impl CoreLoop { return Err(e); } - crate::fail_point!("replay::before_sync_hwm_pass"); + crate::fail_point!(self.fail_scope, "replay::before_sync_hwm_pass"); // Reconstruct sync HWM maps from SyncSeqAdvance records so // post-restart deduplication is correct. Fatal on error — @@ -160,6 +188,135 @@ mod tests { .expect("build record") } + /// A KV put record the apply of the proposal keyed `apply_key` appended. + fn keyed_kv_put_record(lsn: u64, key: &[u8], apply_key: u64) -> WalRecord { + WalRecord::new_stamped( + WalRecordArgs { + record_type: RecordType::Put as u32, + lsn, + tenant_id: 1, + vshard_id: 0, + database_id: 0, + payload: zerompk::to_msgpack_vec(&( + "kv_put", + "cache", + key.to_vec(), + b"v".to_vec(), + 0u64, + None::, + 1u32, + )) + .expect("encode kv put"), + encryption_key: None, + preamble_bytes: None, + }, + nodedb_wal::record::RecordStamp { + apply_key, + event_source: nodedb_wal::NO_EVENT_SOURCE, + commit_hlc: 0, + }, + ) + .expect("build record") + } + + /// The `ChangePosition` marker of the entry at `(epoch, log_index)` whose + /// proposal key is `apply_key`. + fn position_marker(lsn: u64, apply_key: u64, epoch: u64, log_index: u64) -> WalRecord { + let marker = crate::event::cdc::position::ChangePositionMarker { + apply_key, + position: crate::event::cdc::position::ReplicatedPosition { + epoch, + group_id: 9, + log_index, + }, + }; + WalRecord::new(WalRecordArgs { + record_type: RecordType::ChangePosition as u32, + lsn, + tenant_id: 1, + vshard_id: 0, + database_id: 0, + payload: marker.to_bytes().to_vec(), + encryption_key: None, + preamble_bytes: None, + }) + .expect("build marker") + } + + fn kv_version(core: &CoreLoop, key: &[u8]) -> Option { + core.write_index + .key_version(&crate::data::executor::core_loop::write_index::WriteKey { + vshard: crate::types::VShardId::new(0), + db: DatabaseId::DEFAULT, + tenant: TenantId::new(TID), + collection: Box::from("cache"), + key: crate::types::KeyRepr::KvKey(Box::from(key)), + }) + } + + /// Restart replay rebuilds each write's version from the data-group + /// entry its record applied, the version every replica recorded live. + /// A record no entry applied takes its WAL LSN, as it did live. + #[test] + fn replay_records_the_entry_position_each_record_applied() { + let dir = tempfile::tempdir().expect("tempdir"); + let (mut core, _req, _resp) = make_core_with_dir(dir.path()); + let records = vec![ + position_marker(10, 0xA1, 2, 70), + keyed_kv_put_record(11, b"replicated", 0xA1), + keyed_kv_put_record(12, b"local", 0), + ]; + + core.replay_all_wal(&records, 1, &TombstoneSet::new()) + .expect("replay"); + + let entry = nodedb_types::WriteVersion::logged(2, 70); + assert_eq!(kv_version(&core, b"replicated"), Some(entry)); + assert_eq!( + kv_version(&core, b"local"), + Some(nodedb_types::WriteVersion::local_after(entry, 12)) + ); + assert_eq!( + core.write_index.latest(crate::types::VShardId::new(0)), + nodedb_types::WriteVersion::local_after(entry, 12) + ); + assert!( + core.write_index.record_stamps.get(11).is_none(), + "replay forgets the record stamps once it ends" + ); + } + + /// A key whose record an engine checkpoint already held keeps no version + /// after replay. A read older than the WAL's newest record then no longer + /// validates against it. + #[test] + fn replay_bounds_the_versions_a_checkpoint_hid() { + let dir = tempfile::tempdir().expect("tempdir"); + let (mut core, _req, _resp) = make_core_with_dir(dir.path()); + let records = vec![ + position_marker(20, 0xB2, 0, 40), + keyed_kv_put_record(21, b"k", 0xB2), + ]; + core.replay_all_wal(&records, 1, &TombstoneSet::new()) + .expect("replay"); + + let never_replayed = nodedb_types::calvin::ReadKeyIdent::Point( + crate::types::KeyRepr::KvKey(Box::from(b"absent".as_slice())), + ); + let valid = |read| { + core.write_index.read_is_valid( + DatabaseId::DEFAULT, + TenantId::new(TID), + crate::types::VShardId::new(0), + "cache", + &never_replayed, + nodedb_types::WriteVersion::logged(0, read), + ) + }; + assert!(!valid(39), "a read older than the WAL's newest record"); + assert!(valid(40), "a read at the WAL's newest record"); + } + /// Every record replay reads is decided when replay ends, so a checkpoint /// written afterwards names all of them through its prefix, and the /// applied set holds none of them. diff --git a/nodedb/src/data/executor/wal_replay_columnar_image.rs b/nodedb/src/data/executor/wal_replay_columnar_image.rs index 075537626..9902d4821 100644 --- a/nodedb/src/data/executor/wal_replay_columnar_image.rs +++ b/nodedb/src/data/executor/wal_replay_columnar_image.rs @@ -329,7 +329,7 @@ impl CoreLoop { collection, ); self.checkpoint_coordinator.mark_dirty("columnar", written); - self.note_collection_write_lsn(task, collection); + self.note_collection_write(task, collection); } Ok(written) } diff --git a/nodedb/src/data/executor/wal_replay_fts.rs b/nodedb/src/data/executor/wal_replay_fts.rs index 6dc434f35..175e83f26 100644 --- a/nodedb/src/data/executor/wal_replay_fts.rs +++ b/nodedb/src/data/executor/wal_replay_fts.rs @@ -77,6 +77,7 @@ impl CoreLoop { wal_lsn: None, resolved_now_ms: None, commit_hlc: None, + entry_version: None, admission: crate::bridge::envelope::Admission::Exempt( crate::bridge::envelope::ExemptReason::AlreadyOrdered, ), diff --git a/nodedb/src/data/executor/wal_replay_kv_atomic.rs b/nodedb/src/data/executor/wal_replay_kv_atomic.rs index 7b3d64699..15db591b1 100644 --- a/nodedb/src/data/executor/wal_replay_kv_atomic.rs +++ b/nodedb/src/data/executor/wal_replay_kv_atomic.rs @@ -122,7 +122,7 @@ impl CoreLoop { } }; if swapped { - self.note_replay_write_lsn( + self.note_replay_write( database_id, tenant_id, &collection, @@ -180,7 +180,7 @@ impl CoreLoop { &gate, ) { Ok(_) => { - self.note_replay_write_lsn( + self.note_replay_write( database_id, tenant_id, &collection, @@ -313,7 +313,7 @@ impl CoreLoop { self.replay_swap_error("getset", &collection, &key, record_lsn, error); return Some(0); } - self.note_replay_write_lsn( + self.note_replay_write( database_id, tenant_id, &collection, diff --git a/nodedb/src/data/executor/wal_replay_kv_expiry.rs b/nodedb/src/data/executor/wal_replay_kv_expiry.rs index 8c569b995..403e31192 100644 --- a/nodedb/src/data/executor/wal_replay_kv_expiry.rs +++ b/nodedb/src/data/executor/wal_replay_kv_expiry.rs @@ -86,7 +86,7 @@ impl CoreLoop { ); return Some(0); } - self.note_replay_write_lsn( + self.note_replay_write( database_id, tenant_id, &collection, @@ -139,7 +139,7 @@ impl CoreLoop { ); return Some(0); } - self.note_replay_write_lsn( + self.note_replay_write( database_id, tenant_id, &collection, diff --git a/nodedb/src/data/executor/wal_replay_kv_field.rs b/nodedb/src/data/executor/wal_replay_kv_field.rs index ef0a6e167..34be24d81 100644 --- a/nodedb/src/data/executor/wal_replay_kv_field.rs +++ b/nodedb/src/data/executor/wal_replay_kv_field.rs @@ -90,7 +90,7 @@ impl CoreLoop { self.replay_record_unapplied("kv", "field_set_identity", record_lsn, &e.to_string()); return Some(0); } - self.note_replay_write_lsn( + self.note_replay_write( database_id, tenant_id, &collection, diff --git a/nodedb/src/data/executor/wal_replay_kv_incr.rs b/nodedb/src/data/executor/wal_replay_kv_incr.rs index 5a53d7453..ad294bdb7 100644 --- a/nodedb/src/data/executor/wal_replay_kv_incr.rs +++ b/nodedb/src/data/executor/wal_replay_kv_incr.rs @@ -99,7 +99,7 @@ impl CoreLoop { }; let applied = self.log_kv_incr_result(&collection, &key, delta, record_lsn, result); if applied > 0 { - self.note_replay_write_lsn( + self.note_replay_write( database_id, tenant_id, &collection, diff --git a/nodedb/src/data/executor/wal_replay_kv_insert_conflict.rs b/nodedb/src/data/executor/wal_replay_kv_insert_conflict.rs index b989b4f0f..3a4eec607 100644 --- a/nodedb/src/data/executor/wal_replay_kv_insert_conflict.rs +++ b/nodedb/src/data/executor/wal_replay_kv_insert_conflict.rs @@ -172,7 +172,7 @@ impl CoreLoop { ); return 0; } - self.note_replay_write_lsn( + self.note_replay_write( database_id, tenant_id, collection, diff --git a/nodedb/src/data/executor/wal_replay_kv_predicate.rs b/nodedb/src/data/executor/wal_replay_kv_predicate.rs index 4fe215c5d..b8a672a26 100644 --- a/nodedb/src/data/executor/wal_replay_kv_predicate.rs +++ b/nodedb/src/data/executor/wal_replay_kv_predicate.rs @@ -92,7 +92,7 @@ impl CoreLoop { } // The key set is resolved at replay time rather than carried, so the // collection floor is the version this can record. - self.note_replay_write_lsn(database_id, tenant_id, &collection, None, record_lsn); + self.note_replay_write(database_id, tenant_id, &collection, None, record_lsn); Some(written) } @@ -138,7 +138,7 @@ impl CoreLoop { let removed = self .kv_engine .delete(database_id, tenant_id, &collection, &keys, now_ms); - self.note_replay_write_lsn(database_id, tenant_id, &collection, None, record_lsn); + self.note_replay_write(database_id, tenant_id, &collection, None, record_lsn); Some(removed) } } diff --git a/nodedb/src/data/executor/wal_replay_kv_transfer.rs b/nodedb/src/data/executor/wal_replay_kv_transfer.rs index ec0ae0eac..8b8b8183f 100644 --- a/nodedb/src/data/executor/wal_replay_kv_transfer.rs +++ b/nodedb/src/data/executor/wal_replay_kv_transfer.rs @@ -181,14 +181,14 @@ impl CoreLoop { return 0; } } - self.note_replay_write_lsn( + self.note_replay_write( p.database_id, p.tenant_id, p.collection, Some(KeyRepr::KvKey(Box::from(p.source_key))), p.record_lsn, ); - self.note_replay_write_lsn( + self.note_replay_write( p.database_id, p.tenant_id, p.collection, @@ -282,14 +282,14 @@ impl CoreLoop { return (0, 0); } - self.note_replay_write_lsn( + self.note_replay_write( p.database_id, p.tenant_id, p.source_collection, Some(KeyRepr::KvKey(Box::from(p.item_key))), p.record_lsn, ); - self.note_replay_write_lsn( + self.note_replay_write( p.database_id, p.tenant_id, p.dest_collection, diff --git a/nodedb/src/data/executor/wal_replay_kv_ttl.rs b/nodedb/src/data/executor/wal_replay_kv_ttl.rs index 91fcbdfab..251ba7ae0 100644 --- a/nodedb/src/data/executor/wal_replay_kv_ttl.rs +++ b/nodedb/src/data/executor/wal_replay_kv_ttl.rs @@ -277,6 +277,7 @@ mod tests { wal_lsn: None, resolved_now_ms, commit_hlc: None, + entry_version: None, admission: Admission::Exempt(ExemptReason::Read), }) } diff --git a/nodedb/src/data/executor/wal_replay_redo_document.rs b/nodedb/src/data/executor/wal_replay_redo_document.rs index 0d2cf8fef..725a2908b 100644 --- a/nodedb/src/data/executor/wal_replay_redo_document.rs +++ b/nodedb/src/data/executor/wal_replay_redo_document.rs @@ -94,14 +94,6 @@ //! their targets and link their hash chain, exactly as replication does (see //! `handlers::transaction::redo_apply`). With no scope open this arm is plain //! restart replay. -//! -//! ### Calvin records in restart replay -//! -//! A Calvin redo record's stamp carries the sum targets its slice folds, and -//! no later record carries the target rows. Restart replay runs its document -//! rows through the committed path, which folds them at the record's LSN. -//! The fold subtracts the row's prior image, so a row that already holds its -//! post-image folds nothing. use nodedb_types::Surrogate; use nodedb_types::sync::wire::SyncProvenance; @@ -236,7 +228,7 @@ impl CoreLoop { self.observe_bitemporal_stamp(s.sys_from_ms); self.apply_scope.bitemporal_stamps.insert(surrogate_u32, s); } - let folds = self.redo_folds_at(record_lsn, &collection); + let folds = self.redo_folds(); let applied = if folds { self.apply_committed_document_put( CommittedDocWrite { @@ -266,7 +258,7 @@ impl CoreLoop { } if applied { puts += 1; - self.note_replay_write_lsn( + self.note_replay_write( database_id, tenant_id, &collection, @@ -335,7 +327,7 @@ impl CoreLoop { }, ); } - let folds = self.redo_folds_at(record_lsn, &collection); + let folds = self.redo_folds(); let removed = if folds { self.apply_committed_document_delete(CommittedDocWrite { database_id, @@ -359,7 +351,7 @@ impl CoreLoop { } if removed { deletes += 1; - self.note_replay_write_lsn( + self.note_replay_write( database_id, tenant_id, &collection, @@ -380,18 +372,11 @@ impl CoreLoop { } } - /// Whether a document write at `record_lsn` to `collection` runs the - /// committed path, which folds materialized sums: always under a - /// committed-redo apply, and in restart replay when the record's Calvin - /// stamp names sum targets for `collection`. Folding reads the row's - /// prior image, so a replay over a row that already holds the post-image - /// folds nothing. - fn redo_folds_at(&self, record_lsn: u64, collection: &str) -> bool { + /// Whether a document write runs the committed path, which folds + /// materialized sums: only under a committed-redo apply. Restart replay + /// applies the fold target rows from the parts the apply journalled. + fn redo_folds(&self) -> bool { self.redo_apply.scope.is_some() - || self - .redo_apply - .replay_folds_for(record_lsn, collection) - .is_some() } } @@ -416,7 +401,7 @@ fn is_kv_put_record(payload: &[u8]) -> bool { #[cfg(test)] mod tests { use super::*; - use crate::types::{DatabaseId, Lsn, TenantId}; + use crate::types::{DatabaseId, TenantId}; use crate::wal::{RedoRecord, RedoSubRecord}; use nodedb_types::Surrogate; use nodedb_wal::WalRecord; @@ -736,6 +721,7 @@ mod tests { #[test] fn redo_replay_populates_write_version_index() { + use crate::data::executor::core_loop::write_index::tests::{local, replay_home}; use crate::data::executor::core_loop::write_index::{CollKey, WriteKey}; let mut h = make_core(); @@ -762,49 +748,53 @@ mod tests { let tenant = TenantId::new(7); let doc_key = WriteKey { + vshard: replay_home(db, "notes"), db, tenant, collection: Box::from("notes"), key: KeyRepr::Surrogate(surrogate), }; assert_eq!( - h.core.write_index.key_write_lsn(&doc_key), - Some(Lsn::new(1)), + h.core.write_index.key_version(&doc_key), + Some(local(1)), "document redo put must populate the per-key write-version index" ); let doc_coll_key = CollKey { + vshard: replay_home(db, "notes"), db, tenant, collection: Box::from("notes"), }; assert_eq!( - h.core.write_index.collection_write_lsn(&doc_coll_key), - Some(Lsn::new(1)), - "document redo put must advance the collection write-version floor" + h.core.write_index.collection_version(&doc_coll_key), + Some(local(1)), + "document redo put must advance the collection write version" ); let kv_key = WriteKey { + vshard: replay_home(db, "sessions"), db, tenant, collection: Box::from("sessions"), key: KeyRepr::KvKey(Box::from(b"s1".as_slice())), }; assert_eq!( - h.core.write_index.key_write_lsn(&kv_key), - Some(Lsn::new(1)), + h.core.write_index.key_version(&kv_key), + Some(local(1)), "kv redo put must populate the per-key write-version index" ); let kv_coll_key = CollKey { + vshard: replay_home(db, "sessions"), db, tenant, collection: Box::from("sessions"), }; assert_eq!( - h.core.write_index.collection_write_lsn(&kv_coll_key), - Some(Lsn::new(1)), - "kv redo put must advance the collection write-version floor" + h.core.write_index.collection_version(&kv_coll_key), + Some(local(1)), + "kv redo put must advance the collection write version" ); // A later delete at a higher LSN must bump the document key's write @@ -842,8 +832,8 @@ mod tests { .expect("redo replay must succeed"); assert_eq!( - h.core.write_index.key_write_lsn(&doc_key), - Some(Lsn::new(2)), + h.core.write_index.key_version(&doc_key), + Some(local(2)), "redo delete must bump the key's write-version above the prior put" ); } diff --git a/nodedb/src/data/executor/wal_replay_redo_graph.rs b/nodedb/src/data/executor/wal_replay_redo_graph.rs index 9820ff67b..fdb8ae4ca 100644 --- a/nodedb/src/data/executor/wal_replay_redo_graph.rs +++ b/nodedb/src/data/executor/wal_replay_redo_graph.rs @@ -442,6 +442,7 @@ impl CoreLoop { wal_lsn, resolved_now_ms: None, commit_hlc: None, + entry_version: None, admission: crate::bridge::envelope::Admission::Exempt( crate::bridge::envelope::ExemptReason::AlreadyOrdered, ), @@ -582,7 +583,7 @@ mod tests { } #[test] - fn redo_graph_edge_put_records_write_version_floor() { + fn redo_graph_edge_put_records_its_write_version() { use crate::data::executor::core_loop::write_index::{KeyRepr, WriteKey}; let mut h = make_core(); @@ -596,7 +597,10 @@ mod tests { ) .expect("redo replay must succeed"); + // The edge applies on the vShard its record names, at the record's + // LSN: no registered stamp names an entry for it. let write_key = WriteKey { + vshard: crate::types::VShardId::new(record.header.vshard_id), db: DatabaseId::new(0), tenant: crate::types::TenantId::new(7), collection: Box::from("knows"), @@ -607,9 +611,11 @@ mod tests { }, }; assert_eq!( - h.core.write_index.key_write_lsn(&write_key), - Some(Lsn::new(1)), - "graph edge redo replay must record the write-version floor at the record's LSN" + h.core.write_index.key_version(&write_key), + Some(crate::data::executor::core_loop::write_index::tests::local( + 1 + )), + "graph edge redo replay must record the write version at the record's LSN" ); } diff --git a/nodedb/src/data/executor/wal_replay_redo_graph_cut.rs b/nodedb/src/data/executor/wal_replay_redo_graph_cut.rs index 7b025fbb5..c47a406e4 100644 --- a/nodedb/src/data/executor/wal_replay_redo_graph_cut.rs +++ b/nodedb/src/data/executor/wal_replay_redo_graph_cut.rs @@ -153,6 +153,7 @@ mod tests { wal_lsn: None, resolved_now_ms: None, commit_hlc: None, + entry_version: None, admission: Admission::Exempt(ExemptReason::Read), }) } diff --git a/nodedb/src/data/executor/wal_replay_row_image_tests.rs b/nodedb/src/data/executor/wal_replay_row_image_tests.rs index 7f70009ae..9be069945 100644 --- a/nodedb/src/data/executor/wal_replay_row_image_tests.rs +++ b/nodedb/src/data/executor/wal_replay_row_image_tests.rs @@ -119,6 +119,7 @@ fn request(plan: PhysicalPlan, wal_lsn: Option) -> Request { wal_lsn, resolved_now_ms: None, commit_hlc: None, + entry_version: None, admission: Admission::Admitted, } } diff --git a/nodedb/src/data/executor/wal_replay_spatial.rs b/nodedb/src/data/executor/wal_replay_spatial.rs index cb24fc978..e6c9dc85c 100644 --- a/nodedb/src/data/executor/wal_replay_spatial.rs +++ b/nodedb/src/data/executor/wal_replay_spatial.rs @@ -105,6 +105,7 @@ impl CoreLoop { wal_lsn: None, resolved_now_ms: None, commit_hlc: None, + entry_version: None, admission: crate::bridge::envelope::Admission::Exempt( crate::bridge::envelope::ExemptReason::AlreadyOrdered, ), diff --git a/nodedb/src/data/executor/wal_replay_vector.rs b/nodedb/src/data/executor/wal_replay_vector.rs index 25ff6011c..28aec74ed 100644 --- a/nodedb/src/data/executor/wal_replay_vector.rs +++ b/nodedb/src/data/executor/wal_replay_vector.rs @@ -198,6 +198,7 @@ impl CoreLoop { pk_bytes: None, provenance: provenance.clone(), }), + Self::replay_record_lsn(record_lsn), ); let response = self.execute_vector_insert( crate::data::executor::handlers::vector::VectorInsertParams { diff --git a/nodedb/src/data/executor/wal_replay_vector_delete.rs b/nodedb/src/data/executor/wal_replay_vector_delete.rs index c705f410b..bf3ddcb57 100644 --- a/nodedb/src/data/executor/wal_replay_vector_delete.rs +++ b/nodedb/src/data/executor/wal_replay_vector_delete.rs @@ -89,6 +89,7 @@ impl CoreLoop { provenance: provenance.clone(), }, ), + Self::replay_record_lsn(record_lsn), ); let response = self.execute_vector_delete_by_surrogate( &task, diff --git a/nodedb/src/data/executor/wal_replay_vector_direct.rs b/nodedb/src/data/executor/wal_replay_vector_direct.rs index 86cfd7b54..a89d79d8d 100644 --- a/nodedb/src/data/executor/wal_replay_vector_direct.rs +++ b/nodedb/src/data/executor/wal_replay_vector_direct.rs @@ -81,6 +81,7 @@ impl CoreLoop { rls_filters: Vec::new(), rls_write_check: nodedb_types::RlsWriteCheck::already_decided_elsewhere(), }), + Self::replay_record_lsn(record_lsn), ); let response = self.execute_vector_direct_delete( crate::data::executor::handlers::vector_direct_delete::VectorDirectDeleteParams { @@ -162,6 +163,7 @@ impl CoreLoop { field: field.clone(), restart_identity: false, }), + Self::replay_record_lsn(record_lsn), ); let response = self.execute_vector_direct_truncate(&task, tenant_id, &collection, &field); if response.status != Status::Ok { @@ -250,6 +252,7 @@ impl CoreLoop { rls_filters: Vec::new(), rls_write_check: nodedb_types::RlsWriteCheck::already_decided_elsewhere(), }), + Self::replay_record_lsn(record_lsn), ); let response = self.execute_vector_direct_update( crate::data::executor::handlers::vector_direct_update::VectorDirectUpdateParams { diff --git a/nodedb/src/data/executor/wal_replay_vector_extended.rs b/nodedb/src/data/executor/wal_replay_vector_extended.rs index a58bed7f2..30b0ea471 100644 --- a/nodedb/src/data/executor/wal_replay_vector_extended.rs +++ b/nodedb/src/data/executor/wal_replay_vector_extended.rs @@ -253,6 +253,7 @@ impl CoreLoop { on_conflict_updates: on_conflict_updates.clone(), rls_write_check: nodedb_types::RlsWriteCheck::already_decided_elsewhere(), }), + Self::replay_record_lsn(record_lsn), ); let response = self.execute_vector_direct_upsert( crate::data::executor::handlers::vector_upsert::VectorDirectUpsertParams { @@ -351,6 +352,7 @@ impl CoreLoop { count, dim, }), + Self::replay_record_lsn(record_lsn), ); let response = self.execute_multi_vector_insert( crate::data::executor::handlers::vector_multi::MultiVectorInsertParams { @@ -440,6 +442,7 @@ impl CoreLoop { field_name: field_name.clone(), document_surrogate, }), + Self::replay_record_lsn(record_lsn), ); // The delete is idempotent: an already-absent document answers Ok. // Advance the watermark regardless so a later checkpoint records the diff --git a/nodedb/src/data/executor/wal_replay_vector_resolved.rs b/nodedb/src/data/executor/wal_replay_vector_resolved.rs index c396686d0..eafbc25d5 100644 --- a/nodedb/src/data/executor/wal_replay_vector_resolved.rs +++ b/nodedb/src/data/executor/wal_replay_vector_resolved.rs @@ -94,6 +94,7 @@ impl CoreLoop { response_payload: Vec::new(), rls_write_check: rls_write_check.clone(), }), + Self::replay_record_lsn(record_lsn), ); let index = VectorResolvedIndexSpec { collection: &collection, diff --git a/nodedb/src/data/executor/wal_replay_vector_sparse.rs b/nodedb/src/data/executor/wal_replay_vector_sparse.rs index a09953701..280d2082e 100644 --- a/nodedb/src/data/executor/wal_replay_vector_sparse.rs +++ b/nodedb/src/data/executor/wal_replay_vector_sparse.rs @@ -100,6 +100,7 @@ impl CoreLoop { doc_id: doc_id.clone(), entries: entries.clone(), }), + Self::replay_record_lsn(record_lsn), ); let response = self.execute_sparse_insert( &task, @@ -170,6 +171,7 @@ impl CoreLoop { field_name: field_name.clone(), doc_id: doc_id.clone(), }), + Self::replay_record_lsn(record_lsn), ); // An absent document yields NotFound; that is an expected idempotent // no-op on replay, not a failure. diff --git a/nodedb/src/data/executor/wal_replay_vector_task.rs b/nodedb/src/data/executor/wal_replay_vector_task.rs index 71d4aa1bc..4cce0934f 100644 --- a/nodedb/src/data/executor/wal_replay_vector_task.rs +++ b/nodedb/src/data/executor/wal_replay_vector_task.rs @@ -9,16 +9,26 @@ use crate::types::{DatabaseId, ReadConsistency}; use super::core_loop::CoreLoop; impl CoreLoop { + /// The LSN a replayed record hands its task. A `record_lsn` of 0 means + /// the record has no durable LSN and yields `None`. + pub(in crate::data::executor) fn replay_record_lsn( + record_lsn: u64, + ) -> Option { + (record_lsn != 0).then(|| crate::types::Lsn::new(record_lsn)) + } + /// Build a synthetic `ExecutionTask` for WAL replay. /// /// Mirrors `CoreLoop::replay_task` (`replay_task.rs`). The task carries - /// no meaningful request semantics — it is only needed so that the handler - /// methods can return a typed `Response`. + /// no meaningful request semantics. It lets the handler methods return a + /// typed `Response`. The task carries the record's LSN so the handler + /// records the write's version. pub(in crate::data::executor) fn replay_vector_task( tenant_id: crate::types::TenantId, database_id: DatabaseId, vshard_id: crate::types::VShardId, plan: PhysicalPlan, + wal_lsn: Option, ) -> ExecutionTask { ExecutionTask { request: Request { @@ -38,15 +48,16 @@ impl CoreLoop { user_id: None, statement_digest: None, txn_id: None, - wal_lsn: None, + wal_lsn, resolved_now_ms: None, commit_hlc: None, + entry_version: None, admission: crate::bridge::envelope::Admission::Exempt( crate::bridge::envelope::ExemptReason::AlreadyOrdered, ), }, state: TaskState::Running, - wal_lsn: None, + wal_lsn, resolved_now_ms: None, } } diff --git a/nodedb/src/data/executor/write_set_crash_tests.rs b/nodedb/src/data/executor/write_set_crash_tests.rs index a507478e2..bcfa36b8c 100644 --- a/nodedb/src/data/executor/write_set_crash_tests.rs +++ b/nodedb/src/data/executor/write_set_crash_tests.rs @@ -9,11 +9,13 @@ //! onto an empty base (a point-in-time restore), and the settled WAL the //! event stream is rebuilt from. -use std::collections::HashMap; +use std::collections::{BTreeSet, HashMap}; use std::path::Path; use std::time::{Duration, Instant}; -use nodedb_physical::physical_plan::{DocumentOp, UpdateValue}; +use nodedb_physical::physical_plan::{ + CalvinInstall, CalvinReplySpec, DocumentOp, MetaOp, RedoOrigin, UpdateValue, +}; use nodedb_types::{QualifiedCollection, RlsWriteCheck, Surrogate, Value}; use nodedb_wal::{TombstoneSet, WalRecord}; @@ -23,11 +25,16 @@ use crate::bridge::dispatch::JournalGroup; use crate::bridge::envelope::{ Admission, PhysicalPlan, Priority, Request, Response, Status, WriteSetEntry, }; +use crate::control::cluster::calvin::scheduler::recover_all_applied; +use crate::control::security::catalog::SystemCatalog; use crate::control::server::wal_dispatch::{ GroupOrigin, WalAppendRequest, WriteSetTarget, append_group_origin, append_group_parts, }; use crate::data::executor::core_loop::CoreLoop; use crate::data::executor::core_loop::tests::make_core_with_dir; +use crate::data::executor::handlers::transaction::redo_apply::calvin_fold_tests::{ + SOURCE, balance, calvin_redo, configure_sum, core_with_sum, sum_targets, +}; use crate::data::executor::task::ExecutionTask; use crate::types::{DatabaseId, ReadConsistency, RequestId, TenantId, TraceId, VShardId}; use crate::wal::GroupMembership; @@ -49,10 +56,15 @@ struct Live { impl Live { fn open() -> Self { + Self::open_with(|dir| make_core_with_dir(dir).0) + } + + /// A live node whose core `make` builds over its store directory. + fn open_with(make: impl FnOnce(&Path) -> CoreLoop) -> Self { let wal_dir = tempfile::tempdir().expect("tempdir"); let wal = WalManager::open_for_testing(&wal_dir.path().join("wal")).expect("open wal"); let store_dir = tempfile::tempdir().expect("tempdir"); - let (core, _req, _resp) = make_core_with_dir(store_dir.path()); + let core = make(store_dir.path()); Self { wal, wal_dir, @@ -106,6 +118,12 @@ impl Live { /// `stored`, the core stores its write set. The parts are never /// appended: the crash comes first. fn journalled_until_crash(&mut self, plan: PhysicalPlan, stored: bool) -> GroupOrigin { + self.journalled_in(ORDERS, plan, stored) + } + + /// [`Self::journalled_until_crash`] for a write whose group falls back + /// to `collection`. + fn journalled_in(&mut self, collection: &str, plan: PhysicalPlan, stored: bool) -> GroupOrigin { let origin = self.append_origin(&plan); let task = ExecutionTask::new(request(plan, origin)); self.core.begin_journalled(); @@ -114,7 +132,7 @@ impl Live { if stored { let group = JournalGroup { origin: origin.lsn, - collection: ORDERS.to_string(), + collection: collection.to_string(), apply_key: NO_APPLY_KEY, commit_hlc: None, change_position: None, @@ -179,6 +197,7 @@ fn request(plan: PhysicalPlan, origin: GroupOrigin) -> Request { wal_lsn: Some(origin.lsn), resolved_now_ms: None, commit_hlc: None, + entry_version: None, admission: Admission::Admitted, } } @@ -442,3 +461,81 @@ fn a_settled_group_needs_nothing_at_boot() { "a whole WAL needs no record" ); } + +/// The stamped install of the Calvin slice at `(3, 0)` on vShard 0. Its +/// record writes one source row, and its sum target folds 25 into the +/// target's balance. +fn calvin_install() -> PhysicalPlan { + PhysicalPlan::Meta(MetaOp::ApplyTransactionRedo { + redo: calvin_redo(), + collections: vec![SOURCE.to_string()], + sum_targets: sum_targets(), + origin: RedoOrigin::Commit, + calvin: Some(CalvinInstall { + epoch: 3, + position: 0, + epoch_system_ms: 0, + reply: CalvinReplySpec::Count(Vec::new()), + user_write: true, + }), + }) +} + +/// The target's balance after replaying `records` onto the stores at `dir`, +/// on a core that declares the sum. +fn balance_after_replay(dir: &Path, records: &[WalRecord]) -> Option { + let (mut core, _req, _resp) = make_core_with_dir(dir); + configure_sum(&mut core); + core.replay_all_wal(records, 1, &TombstoneSet::new()) + .expect("replay"); + balance(&core) +} + +/// The Calvin positions boot recovery finds applied on vShard 0 in `wal`. +fn recovered_positions(wal: &WalManager) -> BTreeSet<(u64, u32)> { + let catalog = SystemCatalog::open_in_memory().expect("in-memory catalog"); + recover_all_applied(wal, &catalog, &|_| None) + .expect("recover applied positions") + .remove(&0) + .map(|recovered| recovered.applied_tail) + .unwrap_or_default() +} + +/// A crash cuts the origin of a stamped Calvin install from the WAL after +/// the install folded its sum and stored its write set. Boot journals the +/// stamped record again, then the fold rows as its parts. Recovery finds +/// the position applied, and replay keeps the single-apply total. +#[test] +fn a_crash_that_cuts_a_calvin_redo_origin_journals_it_again() { + let mut live = Live::open_with(|dir| core_with_sum(dir).0); + let cut = live.wal_image(); + live.journalled_in(SOURCE, calvin_install(), true); + assert_eq!(balance(&live.core).as_deref(), Some("125"), "the live fold"); + let stores = live.crash_image(); + + let cut_wal = WalManager::open_for_testing(&cut.path().join("wal")).expect("open cut wal"); + assert!( + recovered_positions(&cut_wal).is_empty(), + "the crash cuts the stamped record" + ); + let records = settle(&cut_wal, stores.path()); + assert!( + recovered_positions(&cut_wal).contains(&(3, 0)), + "recovery finds the stamp of the record boot journals" + ); + assert!( + membership(&records).broken_through(u64::MAX).is_empty(), + "the journalled record's group is whole" + ); + assert_eq!( + balance_after_replay(stores.path(), &records).as_deref(), + Some("125"), + "restart replay onto the cut stores keeps the single-apply total" + ); + let base = tempfile::tempdir().expect("tempdir"); + assert_eq!( + balance_after_replay(base.path(), &records).as_deref(), + Some("125"), + "a point-in-time restore reaches the single-apply total" + ); +} diff --git a/nodedb/src/data/runtime/event_loop.rs b/nodedb/src/data/runtime/event_loop.rs index 08ba80cc5..35b7b03a1 100644 --- a/nodedb/src/data/runtime/event_loop.rs +++ b/nodedb/src/data/runtime/event_loop.rs @@ -201,8 +201,8 @@ fn drain_and_reject(core: &mut CoreLoop, core_id: usize) { error_code: Some(Box::new(ErrorCode::Internal { detail: format!("core-{core_id} is degraded after repeated panics"), })), - read_set_valid: None, - read_version_lsn: crate::types::Lsn::ZERO, + stage_vote: None, + read_versions: crate::types::ReadVersions::new(), write_set: Vec::new(), }; if let Err(e) = core diff --git a/nodedb/src/data/runtime/params.rs b/nodedb/src/data/runtime/params.rs index 144ae55ca..a9cf895ee 100644 --- a/nodedb/src/data/runtime/params.rs +++ b/nodedb/src/data/runtime/params.rs @@ -14,6 +14,9 @@ use super::config::CoreCompactionConfig; /// Parameters for [`spawn_core`](super::spawn_core). pub struct SpawnCoreParams<'a> { pub core_id: usize, + /// The node this core runs on. Its fail points evaluate in this node's + /// scope. + pub node_id: u64, pub request_rx: Consumer, pub response_tx: Producer, pub data_dir: &'a Path, diff --git a/nodedb/src/data/runtime/spawn.rs b/nodedb/src/data/runtime/spawn.rs index 8d143e181..b116b2caa 100644 --- a/nodedb/src/data/runtime/spawn.rs +++ b/nodedb/src/data/runtime/spawn.rs @@ -29,6 +29,7 @@ pub fn spawn_core( ) -> std::io::Result<(JoinHandle<()>, EventFdNotifier)> { let SpawnCoreParams { core_id, + node_id, request_rx, response_tx, data_dir, @@ -77,6 +78,7 @@ pub fn spawn_core( array_catalog, ) .expect("failed to open CoreLoop engines"); + core.set_fail_scope(node_id); wire_core_dependencies( &mut core, diff --git a/nodedb/src/diag/context/cut_barrier.rs b/nodedb/src/diag/context/cut_barrier.rs new file mode 100644 index 000000000..4d464fd68 --- /dev/null +++ b/nodedb/src/diag/context/cut_barrier.rs @@ -0,0 +1,48 @@ +// SPDX-License-Identifier: BUSL-1.1 + +//! Forensic payloads for a backup cut's data-group barriers. + +use faultbox::DomainContext; +use faultbox::serde_json::{Value, json}; + +/// A cut's barrier driver stopped with a data group of this node that never +/// applied the cut's barrier. +pub(in crate::diag) struct CutBarrierNotPlaced { + pub group_id: u64, + /// The cut's watermark. + pub hlc: u64, + pub restore_point: u64, + /// Whether this node led the group when the driver stopped. + pub led_here: bool, + /// The vShards of the group whose scheduler here had not passed the + /// cut's marker. + pub lagging: Vec, +} + +impl DomainContext for CutBarrierNotPlaced { + fn domain_kind(&self) -> &'static str { + "nodedb.cut_barrier_not_placed" + } + + fn grouping_key(&self) -> String { + // One report per group: the watermark and the lagging vShards + // change with every cut. + format!("cut_barrier_not_placed:{}", self.group_id) + } + + fn to_json(&self) -> Value { + json!({ + "group_id": self.group_id, + "hlc": self.hlc, + "restore_point": self.restore_point, + "led_here": self.led_here, + "lagging_vshards": self.lagging, + "effect": "the cut has no barrier in this group. The backup or restore point \ + that took it failed. Calvin redo the leader held for the cut was \ + released without a barrier", + "operator_action": "check the group's leadership and the lagging vShards' \ + Calvin schedulers: a halted scheduler never passes a cut \ + marker. Retry the backup or restore point once they run", + }) + } +} diff --git a/nodedb/src/diag/context/data_plane.rs b/nodedb/src/diag/context/data_plane.rs index ef97848d4..f1859a5ee 100644 --- a/nodedb/src/diag/context/data_plane.rs +++ b/nodedb/src/diag/context/data_plane.rs @@ -109,9 +109,9 @@ pub(in crate::diag) struct CalvinApplyHalted<'a> { pub vshard_id: u32, pub epoch: u64, pub position: u32, - /// Halt reason label (`dispatch_refused`, `flush_failed`, ...). + /// Halt reason label (`dispatch_refused`, `redo_apply_failed`, ...). pub reason: &'a str, - /// Sub-operation of the txn that failed (`stage`, `flush`, ...). + /// Sub-operation of the txn that failed (`stage`, `redo_propose`, ...). pub step: &'a str, pub error: &'a str, } diff --git a/nodedb/src/diag/context/mod.rs b/nodedb/src/diag/context/mod.rs index 1a26fa9f0..695e302c4 100644 --- a/nodedb/src/diag/context/mod.rs +++ b/nodedb/src/diag/context/mod.rs @@ -11,6 +11,7 @@ mod columnar; mod continuous_agg; mod crdt; mod crdt_dead_letter_store; +mod cut_barrier; mod data_plane; mod event_image; mod index_rebuild; @@ -20,6 +21,7 @@ mod outcome_floor; mod quota; mod raft_apply; mod recovery; +mod redo_stream; mod retention; mod vector; mod vector_build; @@ -35,6 +37,7 @@ pub(in crate::diag) use crdt::{CrdtDeadLetterNotEnqueued, HistoryCompactionNotAp pub(in crate::diag) use crdt_dead_letter_store::{ CrdtDeadLetterNotRestored, CrdtDeadLetterNotStored, }; +pub(in crate::diag) use cut_barrier::CutBarrierNotPlaced; pub use data_plane::LostResponseWrite; pub(in crate::diag) use data_plane::{ CalvinApplyHalted, CalvinCompletionTimeout, CoreFailStopped, DataPlaneResponseLost, @@ -50,8 +53,11 @@ pub(in crate::diag) use quota::{ QuotaRowNotInstalled, QuotaRowWriteFailed, QuotaScopePurgeIncomplete, QuotaScopeReplayAborted, ScopeQuotaNotInstalled, }; -pub(in crate::diag) use raft_apply::{RaftEntryReapplied, ReplicatedWriteParked}; +pub(in crate::diag) use raft_apply::{CalvinBarrierLogStoreFailed, RaftEntryReapplied}; pub(in crate::diag) use recovery::{ReplayRecordUnapplied, WalArchivalFailedTruncationHeld}; +pub(in crate::diag) use redo_stream::{ + RedoAbandonGivenUp, RedoSnapshotDebtNotRecorded, RedoStreamLostHere, +}; pub(in crate::diag) use retention::RetentionAutowireOrphaned; pub(in crate::diag) use vector::VectorIndexNotApplied; pub(in crate::diag) use vector_build::{VectorBuildNotInstalled, VectorBuilderUnavailable}; diff --git a/nodedb/src/diag/context/raft_apply.rs b/nodedb/src/diag/context/raft_apply.rs index e3f06c015..13178ca52 100644 --- a/nodedb/src/diag/context/raft_apply.rs +++ b/nodedb/src/diag/context/raft_apply.rs @@ -41,39 +41,55 @@ impl DomainContext for RaftEntryReapplied { } } -/// A replicated write waited on a core for a staged Calvin transaction that -/// owns its rows. The data group's apply loop awaits it, so every later entry -/// of every group waits too. -pub(in crate::diag) struct ReplicatedWriteParked { - pub core_id: usize, - pub collection: String, - pub owner_epoch: u64, - pub owner_position: u32, - pub owner_vshard: u32, +/// The catalog refused a read or write of a vShard's stored dependent-read +/// barrier log. +pub(in crate::diag) struct CalvinBarrierLogStoreFailed<'a> { + pub vshard_id: u32, + /// The txn's `(epoch, position)`. A removal of every row of the vShard + /// names none. + pub txn: Option<(u64, u32)>, + /// What the store did: `save`, `load` or `remove`. + pub op: &'static str, + pub error_class: &'a str, } -impl DomainContext for ReplicatedWriteParked { +impl DomainContext for CalvinBarrierLogStoreFailed<'_> { fn domain_kind(&self) -> &'static str { - "nodedb.replicated_write_parked" + "nodedb.calvin_barrier_log_store_failed" } fn grouping_key(&self) -> String { - "replicated_write_parked".to_string() + // One report per vShard and operation: the txn changes per event. + format!( + "calvin_barrier_log_store_failed:{}:{}", + self.op, self.vshard_id + ) } fn to_json(&self) -> Value { + let effect = match self.op { + "save" => { + "the barrier event stays in memory, and every later event of the \ + txn too. The entry does not count as durably applied, so a restart \ + delivers it again from the log" + } + "load" => { + "the scheduler keeps the txn's barrier open and reads the row \ + again on its next pass" + } + _ => { + "the row of a finished txn stays stored. The scheduler's next \ + stall-tick sweep removes it" + } + }; json!({ - "core_id": self.core_id, - "collection": self.collection, - "owner_epoch": self.owner_epoch, - "owner_position": self.owner_position, - "owner_vshard": self.owner_vshard, - "why_fatal": "the data-group apply loop applies committed entries one at a \ - time. A parked replicated write holds it until the Calvin \ - transaction flushes or drops, so replicated writes to unrelated \ - collections wait behind an external event", - "operator_action": "check the named Calvin transaction's flush; the parked \ - write is refused at its request deadline", + "vshard_id": self.vshard_id, + "txn": self.txn, + "op": self.op, + "error_class": self.error_class, + "effect": effect, + "operator_action": "check the system catalog's disk: free space, I/O errors, \ + and the redb file's permissions", }) } } diff --git a/nodedb/src/diag/context/redo_stream.rs b/nodedb/src/diag/context/redo_stream.rs new file mode 100644 index 000000000..c3ae03284 --- /dev/null +++ b/nodedb/src/diag/context/redo_stream.rs @@ -0,0 +1,130 @@ +// SPDX-License-Identifier: BUSL-1.1 + +//! Forensic payloads for chunked redo streams: an abandon that stopped +//! retrying, and a stream entry held on a replica that dropped the stream. + +use faultbox::DomainContext; +use faultbox::serde_json::{Value, json}; + +/// The abandoner stopped retrying a session stream's abandon while the +/// stream was open on this node. +pub(in crate::diag) struct RedoAbandonGivenUp { + /// The stream, as its `Debug` text. + pub stream: String, + pub vshard: u32, + /// The data group whose log carries the stream. + pub group_id: u64, + /// The Raft term of the stream's first chunk entry. + pub term: u64, + /// The stream's declared byte length. + pub len: u64, + /// The last abandon proposal's error, when one ran. + pub last_error: Option, +} + +impl DomainContext for RedoAbandonGivenUp { + fn domain_kind(&self) -> &'static str { + "nodedb.redo_abandon_given_up" + } + + fn grouping_key(&self) -> String { + // One bug class: an abandon that never applied before the node + // stopped. The stream and its group are the occurrence. + "redo_abandon_given_up".to_string() + } + + fn to_json(&self) -> Value { + json!({ + "stream": self.stream, + "vshard": self.vshard, + "group_id": self.group_id, + "term": self.term, + "len": self.len, + "last_error": self.last_error, + "impact": "every replica holds the stream's chunks and a WAL floor at its \ + first chunk record. They stay until the group applies an entry \ + of a later term, so the WAL on each replica cannot truncate past \ + that record meanwhile", + "operator_action": "a leadership transfer of the named group ends the term \ + and drops the stream on every replica. The last error \ + names why the abandon proposal did not apply", + }) + } +} + +/// A chunk or final entry of a redo stream refused on a replica that owes +/// its group a snapshot install: this node dropped the group's streams when +/// it left the group, and a remount resumed the group's old log. +pub(in crate::diag) struct RedoStreamLostHere { + /// The data group whose log carries the stream. + pub group_id: u64, + /// The held entry's index in the group's log. + pub log_index: u64, + /// The stream, as its `Debug` text. + pub stream: String, + pub vshard: u32, + /// `chunk` or `final`. + pub entry: &'static str, + /// Why this replica's stream refused the entry. + pub refusal: String, +} + +impl DomainContext for RedoStreamLostHere { + fn domain_kind(&self) -> &'static str { + "nodedb.redo_stream_lost_here" + } + + fn grouping_key(&self) -> String { + // One bug class: a log resumed past streams this replica dropped. + // The group, stream, and index are the occurrence. + "redo_stream_lost_here".to_string() + } + + fn to_json(&self) -> Value { + json!({ + "group_id": self.group_id, + "log_index": self.log_index, + "stream": self.stream, + "vshard": self.vshard, + "entry": self.entry, + "refusal": self.refusal, + "impact": "the other replicas can install the stream's redo, which this replica \ + lacks. Its durable floor holds at the entry, so a restart applies the \ + entry again. Later entries of the group apply here on the state \ + without the redo until a snapshot of the group installs", + "operator_action": "the node's membership pass makes the group's replica here \ + refuse log entries, and moves a leadership it holds. The \ + group's leader then sends a snapshot. When the group has no \ + other replica, no snapshot comes: restore the group from a \ + backup", + }) + } +} + +/// The membership pass failed to persist that data groups this node left +/// owe a snapshot install, so their redo streams stay. +pub(in crate::diag) struct RedoSnapshotDebtNotRecorded { + /// The catalog error. + pub error: String, +} + +impl DomainContext for RedoSnapshotDebtNotRecorded { + fn domain_kind(&self) -> &'static str { + "nodedb.redo_snapshot_debt_not_recorded" + } + + fn grouping_key(&self) -> String { + // The pass retries on every tick: one report per root cause. + "redo_snapshot_debt_not_recorded".to_string() + } + + fn to_json(&self) -> Value { + json!({ + "error": self.error, + "impact": "the left groups' redo streams and their WAL floor holds stay on this \ + node, so the WAL cannot truncate past their first chunk records", + "operator_action": "the error names why the system catalog refused the write. \ + The pass retries on every membership tick", + }) + } +} diff --git a/nodedb/src/diag/mod.rs b/nodedb/src/diag/mod.rs index c2f3ec656..82573cd7b 100644 --- a/nodedb/src/diag/mod.rs +++ b/nodedb/src/diag/mod.rs @@ -11,16 +11,17 @@ mod recording; pub use context::{DATABASE_SCOPE, IlpFlushOutcome, LostResponseWrite, TENANT_SCOPE}; pub use recording::{ IndexRebuildTarget, VectorBuildTarget, batch_insert_without_surrogates, calvin_apply_halted, - calvin_completion_timeout, catalog_apply_orphan_row, collection_purge_row_missing, - columnar_segment_corrupt, consumer_group_offsets_retained, continuous_aggregate_not_applied, - crdt_dead_letter_not_enqueued, crdt_dead_letter_not_restored, crdt_dead_letter_not_stored, - data_plane_core_fail_stopped, data_plane_response_lost, data_plane_responses_lost, - descriptor_lease_not_renewed, entry_kind, fts_index_update_failed, - history_compaction_not_applied, ilp_invalid_utf8_drop, ilp_line_read_drop, - index_rebuild_not_installed, metadata_apply_wedged, orphaned_index_entry_after_delete, - quota_row_invalid, quota_row_undecodable, quota_row_write_failed, quota_scope_purge_incomplete, - quota_scope_replay_aborted, raft_entries_reapplied, raft_entry_reapplied, - replay_record_unapplied, replicated_write_parked, replicated_writes_parked, + calvin_barrier_log_store_failed, calvin_completion_timeout, catalog_apply_orphan_row, + collection_purge_row_missing, columnar_segment_corrupt, consumer_group_offsets_retained, + continuous_aggregate_not_applied, crdt_dead_letter_not_enqueued, crdt_dead_letter_not_restored, + crdt_dead_letter_not_stored, cut_barrier_not_placed, data_plane_core_fail_stopped, + data_plane_response_lost, data_plane_responses_lost, descriptor_lease_not_renewed, entry_kind, + fts_index_update_failed, history_compaction_not_applied, ilp_invalid_utf8_drop, + ilp_line_read_drop, index_rebuild_not_installed, metadata_apply_wedged, + orphaned_index_entry_after_delete, quota_row_invalid, quota_row_undecodable, + quota_row_write_failed, quota_scope_purge_incomplete, quota_scope_replay_aborted, + raft_entries_reapplied, raft_entry_reapplied, redo_abandon_given_up, + redo_snapshot_debt_not_recorded, redo_stream_lost_here, replay_record_unapplied, retention_autowire_orphaned, scope_quota_not_installed, strict_row_image_unrendered, strict_row_undecodable, synonym_group_not_applied, timeseries_partition_unreadable, timeseries_row_image_undecodable, vector_build_failed, vector_builder_disconnected, diff --git a/nodedb/src/diag/recording/cut_barrier.rs b/nodedb/src/diag/recording/cut_barrier.rs new file mode 100644 index 000000000..fdf0812a4 --- /dev/null +++ b/nodedb/src/diag/recording/cut_barrier.rs @@ -0,0 +1,32 @@ +// SPDX-License-Identifier: BUSL-1.1 + +//! Capture sites of a backup cut's data-group barriers. + +use faultbox::{Capture, EventKind}; + +use crate::diag::context; + +/// Report that a cut's barrier driver stopped before `group_id` applied the +/// cut's barrier on this node. Called only from the driver, the one site +/// that sees the cut's ordering window close. +pub fn cut_barrier_not_placed( + group_id: u64, + hlc: u64, + restore_point: u64, + led_here: bool, + lagging: Vec, +) { + let ctx = context::CutBarrierNotPlaced { + group_id, + hlc, + restore_point, + led_here, + lagging, + }; + let _ = Capture::new( + EventKind::Error, + "backup cut barrier not placed in a data group", + ) + .domain(&ctx) + .emit(); +} diff --git a/nodedb/src/diag/recording/mod.rs b/nodedb/src/diag/recording/mod.rs index c34f9d202..4a9e24f17 100644 --- a/nodedb/src/diag/recording/mod.rs +++ b/nodedb/src/diag/recording/mod.rs @@ -12,6 +12,7 @@ mod catalog; mod columnar; mod continuous_agg; mod crdt; +mod cut_barrier; mod data_plane; mod event_image; mod index_rebuild; @@ -21,6 +22,7 @@ mod outcome_floor; mod quota; mod raft_apply; mod recovery; +mod redo_stream; mod retention; mod shared; mod vector; @@ -36,6 +38,7 @@ pub use crdt::{ crdt_dead_letter_not_enqueued, crdt_dead_letter_not_restored, crdt_dead_letter_not_stored, history_compaction_not_applied, }; +pub use cut_barrier::cut_barrier_not_placed; pub use data_plane::{ calvin_apply_halted, calvin_completion_timeout, data_plane_core_fail_stopped, data_plane_response_lost, data_plane_responses_lost, @@ -50,13 +53,16 @@ pub use quota::{ quota_scope_replay_aborted, scope_quota_not_installed, }; pub use raft_apply::{ - raft_entries_reapplied, raft_entry_reapplied, replicated_write_parked, replicated_writes_parked, + calvin_barrier_log_store_failed, raft_entries_reapplied, raft_entry_reapplied, }; pub use recovery::{ batch_insert_without_surrogates, fts_index_update_failed, orphaned_index_entry_after_delete, replay_record_unapplied, strict_row_undecodable, wal_archival_failed_truncation_held, write_acked_without_durability, }; +pub use redo_stream::{ + redo_abandon_given_up, redo_snapshot_debt_not_recorded, redo_stream_lost_here, +}; pub use retention::retention_autowire_orphaned; pub use shared::entry_kind; pub use vector::vector_index_not_applied; diff --git a/nodedb/src/diag/recording/raft_apply.rs b/nodedb/src/diag/recording/raft_apply.rs index 55152d53e..8799ddacf 100644 --- a/nodedb/src/diag/recording/raft_apply.rs +++ b/nodedb/src/diag/recording/raft_apply.rs @@ -4,28 +4,21 @@ use std::sync::atomic::{AtomicU64, Ordering}; -use faultbox::{Capture, EventKind}; +use faultbox::{Capture, EventKind, error_chain_of}; + +use super::shared::error_class; use crate::diag::context; /// Count of committed Raft entries the apply loop applied a second time. static RAFT_ENTRIES_REAPPLIED: AtomicU64 = AtomicU64::new(0); -/// Count of replicated writes parked behind a staged Calvin transaction. -static REPLICATED_WRITES_PARKED: AtomicU64 = AtomicU64::new(0); - /// Read the count of committed Raft entries applied twice. Exposed for the /// metrics exporter and tests. pub fn raft_entries_reapplied() -> u64 { RAFT_ENTRIES_REAPPLIED.load(Ordering::Relaxed) } -/// Read the count of replicated writes parked behind a staged Calvin -/// transaction. Exposed for the metrics exporter and tests. -pub fn replicated_writes_parked() -> u64 { - REPLICATED_WRITES_PARKED.load(Ordering::Relaxed) -} - /// Report an apply of a Raft log index the apply loop already applied. /// Called from the apply loop, the one site that sees every applied index. pub fn raft_entry_reapplied(group_id: u64, log_index: u64, highest_applied: u64) { @@ -44,22 +37,26 @@ pub fn raft_entry_reapplied(group_id: u64, log_index: u64, highest_applied: u64) .emit(); } -/// Report a replicated write a core parked behind a staged Calvin -/// transaction. Called from the core's Calvin fence when it parks a write -/// whose order Raft already fixed. -pub fn replicated_write_parked(core_id: usize, collection: &str, owner: (u64, u32, u32)) { - REPLICATED_WRITES_PARKED.fetch_add(1, Ordering::Relaxed); - let ctx = context::ReplicatedWriteParked { - core_id, - collection: collection.to_owned(), - owner_epoch: owner.0, - owner_position: owner.1, - owner_vshard: owner.2, +/// Report a catalog error on a vShard's stored dependent-read barrier log. +/// Called from the sites that read and write the rows. `op` names the operation: `save`, `load` or `remove`. +pub fn calvin_barrier_log_store_failed( + vshard_id: u32, + txn: Option<(u64, u32)>, + op: &'static str, + err: &crate::Error, +) { + let class = error_class(err); + let ctx = context::CalvinBarrierLogStoreFailed { + vshard_id, + txn, + op, + error_class: &class, }; let _ = Capture::new( EventKind::Error, - "replicated write parked behind a staged Calvin transaction", + "calvin dependent-read barrier log store failed", ) + .error_chain(error_chain_of(err)) .domain(&ctx) .emit(); } diff --git a/nodedb/src/diag/recording/redo_stream.rs b/nodedb/src/diag/recording/redo_stream.rs new file mode 100644 index 000000000..e0c73936e --- /dev/null +++ b/nodedb/src/diag/recording/redo_stream.rs @@ -0,0 +1,76 @@ +// SPDX-License-Identifier: BUSL-1.1 + +//! Capture sites for chunked redo streams: an abandon that stopped retrying, +//! and a stream entry held on a replica that dropped the stream. + +use faultbox::{Capture, EventKind, error_chain_of}; + +use crate::control::wal_replication::transaction_redo::chunks::{DueAbandon, RedoChunkError}; +use crate::diag::context; +use crate::wal::RedoStreamId; + +/// Report a session stream left open when the abandoner stopped. Called from +/// the abandoner's exit, the one site that gives the abandon up. +pub fn redo_abandon_given_up(abandon: &DueAbandon, last_error: Option<&crate::Error>) { + let ctx = context::RedoAbandonGivenUp { + stream: format!("{:?}", abandon.stream), + vshard: abandon.stream.vshard(), + group_id: abandon.group_id, + term: abandon.term, + len: abandon.len, + last_error: last_error.map(ToString::to_string), + }; + let _ = Capture::new( + EventKind::Error, + "redo stream abandon given up: the stream stays open until its group's next term", + ) + .domain(&ctx) + .emit(); +} + +/// Report a refusal of `stream`'s `entry` (`chunk` or `final`) at log index +/// `log_index` of `group_id`, on a replica that dropped the group's streams +/// when it left the group. The other replicas can hold the stream and +/// install its redo. Called from the data-group apply loop, the one site +/// that holds the entry. +pub fn redo_stream_lost_here( + group_id: u64, + log_index: u64, + stream: RedoStreamId, + entry: &'static str, + refusal: &RedoChunkError, +) { + let ctx = context::RedoStreamLostHere { + group_id, + log_index, + stream: format!("{stream:?}"), + vshard: stream.vshard(), + entry, + refusal: refusal.to_string(), + }; + let _ = Capture::new( + EventKind::InvariantViolation, + "redo stream entry refused on a replica that dropped the stream: held until a snapshot \ + of the group installs", + ) + .domain(&ctx) + .error_chain(error_chain_of(refusal)) + .emit(); +} + +/// Report that the snapshot debt of data groups this node left did not +/// persist. Their streams stay, with their WAL floor holds, until a later +/// membership pass records it. Called from that pass, the one site that +/// writes the debt. +pub fn redo_snapshot_debt_not_recorded(error: &crate::Error) { + let ctx = context::RedoSnapshotDebtNotRecorded { + error: error.to_string(), + }; + let _ = Capture::new( + EventKind::Error, + "snapshot debt of left data groups did not persist: their redo streams stay held", + ) + .domain(&ctx) + .error_chain(error_chain_of(error)) + .emit(); +} diff --git a/nodedb/src/engine/array/store/catalog/scan.rs b/nodedb/src/engine/array/store/catalog/scan.rs index 9289ad5f3..d44637572 100644 --- a/nodedb/src/engine/array/store/catalog/scan.rs +++ b/nodedb/src/engine/array/store/catalog/scan.rs @@ -208,15 +208,18 @@ impl ArrayStore { valid_at_ms: Option, ) -> nodedb_array::ArrayResult { use nodedb_array::query::ceiling::CeilingParams; - // Find the hilbert_prefix for this coord. - let hilbert_prefix = { - use nodedb_array::tile::tile_id_for_cell; - let tile = tile_id_for_cell(&self.schema, coord, 0).map_err(|e| { - nodedb_array::ArrayError::SegmentCorruption { + // Find the hilbert_prefix for this coord. A coordinate outside the + // domain has no tile: a cell write refuses it, so no cell is there. + let hilbert_prefix = match nodedb_array::tile::tile_id_for_cell(&self.schema, coord, 0) { + Ok(tile) => tile.hilbert_prefix, + Err(nodedb_array::ArrayError::CoordOutOfDomain { .. }) => { + return Ok(CeilingResult::NotFound); + } + Err(e) => { + return Err(nodedb_array::ArrayError::SegmentCorruption { detail: format!("ceiling_for_coord: tile id: {e}"), - } - })?; - tile.hilbert_prefix + }); + } }; let versions = self.cell_versions_for_coord(hilbert_prefix, coord, system_as_of)?; let params = CeilingParams { diff --git a/nodedb/src/error_from.rs b/nodedb/src/error_from.rs index fb0d65b0e..2cc32a820 100644 --- a/nodedb/src/error_from.rs +++ b/nodedb/src/error_from.rs @@ -250,9 +250,15 @@ impl From for Error { descriptor: collection, } } - TypedClusterError::DeadlineExceeded { .. } => Error::DeadlineExceeded { - request_id: crate::types::RequestId::new(0), - }, + TypedClusterError::DeadlineExceeded { elapsed_ms } => { + tracing::warn!( + elapsed_ms, + "a remote node reported the request's deadline exceeded" + ); + Error::DeadlineExceeded { + request_id: crate::types::RequestId::new(0), + } + } // A remote shard's Data-Plane verdict, verbatim: rebuilding the // code keeps the SQLSTATE `error_classify` renders locally. TypedClusterError::DataPlane { code } => Error::DataPlane(code.into()), diff --git a/nodedb/src/error_from_data_plane.rs b/nodedb/src/error_from_data_plane.rs index e8cd847ed..8e11b6993 100644 --- a/nodedb/src/error_from_data_plane.rs +++ b/nodedb/src/error_from_data_plane.rs @@ -55,6 +55,12 @@ pub(crate) fn data_plane_code_to_public(code: ErrorCode) -> NodeDbError { PublicCode::WRITE_CONFLICT, format!("write refused without applying, retry: {reason}"), ), + // The core applied nothing, and another replica or a restart serves + // the request: the retriable class. + ErrorCode::CoreFailStopped { core_id, detail } => NodeDbError::from_wire( + PublicCode::WRITE_CONFLICT, + format!("core {core_id} is fail-stopped and applied nothing, retry: {detail}"), + ), // The Data Plane cannot distinguish an absent collection from an // absent row through this code, and `document_not_found` is the // narrower of the two claims: it never asserts the collection is diff --git a/nodedb/src/event/bus.rs b/nodedb/src/event/bus.rs index a66633712..7579f3c26 100644 --- a/nodedb/src/event/bus.rs +++ b/nodedb/src/event/bus.rs @@ -27,20 +27,6 @@ use super::types::WriteEvent; /// Default ring buffer capacity per core (must be power of two). const DEFAULT_EVENT_BUS_CAPACITY: usize = 65_536; -/// Whether a test armed fail point `event::bus::drop::` for -/// `event`'s collection. The event is then lost as a full ring loses it: its -/// sequence is spent and the Event Plane recovers it from the WAL. -#[cfg(feature = "failpoints")] -fn dropped_by_fail_point(event: &WriteEvent) -> bool { - crate::fail_point::eval_fail(&format!("event::bus::drop::{}", event.collection)).is_some() -} - -/// Without the `failpoints` feature no event is dropped here. -#[cfg(not(feature = "failpoints"))] -fn dropped_by_fail_point(_event: &WriteEvent) -> bool { - false -} - /// The producer half given to a Data Plane core. /// /// `Send` at the trait level, but **logically owned by exactly one @@ -69,6 +55,9 @@ pub struct EventProducer { /// disconnect exactly once per producer instead of spamming /// a warning for every dropped event. disconnect_logged: AtomicBool, + /// The scope of the ring-drop fail point: the owning core's node. + #[cfg(feature = "failpoints")] + fail_scope: crate::fail_point::FailScope, } impl EventProducer { @@ -84,7 +73,7 @@ impl EventProducer { pub fn emit(&mut self, mut event: WriteEvent) -> bool { self.numbering.stamp(&mut event); let sequence = event.sequence; - let pushed = if dropped_by_fail_point(&event) { + let pushed = if self.dropped_by_fail_point(&event) { false } else { self.push(event) @@ -95,6 +84,31 @@ impl EventProducer { pushed } + /// Scope this producer's ring-drop fail point to the owning core's node. + #[cfg(feature = "failpoints")] + pub fn set_fail_scope(&mut self, scope: crate::fail_point::FailScope) { + self.fail_scope = scope; + } + + /// Whether a test armed fail point `event::bus::drop::` for + /// `event`'s collection on this producer's node. The event is then lost + /// as a full ring loses it: its sequence is spent and the Event Plane + /// recovers it from the WAL. + #[cfg(feature = "failpoints")] + fn dropped_by_fail_point(&self, event: &WriteEvent) -> bool { + crate::fail_point::eval_fail( + self.fail_scope, + &format!("event::bus::drop::{}", event.collection), + ) + .is_some() + } + + /// Without the `failpoints` feature no event is dropped here. + #[cfg(not(feature = "failpoints"))] + fn dropped_by_fail_point(&self, _event: &WriteEvent) -> bool { + false + } + /// Update the backpressure state, then push `event` onto the ring. fn push(&mut self, event: WriteEvent) -> bool { let util = self.inner.utilization(); @@ -242,6 +256,8 @@ pub fn create_event_bus_with_capacity( progress: Arc::clone(&progress), numbering: RecordNumbering::new(), disconnect_logged: AtomicBool::new(false), + #[cfg(feature = "failpoints")] + fail_scope: crate::fail_point::FailScope::Any, }); consumers.push(EventConsumerRx { diff --git a/nodedb/src/event/cdc/consume/remote.rs b/nodedb/src/event/cdc/consume/remote.rs index a65e5ca48..3cdcff989 100644 --- a/nodedb/src/event/cdc/consume/remote.rs +++ b/nodedb/src/event/cdc/consume/remote.rs @@ -19,7 +19,6 @@ use crate::control::server::shared::ddl::neutral::consumer_group::identity::cano use crate::control::state::SharedState; use crate::event::cdc::event::CdcEvent; use crate::event::cdc::offset::CdcOffset; -use crate::event::cdc::position::vshard_of_partition; use super::error::ConsumeError; use super::params::{ConsumeParams, ConsumeResult, batch_tails}; @@ -133,7 +132,7 @@ fn partition_members( .as_ref() .ok_or(ConsumeError::NoClusterRouting)?; let routing = routing_lock.read().unwrap_or_else(|p| p.into_inner()); - let Ok(group_id) = routing.group_for_vshard(vshard_of_partition(partition_id)) else { + let Ok(group_id) = routing.group_for_vshard(partition_id) else { return Ok(None); }; Ok(routing.group_info(group_id).map(|group| PartitionMembers { diff --git a/nodedb/src/event/cdc/consume/sink.rs b/nodedb/src/event/cdc/consume/sink.rs index bb5e9250f..b959c3263 100644 --- a/nodedb/src/event/cdc/consume/sink.rs +++ b/nodedb/src/event/cdc/consume/sink.rs @@ -18,7 +18,6 @@ use std::sync::Arc; use crate::control::state::SharedState; use crate::event::cdc::event::CdcEvent; -use crate::event::cdc::position::vshard_of_partition; use super::error::ConsumeError; use super::local::consume_stream; @@ -76,9 +75,9 @@ impl ReadPlan { } } - /// The data group of `partition`. + /// The data group of `partition`, a vShard. fn group_of(&self, partition: u32) -> Option { - let vshard = usize::try_from(vshard_of_partition(partition)).ok()?; + let vshard = usize::try_from(partition).ok()?; self.vshard_group.get(vshard).copied() } @@ -238,7 +237,6 @@ fn empty() -> ConsumeResult { #[cfg(test)] mod tests { use super::*; - use crate::event::cdc::position::calvin_partition; fn event(partition: u32) -> CdcEvent { CdcEvent { @@ -278,9 +276,8 @@ mod tests { assert!(plan.keeps(&plan.local, &event(0))); assert!(!plan.keeps(&plan.local, &event(1))); assert!(plan.keeps(&plan.remote[&2], &event(1))); - // A Calvin partition belongs to its vShard's group. - assert!(plan.keeps(&plan.remote[&3], &event(calvin_partition(2)))); - assert!(!plan.keeps(&plan.remote[&2], &event(calvin_partition(2)))); + assert!(plan.keeps(&plan.remote[&3], &event(2))); + assert!(!plan.keeps(&plan.remote[&2], &event(2))); } #[test] diff --git a/nodedb/src/event/cdc/offset.rs b/nodedb/src/event/cdc/offset.rs index 6595bbb56..32e10d562 100644 --- a/nodedb/src/event/cdc/offset.rs +++ b/nodedb/src/event/cdc/offset.rs @@ -11,13 +11,12 @@ //! metadata entry is the one event that raises it. No path proposes one, so //! every position has epoch `0`. //! - `index` names the write that produced the event. In a cluster it is the -//! Raft log index of the data-group entry, or the Calvin sequencer epoch of a -//! Calvin transaction. On a single node it is the WAL LSN of the write's -//! record. On a durable topic it is the message's log position. -//! - `sequence` orders the events of one write. Its low 32 bits carry -//! `2 × ordinal` for a data event and `2 × ordinal + 1` for that event's -//! late-data correction. Its high 32 bits carry the Calvin transaction's -//! position within its sequencer epoch, `0` for every other write. +//! Raft log index of the data-group entry. On a single node it is the WAL +//! LSN of the write's record. On a durable topic it is the message's log +//! position. +//! - `sequence` orders the events of one write. It carries `2 × ordinal` for +//! a data event and `2 × ordinal + 1` for that event's late-data +//! correction. //! //! Every replica applies a write at the same position, so a cursor from one //! node is valid on every node. @@ -29,6 +28,7 @@ use serde::{Deserialize, Serialize}; /// Bits of `sequence` that carry the ordinal and correction flag. const ORDINAL_BITS: u32 = 32; +/// The largest sequence a data event or its correction takes. const ORDINAL_MASK: u64 = (1 << ORDINAL_BITS) - 1; /// A lossless position in a CDC stream partition. @@ -104,25 +104,14 @@ impl CdcOffset { } /// The position of the data event with ordinal `ordinal` (1-based) in the - /// write at `(epoch, index)`, whose sequence carries `base` in its high - /// bits. - pub const fn data_event_in(epoch: u64, index: u64, base: u64, ordinal: u64) -> Self { - let low = if ordinal > (ORDINAL_MASK >> 1) { + /// write at `(epoch, index)`. + pub const fn data_event(epoch: u64, index: u64, ordinal: u64) -> Self { + let sequence = if ordinal > (ORDINAL_MASK >> 1) { ORDINAL_MASK - 1 } else { ordinal * 2 }; - Self::at(epoch, index, (base << ORDINAL_BITS) | low) - } - - /// [`Self::data_event_in`] with base `0`. - pub const fn data_event(epoch: u64, index: u64, ordinal: u64) -> Self { - Self::data_event_in(epoch, index, 0, ordinal) - } - - /// The high bits of the sequence: the Calvin position, `0` otherwise. - pub const fn base(self) -> u64 { - self.sequence >> ORDINAL_BITS + Self::at(epoch, index, sequence) } /// The ordinal of the data event this position belongs to. A correction @@ -211,16 +200,6 @@ mod tests { assert_eq!(second.ordinal(), 2); } - #[test] - fn calvin_positions_order_by_transaction_then_ordinal() { - let early = CdcOffset::data_event_in(0, 7, 3, 9); - let late = CdcOffset::data_event_in(0, 7, 4, 1); - assert!(early < late); - assert_eq!(late.base(), 4); - assert_eq!(late.ordinal(), 1); - assert!(early.correction() < late); - } - #[test] fn equal_positions_from_two_replicas_compare_equal() { let leader = CdcOffset::data_event(2, 42, 3); diff --git a/nodedb/src/event/cdc/position/ledger.rs b/nodedb/src/event/cdc/position/ledger.rs index a923aaea1..ac811f0e2 100644 --- a/nodedb/src/event/cdc/position/ledger.rs +++ b/nodedb/src/event/cdc/position/ledger.rs @@ -4,15 +4,14 @@ //! //! A Data-Plane change event names its write by the local WAL LSN of the //! write's record. That LSN differs on every replica. The Raft log position -//! of the entry the record applies, or the Calvin sequencer position of the -//! transaction it applies, is the same on every replica, so the CDC router -//! positions events by it. +//! of the entry the record applies is the same on every replica, so the CDC +//! router positions events by it. A committed Calvin slice installs from a +//! data-group entry, so its records take that entry's position too. //! //! The write path records `record LSN -> position` before it hands the write //! to a core, so the entry exists before any event of the write can reach //! the Event Plane. Boot rebuilds the ledger from the WAL: a Raft entry's -//! records link to its `ChangePosition` marker by proposal key, and a Calvin -//! transaction's redo record carries its sequencer position itself. A WAL +//! records link to its `ChangePosition` marker by proposal key. A WAL //! catch-up that rebuilds an event after a restart therefore positions it as //! the ring did. @@ -23,7 +22,7 @@ use std::sync::atomic::{AtomicBool, Ordering}; use nodedb_wal::WalRecord; use nodedb_wal::record::RecordType; -use super::marker::{CalvinPosition, ChangePositionMarker, ReplicatedPosition, WritePosition}; +use super::marker::{ChangePositionMarker, ReplicatedPosition}; /// Records the ledger keeps before it evicts the lowest LSN. An event whose /// record was evicted before the Event Plane routed it takes its partition's @@ -33,7 +32,7 @@ pub const CHANGE_POSITION_CAPACITY: usize = 1 << 18; /// Local record LSN to replicated position, bounded. #[derive(Debug)] pub struct ChangePositionLedger { - positions: RwLock>, + positions: RwLock>, /// Record LSN to the commit HLC (wall nanoseconds) of the write the /// record belongs to. Every replica holds the same value for a /// replicated write, so change events date by it. @@ -64,16 +63,6 @@ impl ChangePositionLedger { /// `position`. pub fn record(&self, record_lsn: u64, position: ReplicatedPosition) { self.mark_replicated(); - self.insert(record_lsn, WritePosition::Raft(position)); - } - - /// Record that the WAL record at `record_lsn` applies the Calvin - /// transaction at `position`. - pub fn record_calvin(&self, record_lsn: u64, position: CalvinPosition) { - self.insert(record_lsn, WritePosition::Calvin(position)); - } - - fn insert(&self, record_lsn: u64, position: WritePosition) { let mut positions = self.positions.write().unwrap_or_else(|p| p.into_inner()); positions.insert(record_lsn, position); while positions.len() > self.capacity { @@ -102,7 +91,7 @@ impl ChangePositionLedger { } /// The replicated position of the record at `record_lsn`, if known. - pub fn get(&self, record_lsn: u64) -> Option { + pub fn get(&self, record_lsn: u64) -> Option { self.positions .read() .unwrap_or_else(|p| p.into_inner()) @@ -125,9 +114,7 @@ impl ChangePositionLedger { /// order. Returns how many records it positioned. /// /// A record whose header key names a preceding `ChangePosition` marker - /// takes that marker's Raft position. Otherwise a `TransactionRedo` - /// record that a Calvin flush wrote takes its stamp's sequencer - /// position. + /// takes that marker's Raft position. pub fn recover(&self, records: &[WalRecord]) -> usize { let mut by_key: HashMap = HashMap::new(); let mut positioned = 0; @@ -156,13 +143,6 @@ impl ChangePositionLedger { { self.record(record.header.lsn, *position); positioned += 1; - continue; - } - if record_type == Some(RecordType::TransactionRedo) - && let Some(position) = calvin_position(record) - { - self.record_calvin(record.header.lsn, position); - positioned += 1; } } if !by_key.is_empty() { @@ -183,16 +163,6 @@ impl ChangePositionLedger { } } -/// The sequencer position a Calvin flush stamped on a redo record. -fn calvin_position(record: &WalRecord) -> Option { - let redo = crate::wal::RedoRecord::from_bytes(&record.payload).ok()?; - let stamp = redo.calvin_stamp?; - Some(CalvinPosition { - sequencer_epoch: stamp.epoch, - position: stamp.position, - }) -} - #[cfg(test)] mod tests { use super::*; @@ -205,8 +175,8 @@ mod tests { } } - fn raft(log_index: u64) -> Option { - Some(WritePosition::Raft(position(log_index))) + fn raft(log_index: u64) -> Option { + Some(position(log_index)) } #[test] @@ -217,12 +187,6 @@ mod tests { assert!(ledger.is_replicated()); assert_eq!(ledger.get(100), raft(7)); assert_eq!(ledger.get(101), None); - let calvin = CalvinPosition { - sequencer_epoch: 4, - position: 2, - }; - ledger.record_calvin(102, calvin); - assert_eq!(ledger.get(102), Some(WritePosition::Calvin(calvin))); } fn record(record_type: RecordType, lsn: u64, apply_key: u64, payload: Vec) -> WalRecord { diff --git a/nodedb/src/event/cdc/position/marker.rs b/nodedb/src/event/cdc/position/marker.rs index 8150fac4b..fc6563258 100644 --- a/nodedb/src/event/cdc/position/marker.rs +++ b/nodedb/src/event/cdc/position/marker.rs @@ -13,22 +13,6 @@ pub struct ReplicatedPosition { pub log_index: u64, } -/// Where a Calvin transaction sits in the sequencer log. Every replica's -/// scheduler applies the transaction at the same value. -#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)] -pub struct CalvinPosition { - pub sequencer_epoch: u64, - /// Zero-based position within the sequencer epoch's batch. - pub position: u32, -} - -/// The replicated position of one locally applied write. -#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)] -pub enum WritePosition { - Raft(ReplicatedPosition), - Calvin(CalvinPosition), -} - /// Byte length of an encoded [`ChangePositionMarker`]. pub const MARKER_LEN: usize = 32; diff --git a/nodedb/src/event/cdc/position/mod.rs b/nodedb/src/event/cdc/position/mod.rs index 2a3c9078f..950a1acd4 100644 --- a/nodedb/src/event/cdc/position/mod.rs +++ b/nodedb/src/event/cdc/position/mod.rs @@ -4,15 +4,10 @@ pub mod availability; pub mod epoch; pub mod ledger; pub mod marker; -pub mod partition; pub mod sequencer; pub use availability::AvailabilityFloors; pub use epoch::{entry_position, record_install_floor}; pub use ledger::{CHANGE_POSITION_CAPACITY, ChangePositionLedger}; -pub use marker::{ - CalvinPosition, ChangePositionMarker, MARKER_LEN, MarkerDecodeError, ReplicatedPosition, - WritePosition, -}; -pub use partition::{CALVIN_PARTITION_BASE, calvin_partition, vshard_of_partition}; +pub use marker::{ChangePositionMarker, MARKER_LEN, MarkerDecodeError, ReplicatedPosition}; pub use sequencer::{IndexSource, PartitionTail, PositionSequencer}; diff --git a/nodedb/src/event/cdc/position/partition.rs b/nodedb/src/event/cdc/position/partition.rs deleted file mode 100644 index aba4b1a6c..000000000 --- a/nodedb/src/event/cdc/position/partition.rs +++ /dev/null @@ -1,35 +0,0 @@ -// SPDX-License-Identifier: BUSL-1.1 - -//! Change-feed partitions. -//! -//! A vShard's Raft-applied writes and its Calvin transactions come from two -//! separate replicated logs. Every replica applies each log in order, but the -//! two interleave differently on each replica, so they cannot share one -//! position order. Each log therefore has its own partition: partition `v` -//! carries vShard `v`'s Raft-applied writes, and partition -//! `CALVIN_PARTITION_BASE + v` carries its Calvin transactions. - -/// First partition of the Calvin partitions. -pub const CALVIN_PARTITION_BASE: u32 = 1 << 16; - -/// The partition of vShard `vshard`'s Calvin transactions. -pub const fn calvin_partition(vshard: u32) -> u32 { - CALVIN_PARTITION_BASE + vshard -} - -/// The vShard a partition belongs to. -pub const fn vshard_of_partition(partition: u32) -> u32 { - partition % CALVIN_PARTITION_BASE -} - -#[cfg(test)] -mod tests { - use super::*; - - #[test] - fn a_calvin_partition_maps_back_to_its_vshard() { - assert_eq!(vshard_of_partition(calvin_partition(17)), 17); - assert_eq!(vshard_of_partition(17), 17); - assert_ne!(calvin_partition(17), 17); - } -} diff --git a/nodedb/src/event/cdc/position/sequencer.rs b/nodedb/src/event/cdc/position/sequencer.rs index 660ff943d..c69af4324 100644 --- a/nodedb/src/event/cdc/position/sequencer.rs +++ b/nodedb/src/event/cdc/position/sequencer.rs @@ -24,9 +24,8 @@ use crate::event::cdc::offset::CdcOffset; /// Where the position of an event comes from. #[derive(Debug, Clone, Copy, PartialEq, Eq)] pub enum IndexSource { - /// A replicated write: a Raft entry at `(epoch, index)`, or a Calvin - /// transaction at sequencer epoch `index` and batch position `base`. - Replicated { epoch: u64, index: u64, base: u64 }, + /// A replicated write: a Raft entry at `(epoch, index)`. + Replicated { epoch: u64, index: u64 }, /// The event's write has no known replicated position on a node that /// positions by it. The event joins the partition's current write. Unmapped, @@ -73,28 +72,25 @@ impl PositionSequencer { }) }); let last = tail.position; - let (epoch, index, base, continues) = match source { - IndexSource::Replicated { epoch, index, base } => ( + let (epoch, index, continues) = match source { + IndexSource::Replicated { epoch, index } => ( epoch, index, - base, - (epoch, index, base) == (last.epoch, last.index, last.base()) - && record_lsn == tail.record_lsn, + (epoch, index) == (last.epoch, last.index) && record_lsn == tail.record_lsn, ), IndexSource::Local(index) => ( 0, index, - 0, (0, index) == (last.epoch, last.index) && record_lsn == tail.record_lsn, ), - IndexSource::Unmapped => (last.epoch, last.index, last.base(), true), + IndexSource::Unmapped => (last.epoch, last.index, true), }; let ordinal = if continues { last.ordinal().saturating_add(1) } else { 1 }; - let position = CdcOffset::data_event_in(epoch, index, base, ordinal); + let position = CdcOffset::data_event(epoch, index, ordinal); *tail = PartitionTail { position, record_lsn, @@ -111,46 +107,23 @@ mod tests { None } + fn entry(index: u64) -> IndexSource { + IndexSource::Replicated { epoch: 0, index } + } + #[test] fn events_of_one_write_take_consecutive_ordinals() { let sequencer = PositionSequencer::new(); assert_eq!( - sequencer.next( - 0, - IndexSource::Replicated { - epoch: 0, - index: 7, - base: 0 - }, - 70, - none - ), + sequencer.next(0, entry(7), 70, none), CdcOffset::data_event(0, 7, 1) ); assert_eq!( - sequencer.next( - 0, - IndexSource::Replicated { - epoch: 0, - index: 7, - base: 0 - }, - 70, - none - ), + sequencer.next(0, entry(7), 70, none), CdcOffset::data_event(0, 7, 2) ); assert_eq!( - sequencer.next( - 0, - IndexSource::Replicated { - epoch: 0, - index: 8, - base: 0 - }, - 71, - none - ), + sequencer.next(0, entry(8), 71, none), CdcOffset::data_event(0, 8, 1) ); } @@ -158,27 +131,9 @@ mod tests { #[test] fn partitions_number_independently() { let sequencer = PositionSequencer::new(); - sequencer.next( - 0, - IndexSource::Replicated { - epoch: 0, - index: 7, - base: 0, - }, - 70, - none, - ); + sequencer.next(0, entry(7), 70, none); assert_eq!( - sequencer.next( - 1, - IndexSource::Replicated { - epoch: 0, - index: 7, - base: 0 - }, - 70, - none - ), + sequencer.next(1, entry(7), 70, none), CdcOffset::data_event(0, 7, 1) ); } @@ -186,16 +141,7 @@ mod tests { #[test] fn an_unmapped_event_joins_the_current_write() { let sequencer = PositionSequencer::new(); - sequencer.next( - 0, - IndexSource::Replicated { - epoch: 0, - index: 9, - base: 0, - }, - 90, - none, - ); + sequencer.next(0, entry(9), 90, none); assert_eq!( sequencer.next(0, IndexSource::Unmapped, 95, none), CdcOffset::data_event(0, 9, 2) @@ -214,16 +160,7 @@ mod tests { }) }; assert_eq!( - sequencer.next( - 0, - IndexSource::Replicated { - epoch: 0, - index: 12, - base: 0 - }, - 500, - seed - ), + sequencer.next(0, entry(12), 500, seed), CdcOffset::data_event(0, 12, 3) ); } @@ -239,29 +176,11 @@ mod tests { }; // Entry 12 applied again after a restart writes record 900. assert_eq!( - sequencer.next( - 0, - IndexSource::Replicated { - epoch: 0, - index: 12, - base: 0 - }, - 900, - seed - ), + sequencer.next(0, entry(12), 900, seed), CdcOffset::data_event(0, 12, 1) ); assert_eq!( - sequencer.next( - 0, - IndexSource::Replicated { - epoch: 0, - index: 12, - base: 0 - }, - 900, - none - ), + sequencer.next(0, entry(12), 900, none), CdcOffset::data_event(0, 12, 2) ); } @@ -275,26 +194,8 @@ mod tests { for _ in 0..events { // Each replica holds the entry at its own local WAL LSN. assert_eq!( - leader.next( - 2, - IndexSource::Replicated { - epoch: 0, - index, - base: 0 - }, - index * 10, - none - ), - follower.next( - 2, - IndexSource::Replicated { - epoch: 0, - index, - base: 0 - }, - index * 1_000, - none - ) + leader.next(2, entry(index), index * 10, none), + follower.next(2, entry(index), index * 1_000, none) ); } } @@ -303,19 +204,13 @@ mod tests { #[test] fn a_group_move_never_makes_positions_go_backwards() { let sequencer = PositionSequencer::new(); - let old_group = |index| IndexSource::Replicated { - epoch: 0, - index, - base: 0, - }; - let before = sequencer.next(4, old_group(9_000), 1, none); + let before = sequencer.next(4, entry(9_000), 1, none); // The new group numbers its log from a low index, in a higher epoch. let after = sequencer.next( 4, IndexSource::Replicated { epoch: 55, index: 3, - base: 0, }, 2, none, @@ -323,21 +218,4 @@ mod tests { assert!(after > before); assert_eq!(after, CdcOffset::data_event(55, 3, 1)); } - - #[test] - fn calvin_transactions_of_one_epoch_number_by_position() { - let sequencer = PositionSequencer::new(); - let calvin = |position| IndexSource::Replicated { - epoch: 0, - index: 12, - base: position, - }; - let first = sequencer.next(9, calvin(0), 100, none); - let second = sequencer.next(9, calvin(0), 100, none); - let next_txn = sequencer.next(9, calvin(1), 101, none); - assert_eq!(first, CdcOffset::data_event_in(0, 12, 0, 1)); - assert_eq!(second, CdcOffset::data_event_in(0, 12, 0, 2)); - assert_eq!(next_txn, CdcOffset::data_event_in(0, 12, 1, 1)); - assert!(first < second && second < next_txn); - } } diff --git a/nodedb/src/event/cdc/router.rs b/nodedb/src/event/cdc/router.rs index 685dff8c3..ce9833a53 100644 --- a/nodedb/src/event/cdc/router.rs +++ b/nodedb/src/event/cdc/router.rs @@ -23,7 +23,6 @@ use super::lag_warner::{CdcLagWarner, DEFAULT_THRESHOLD}; use super::offset::CdcOffset; use super::position::{ AvailabilityFloors, ChangePositionLedger, IndexSource, PartitionTail, PositionSequencer, - WritePosition, calvin_partition, }; use super::registry::StreamRegistry; use super::stream_def::{ChangeStreamDef, LateDataPolicy}; @@ -254,24 +253,15 @@ impl CdcRouter { /// The partition and position source of a write of `vshard` at local /// record `record_lsn`: the replicated position the record applies, when - /// the ledger knows it. A Calvin transaction goes to the vShard's Calvin - /// partition. The trigger action lane positions its events by it too. + /// the ledger knows it. The partition is the vShard. The trigger action + /// lane positions its events by it too. pub(crate) fn index_source(&self, vshard: u32, record_lsn: u64) -> (u32, IndexSource) { match self.positions.get(record_lsn) { - Some(WritePosition::Raft(position)) => ( + Some(position) => ( vshard, IndexSource::Replicated { epoch: position.epoch, index: position.log_index, - base: 0, - }, - ), - Some(WritePosition::Calvin(position)) => ( - calvin_partition(vshard), - IndexSource::Replicated { - epoch: 0, - index: position.sequencer_epoch, - base: u64::from(position.position), }, ), None if self.positions.is_replicated() => (vshard, IndexSource::Unmapped), @@ -715,45 +705,6 @@ mod tests { assert_eq!(positions(&leader, "s"), positions(&follower, "s")); } - #[test] - fn replicas_number_a_calvin_transaction_alike_in_its_own_partition() { - use crate::event::cdc::position::{CalvinPosition, calvin_partition}; - let replica = || { - let registry = Arc::new(StreamRegistry::new()); - registry.register(sample_def("s", "orders")); - CdcRouter::new(registry) - }; - let leader = replica(); - let follower = replica(); - let wt = test_tracker(); - let txn = CalvinPosition { - sequencer_epoch: 31, - position: 2, - }; - for (router, local_lsn) in [(&leader, 300), (&follower, 8_000)] { - router.positions().record_calvin(local_lsn, txn); - router.route_event(&write_at(local_lsn, 1), &wt); - router.route_event(&write_at(local_lsn, 2), &wt); - } - let seen = |router: &CdcRouter| { - router - .get_buffer(DatabaseId::new(7), 1, "s") - .expect("buffer") - .read_from(CdcOffset::ZERO, 16) - .iter() - .map(|event| (event.partition, event.position())) - .collect::>() - }; - assert_eq!( - seen(&leader), - vec![ - (calvin_partition(0), CdcOffset::data_event_in(0, 31, 2, 1)), - (calvin_partition(0), CdcOffset::data_event_in(0, 31, 2, 2)), - ] - ); - assert_eq!(seen(&leader), seen(&follower)); - } - #[test] fn a_node_without_raft_positions_by_local_wal_lsn() { let registry = Arc::new(StreamRegistry::new()); diff --git a/nodedb/src/event/cross_shard/types.rs b/nodedb/src/event/cross_shard/types.rs index 0efd27f9b..2d02699da 100644 --- a/nodedb/src/event/cross_shard/types.rs +++ b/nodedb/src/event/cross_shard/types.rs @@ -96,9 +96,9 @@ impl CrossShardWriteResponse { pub struct NotifyBroadcastMsg { /// The forwarding node. pub source_node: u64, - /// [`NOTIFY_PARTITION_GROUP`] or [`NOTIFY_PARTITION_CALVIN`]. + /// [`NOTIFY_PARTITION_GROUP`]. A receiver drops a run of any other kind. pub partition_kind: u8, - /// The data group, or the Calvin vShard. + /// The data group. pub partition_id: u64, pub after: crate::event::cdc::CdcOffset, pub through: crate::event::cdc::CdcOffset, @@ -107,8 +107,6 @@ pub struct NotifyBroadcastMsg { /// `NotifyBroadcastMsg::partition_kind` of a data group's feed. pub const NOTIFY_PARTITION_GROUP: u8 = 0; -/// `NotifyBroadcastMsg::partition_kind` of a vShard's Calvin feed. -pub const NOTIFY_PARTITION_CALVIN: u8 = 1; /// One forwarded change at its position. #[derive(Debug, Clone, zerompk::ToMessagePack, zerompk::FromMessagePack)] diff --git a/nodedb/src/event/topic/committed/deliver.rs b/nodedb/src/event/topic/committed/deliver.rs index 25f7eb0cf..e890dcc88 100644 --- a/nodedb/src/event/topic/committed/deliver.rs +++ b/nodedb/src/event/topic/committed/deliver.rs @@ -349,7 +349,6 @@ mod tests { partition: PARTITION, epoch: 0, index, - base: 0, }); message } diff --git a/nodedb/src/event/topic/committed/key.rs b/nodedb/src/event/topic/committed/key.rs index e82a2d956..6738322e0 100644 --- a/nodedb/src/event/topic/committed/key.rs +++ b/nodedb/src/event/topic/committed/key.rs @@ -6,13 +6,11 @@ //! A message is named by the change-feed partition of the record that carries //! it and a position in that partition: the record's replicated position with //! the message's ordinal. The apply that installs the record stamps that -//! position on the message itself ([`crate::wal::PublishPosition`]): a -//! Raft-applied record takes its entry's `(epoch, log index)`, and a Calvin -//! record takes its sequencer epoch and batch position. +//! position on the message itself ([`crate::wal::PublishPosition`]): the +//! record takes its data-group entry's `(epoch, log index)`. use crate::control::state::SharedState; use crate::event::cdc::CdcOffset; -use crate::event::cdc::position::vshard_of_partition; use crate::event::topic::types::PublishOrigin; use crate::wal::RedoPublish; @@ -53,12 +51,7 @@ pub(super) fn origin_of_event(ordinal: u32, publish: &RedoPublish) -> Option Option<(u64 let group_id = routing .read() .unwrap_or_else(|p| p.into_inner()) - .group_for_vshard(vshard_of_partition(partition)) + .group_for_vshard(partition) .ok()?; gate.leader_lease_term(group_id) .map(|term| (group_id, term)) @@ -108,7 +101,6 @@ mod tests { partition: 9, epoch: 2, index: 500, - base: 0, }); assert_eq!( origin_of_event(0, &stamped), diff --git a/nodedb/src/event/topic/publish.rs b/nodedb/src/event/topic/publish.rs index fce03896c..04d18d751 100644 --- a/nodedb/src/event/topic/publish.rs +++ b/nodedb/src/event/topic/publish.rs @@ -100,15 +100,19 @@ async fn publish_replicated( entry: ReplicatedEntry, topic_name: &str, ) -> Result { - let (payload, _) = - crate::control::wal_replication::propose_replicated_entry(state, proposer, entry) - .await - .map_err(|error| match error { - crate::Error::UndefinedObject { kind: "topic", .. } => { - PublishError::TopicNotFound(topic_name.to_string()) - } - other => PublishError::Persistence(other.to_string()), - })?; + let (payload, _) = crate::control::wal_replication::propose_replicated_entry( + state, + proposer, + entry, + crate::control::wal_replication::statement_propose_deadline(state), + ) + .await + .map_err(|error| match error { + crate::Error::UndefinedObject { kind: "topic", .. } => { + PublishError::TopicNotFound(topic_name.to_string()) + } + other => PublishError::Persistence(other.to_string()), + })?; zerompk::from_msgpack::(&payload) .map_err(|error| PublishError::Persistence(format!("publish result: {error}"))) } diff --git a/nodedb/src/event/trigger/lane/fire.rs b/nodedb/src/event/trigger/lane/fire.rs index 57ff6ab33..454367518 100644 --- a/nodedb/src/event/trigger/lane/fire.rs +++ b/nodedb/src/event/trigger/lane/fire.rs @@ -288,7 +288,7 @@ async fn fire_partition( if delivery_lease(state, partition) != Some(lease) { break; } - let event = match action.to_event(partition, position) { + let event = match action.to_event(position) { Ok(event) => event, // A held event that does not rebuild fires nothing on any owner. Err(error) => { diff --git a/nodedb/src/event/trigger/lane/held.rs b/nodedb/src/event/trigger/lane/held.rs index b353141a3..96285a044 100644 --- a/nodedb/src/event/trigger/lane/held.rs +++ b/nodedb/src/event/trigger/lane/held.rs @@ -111,13 +111,13 @@ impl HeldAction { } /// The event the lane fires the actions of: this held event at - /// `position` of `partition`. + /// `position` of its partition. /// /// Its `lsn` and `sequence` carry the position's action identity, which /// every replica shares. The actions name their source write by them, so /// a body fired again by a later owner carries the same key and applies /// once. - pub fn to_event(&self, partition: u32, position: CdcOffset) -> crate::Result { + pub fn to_event(&self, position: CdcOffset) -> crate::Result { let source = EventSource::from_wal_code(self.source).ok_or_else(|| crate::Error::Internal { detail: format!( @@ -131,7 +131,7 @@ impl HeldAction { self.collection, self.op ), })?; - let (source_lsn, source_sequence) = action_identity(partition, position); + let (source_lsn, source_sequence) = action_identity(position); Ok(WriteEvent { sequence: source_sequence, collection: Arc::from(self.collection.as_str()), @@ -189,7 +189,7 @@ mod tests { let back = HeldAction::from_bytes(&held.to_bytes().expect("encode")).expect("decode"); assert_eq!(back, held); let fired = back - .to_event(7, CdcOffset::data_event(0, 12, 1)) + .to_event(CdcOffset::data_event(0, 12, 1)) .expect("rebuild"); assert_eq!(fired.op, WriteOp::Update); assert_eq!(fired.row_id.as_str(), "o-1"); diff --git a/nodedb/src/event/trigger/lane/position.rs b/nodedb/src/event/trigger/lane/position.rs index 77d308374..dc3c42756 100644 --- a/nodedb/src/event/trigger/lane/position.rs +++ b/nodedb/src/event/trigger/lane/position.rs @@ -4,8 +4,8 @@ //! //! The lane names an event as the CDC router does: by the change-feed //! partition of its write and a position in that partition. The position is -//! the write's replicated position (a Raft entry, or a Calvin transaction) -//! with the event's ordinal among the write's firing events. One partition's +//! the write's replicated position (a Raft entry) with the event's ordinal +//! among the write's firing events. One partition's //! events come from one Data-Plane core in apply order, so every replica //! numbers them alike. //! @@ -13,15 +13,10 @@ //! holds an action for its collection. The numbering then never depends on //! when a replica applied a trigger's DDL. -use crate::event::cdc::position::{CALVIN_PARTITION_BASE, PartitionTail, PositionSequencer}; +use crate::event::cdc::position::{PartitionTail, PositionSequencer}; use crate::event::cdc::{CdcOffset, CdcRouter}; use crate::event::types::WriteEvent; -/// The bit of an action identity's source LSN that marks a Calvin -/// partition. A Raft log index and a Calvin sequencer epoch share one number -/// space, so the bit keeps their events apart. Neither reaches `2^63`. -const CALVIN_IDENTITY_BIT: u64 = 1 << 63; - /// The position allocator of the lane's firing events. #[derive(Debug, Default)] pub struct ActionPositions { @@ -58,31 +53,21 @@ impl ActionPositions { } /// The `(source_lsn, source_sequence)` a fired action names its event by: -/// the position's index and sequence, with the Calvin bit set for a Calvin -/// partition. Every replica derives the same pair. +/// the position's index and sequence. Every replica derives the same pair. /// /// Every position has epoch `0` (see `cdc::offset`), so the pair names the /// event within its vShard. -pub fn action_identity(partition: u32, position: CdcOffset) -> (u64, u64) { - let kind = if partition >= CALVIN_PARTITION_BASE { - CALVIN_IDENTITY_BIT - } else { - 0 - }; - (position.index | kind, position.sequence) +pub fn action_identity(position: CdcOffset) -> (u64, u64) { + (position.index, position.sequence) } #[cfg(test)] mod tests { use super::*; - use crate::event::cdc::position::calvin_partition; #[test] - fn a_raft_and_a_calvin_event_at_one_index_have_distinct_identities() { + fn an_action_identity_is_the_index_and_sequence() { let position = CdcOffset::data_event(0, 12, 1); - let raft = action_identity(3, position); - let calvin = action_identity(calvin_partition(3), position); - assert_eq!(raft, (12, position.sequence)); - assert_ne!(raft, calvin); + assert_eq!(action_identity(position), (12, position.sequence)); } } diff --git a/nodedb/src/event/trigger/lane/snapshot.rs b/nodedb/src/event/trigger/lane/snapshot.rs index ae1171a86..d2161cda1 100644 --- a/nodedb/src/event/trigger/lane/snapshot.rs +++ b/nodedb/src/event/trigger/lane/snapshot.rs @@ -18,7 +18,6 @@ use std::time::Duration; use crate::control::state::SharedState; use crate::event::cdc::CdcOffset; -use crate::event::cdc::position::calvin_partition; use crate::event::topic::committed::cursor::{delivered_through, raise_delivered_here}; use crate::event::topic::types::PublishOrigin; use crate::types::snapshot::{GroupEventLane, HeldAtPosition, PartitionCursor}; @@ -35,13 +34,9 @@ fn lane_error(detail: impl std::fmt::Display) -> crate::Error { } } -/// Every partition of `vshards`: each vShard's Raft partition and its Calvin -/// partition, in order. +/// Every partition of `vshards`, in order: each vShard is one partition. fn partitions_of(vshards: &HashSet) -> Vec { - let mut partitions: Vec = vshards - .iter() - .flat_map(|vshard| [*vshard, calvin_partition(*vshard)]) - .collect(); + let mut partitions: Vec = vshards.iter().copied().collect(); partitions.sort_unstable(); partitions } @@ -104,10 +99,8 @@ pub async fn capture(state: &SharedState, vshards: &HashSet) -> crate::Resu /// Install `lane`, a snapshot of `vshards` cut at `cut_index`, on this node. /// -/// A Raft partition's rows at or below the cut are the builder's: this node -/// never applies those entries. A Calvin partition's rows join this node's, -/// since its Calvin applies continue apart from the data group. Each cursor -/// rises to the builder's. +/// A partition's rows at or below the cut are the builder's: this node never +/// applies those entries. Each cursor rises to the builder's. pub fn install( state: &SharedState, vshards: &HashSet, @@ -131,13 +124,11 @@ pub fn install( (CdcOffset::at(*epoch, *index, *sequence), bytes.clone()) }) .collect(); - let through = vshards - .contains(&partition) - .then_some(CdcOffset::whole_write(0, cut_index)); - ledgers - .actions - .ledger - .replace_through(partition, through, &rows)?; + ledgers.actions.ledger.replace_through( + partition, + Some(CdcOffset::whole_write(0, cut_index)), + &rows, + )?; } for (partition, epoch, index, sequence, bytes) in held_publishes { let publish: RedoPublish = zerompk::from_msgpack(&bytes).map_err(lane_error)?; @@ -162,12 +153,9 @@ mod tests { use super::*; #[test] - fn a_group_covers_the_raft_and_calvin_partitions_of_its_vshards() { + fn a_group_covers_the_partitions_of_its_vshards_in_order() { let vshards: HashSet = [3, 1].into_iter().collect(); - assert_eq!( - partitions_of(&vshards), - vec![1, 3, calvin_partition(1), calvin_partition(3)] - ); + assert_eq!(partitions_of(&vshards), vec![1, 3]); } #[test] diff --git a/nodedb/src/event/wal_replay.rs b/nodedb/src/event/wal_replay.rs index 388d1369f..a3cceeb54 100644 --- a/nodedb/src/event/wal_replay.rs +++ b/nodedb/src/event/wal_replay.rs @@ -726,16 +726,6 @@ mod tests { assert_eq!(seq, 1, "the VectorPut sub-op did not consume a sequence"); } - /// A `CalvinApplied` payload-free marker emits no events — its base writes, - /// if any, ride a separate `TransactionRedo`. - #[test] - fn calvin_applied_marker_emits_no_events() { - let record = make_record(RecordType::CalvinApplied, &[], 1, 0, 302); - let mut seq = 0u64; - assert!(record_to_events(&record, &mut seq).is_empty()); - assert_eq!(seq, 0); - } - /// A malformed `TransactionRedo` payload is skipped (logged, no panic) and /// produces no events. #[test] diff --git a/nodedb/src/event/wal_replay_kind.rs b/nodedb/src/event/wal_replay_kind.rs index 212e2627d..c2486408f 100644 --- a/nodedb/src/event/wal_replay_kind.rs +++ b/nodedb/src/event/wal_replay_kind.rs @@ -48,12 +48,6 @@ pub(super) fn row_kind(record_type: RecordType) -> Option { // the same events. A columnar insert rides the same record type and // rebuilds none: its forward path emits no WriteEvent. RecordType::TimeseriesBatch => Some(RowRecord::Timeseries), - // `CalvinApplied` is a payload-free applied-marker: it records that a - // sequencer `(epoch, position)` was applied, but carries no writes. Its - // base writes, if any, ride a separate `TransactionRedo`; a pure-read or - // CRDT-only commit has no base WriteEvents at all (CRDT effects ride - // `CrdtDelta` records). Nothing to emit. - RecordType::CalvinApplied => None, // The records below carry NO forward-path Data-Plane WriteEvent, so // there is nothing for replay to reconstruct. `record_to_events` // reconstructs exactly the forward WriteEvent stream the Data Plane @@ -139,6 +133,9 @@ pub(super) fn row_kind(record_type: RecordType) -> Option { // emits no per-edge event, so replay rebuilds none. | RecordType::GraphEdgeCut // A snapshot install marker writes no row. - | RecordType::SnapshotInstalled => None, + | RecordType::SnapshotInstalled + // A redo chunk holds bytes of a redo stream. The stream's final + // record carries the rows and their events. + | RecordType::RedoChunk => None, } } diff --git a/nodedb/src/main_boot/post_open.rs b/nodedb/src/main_boot/post_open.rs index a33a68987..2ddc723fd 100644 --- a/nodedb/src/main_boot/post_open.rs +++ b/nodedb/src/main_boot/post_open.rs @@ -1,8 +1,8 @@ // SPDX-License-Identifier: BUSL-1.1 //! Post-`SharedState::open` catalog steps: fire the catalog-open gate, -//! replay the surrogate WAL, read back the cut barriers, and bootstrap the -//! superuser credential. +//! replay the surrogate WAL, rebuild the open redo streams, read back the cut +//! barriers, and bootstrap the superuser credential. use std::sync::Arc; @@ -26,6 +26,16 @@ pub(crate) async fn run( // Replay surrogate WAL records into the in-memory registry. bootstrap::credentials::replay_surrogate_wal(shared, wal_records, replay_tombstones); + // Open chunked redo streams, before any data group applies an entry. + let streams = shared.redo_chunks.rebuild(wal_records)?; + if streams.open > 0 { + tracing::info!( + open = streams.open, + closed = streams.closed, + "chunked redo streams rebuilt from the WAL" + ); + } + // Cut barriers, for the apply loop's commit HLC floors. shared .pitr diff --git a/nodedb/src/main_boot/shutdown_wiring.rs b/nodedb/src/main_boot/shutdown_wiring.rs index c6a713093..c9f2b6e0b 100644 --- a/nodedb/src/main_boot/shutdown_wiring.rs +++ b/nodedb/src/main_boot/shutdown_wiring.rs @@ -62,7 +62,7 @@ pub(crate) fn wire_shutdown_bus( // Final WAL fsync. Every append before this phase is buffered until an // fsync covers it, and a write whose caller never awaited durability - // (a Calvin applied marker, a background maintenance record) has only + // (a background maintenance record) has only // this fsync between it and a graceful exit. Registered at startup so the // bus cannot pass `WalFsync` without it. spawn_wal_fsync_barrier(Arc::clone(shared), &shutdown_bus); diff --git a/nodedb/src/types/mod.rs b/nodedb/src/types/mod.rs index 8ef167f00..22ba096bb 100644 --- a/nodedb/src/types/mod.rs +++ b/nodedb/src/types/mod.rs @@ -5,6 +5,7 @@ pub mod core_route; pub mod hash_chain; pub mod id; pub mod lsn; +pub mod read_versions; pub mod record_home; pub mod replay_stamp; pub mod snapshot; @@ -15,9 +16,10 @@ pub use core_route::core_for_vshard; pub use id::{DatabaseId, DocumentId, RequestId, TenantId, TxnId, VShardId}; pub use lsn::Lsn; pub use nodedb_types::{KeyRepr, SpanId, TraceId}; +pub use read_versions::ReadVersions; pub use record_home::{HomedRecord, RecordHomes}; pub use snapshot::{ ArrayCellsBlob, SurrogateBindEntry, TenantDataSnapshot, TsFlushedCollectionBlob, TsFlushedPartitionBlob, }; -pub use snapshot_calvin::{GroupCalvinCut, VShardCalvinState}; +pub use snapshot_calvin::{CutBarrierLog, GroupCalvinCut, VShardCalvinState}; diff --git a/nodedb/src/types/read_versions.rs b/nodedb/src/types/read_versions.rs new file mode 100644 index 000000000..3a6d9e0e0 --- /dev/null +++ b/nodedb/src/types/read_versions.rs @@ -0,0 +1,115 @@ +// SPDX-License-Identifier: BUSL-1.1 + +//! The write versions one read observed, one per vShard it read. + +use smallvec::SmallVec; + +use nodedb_types::{ShardVersion, WriteVersion}; + +use super::VShardId; + +/// The versions one read observed, one per vShard. +/// +/// A version positions a write within its vShard only, so each one travels +/// with the vShard it belongs to. A read of one collection observes one +/// vShard, so the common case holds one version without allocating. +#[derive(Debug, Clone, Default, PartialEq, Eq)] +pub struct ReadVersions(SmallVec<[ShardVersion; 1]>); + +impl ReadVersions { + /// No versions: the response observed nothing a read validates. + pub fn new() -> Self { + Self::default() + } + + /// One vShard observed at `version`. + pub fn single(vshard: VShardId, version: WriteVersion) -> Self { + let mut versions = Self::new(); + versions.note(vshard, version); + versions + } + + /// Note that the read observed `vshard` at `version`. + /// + /// Two answers for one vShard keep the higher. Only the core that holds a + /// vShard records versions of it, so every other answer is lower. + pub fn note(&mut self, vshard: VShardId, version: WriteVersion) { + self.note_raw(vshard.as_u32(), version); + } + + fn note_raw(&mut self, vshard: u32, version: WriteVersion) { + match self.0.iter_mut().find(|shard| shard.vshard == vshard) { + Some(shard) => shard.version = shard.version.max(version), + None => self.0.push(ShardVersion { vshard, version }), + } + } + + /// Fold `other` in, keeping the higher version per vShard. + pub fn merge(&mut self, other: &ReadVersions) { + for shard in &other.0 { + self.note_raw(shard.vshard, shard.version); + } + } + + /// The version the read observed on `vshard`, if it reported one. + pub fn of(&self, vshard: VShardId) -> Option { + self.0 + .iter() + .find(|shard| shard.vshard == vshard.as_u32()) + .map(|shard| shard.version) + } + + /// Every observed vShard, in the order first noted. + pub fn iter(&self) -> impl Iterator { + self.0.iter() + } + + pub fn is_empty(&self) -> bool { + self.0.is_empty() + } + + /// The versions as they travel between nodes. + pub fn to_wire(&self) -> Vec { + self.0.to_vec() + } + + /// The versions a node sent. + pub fn from_wire(versions: &[ShardVersion]) -> Self { + let mut read = Self::new(); + for shard in versions { + read.note_raw(shard.vshard, shard.version); + } + read + } +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn a_vshard_noted_twice_keeps_the_higher_version() { + let mut versions = ReadVersions::single(VShardId::new(3), WriteVersion::logged(0, 9)); + versions.note(VShardId::new(3), WriteVersion::logged(0, 4)); + versions.note(VShardId::new(5), WriteVersion::logged(1, 2)); + assert_eq!( + versions.of(VShardId::new(3)), + Some(WriteVersion::logged(0, 9)) + ); + assert_eq!( + versions.of(VShardId::new(5)), + Some(WriteVersion::logged(1, 2)) + ); + assert_eq!(versions.of(VShardId::new(6)), None); + } + + #[test] + fn versions_survive_the_wire() { + let mut versions = ReadVersions::single(VShardId::new(3), WriteVersion::logged(0, 9)); + versions.merge(&ReadVersions::single( + VShardId::new(8), + WriteVersion::logged(2, 1), + )); + assert_eq!(ReadVersions::from_wire(&versions.to_wire()), versions); + } +} diff --git a/nodedb/src/types/snapshot.rs b/nodedb/src/types/snapshot.rs index f8a3a21c7..6220622a8 100644 --- a/nodedb/src/types/snapshot.rs +++ b/nodedb/src/types/snapshot.rs @@ -287,6 +287,14 @@ pub struct TenantDataSnapshot { #[msgpack(default)] #[serde(default)] pub group_calvin: Option, + + /// The open chunked redo streams of a Raft group snapshot's group. A + /// follower caught up by the snapshot never applies the chunks it covers, + /// so it holds them from here for each stream's final entry. Empty in a + /// backup and a per-core part. + #[msgpack(default)] + #[serde(default)] + pub group_redo_streams: Vec, } /// One `(partition, epoch, index, sequence)` position with its held row. diff --git a/nodedb/src/types/snapshot_calvin.rs b/nodedb/src/types/snapshot_calvin.rs index 2783d91c1..585278355 100644 --- a/nodedb/src/types/snapshot_calvin.rs +++ b/nodedb/src/types/snapshot_calvin.rs @@ -33,6 +33,10 @@ pub struct GroupCalvinCut { /// The applied state of each group vShard whose scheduler runs on the /// builder. pub vshards: Vec, + /// The stored dependent-read barrier log of each unfinished txn of a + /// group vShard. It holds every barrier entry of the group's log at or + /// below the snapshot's index, which the receiver never applies. + pub barrier_logs: Vec, } /// One vShard's applied Calvin positions in a [`GroupCalvinCut`]. @@ -56,3 +60,24 @@ pub struct VShardCalvinState { /// The applied `(epoch, position)` pairs above the watermark. pub tail: Vec<(u64, u32)>, } + +/// One txn's stored dependent-read barrier log in a [`GroupCalvinCut`]. +#[derive( + Debug, + Clone, + Default, + PartialEq, + Eq, + serde::Serialize, + serde::Deserialize, + zerompk::ToMessagePack, + zerompk::FromMessagePack, +)] +#[msgpack(map)] +pub struct CutBarrierLog { + pub vshard_id: u32, + pub epoch: u64, + pub position: u32, + /// The encoded barrier log. + pub log: Vec, +} diff --git a/nodedb/src/wal/manager/append_metadata.rs b/nodedb/src/wal/manager/append_metadata.rs index 7648f4e35..cf2799ea7 100644 --- a/nodedb/src/wal/manager/append_metadata.rs +++ b/nodedb/src/wal/manager/append_metadata.rs @@ -1,8 +1,7 @@ // SPDX-License-Identifier: BUSL-1.1 //! WAL appends for node-global metadata: temporal-purge audit, surrogate -//! allocation/binding, Calvin epoch tracking, applied Raft proposals, and -//! sync watermarks. +//! allocation/binding, applied Raft proposals, and sync watermarks. use nodedb_wal::record::RecordType; @@ -86,31 +85,6 @@ impl WalAppender<'_> { ) } - /// Append a `CalvinApplied` record after a Calvin executor successfully - /// commits a `MetaOp::CalvinExecute` batch. - /// - /// The scheduler's restart path scans these records to compute - /// `last_applied_epoch` for a given vshard without re-reading the full - /// Raft sequencer log. `vshard_id` is stored in the payload (not in the - /// WAL record header's `vshard_id` field) so it can be decoded during - /// a scan of all records regardless of which vshard they were written on. - pub fn append_calvin_applied( - &self, - vshard_id: crate::types::VShardId, - epoch: u64, - position: u32, - ) -> crate::Result { - let payload = - nodedb_wal::CalvinAppliedPayload::new(epoch, position, vshard_id.as_u32()).to_bytes(); - self.append_record( - RecordType::CalvinApplied, - TenantId::new(0), - vshard_id, - DatabaseId::DEFAULT, - &payload, - ) - } - /// Append a payload-free `ProposalApplied` marker carrying this /// appender's apply key. An apply that writes no record of its own /// appends it in place of its forward record, so the proposal's key still diff --git a/nodedb/src/wal/manager/append_redo_chunk.rs b/nodedb/src/wal/manager/append_redo_chunk.rs new file mode 100644 index 000000000..10fafe9ad --- /dev/null +++ b/nodedb/src/wal/manager/append_redo_chunk.rs @@ -0,0 +1,165 @@ +// SPDX-License-Identifier: BUSL-1.1 + +//! WAL appends for chunked transaction redo streams. +//! +//! A chunk past the WAL record limit spans several `RedoChunk` pieces. Only +//! the last piece carries the chunk entry's apply key. The WAL is written in +//! order, so a durable last piece proves every earlier piece durable, and the +//! proposal ledger never counts a chunk applied whose bytes are partial. + +use nodedb_wal::record::RecordType; + +use super::appender::NO_APPLY_KEY; +use super::core::WalManager; +use crate::types::{DatabaseId, Lsn, TenantId, VShardId}; +use crate::wal::redo::{RedoChunkHeader, RedoChunkPiece, RedoChunkRecord, RedoStreamId}; + +/// Room one piece record keeps for its header beside the chunk bytes. +const PIECE_HEADER_BYTES: usize = 256; + +/// The header fields of every record a stream appends. +#[derive(Debug, Clone, Copy)] +pub struct RedoChunkPlacement { + pub tenant_id: TenantId, + pub vshard_id: VShardId, + pub database_id: DatabaseId, +} + +/// The LSNs of the first and the last record one chunk appended. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub struct ChunkLsns { + pub first: Lsn, + pub last: Lsn, +} + +impl WalManager { + /// Append chunk `header.index` of its stream, holding `bytes`. The last + /// piece carries `apply_key`, every other piece [`NO_APPLY_KEY`]. + pub fn append_redo_chunk( + &self, + apply_key: u64, + placement: RedoChunkPlacement, + header: RedoChunkHeader, + bytes: &[u8], + ) -> crate::Result { + let room = self + .max_payload() + .checked_sub(PIECE_HEADER_BYTES) + .filter(|room| *room > 0) + .ok_or_else(|| crate::Error::Internal { + detail: format!( + "WAL record limit {} leaves no room for a redo chunk piece", + self.max_payload() + ), + })?; + let parts: Vec<&[u8]> = if bytes.is_empty() { + vec![bytes] + } else { + bytes.chunks(room).collect() + }; + let pieces = u32::try_from(parts.len()).map_err(|_| crate::Error::Internal { + detail: format!("a redo chunk spans {} WAL pieces", parts.len()), + })?; + let mut first = None; + let mut last = Lsn::new(0); + for (piece, part) in (0u32..).zip(parts) { + let key = if piece + 1 == pieces { + apply_key + } else { + NO_APPLY_KEY + }; + let record = RedoChunkRecord::Piece(RedoChunkPiece { + header, + piece, + pieces, + bytes: part.to_vec(), + }); + last = self.appender(key).append_record( + RecordType::RedoChunk, + placement.tenant_id, + placement.vshard_id, + placement.database_id, + &record.to_bytes()?, + )?; + first.get_or_insert(last); + } + Ok(ChunkLsns { + first: first.unwrap_or(last), + last, + }) + } + + /// Append the close of `stream`: boot opens it no more. + pub fn append_redo_stream_closed( + &self, + apply_key: u64, + placement: RedoChunkPlacement, + stream: RedoStreamId, + ) -> crate::Result { + self.appender(apply_key).append_record( + RecordType::RedoChunk, + placement.tenant_id, + placement.vshard_id, + placement.database_id, + &RedoChunkRecord::Closed { stream }.to_bytes()?, + ) + } +} + +#[cfg(test)] +mod tests { + use super::*; + + fn placement() -> RedoChunkPlacement { + RedoChunkPlacement { + tenant_id: TenantId::new(1), + vshard_id: VShardId::new(4), + database_id: DatabaseId::DEFAULT, + } + } + + fn header() -> RedoChunkHeader { + RedoChunkHeader { + stream: RedoStreamId::Session { + vshard: 4, + idempotency_key: 77, + }, + group_id: 1, + term: 2, + len: 0, + index: 0, + } + } + + #[test] + fn a_chunk_past_the_record_limit_keys_only_its_last_piece() { + let dir = tempfile::tempdir().expect("tempdir"); + let wal = WalManager::open_for_testing(&dir.path().join("wal")).expect("open wal"); + let bytes: Vec = (0..wal.max_payload() * 2).map(|i| i as u8).collect(); + let mut head = header(); + head.len = bytes.len() as u64; + let lsns = wal + .append_redo_chunk(0xC0FFEE, placement(), head, &bytes) + .expect("append"); + wal.sync().expect("sync"); + assert!(lsns.last > lsns.first); + + let records = wal.replay().expect("replay"); + let mut joined = Vec::new(); + let mut keys = Vec::new(); + for record in &records { + let RedoChunkRecord::Piece(piece) = + RedoChunkRecord::from_bytes(&record.payload).expect("decode") + else { + panic!("expected a piece"); + }; + assert_eq!(piece.pieces as usize, records.len()); + joined.extend_from_slice(&piece.bytes); + keys.push(record.apply_key()); + } + assert_eq!(joined, bytes); + let (last, rest) = keys.split_last().expect("pieces"); + assert_eq!(*last, 0xC0FFEE); + assert!(rest.iter().all(|key| *key == NO_APPLY_KEY)); + } +} diff --git a/nodedb/src/wal/manager/core.rs b/nodedb/src/wal/manager/core.rs index 0dd08b61a..03ad79e25 100644 --- a/nodedb/src/wal/manager/core.rs +++ b/nodedb/src/wal/manager/core.rs @@ -49,6 +49,8 @@ pub struct WalManager { /// Commit-time anchors of this WAL, stamped from the node's HLC. The writer /// records one per fsynced batch. Replay records the persisted ones. pub(super) time_anchors: Arc, + /// Floors truncation keeps for holders that read records back at boot. + pub(super) floor_holds: super::floor_holds::WalFloorHolds, } impl WalManager { @@ -76,6 +78,11 @@ impl WalManager { Ok(self.crdt_signing_root) } + /// The floors truncation of this WAL keeps. + pub fn floor_holds(&self) -> &super::floor_holds::WalFloorHolds { + &self.floor_holds + } + /// This WAL's commit-time anchors. pub fn time_anchors(&self) -> &Arc { &self.time_anchors @@ -197,6 +204,7 @@ impl WalManager { commit_lock: tokio::sync::Mutex::new(()), durable_notify: tokio::sync::Notify::new(), time_anchors, + floor_holds: super::floor_holds::WalFloorHolds::default(), }) } diff --git a/nodedb/src/wal/manager/floor_holds.rs b/nodedb/src/wal/manager/floor_holds.rs new file mode 100644 index 000000000..fbeb65122 --- /dev/null +++ b/nodedb/src/wal/manager/floor_holds.rs @@ -0,0 +1,115 @@ +// SPDX-License-Identifier: BUSL-1.1 + +//! Floors WAL truncation must keep. +//! +//! A holder that reads records back after a restart takes a [`WalFloorHold`] +//! at the lowest LSN it needs. [`super::WalManager::truncate_before`] never +//! deletes a segment holding a record at or above the lowest held LSN. A +//! hold lasts until its guard drops. + +use std::collections::BTreeMap; +use std::sync::atomic::{AtomicU64, Ordering}; +use std::sync::{Arc, Mutex}; + +use crate::types::Lsn; + +/// The held floors of one WAL. +#[derive(Default)] +pub struct WalFloorHolds { + inner: Arc, +} + +#[derive(Default)] +struct HoldsInner { + next_id: AtomicU64, + /// Hold id to the LSN it holds. + held: Mutex>, +} + +/// One held floor. Dropping it releases the floor. +pub struct WalFloorHold { + inner: Arc, + id: u64, + lsn: Lsn, +} + +impl std::fmt::Debug for WalFloorHold { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + f.debug_struct("WalFloorHold") + .field("id", &self.id) + .field("lsn", &self.lsn) + .finish() + } +} + +impl WalFloorHolds { + /// Hold truncation below `lsn` until the returned guard drops. + pub fn hold(&self, lsn: Lsn) -> WalFloorHold { + let id = self.inner.next_id.fetch_add(1, Ordering::Relaxed); + self.inner + .held + .lock() + .unwrap_or_else(|p| p.into_inner()) + .insert(id, lsn.as_u64()); + WalFloorHold { + inner: Arc::clone(&self.inner), + id, + lsn, + } + } + + /// The lowest held LSN, or `None` when nothing is held. + pub fn lowest(&self) -> Option { + self.inner + .held + .lock() + .unwrap_or_else(|p| p.into_inner()) + .values() + .min() + .map(|lsn| Lsn::new(*lsn)) + } + + /// `lsn`, lowered to the lowest held LSN. + pub fn clamp(&self, lsn: Lsn) -> Lsn { + match self.lowest() { + Some(held) if held < lsn => held, + _ => lsn, + } + } +} + +impl WalFloorHold { + /// The LSN this guard holds. + pub fn lsn(&self) -> Lsn { + self.lsn + } +} + +impl Drop for WalFloorHold { + fn drop(&mut self) { + self.inner + .held + .lock() + .unwrap_or_else(|p| p.into_inner()) + .remove(&self.id); + } +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn the_lowest_live_hold_clamps_truncation() { + let holds = WalFloorHolds::default(); + assert_eq!(holds.clamp(Lsn::new(50)), Lsn::new(50)); + let high = holds.hold(Lsn::new(30)); + let low = holds.hold(Lsn::new(10)); + assert_eq!(holds.clamp(Lsn::new(50)), Lsn::new(10)); + assert_eq!(holds.clamp(Lsn::new(5)), Lsn::new(5)); + drop(low); + assert_eq!(holds.clamp(Lsn::new(50)), Lsn::new(30)); + drop(high); + assert_eq!(holds.lowest(), None); + } +} diff --git a/nodedb/src/wal/manager/mod.rs b/nodedb/src/wal/manager/mod.rs index 66227d851..2f83a3ef1 100644 --- a/nodedb/src/wal/manager/mod.rs +++ b/nodedb/src/wal/manager/mod.rs @@ -4,6 +4,7 @@ pub mod append; pub mod append_batch; pub mod append_index; pub mod append_metadata; +pub mod append_redo_chunk; pub mod append_restore_point; pub mod append_transaction; pub mod append_truncate; @@ -14,8 +15,11 @@ pub mod core; pub mod durable_commit; pub mod effect_stamp; pub mod encryption; +pub mod floor_holds; pub mod ops; pub mod replay; +pub use append_redo_chunk::{ChunkLsns, RedoChunkPlacement}; pub use appender::{AppendSink, NO_APPLY_KEY, RecordedAppend, WalAppender}; pub use core::WalManager; +pub use floor_holds::{WalFloorHold, WalFloorHolds}; diff --git a/nodedb/src/wal/manager/ops.rs b/nodedb/src/wal/manager/ops.rs index 3a1b0cff2..1d4295027 100644 --- a/nodedb/src/wal/manager/ops.rs +++ b/nodedb/src/wal/manager/ops.rs @@ -15,9 +15,18 @@ impl WalManager { &self, checkpoint_lsn: Lsn, ) -> crate::Result { + // A held floor keeps every record a holder reads back at boot. + let bound = self.floor_holds.clamp(checkpoint_lsn); + if bound < checkpoint_lsn { + tracing::debug!( + checkpoint_lsn = checkpoint_lsn.as_u64(), + held_lsn = bound.as_u64(), + "WAL truncation held below the checkpoint" + ); + } let wal = self.wal.lock().unwrap_or_else(|p| p.into_inner()); let result = wal - .truncate_before(checkpoint_lsn.as_u64()) + .truncate_before(bound.as_u64()) .map_err(crate::Error::Wal)?; if result.segments_deleted > 0 { diff --git a/nodedb/src/wal/mod.rs b/nodedb/src/wal/mod.rs index 229e71169..c9d77ce73 100644 --- a/nodedb/src/wal/mod.rs +++ b/nodedb/src/wal/mod.rs @@ -21,11 +21,12 @@ pub(crate) use crdt_payload::{ }; pub use manager::WalManager; pub use redo::{ - CalvinStamp, CapturedEntry, CascadedEdge, ContinuedRedo, CrossShardAppliedKey, EVERY_ROW, - EdgeCutRedo, EdgeDeleteRedo, EdgePutRedo, GroupMembership, NodeCascadeRedo, OriginAppend, - PublishPosition, RedoPublish, RedoRecord, RedoRowChange, RedoRowKind, RedoRowSource, - RedoSubRecord, RowSourceIndex, SplitRedo, WriteGroup, WriteGroupRecord, WriteSetCapture, - split_redo, + CalvinStamp, CapturedEntry, CarriedRedoStream, CascadedEdge, ContinuedRedo, + CrossShardAppliedKey, EVERY_ROW, EdgeCutRedo, EdgeDeleteRedo, EdgePutRedo, GroupMembership, + NodeCascadeRedo, OriginAppend, PublishPosition, RedoChunkHeader, RedoChunkPiece, + RedoChunkRecord, RedoPublish, RedoRecord, RedoRowChange, RedoRowKind, RedoRowSource, + RedoStreamId, RedoSubRecord, RowSourceIndex, SplitRedo, WriteGroup, WriteGroupRecord, + WriteSetCapture, split_redo, }; pub use replay::SyncHwmReplayMaps; pub use replay::SyncHwmReplayStats; diff --git a/nodedb/src/wal/redo/chunk.rs b/nodedb/src/wal/redo/chunk.rs new file mode 100644 index 000000000..7e0127d8e --- /dev/null +++ b/nodedb/src/wal/redo/chunk.rs @@ -0,0 +1,182 @@ +// SPDX-License-Identifier: BUSL-1.1 + +//! Chunked transaction redo: the stream a redo's chunks share, and the WAL +//! record of one chunk. +//! +//! A committed redo past the redo entry limit travels through its data-group +//! log as chunk entries followed by a final entry. Every replica appends each +//! chunk it applies as `RedoChunk` records and holds the bytes until the final +//! entry assembles them. Boot rebuilds the open streams from these records. + +use serde::{Deserialize, Serialize}; + +/// The stream one chunked redo's chunks and final entry share. +#[derive( + Debug, + Clone, + Copy, + PartialEq, + Eq, + PartialOrd, + Ord, + Hash, + Serialize, + Deserialize, + zerompk::ToMessagePack, + zerompk::FromMessagePack, +)] +pub enum RedoStreamId { + /// A committed Calvin slice. Each proposal attempt of the position + /// opens its own stream. + Calvin { + vshard: u32, + epoch: u64, + position: u32, + attempt: u32, + }, + /// A session COMMIT. The final entry's idempotency key names the stream. + Session { vshard: u32, idempotency_key: u64 }, +} + +impl RedoStreamId { + /// The vShard whose data-group log carries the stream. + pub fn vshard(&self) -> u32 { + match self { + Self::Calvin { vshard, .. } | Self::Session { vshard, .. } => *vshard, + } + } +} + +/// Where one chunk sits in its stream. +#[derive( + Debug, + Clone, + Copy, + PartialEq, + Eq, + Serialize, + Deserialize, + zerompk::ToMessagePack, + zerompk::FromMessagePack, +)] +pub struct RedoChunkHeader { + pub stream: RedoStreamId, + /// The data group whose log carried the chunk. + pub group_id: u64, + /// The Raft term of the stream's first chunk entry. + pub term: u64, + /// The byte length of the whole stream, as every chunk declares it. + pub len: u64, + /// The chunk's place in the stream, from `0`. + pub index: u32, +} + +/// One WAL piece of one chunk. +#[derive( + Debug, + Clone, + PartialEq, + Eq, + Serialize, + Deserialize, + zerompk::ToMessagePack, + zerompk::FromMessagePack, +)] +pub struct RedoChunkPiece { + pub header: RedoChunkHeader, + /// The piece's place in the chunk, from `0`. + pub piece: u32, + /// How many pieces the chunk spans. + pub pieces: u32, + pub bytes: Vec, +} + +/// The payload of a `RedoChunk` WAL record. +#[derive( + Debug, + Clone, + PartialEq, + Eq, + Serialize, + Deserialize, + zerompk::ToMessagePack, + zerompk::FromMessagePack, +)] +pub enum RedoChunkRecord { + /// One piece of one chunk. + Piece(RedoChunkPiece), + /// The stream closed without its final entry. Boot opens it no more. + Closed { stream: RedoStreamId }, +} + +impl RedoChunkRecord { + pub fn to_bytes(&self) -> crate::Result> { + zerompk::to_msgpack_vec(self).map_err(|e| crate::Error::Serialization { + format: "msgpack".into(), + detail: format!("redo chunk record encode: {e}"), + }) + } + + pub fn from_bytes(bytes: &[u8]) -> crate::Result { + zerompk::from_msgpack(bytes).map_err(|e| crate::Error::Serialization { + format: "msgpack".into(), + detail: format!("redo chunk record decode: {e}"), + }) + } +} + +/// One open stream a data-group snapshot carries to the follower that +/// installs it. +#[derive( + Debug, + Clone, + PartialEq, + Eq, + Serialize, + Deserialize, + zerompk::ToMessagePack, + zerompk::FromMessagePack, +)] +pub struct CarriedRedoStream { + pub stream: RedoStreamId, + pub term: u64, + pub len: u64, + /// The chunks the stream holds, in index order. + pub chunks: Vec>, +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn a_chunk_record_round_trips() { + let record = RedoChunkRecord::Piece(RedoChunkPiece { + header: RedoChunkHeader { + stream: RedoStreamId::Session { + vshard: 3, + idempotency_key: 0xAB, + }, + group_id: 2, + term: 5, + len: 10, + index: 1, + }, + piece: 0, + pieces: 2, + bytes: vec![1, 2, 3], + }); + let bytes = record.to_bytes().expect("encode"); + assert_eq!(RedoChunkRecord::from_bytes(&bytes).expect("decode"), record); + let closed = RedoChunkRecord::Closed { + stream: RedoStreamId::Calvin { + vshard: 1, + epoch: 9, + position: 4, + attempt: 2, + }, + }; + let bytes = closed.to_bytes().expect("encode"); + assert_eq!(RedoChunkRecord::from_bytes(&bytes).expect("decode"), closed); + } +} diff --git a/nodedb/src/wal/redo/mod.rs b/nodedb/src/wal/redo/mod.rs index 6db7d70a2..685e575d4 100644 --- a/nodedb/src/wal/redo/mod.rs +++ b/nodedb/src/wal/redo/mod.rs @@ -1,6 +1,7 @@ // SPDX-License-Identifier: BUSL-1.1 pub mod capture; +pub mod chunk; pub mod continuation; pub mod group; pub mod record; @@ -9,6 +10,9 @@ pub mod row_changes; pub mod row_sources; pub use capture::{CapturedEffect, CapturedEntry, OriginAppend, WriteSetCapture}; +pub use chunk::{ + CarriedRedoStream, RedoChunkHeader, RedoChunkPiece, RedoChunkRecord, RedoStreamId, +}; pub use continuation::{ContinuedRedo, SplitRedo, split_redo}; pub use group::{CascadedEdge, GroupMembership, NodeCascadeRedo, WriteGroup, WriteGroupRecord}; pub use record::{ diff --git a/nodedb/src/wal/redo/record.rs b/nodedb/src/wal/redo/record.rs index 1de22dbd1..ddebbd4c6 100644 --- a/nodedb/src/wal/redo/record.rs +++ b/nodedb/src/wal/redo/record.rs @@ -109,10 +109,8 @@ pub struct RedoPublish { } /// The replicated position of the record that carries a committed message: -/// its change-feed partition, and `(epoch, index, base)` as the partition -/// orders writes. A Raft-applied record takes its entry's `(epoch, log -/// index)` with base `0`. A Calvin record takes its sequencer epoch as the -/// index and its batch position as the base. +/// its change-feed partition (the vShard), and its data-group entry's +/// `(epoch, log index)`. #[derive( Debug, Clone, @@ -129,7 +127,6 @@ pub struct PublishPosition { pub partition: u32, pub epoch: u64, pub index: u64, - pub base: u64, } /// Identity of one cross-shard trigger request: the source write's position @@ -175,9 +172,9 @@ pub struct RedoSubRecord { pub payload: Vec, } -/// Calvin sequencer coordinates that a [`RedoRecord`] may carry to double as an -/// applied-marker, and what the vShard's slice folds. Mirrors -/// `nodedb_wal::CalvinAppliedPayload` in its coordinates. +/// Calvin sequencer coordinates a [`RedoRecord`] carries when it installs a +/// committed Calvin slice. The record doubles as the position's applied +/// marker: boot recovery reads the stamp. #[derive( Debug, Clone, @@ -196,17 +193,6 @@ pub struct CalvinStamp { pub position: u32, /// The vshard that applied this transaction. pub vshard_id: u32, - /// Every collection the vShard's slice writes. - #[serde(default, skip_serializing_if = "Vec::is_empty")] - #[msgpack(default)] - pub collections: Vec, - /// The materialized-sum targets the slice's document writes fold into, - /// keyed by source collection. The live install and restart replay both - /// fold from them at the record's LSN, so no later record carries the - /// target rows. - #[serde(default, skip_serializing_if = "Vec::is_empty")] - #[msgpack(default)] - pub sum_targets: Vec, } /// Payload of a graph edge upsert: the bytes of an autocommit @@ -437,7 +423,6 @@ mod tests { partition: 3, epoch: 0, index: 41, - base: 0, }), }], row_changes: Vec::new(), @@ -470,8 +455,6 @@ mod tests { epoch: 42, position: 7, vshard_id: 3, - collections: Vec::new(), - sum_targets: Vec::new(), }), cross_shard_applied: None, row_sources: Vec::new(), diff --git a/nodedb/src/wal/redo/replay.rs b/nodedb/src/wal/redo/replay.rs index 2a70e607f..006a92dca 100644 --- a/nodedb/src/wal/redo/replay.rs +++ b/nodedb/src/wal/redo/replay.rs @@ -43,7 +43,9 @@ //! `replay_graph_redo` arms in `crate::data::executor`). //! //! `calvin_stamp` is ignored here: the Calvin recovery scan reads it, and it -//! does not gate engine replay. +//! does not gate engine replay. A Calvin slice's materialized-sum fold rows +//! journal as parts of the record's group, so replay applies them like any +//! other write's rows. //! //! The committed-redo apply (`handlers::transaction::redo_apply`) drives this //! same entry point with a one-record slice for every committed transaction, @@ -85,20 +87,12 @@ use crate::data::executor::core_loop::CoreLoop; /// on its core. Replay then applies a write's rows at the write's place in /// the order, wherever its parts sit in the WAL: a part boot journals at the /// tail never lands over a later write to the same row. -/// -/// Also returns the materialized-sum targets every Calvin record's stamp -/// carries, keyed by the record's LSN. -fn reconstitute_with_folds(records: &[WalRecord]) -> crate::Result { - let mut out = Reconstituted::default(); +fn reconstitute_redo_records(records: &[WalRecord]) -> crate::Result> { + let mut out = Vec::new(); for record in records { let (lsn, ops) = match RecordType::from_raw(record.logical_record_type()) { Some(RecordType::TransactionRedo) => { let redo = RedoRecord::from_bytes(&record.payload)?; - if let Some(stamp) = redo.calvin_stamp - && !stamp.sum_targets.is_empty() - { - out.folds.insert(record.header.lsn, stamp.sum_targets); - } (record.header.lsn, redo.ops) } Some(RecordType::WriteGroup) => { @@ -108,7 +102,7 @@ fn reconstitute_with_folds(records: &[WalRecord]) -> crate::Result continue, }; for sub in ops { - out.ops.push(WalRecord::new(WalRecordArgs { + out.push(WalRecord::new(WalRecordArgs { record_type: sub.record_type, lsn, tenant_id: record.header.tenant_id, @@ -123,18 +117,6 @@ fn reconstitute_with_folds(records: &[WalRecord]) -> crate::Result, - folds: RedoFolds, -} - -/// Materialized-sum targets per redo record LSN. -type RedoFolds = - std::collections::HashMap>; - /// Merge the standalone records with the reconstituted redo sub-records into /// one LSN-ordered sequence. /// @@ -198,25 +180,15 @@ impl CoreLoop { num_cores: usize, tombstones: &nodedb_wal::TombstoneSet, ) -> crate::Result<()> { - let Reconstituted { - ops: redo_ops, - folds, - } = reconstitute_with_folds(records)?; + let redo_ops = reconstitute_redo_records(records)?; let ordered = merge_by_lsn(records, &redo_ops); - // A committed-redo apply folds from its open scope. Restart replay - // folds from each Calvin record's stamp. - let restart = self.begin_replay_folds(folds); self.replay_vector_wal(&ordered, num_cores, tombstones); - crate::fail_point!("replay::between_engine_passes"); + crate::fail_point!(self.fail_scope(), "replay::between_engine_passes"); self.replay_vector_extended_wal(&ordered, num_cores, tombstones); // Every document row the WAL names, standalone and redo alike, in LSN - // order. A Calvin record's rows fold into their sum targets at the - // record's LSN, so the fold scope closes right after this arm. + // order. self.replay_document_redo(&ordered, num_cores, tombstones); - if restart { - self.end_replay_folds(); - } self.replay_kv_wal(&ordered, num_cores, tombstones); self.replay_timeseries_wal(&ordered, num_cores, tombstones); self.replay_array_wal(&ordered, num_cores, tombstones); @@ -231,7 +203,7 @@ impl CoreLoop { // a WAL record is their only durable backing. self.replay_graph_node_label_wal(records, num_cores); - crate::fail_point!("replay::between_standalone_and_redo"); + crate::fail_point!(self.fail_scope(), "replay::between_standalone_and_redo"); // Every graph edge version the WAL names, standalone and redo alike, in // LSN order. The edges' CSR is rebuilt from the `EdgeStore` at open, so @@ -271,11 +243,6 @@ mod tests { use super::*; use crate::wal::{RedoRecord, RedoSubRecord}; - /// The reconstituted records alone, without the fold targets. - fn reconstitute_redo_records(records: &[WalRecord]) -> crate::Result> { - Ok(super::reconstitute_with_folds(records)?.ops) - } - fn redo_wal_record(lsn: u64, tenant_id: u64, vshard_id: u32, record: &RedoRecord) -> WalRecord { WalRecord::new(WalRecordArgs { record_type: RecordType::TransactionRedo as u32, diff --git a/nodedb/src/wal/replay/surrogate/dispatch.rs b/nodedb/src/wal/replay/surrogate/dispatch.rs index 23c6eba88..524302d13 100644 --- a/nodedb/src/wal/replay/surrogate/dispatch.rs +++ b/nodedb/src/wal/replay/surrogate/dispatch.rs @@ -102,7 +102,6 @@ pub fn replay_surrogate_records( | RecordType::CollectionTombstoned | RecordType::TimeAnchor | RecordType::TemporalPurge - | RecordType::CalvinApplied // SyncSeqAdvance: not relevant to surrogate replay; the sync // idempotency replay pass handles HWM reconstruction. | RecordType::SyncSeqAdvance @@ -129,7 +128,10 @@ pub fn replay_surrogate_records( | RecordType::SnapshotInstalled // A write group carries row images, whose surrogates were bound // before the write that stored them. - | RecordType::WriteGroup => {} + | RecordType::WriteGroup + // A redo chunk holds stream bytes; its final record binds nothing + // the carried identities did not bind first. + | RecordType::RedoChunk => {} } } Ok(stats) diff --git a/nodedb/src/wal/replay/sync_hwm/advance.rs b/nodedb/src/wal/replay/sync_hwm/advance.rs index 34dc15918..07087e37b 100644 --- a/nodedb/src/wal/replay/sync_hwm/advance.rs +++ b/nodedb/src/wal/replay/sync_hwm/advance.rs @@ -117,7 +117,6 @@ pub fn replay_sync_hwm_records( | RecordType::CollectionTombstoned | RecordType::TimeAnchor | RecordType::TemporalPurge - | RecordType::CalvinApplied | RecordType::SurrogateAlloc | RecordType::SurrogateBind | RecordType::FtsIndex @@ -137,7 +136,8 @@ pub fn replay_sync_hwm_records( | RecordType::GraphNodeCascade | RecordType::GraphEdgeCut | RecordType::SnapshotInstalled - | RecordType::WriteGroup => {} + | RecordType::WriteGroup + | RecordType::RedoChunk => {} } } diff --git a/nodedb/tests/calvin_hold_liveness.rs b/nodedb/tests/calvin_hold_liveness.rs index 699e140d9..af52f6eff 100644 --- a/nodedb/tests/calvin_hold_liveness.rs +++ b/nodedb/tests/calvin_hold_liveness.rs @@ -1,12 +1,14 @@ // SPDX-License-Identifier: BUSL-1.1 -//! A held Calvin flush never stalls replicated writes to other collections. +//! A held Calvin redo proposal never stalls replicated writes to other +//! collections. //! //! Transaction A writes two collections on two vShards, so it commits -//! through the Calvin scheduler. Its flush on one vShard is held at a fail -//! point, so A stays staged: its rows are owned until the flush. A replicated -//! autocommit write to a third collection shares no row and no collection -//! with A. It must apply and be acknowledged while A's flush is held. +//! through the Calvin scheduler. The redo proposal of its slice on one vShard +//! is held at a fail point, so A stays staged and holds its locks. A +//! replicated autocommit write to a third collection shares no row and no +//! collection with A. It must apply and be acknowledged while A's redo is +//! held. //! //! Requires `--features failpoints`. @@ -23,27 +25,27 @@ use crash_harness::vshards::names_on_distinct_vshards; /// How long the Calvin sequencer may take to elect its leader after a boot. const CALVIN_READY_TIMEOUT: Duration = Duration::from_secs(30); -/// How long the test waits for A's flush to be held. +/// How long the test waits for A's redo proposal to be held. const HOLD_DEADLINE: Duration = Duration::from_secs(30); /// How long the unrelated write may take. Well under the 30s request -/// deadline a write parked behind A would wait out. +/// deadline a write waiting behind A would wait out. const UNRELATED_WRITE_BUDGET: Duration = Duration::from_secs(10); -/// The scheduler's log line for a flush held at the fail point. -const FLUSH_HELD: &str = "calvin: flush held at a fail point"; +/// The scheduler's log line for a redo proposal held at the fail point. +const REDO_HELD: &str = "calvin: redo proposal held at a fail point"; const LOG_DIRECTIVES: &str = "warn,nodedb::control::cluster::calvin::scheduler::driver::core=info"; #[tokio::test(flavor = "multi_thread")] -async fn a_replicated_write_to_an_unrelated_collection_applies_while_a_calvin_flush_is_held() { +async fn a_replicated_write_to_an_unrelated_collection_applies_while_a_calvin_redo_is_held() { let [held, peer, other] = names_on_distinct_vshards(["hold_held", "hold_peer", "hold_other"]); let h = CrashHarness::new(); - let release = h.data_dir().join("release-held-flush"); + let release = h.data_dir().join("release-held-redo"); let mut h = h.with_env("RUST_LOG", LOG_DIRECTIVES).with_env( "NODEDB_FAILPOINTS", &format!( - "calvin::before_flush::{held}=wait_file({})", + "calvin::before_redo_propose::{held}=wait_file({})", release.display() ), ); @@ -58,7 +60,7 @@ async fn a_replicated_write_to_an_unrelated_collection_applies_while_a_calvin_fl } // Transaction A, on its own connection: its COMMIT waits for the held - // flush. + // redo. let conn_str = h.pgwire_conn_str(); let statements = [ "BEGIN".to_string(), @@ -81,10 +83,10 @@ async fn a_replicated_write_to_an_unrelated_collection_applies_while_a_calvin_fl }); let deadline = Instant::now() + HOLD_DEADLINE; - while count_lines(&boot_section(&h.server_log(), 1), &[FLUSH_HELD]) == 0 { + while count_lines(&boot_section(&h.server_log(), 1), &[REDO_HELD]) == 0 { assert!( Instant::now() < deadline && !txn.is_finished(), - "transaction A's flush was never held (A finished: {}).{}\n{}", + "transaction A's redo was never held (A finished: {}).{}\n{}", txn.is_finished(), h.keep_data_dir_note(), crash_harness::diagnostics::log_tail_section(&h.server_log()) @@ -99,7 +101,7 @@ async fn a_replicated_write_to_an_unrelated_collection_applies_while_a_calvin_fl .await .unwrap_or_else(|_| { panic!( - "a replicated write to {other} stalled behind transaction A's held flush, \ + "a replicated write to {other} stalled behind transaction A's held redo, \ though it shares no row and no collection with A" ) }); diff --git a/nodedb/tests/crash_calvin_redo_cut.rs b/nodedb/tests/crash_calvin_redo_cut.rs new file mode 100644 index 000000000..1384dfe47 --- /dev/null +++ b/nodedb/tests/crash_calvin_redo_cut.rs @@ -0,0 +1,246 @@ +// SPDX-License-Identifier: BUSL-1.1 + +//! A Calvin install cut by a crash after its write set is stored applies +//! once. +//! +//! The source collection carries two materialized sums. One target shares +//! the source's vShard, so the source slice's install folds it inline. The +//! other target homes on another vShard, so its balance travels as an +//! `ApplyBalanceDelta` slice. One source `INSERT` therefore commits through +//! the Calvin scheduler, and each slice installs from a stamped redo entry. +//! +//! The process aborts at `core::after_capture::`: the core stored the +//! source install's write set, and its parts are not journalled or synced. +//! Boot journals the install from the stored write set. The stamp of that +//! record marks the position applied, so a redelivered copy of the entry +//! installs nothing, and both balances hold the single-apply total. +//! +//! The checkpoint interval is pushed beyond the test's runtime, so the values +//! read after the crash come from WAL replay alone. +//! +//! Requires `--features failpoints`. + +#![cfg(feature = "failpoints")] + +mod crash_harness; + +use std::collections::BTreeSet; +use std::time::{Duration, Instant}; + +use crash_harness::log_fields::same_value; +use crash_harness::vshards::names_on_distinct_vshards; +use crash_harness::{CrashHarness, diagnostics}; +use nodedb::control::cluster::calvin::scheduler::{CalvinAppliedLedgers, recover_all_applied}; +use nodedb::control::security::catalog::SystemCatalog; +use nodedb::wal::WalManager; +use nodedb_types::id::DatabaseId; + +/// A checkpoint between the crash and the reads holds the balances +/// independent of the WAL, and a doubled fold then hides in it. +const NO_CHECKPOINT_SECS: &str = "3600"; + +/// How long the Calvin sequencer may take to elect its leader after a boot. +const CALVIN_READY_TIMEOUT: Duration = Duration::from_secs(30); + +/// How long the process may take to abort once the insert is sent. +const CRASH_TIMEOUT: Duration = Duration::from_secs(60); + +/// How long a balance may take to move off its seed after the restart. +const SETTLE_TIMEOUT: Duration = Duration::from_secs(30); + +const ACCOUNT: &str = "acc-1"; +const SEED: &str = "100"; +const AMOUNT: &str = "25"; +const SINGLE_APPLY: &str = "125"; + +fn vshard_of(name: &str) -> u32 { + nodedb_types::CollectionKey::from_bare(DatabaseId::DEFAULT, name) + .vshard() + .as_u32() +} + +/// The first name `_` that homes on `vshard`. +fn name_on_vshard(prefix: &str, vshard: u32) -> String { + (0..1u32 << 16) + .map(|i| format!("{prefix}_{i}")) + .find(|name| vshard_of(name) == vshard) + .unwrap_or_else(|| panic!("no {prefix} name on vShard {vshard} in 65536 tries")) +} + +/// The balance of the account in `target`, once it moved off the seed or +/// `SETTLE_TIMEOUT` passed. +async fn settled_balance(h: &CrashHarness, target: &str) -> String { + let sql = format!("SELECT balance FROM {target} WHERE id = '{ACCOUNT}'"); + let deadline = Instant::now() + SETTLE_TIMEOUT; + loop { + let read = h.query_col_idx(&sql, 0).await; + assert_eq!(read.len(), 1, "{target} holds one account row: {read:?}"); + if !same_value(&read[0], SEED) || Instant::now() >= deadline { + return read[0].clone(); + } + tokio::time::sleep(Duration::from_millis(100)).await; + } +} + +/// The applied ledgers boot fills from the stopped server's WAL and the +/// state its catalog saved. +fn boot_ledgers(h: &CrashHarness) -> CalvinAppliedLedgers { + let wal = WalManager::open_for_testing(&h.data_dir().join("wal")).expect("open the WAL"); + let catalog = SystemCatalog::open(&h.data_dir().join("system.redb")).expect("open the catalog"); + let ledgers = CalvinAppliedLedgers::default(); + let recovered = + recover_all_applied(&wal, &catalog, &|_| None).expect("recover applied positions"); + for (vshard_id, state) in recovered { + ledgers.install(vshard_id, state.fully_applied_epoch, state.applied_tail); + } + ledgers +} + +#[tokio::test(flavor = "multi_thread")] +async fn a_calvin_install_cut_after_its_write_set_is_stored_applies_once() { + let [source, remote] = names_on_distinct_vshards(["cut_entries", "cut_remote"]); + let local = name_on_vshard("cut_local", vshard_of(&source)); + assert_ne!(vshard_of(&source), vshard_of(&remote)); + + let mut h = CrashHarness::new().with_env("NODEDB_CHECKPOINT_INTERVAL_SECS", NO_CHECKPOINT_SECS); + h.spawn(); + h.wait_ready(); + h.wait_for_calvin_ready(CALVIN_READY_TIMEOUT).await; + for target in [&local, &remote] { + h.exec(&format!( + "CREATE COLLECTION {target} (id TEXT PRIMARY KEY, owner TEXT) \ + WITH (engine='document_strict')" + )) + .await; + } + h.exec(&format!( + "CREATE COLLECTION {source} (id TEXT PRIMARY KEY, account_id TEXT, amount TEXT) \ + WITH (engine='document_strict')" + )) + .await; + for target in [&local, &remote] { + h.exec(&format!( + "ALTER COLLECTION {target} ADD COLUMN balance TEXT \ + MATERIALIZED_SUM SOURCE {source} \ + ON {source}.account_id = {target}.id VALUE {source}.amount" + )) + .await; + h.exec(&format!( + "INSERT INTO {target} (id, owner, balance) VALUES ('{ACCOUNT}', 'alice', '{SEED}')" + )) + .await; + } + + // The next boot aborts once a write to the source stored its write set. + // The abort matches only a journalled write, so boot itself passes it. + h.kill_9(); + h.set_env( + "NODEDB_FAILPOINTS", + &format!("core::after_capture::{source}=abort"), + ); + h.reopen(); + h.wait_for_calvin_ready(CALVIN_READY_TIMEOUT).await; + + // The insert's client loses its connection with the process, so its + // result says nothing. + let conn_str = h.pgwire_conn_str(); + let insert = format!( + "INSERT INTO {source} (id, account_id, amount) VALUES ('e1', '{ACCOUNT}', '{AMOUNT}')" + ); + let insert_task = tokio::spawn(async move { + let (client, connection) = tokio_postgres::connect(&conn_str, tokio_postgres::NoTls) + .await + .map_err(|e| e.to_string())?; + tokio::spawn(connection); + client + .simple_query(&insert) + .await + .map(|_| ()) + .map_err(|e| e.to_string()) + }); + h.await_self_crash(CRASH_TIMEOUT); + let marker = format!("fail_point aborting process: core::after_capture::{source}"); + assert!( + h.server_log().contains(&marker), + "the process exited, but not after the source install stored its write set.{}\n{}", + h.keep_data_dir_note(), + diagnostics::log_tail_section(&h.server_log()) + ); + let _ = insert_task.await; + + h.clear_env("NODEDB_FAILPOINTS"); + h.reopen(); + h.wait_for_calvin_ready(CALVIN_READY_TIMEOUT).await; + + for target in [&local, &remote] { + let balance = settled_balance(&h, target).await; + assert!( + same_value(&balance, SINGLE_APPLY), + "{target} holds balance {balance}, not the single-apply total {SINGLE_APPLY}.{}\n{}", + h.keep_data_dir_note(), + diagnostics::log_tail_section(&h.server_log()) + ); + } + assert_eq!( + h.query_col_idx(&format!("SELECT id FROM {source}"), 0) + .await, + vec!["e1".to_string()], + "the source row is stored once" + ); + + // The ledgers a boot fills from this data directory report the + // transaction's position applied on both of its vShards. + h.kill_9(); + let ledgers = boot_ledgers(&h); + let tail = |vshard: u32| -> BTreeSet<(u64, u32)> { + ledgers + .get(vshard) + .map(|ledger| ledger.snapshot().1) + .unwrap_or_default() + }; + let shared: Vec<(u64, u32)> = tail(vshard_of(&source)) + .intersection(&tail(vshard_of(&remote))) + .copied() + .collect(); + assert_eq!( + shared.len(), + 1, + "the cross-shard transaction is the one position applied on both vShards: {shared:?}" + ); + let (epoch, position) = shared[0]; + for vshard in [vshard_of(&source), vshard_of(&remote)] { + assert!( + ledgers + .get(vshard) + .is_some_and(|ledger| ledger.is_applied(epoch, position)), + "the ledger of vShard {vshard} reports ({epoch}, {position}) applied" + ); + } + + // One more boot replays the WAL and takes every redelivered entry again. + // A second install of either slice folds a second time, and the reads + // below then show the doubled total. + h.reopen(); + h.wait_for_calvin_ready(CALVIN_READY_TIMEOUT).await; + for target in [&local, &remote] { + let read = h + .query_col_idx( + &format!("SELECT balance FROM {target} WHERE id = '{ACCOUNT}'"), + 0, + ) + .await; + assert!( + read.len() == 1 && same_value(&read[0], SINGLE_APPLY), + "after a second restart {target} holds {read:?}, not the single-apply total \ + {SINGLE_APPLY}: a copy of the entry installed again.{}\n{}", + h.keep_data_dir_note(), + diagnostics::log_tail_section(&h.server_log()) + ); + } + assert_eq!( + h.query_col_idx(&format!("SELECT id FROM {source}"), 0) + .await, + vec!["e1".to_string()], + "after a second restart the source row is stored once" + ); +} diff --git a/nodedb/tests/crash_harness/vshards.rs b/nodedb/tests/crash_harness/vshards.rs index e3b55cf92..f9187388f 100644 --- a/nodedb/tests/crash_harness/vshards.rs +++ b/nodedb/tests/crash_harness/vshards.rs @@ -40,19 +40,20 @@ pub fn single_node_data_group(name: &str) -> u32 { /// The data Raft groups a single-node server applies the cell writes of one /// `INT64` dimension `[lo..hi]` array in. /// -/// An array cell write routes by its coordinate, never by the array's name. -/// The coordinate's Hilbert prefix picks a bucket from its top `prefix_bits` -/// bits, and the bucket owns vShard `bucket * (1024 >> prefix_bits)`. At the -/// default of 8 bits every array vShard is a multiple of 4, so every cell of -/// every array applies in data group 1. Only `prefix_bits` of 9 or more -/// spreads cells over more than one group. +/// An array cell write routes by its tile, never by the array's name. The +/// tile's key picks a bucket from its top `prefix_bits` bits, and the bucket +/// owns vShard `bucket * (1024 >> prefix_bits)`. At the default of 8 bits +/// every array vShard is a multiple of 4, so every cell of every array +/// applies in data group 1. Only `prefix_bits` of 9 or more spreads cells +/// over more than one group. `tile_extent` must match the array's +/// `TILE_EXTENTS`. pub struct ArrayCellRoute { schema: nodedb_array::ArraySchema, prefix_bits: u8, } impl ArrayCellRoute { - pub fn new(lo: i64, hi: i64, prefix_bits: u8) -> Self { + pub fn new(lo: i64, hi: i64, tile_extent: u64, prefix_bits: u8) -> Self { use nodedb_array::schema::{ArraySchemaBuilder, AttrSpec, AttrType, DimSpec, DimType}; use nodedb_array::types::domain::{Domain, DomainBound}; let schema = ArraySchemaBuilder::new("route") @@ -62,7 +63,7 @@ impl ArrayCellRoute { Domain::new(DomainBound::Int64(lo), DomainBound::Int64(hi)), )) .attr(AttrSpec::new("v", AttrType::Float64, true)) - .tile_extents(vec![1]) + .tile_extents(vec![tile_extent]) .build() .expect("a one-dimension routing schema"); Self { @@ -74,7 +75,7 @@ impl ArrayCellRoute { /// The data group the cell at `coord` applies in. pub fn group(&self, coord: i64) -> u32 { use nodedb_array::types::coord::value::CoordValue; - let prefix = nodedb_array::encode_hilbert_prefix(&self.schema, &[CoordValue::Int64(coord)]) + let prefix = nodedb_array::cell_tile_prefix(&self.schema, &[CoordValue::Int64(coord)]) .unwrap_or_else(|e| panic!("coordinate {coord} outside the routing schema: {e}")); let vshard = nodedb_cluster::distributed_array::array_vshard_for_tile(prefix, self.prefix_bits) diff --git a/nodedb/tests/crash_redo_chunk_stream.rs b/nodedb/tests/crash_redo_chunk_stream.rs new file mode 100644 index 000000000..2eb760a61 --- /dev/null +++ b/nodedb/tests/crash_redo_chunk_stream.rs @@ -0,0 +1,153 @@ +// SPDX-License-Identifier: BUSL-1.1 + +//! A chunked session redo survives a crash inside its stream. +//! +//! The server boots with `tuning.calvin.max_redo_entry_bytes` at its floor, +//! so a commit of a few hundred kilobytes travels as chunk entries and a +//! final entry. Each test kills the server at a fail point inside the stream +//! and restarts it on the same data directory: +//! +//! - after every chunk is durable and before the final entry applies: boot +//! rebuilds the stream from its WAL records, and the re-delivered final +//! entry installs every row once; +//! - after the first chunk is durable: no final entry exists, so no row +//! installs, and the stream drops at the group's next term. A later +//! chunked commit installs as usual. +//! +//! Requires `--features failpoints`. + +#![cfg(feature = "failpoints")] + +mod crash_harness; + +use std::time::Duration; + +use crash_harness::{CrashHarness, diagnostics}; + +/// The redo entry limit the server boots with: the smallest the config takes. +const ENTRY_LIMIT: usize = 64 * 1024; + +/// Rows one statement inserts. +const ROWS_PER_STATEMENT: usize = 100; + +/// Statements one transaction runs. With 400-byte values the redo holds +/// several times `ENTRY_LIMIT`. +const STATEMENTS: usize = 6; + +/// Bounded wait for the injected abort. +const CRASH_TIMEOUT: Duration = Duration::from_secs(60); + +/// A harness whose server reads a config file with the lowered entry limit. +fn harness() -> CrashHarness { + let h = CrashHarness::new(); + let config = h.data_dir().join("redo_chunk.toml"); + std::fs::write( + &config, + format!("[tuning.calvin]\nmax_redo_entry_bytes = {ENTRY_LIMIT}\n"), + ) + .expect("write config file"); + let path = config.to_string_lossy().to_string(); + h.with_env("NODEDB_CONFIG", &path) +} + +async fn count(h: &CrashHarness, coll: &str) -> usize { + let rows = h + .query_col_idx(&format!("SELECT COUNT(*) FROM {coll}"), 0) + .await; + assert_eq!(rows.len(), 1, "expected one COUNT(*) row, got {rows:?}"); + rows[0].parse().expect("COUNT(*) is a number") +} + +/// Run one large transaction on its own connection. Returns whether COMMIT +/// answered success: a server that aborts mid-commit answers nothing. +async fn commit_large_transaction(h: &CrashHarness, coll: &str, prefix: &str) -> bool { + let (client, connection) = tokio_postgres::connect(&h.pgwire_conn_str(), tokio_postgres::NoTls) + .await + .expect("connect"); + let conn = tokio::spawn(async move { + let _ = connection.await; + }); + let filler = "x".repeat(400); + client.simple_query("BEGIN").await.expect("BEGIN"); + for statement in 0..STATEMENTS { + let values: Vec = (0..ROWS_PER_STATEMENT) + .map(|row| { + let id = statement * ROWS_PER_STATEMENT + row; + format!("('{prefix}{id}', '{filler}')") + }) + .collect(); + client + .simple_query(&format!( + "INSERT INTO {coll} (id, value) VALUES {}", + values.join(", ") + )) + .await + .expect("INSERT inside the transaction"); + } + let committed = client.simple_query("COMMIT").await.is_ok(); + drop(client); + let _ = conn.await; + committed +} + +/// Arm `fail_point`, crash the server inside a chunked commit, and restart it +/// without the fail point. +async fn crash_inside_a_chunked_commit(h: &mut CrashHarness, fail_point: &str) { + h.set_env("NODEDB_FAILPOINTS", &format!("{fail_point}=abort")); + h.spawn(); + h.wait_ready(); + h.exec( + "CREATE COLLECTION chunk_crash (id STRING PRIMARY KEY, value STRING) \ + WITH (engine='document_strict')", + ) + .await; + let committed = commit_large_transaction(h, "chunk_crash", "a").await; + assert!( + !committed, + "COMMIT answered success, but the server was armed to abort at `{fail_point}`" + ); + h.await_self_crash(CRASH_TIMEOUT); + let log = h.server_log(); + assert!( + log.contains(&format!("fail_point aborting process: {fail_point}")), + "the server exited, but not at the armed fail point `{fail_point}`, so this test \ + proves nothing about a crash inside a redo stream.{}\n{}", + h.keep_data_dir_note(), + diagnostics::log_tail_section(&log) + ); + h.clear_env("NODEDB_FAILPOINTS"); + h.reopen(); +} + +#[tokio::test(flavor = "multi_thread")] +async fn a_crash_before_the_final_entry_applies_installs_the_whole_redo_once() { + let mut h = harness(); + crash_inside_a_chunked_commit(&mut h, "redo_chunk::before_final_install").await; + let expected = STATEMENTS * ROWS_PER_STATEMENT; + assert_eq!( + count(&h, "chunk_crash").await, + expected, + "the committed final entry must install every row of its rebuilt stream once\n{}", + diagnostics::log_tail_section(&h.server_log()) + ); +} + +#[tokio::test(flavor = "multi_thread")] +async fn a_crash_between_chunk_entries_installs_nothing_and_frees_the_stream() { + let mut h = harness(); + crash_inside_a_chunked_commit(&mut h, "redo_chunk::after_chunk_durable").await; + assert_eq!( + count(&h, "chunk_crash").await, + 0, + "a stream with no final entry must install no row" + ); + assert!( + commit_large_transaction(&h, "chunk_crash", "b").await, + "a chunked commit after the crash must succeed\n{}", + diagnostics::log_tail_section(&h.server_log()) + ); + assert_eq!( + count(&h, "chunk_crash").await, + STATEMENTS * ROWS_PER_STATEMENT + ); +} diff --git a/nodedb/tests/crash_replay_stamp/engines.rs b/nodedb/tests/crash_replay_stamp/engines.rs index d617552df..bd04a1c81 100644 --- a/nodedb/tests/crash_replay_stamp/engines.rs +++ b/nodedb/tests/crash_replay_stamp/engines.rs @@ -63,9 +63,9 @@ async fn a_columnar_write_in_flight_at_a_checkpoint_survives_kill_9() { /// That checkpoint flushes the held array with a stamp naming B writes above /// A. Boot 3 replays A below that stamp's highest LSN (`in_flight`). /// -/// An array cell write routes by its coordinate, not by the array's name (see -/// [`ArrayCellRoute`]). Both arrays take `prefix_bits = 10`, so their cells -/// spread over every data group. Each B cell sits at a coordinate whose group +/// An array cell write routes by its tile, not by the array's name (see +/// [`ArrayCellRoute`]). Both arrays take `prefix_bits = 10`, so the tiles of +/// the applied array spread over every data group. Each B cell sits at a coordinate whose group /// is not the group of A's cell. #[tokio::test(flavor = "multi_thread")] async fn an_array_write_in_flight_at_a_checkpoint_survives_kill_9() { @@ -107,10 +107,11 @@ const ARRAY_PREFIX_BITS: u8 = 10; /// The coordinate of B number `n` in the array case: the `n`th coordinate of /// `[0..1023]` whose cell applies in another data group than A's cell, coord 1 -/// of `[0..15]`. The domains match the case's `CREATE ARRAY` statements. +/// of `[0..15]`. The domains and tile extents match the case's `CREATE +/// ARRAY` statements. fn applied_array_coord(n: usize) -> i64 { - let held_group = ArrayCellRoute::new(0, 15, ARRAY_PREFIX_BITS).group(1); - let applied = ArrayCellRoute::new(0, 1023, ARRAY_PREFIX_BITS); + let held_group = ArrayCellRoute::new(0, 15, 16, ARRAY_PREFIX_BITS).group(1); + let applied = ArrayCellRoute::new(0, 1023, 64, ARRAY_PREFIX_BITS); (0..=1023) .filter(|coord| applied.group(*coord) != held_group) .nth(n) diff --git a/nodedb/tests/crash_replay_stamp_calvin.rs b/nodedb/tests/crash_replay_stamp_calvin.rs index 7dc7dd643..d15ef09bf 100644 --- a/nodedb/tests/crash_replay_stamp_calvin.rs +++ b/nodedb/tests/crash_replay_stamp_calvin.rs @@ -1,29 +1,29 @@ // SPDX-License-Identifier: BUSL-1.1 -//! A Calvin transaction's redo record still in flight when a checkpoint is -//! written survives a crash. +//! A Calvin slice's stamped redo still in flight when a checkpoint is written +//! survives a crash. //! //! The server runs the default single-node Calvin stack. A transaction that -//! writes two collections on different vShards commits through the Calvin -//! scheduler. Each vShard's scheduler appends the transaction's redo record, -//! then sends a flush that installs it on the core. The sequence: +//! writes two collections in different data groups commits through the Calvin +//! scheduler. On the COMMIT verdict each slice's leader proposes the slice's +//! stamped redo to its data group, and the group's apply installs it. The +//! sequence: //! -//! 1. Transaction A writes the held collection and a peer collection on -//! another vShard. The held vShard's scheduler appends A's redo record and -//! holds its flush at a fail point. No core installed the record. -//! 2. Writes B, autocommit `INSERT`s into a third collection, apply with -//! higher LSNs until a KV checkpoint's replay stamp names one of them above -//! its prefix. That proves the checkpoint was written while A's record was -//! appended and not applied. +//! 1. Transaction A writes the held collection and a peer collection. The +//! held group's apply appends A's stamped redo record and parks it at +//! `funnel::before_dispatch::`. No core installed the record. +//! 2. Writes B, autocommit `INSERT`s into a third collection in another data +//! group, apply with higher LSNs until a KV checkpoint's replay stamp names +//! one of them above its prefix. That proves the checkpoint was written +//! while A's record was appended and not applied. //! A's COMMIT stays unacknowledged for the whole hold. It either still //! waits, or it gave up at the server's completion deadline, which is no -//! acknowledgement. A COMMIT that returned success is the bug the hold -//! guards against. -//! 3. The test releases the flush. The core installs A's record, and the -//! process aborts before the flush response leaves. -//! 4. After restart, replay must apply A's record. The Calvin scheduler reads -//! the same record as the transaction's applied marker and never runs the -//! transaction again, so replay is the only way A's row returns. +//! acknowledgement. +//! 3. The test releases the gate. The core installs A's record, and the +//! process aborts before the install's response leaves. +//! 4. After restart, replay must apply A's record. Its stamp marks A's +//! position applied, so the scheduler never runs A again and the data +//! group's re-delivered entry installs nothing. //! //! Requires `--features failpoints`. @@ -33,47 +33,39 @@ mod crash_harness; use std::time::{Duration, Instant}; -use crash_harness::log_fields::{boot_section, count_lines, log_field}; -use crash_harness::vshards::names_on_distinct_vshards; +use crash_harness::log_fields::{boot_section, log_field}; +use crash_harness::vshards::names_in_distinct_data_groups; use crash_harness::{CrashHarness, diagnostics}; -/// How long the test waits for the held flush, and for a checkpoint whose +/// How long the test waits for the held install, and for a checkpoint whose /// stamp names a B: thirty checkpoint cycles at one per second. const CHECKPOINT_DEADLINE: Duration = Duration::from_secs(30); -/// How long the process may take to abort once the flush is released. The -/// scheduler sends a held flush again on its next pass. +/// How long the process may take to abort once the gate is released. const CRASH_TIMEOUT: Duration = Duration::from_secs(60); /// How long the Calvin sequencer may take to elect its leader after a boot. const CALVIN_READY_TIMEOUT: Duration = Duration::from_secs(30); -/// The scheduler's log line for a flush held at the fail point. -const FLUSH_HELD: &str = "calvin: flush held at a fail point"; - /// The error a COMMIT returns once its completion deadline passes with no -/// acknowledgement. The redo record stays appended and the flush stays held. +/// acknowledgement. The redo record stays appended and the install stays held. const COMMIT_DEADLINE: &str = "timed out waiting for Calvin transaction completion"; -const LOG_DIRECTIVES: &str = "warn,nodedb::data::executor::kv_checkpoint=info,\ - nodedb::control::cluster::calvin::scheduler::driver::core=info"; +const LOG_DIRECTIVES: &str = "warn,nodedb::data::executor::kv_checkpoint=info"; + +/// The boot that holds A's install. +const HOLD_BOOT: u32 = 2; + +/// The boot after the crash. +const RESTORE_BOOT: u32 = 3; #[tokio::test(flavor = "multi_thread")] async fn a_calvin_redo_in_flight_at_a_checkpoint_survives_kill_9() { let [held, peer, applied] = - names_on_distinct_vshards(["calvin_held", "calvin_peer", "calvin_applied"]); - let h = CrashHarness::new(); - let release = h.data_dir().join("release-held-flush"); - let mut h = h + names_in_distinct_data_groups(["calvin_held", "calvin_peer", "calvin_applied"]); + let mut h = CrashHarness::new() .with_env("NODEDB_CHECKPOINT_INTERVAL_SECS", "1") - .with_env("RUST_LOG", LOG_DIRECTIVES) - .with_env( - "NODEDB_FAILPOINTS", - &format!( - "calvin::before_flush::{held}=wait_file({}),core::after_apply::{held}=abort", - release.display() - ), - ); + .with_env("RUST_LOG", LOG_DIRECTIVES); h.spawn(); h.wait_ready(); h.wait_for_calvin_ready(CALVIN_READY_TIMEOUT).await; @@ -84,7 +76,23 @@ async fn a_calvin_redo_in_flight_at_a_checkpoint_survives_kill_9() { .await; } - // Transaction A. Its COMMIT waits for the held flush, so it runs on its + // The next boot arms the gate and the abort, keyed to the held + // collection. Both match only a request carrying a WAL LSN, so boot + // itself passes them. + h.kill_9(); + let release = h.data_dir().join("release-held-install"); + let parked = h.data_dir().join("release-held-install.parked"); + h.set_env( + "NODEDB_FAILPOINTS", + &format!( + "funnel::before_dispatch::{held}=wait_file({}),core::after_apply::{held}=abort", + release.display() + ), + ); + h.reopen(); + h.wait_for_calvin_ready(CALVIN_READY_TIMEOUT).await; + + // Transaction A. Its COMMIT waits for the held install, so it runs on its // own connection. let conn_str = h.pgwire_conn_str(); let statements = [ @@ -110,13 +118,13 @@ async fn a_calvin_redo_in_flight_at_a_checkpoint_survives_kill_9() { Ok::<(), String>(()) }); - // A's redo record is appended before its flush is sent, so a held flush - // proves the record is in the WAL and not installed. + // The gate parks a write after its record is appended, so a parked + // install proves A's stamped record is in the WAL and not installed. let deadline = Instant::now() + CHECKPOINT_DEADLINE; - while count_lines(&boot_section(&h.server_log(), 1), &[FLUSH_HELD]) == 0 { + while !parked.exists() { assert!( Instant::now() < deadline && !txn_task.is_finished(), - "the flush of transaction A was never held: A did not commit through the \ + "the install of transaction A was never held: A did not commit through the \ Calvin scheduler.{}\n{}", h.keep_data_dir_note(), diagnostics::log_tail_section(&h.server_log()) @@ -134,7 +142,7 @@ async fn a_calvin_redo_in_flight_at_a_checkpoint_survives_kill_9() { .await; written += 1; tokio::time::sleep(Duration::from_millis(200)).await; - let log = boot_section(&h.server_log(), 1); + let log = boot_section(&h.server_log(), HOLD_BOOT); if log_field(&log, "KV checkpoint published", "applied_ranges") .iter() .any(|n| *n > 0) @@ -144,7 +152,7 @@ async fn a_calvin_redo_in_flight_at_a_checkpoint_survives_kill_9() { assert!( Instant::now() < deadline, "no KV checkpoint named an applied LSN above its prefix within \ - {CHECKPOINT_DEADLINE:?} while A's flush was held.{}\n{}", + {CHECKPOINT_DEADLINE:?} while A's install was held.{}\n{}", h.keep_data_dir_note(), diagnostics::log_tail_section(&h.server_log()) ); @@ -157,7 +165,7 @@ async fn a_calvin_redo_in_flight_at_a_checkpoint_survives_kill_9() { match outcome { Err(error) if error.contains(COMMIT_DEADLINE) => None, other => panic!( - "transaction A finished before its flush was released: {other:?}.{}\n{}", + "transaction A finished before its install was released: {other:?}.{}\n{}", h.keep_data_dir_note(), diagnostics::log_tail_section(&h.server_log()) ), @@ -169,14 +177,14 @@ async fn a_calvin_redo_in_flight_at_a_checkpoint_survives_kill_9() { let mut live = h.query_col_idx(&read_applied, 0).await; live.sort(); - // Release the flush. The core installs A's record, and the process aborts - // before the flush response leaves. + // Release the gate. The core installs A's record, and the process aborts + // before the install's response leaves. std::fs::write(&release, b"release").expect("create the release file"); h.await_self_crash(CRASH_TIMEOUT); let marker = format!("fail_point aborting process: core::after_apply::{held}"); assert!( h.server_log().contains(&marker), - "the process exited, but not after A's flush installed its record.{}\n{}", + "the process exited, but not after A's install applied its record.{}\n{}", h.keep_data_dir_note(), diagnostics::log_tail_section(&h.server_log()) ); @@ -190,7 +198,7 @@ async fn a_calvin_redo_in_flight_at_a_checkpoint_survives_kill_9() { h.wait_for_calvin_ready(CALVIN_READY_TIMEOUT).await; let proof = log_field( - &boot_section(&h.server_log(), 2), + &boot_section(&h.server_log(), RESTORE_BOOT), "KV checkpoint restored", "applied_ranges", ); diff --git a/nodedb/tests/inproc/cases/calvin_determinism_contract.rs b/nodedb/tests/inproc/cases/calvin_determinism_contract.rs index 01b75b91e..35e743ddb 100644 --- a/nodedb/tests/inproc/cases/calvin_determinism_contract.rs +++ b/nodedb/tests/inproc/cases/calvin_determinism_contract.rs @@ -84,6 +84,7 @@ fn make_request(plan: PhysicalPlan) -> Request { wal_lsn: None, resolved_now_ms: None, commit_hlc: None, + entry_version: None, admission: nodedb::bridge::envelope::Admission::Admitted, } } diff --git a/nodedb/tests/inproc/cases/calvin_executor_apply.rs b/nodedb/tests/inproc/cases/calvin_executor_apply.rs index a86e88c66..af5748947 100644 --- a/nodedb/tests/inproc/cases/calvin_executor_apply.rs +++ b/nodedb/tests/inproc/cases/calvin_executor_apply.rs @@ -3,9 +3,10 @@ //! Data Plane apply and rollback coverage for a committed Calvin transaction. //! //! These drive the participant's Data Plane steps directly: stage -//! (`CalvinExecuteStatic`), resolve (`CalvinResolve`) and flush the resolved -//! redo record at its LSN (`CalvinFlush`) — the steps the scheduler dispatches -//! once the global verdict is commit. +//! (`CalvinExecuteStatic`), resolve (`CalvinResolve`), and the stamped install +//! of the resolved redo record at its LSN (`ApplyTransactionRedo` with a +//! `CalvinInstall`), which the data group's apply runs once the leader +//! proposed the slice's stamped redo. use std::sync::Arc; use std::time::{Duration, Instant}; @@ -16,7 +17,9 @@ use nodedb::data::executor::core_loop::CoreLoop; use nodedb::types::*; use nodedb::wal::{RedoRecord, RedoSubRecord}; use nodedb_bridge::buffer::{Consumer, Producer, RingBuffer}; -use nodedb_physical::physical_plan::{KvOp, MetaOp, PhysicalPlan}; +use nodedb_physical::physical_plan::{ + CalvinInstall, CalvinReplySpec, CalvinResolved, KvOp, MetaOp, PhysicalPlan, RedoOrigin, +}; use nodedb_types::{OrdinalClock, QualifiedCollection}; // ── Helpers ───────────────────────────────────────────────────────────────── @@ -62,6 +65,7 @@ fn make_request(plan: PhysicalPlan) -> Request { wal_lsn: None, resolved_now_ms: None, commit_hlc: None, + entry_version: None, admission: nodedb::bridge::envelope::Admission::Admitted, } } @@ -117,14 +121,14 @@ fn kv_get(coll: &str, key: &[u8]) -> PhysicalPlan { } /// Stage `plans` as the Calvin transaction at `(epoch, 0)` and resolve it. -/// Returns the resolved redo record. +/// Returns the resolved redo record and the slice's reply spec. fn stage_and_resolve( core: &mut CoreLoop, tx: &mut Producer, rx: &mut Consumer, epoch: u64, plans: Vec, -) -> RedoRecord { +) -> (RedoRecord, CalvinReplySpec) { send_ok( core, tx, @@ -135,7 +139,6 @@ fn stage_and_resolve( tenant_id: TenantId::new(1), plans, epoch_system_ms: 0, - is_group_leader: true, versioned_reads: Vec::new(), body_plans: Vec::new(), }), @@ -146,25 +149,33 @@ fn stage_and_resolve( rx, PhysicalPlan::Meta(MetaOp::CalvinResolve { epoch, position: 0 }), ); - RedoRecord::from_bytes(&redo).expect("decode resolved redo") + let answer: CalvinResolved = zerompk::from_msgpack(&redo).expect("decode resolved answer"); + let record = RedoRecord::from_bytes(&answer.redo).expect("decode resolved redo"); + (record, answer.reply) } -/// Flush `redo` as the Calvin transaction at `(epoch, 0)` with its record at +/// Install `redo` as the stamped slice at `(epoch, 0)` with its record at /// `lsn`. -fn flush( +fn install( core: &mut CoreLoop, tx: &mut Producer, rx: &mut Consumer, epoch: u64, - redo: &RedoRecord, + (redo, reply): &(RedoRecord, CalvinReplySpec), lsn: u64, ) -> nodedb::bridge::envelope::Response { - let mut request = make_request(PhysicalPlan::Meta(MetaOp::CalvinFlush { - epoch, - position: 0, + let mut request = make_request(PhysicalPlan::Meta(MetaOp::ApplyTransactionRedo { redo: redo.to_bytes().expect("encode redo"), collections: vec!["orders".into(), "rollback_coll".into()], sum_targets: Vec::new(), + origin: RedoOrigin::Commit, + calvin: Some(CalvinInstall { + epoch, + position: 0, + epoch_system_ms: 0, + reply: reply.clone(), + user_write: true, + }), })); request.wal_lsn = Some(Lsn::new(lsn)); tx.try_push(BridgeRequest::unfloored(request)).unwrap(); @@ -195,9 +206,9 @@ fn failing_install_sub_record() -> RedoSubRecord { } } -// ── Test 1: successful Calvin flush ────────────────────────────────────────── +// ── Test 1: successful Calvin install ──────────────────────────────────────── -/// A committed Calvin transaction installs its redo record at the flush. The +/// A committed Calvin transaction installs its stamped redo record. The /// written key is readable after it. #[test] fn calvin_static_apply_success() { @@ -210,24 +221,24 @@ fn calvin_static_apply_success() { 1, vec![kv_put("orders", b"k1", b"v1")], ); - let resp = flush(&mut core, &mut tx, &mut rx, 1, &redo, 10); + let resp = install(&mut core, &mut tx, &mut rx, 1, &redo, 10); assert_eq!( resp.status, Status::Ok, - "Calvin flush must succeed; got {:?}", + "Calvin install must succeed; got {:?}", resp.error_code ); let payload = send_ok(&mut core, &mut tx, &mut rx, kv_get("orders", b"k1")); assert!( !payload.is_empty(), - "row must exist after a successful Calvin flush" + "row must exist after a successful Calvin install" ); } // ── Test 2: failing install → error without RollbackFailed ─────────────────── -/// When a sub-record of the redo record fails while it installs, the flush +/// When a sub-record of the redo record fails while it installs, the install /// rolls every write back. The response is `Status::Error` and is NOT /// `RollbackFailed` (the rollback itself succeeded), and the transaction's /// write is gone. @@ -242,13 +253,13 @@ fn calvin_static_apply_failure_rolls_back_cleanly() { 2, vec![kv_put("rollback_coll", b"should_be_gone", b"present")], ); - redo.ops.push(failing_install_sub_record()); - let resp = flush(&mut core, &mut tx, &mut rx, 2, &redo, 20); + redo.0.ops.push(failing_install_sub_record()); + let resp = install(&mut core, &mut tx, &mut rx, 2, &redo, 20); assert_eq!( resp.status, Status::Error, - "a failing sub-record must fail the flush; got {:?}", + "a failing sub-record must fail the install; got {:?}", resp.error_code ); assert!( diff --git a/nodedb/tests/inproc/cases/calvin_executor_caps.rs b/nodedb/tests/inproc/cases/calvin_executor_caps.rs index f4546864a..b07484c8f 100644 --- a/nodedb/tests/inproc/cases/calvin_executor_caps.rs +++ b/nodedb/tests/inproc/cases/calvin_executor_caps.rs @@ -85,6 +85,7 @@ fn submit_rejects_dependent_read_too_large() { ); m }, + expected: BTreeMap::new(), }); let err = inbox.submit(tx).expect_err("should reject: too large"); @@ -122,6 +123,7 @@ fn submit_accepts_dependent_read_within_byte_limit() { ); m }, + expected: BTreeMap::new(), }); inbox.submit(tx).expect("should accept: exactly at limit"); @@ -154,6 +156,7 @@ fn submit_rejects_dependent_read_fanout_too_wide() { } m }, + expected: BTreeMap::new(), }); let err = inbox.submit(tx).expect_err("should reject: too wide"); @@ -196,6 +199,7 @@ fn submit_accepts_dependent_read_exactly_at_fanout_limit() { } m }, + expected: BTreeMap::new(), }); inbox diff --git a/nodedb/tests/inproc/cases/calvin_executor_dependent_read.rs b/nodedb/tests/inproc/cases/calvin_executor_dependent_read.rs index 0d510760f..c4e2b4a0c 100644 --- a/nodedb/tests/inproc/cases/calvin_executor_dependent_read.rs +++ b/nodedb/tests/inproc/cases/calvin_executor_dependent_read.rs @@ -1,37 +1,28 @@ // SPDX-License-Identifier: BUSL-1.1 -//! Integration tests for the Calvin dependent-read path. -//! -//! Uses mock channels to simulate the passive → active read result flow -//! without a real cluster. +//! Integration tests for the Calvin dependent-read barrier's public surface: +//! the dependent transaction class, the barrier log a vShard folds from its +//! data-group log, and the check of the read values against the values the +//! coordinator read. -use std::collections::BTreeMap; -use std::time::Duration; - -use nodedb::control::cluster::calvin::scheduler::driver::barrier::{ - PendingDependentBarrier, ReadResultEvent, -}; -use nodedb::control::cluster::calvin::scheduler::lock_manager::TxnId; -use nodedb_physical::physical_plan::meta::PassiveReadKeyId; -use nodedb_types::{TenantId, Value}; - -use std::collections::BTreeSet; -use std::time::Instant; +use std::collections::{BTreeMap, BTreeSet}; +use nodedb::control::cluster::calvin::scheduler::driver::barrier::expected_reads_drift; +use nodedb::control::cluster::calvin::scheduler::{BarrierEvent, BarrierLog, BarrierOutcome}; use nodedb_cluster::calvin::types::{ - DependentReadSpec, EngineKeySet, PassiveReadKey, ReadWriteSet, SequencedTxn, SortedVec, - TxClass, VersionedReadSet, + DependentReadSpec, EngineKeySet, PassiveReadKey, ReadWriteSet, SortedVec, TxClass, + VersionedReadSet, }; -use nodedb_types::{DatabaseId, QualifiedCollection}; +use nodedb_physical::physical_plan::meta::PassiveReadKeyId; +use nodedb_types::{DatabaseId, QualifiedCollection, TenantId, Value}; fn two_distinct_collections() -> (String, String) { let mut first: Option<(String, u32)> = None; for i in 0u32..512 { let name = format!("col_{i}"); - let vshard = - nodedb_types::CollectionKey::from_bare(nodedb::types::DatabaseId::DEFAULT, &name) - .vshard() - .as_u32(); + let vshard = nodedb_types::CollectionKey::from_bare(DatabaseId::DEFAULT, &name) + .vshard() + .as_u32(); if let Some((ref fname, fv)) = first { if fv != vshard { return (fname.clone(), name); @@ -43,91 +34,24 @@ fn two_distinct_collections() -> (String, String) { panic!("could not find two distinct-vshard collections in 512 tries"); } -fn make_dependent_txn(passive_vshard: u32) -> SequencedTxn { - let (col_a, col_b) = two_distinct_collections(); - let write_set = ReadWriteSet::new(vec![ - EngineKeySet::Document { - collection: col_a.clone(), - surrogates: SortedVec::new(vec![1]), - }, - EngineKeySet::Document { - collection: col_b.clone(), - surrogates: SortedVec::new(vec![2]), - }, - ]); - let spec = DependentReadSpec { - passive_reads: { - let mut m = BTreeMap::new(); - m.insert( - passive_vshard, - vec![PassiveReadKey { - engine_key: EngineKeySet::Document { - collection: "passive_col".to_owned(), - surrogates: SortedVec::new(vec![42u32]), - }, - }], - ); - m - }, - }; - let tx_class = TxClass::new_dependent( - ReadWriteSet::new(vec![]), - write_set, - vec![], - TenantId::new(1), - spec, - VersionedReadSet::default(), +fn row(collection: &str, surrogate: u32) -> PassiveReadKeyId { + PassiveReadKeyId::surrogate( + QualifiedCollection::new(DatabaseId::DEFAULT, collection), + surrogate, ) - .expect("valid dependent TxClass"); - SequencedTxn { - epoch: 1, - position: 0, - tx_class, - epoch_system_ms: 0, - epoch_vshard_txn_count: 0, - lock_owner: None, - } } -#[test] -fn dependent_barrier_completes_after_passive_delivers() { - let txn = make_dependent_txn(7); - let passive_vshard = 7u32; - - let mut waiting_for = BTreeSet::new(); - waiting_for.insert(passive_vshard); - - let timeout_at = Instant::now() + Duration::from_secs(30); - - let mut barrier = PendingDependentBarrier { - txn: txn.clone(), - lock_owner: TxnId::new(txn.epoch, txn.position), - waiting_for, - received: BTreeMap::new(), - timeout_at, - }; - - assert!(!barrier.is_complete()); - - // Simulate the passive vshard delivering its read result. - let key_id = PassiveReadKeyId { - collection: QualifiedCollection::new(DatabaseId::DEFAULT, "passive_col"), - surrogate: 42, - }; - let values = vec![(key_id.clone(), Value::Integer(100))]; - - barrier.waiting_for.remove(&passive_vshard); - barrier.received.insert(passive_vshard, values); - - assert!(barrier.is_complete(), "barrier should be complete now"); - - let injected = barrier.assemble_injected_reads(); - assert_eq!(injected.len(), 1); - assert_eq!(injected.get(&key_id), Some(&Value::Integer(100))); +fn read(passive_vshard: u32, collection: &str, value: &[u8]) -> BarrierEvent { + BarrierEvent::Read { + passive_vshard, + values: vec![(row(collection, 1), Value::Bytes(value.to_vec()))], + } } +/// A dependent class names every passive vShard as a participant, and its +/// write vShards as its active participants. #[test] -fn dependent_barrier_not_complete_with_multiple_passives() { +fn a_dependent_class_participates_its_passive_and_active_vshards() { let (col_a, col_b) = two_distinct_collections(); let write_set = ReadWriteSet::new(vec![ EngineKeySet::Document { @@ -139,136 +63,79 @@ fn dependent_barrier_not_complete_with_multiple_passives() { surrogates: SortedVec::new(vec![2]), }, ]); - + let passive_vshard = 977u32; let spec = DependentReadSpec { - passive_reads: { - let mut m = BTreeMap::new(); - m.insert( - 10u32, - vec![PassiveReadKey { - engine_key: EngineKeySet::Document { - collection: "coll_10".to_owned(), - surrogates: SortedVec::new(vec![1u32]), - }, - }], - ); - m.insert( - 20u32, - vec![PassiveReadKey { - engine_key: EngineKeySet::Document { - collection: "coll_20".to_owned(), - surrogates: SortedVec::new(vec![2u32]), - }, - }], - ); - m - }, + passive_reads: BTreeMap::from([( + passive_vshard, + vec![PassiveReadKey { + engine_key: EngineKeySet::Document { + collection: "passive_col".to_owned(), + surrogates: SortedVec::new(vec![42u32]), + }, + }], + )]), + expected: BTreeMap::from([(row("passive_col", 42), Some(b"v".to_vec()))]), }; - let tx_class = TxClass::new_dependent( ReadWriteSet::new(vec![]), write_set, vec![], TenantId::new(1), + DatabaseId::DEFAULT, spec, VersionedReadSet::default(), ) - .expect("valid"); - - let mut waiting_for = BTreeSet::new(); - waiting_for.insert(10u32); - waiting_for.insert(20u32); - - let mut barrier = PendingDependentBarrier { - txn: SequencedTxn { - epoch: 1, - position: 0, - tx_class, - epoch_system_ms: 0, - epoch_vshard_txn_count: 0, - lock_owner: None, - }, - lock_owner: TxnId::new(1, 0), - waiting_for, - received: BTreeMap::new(), - timeout_at: Instant::now() + Duration::from_secs(30), - }; - - // Only one passive delivers — barrier should still be incomplete. - barrier.waiting_for.remove(&10u32); - barrier.received.insert( - 10u32, - vec![( - PassiveReadKeyId { - collection: QualifiedCollection::new(DatabaseId::DEFAULT, "coll_10"), - surrogate: 1, - }, - Value::Integer(42), - )], - ); + .expect("valid dependent TxClass"); - assert!(!barrier.is_complete(), "still waiting for vshard 20"); + let participants: Vec = tx_class + .participating_vshards() + .iter() + .map(|v| v.as_u32()) + .collect(); + assert!(participants.contains(&passive_vshard)); + let active = tx_class.active_vshards().expect("active vShards"); + assert_eq!(active.len(), 2); + assert!(!active.contains(&passive_vshard)); +} - // Both deliver — now complete. - barrier.waiting_for.remove(&20u32); - barrier.received.insert( - 20u32, - vec![( - PassiveReadKeyId { - collection: QualifiedCollection::new(DatabaseId::DEFAULT, "coll_20"), - surrogate: 2, - }, - Value::Integer(99), - )], +/// The barrier completes once every passive vShard's result is in, and the +/// values it injects are the ones the log carried. +#[test] +fn a_barrier_completes_once_every_passive_result_is_in() { + let passive: BTreeSet = [10, 20].into_iter().collect(); + let mut log = BarrierLog::default(); + log.note(read(10, "coll_10", b"a")); + assert_eq!(log.outcome(&passive), BarrierOutcome::Waiting); + log.note(read(20, "coll_20", b"b")); + assert_eq!(log.outcome(&passive), BarrierOutcome::Complete); + let injected = log.injected_reads(); + assert_eq!( + injected.get(&row("coll_10", 1)), + Some(&Value::Bytes(b"a".to_vec())) ); - - assert!(barrier.is_complete()); - let injected = barrier.assemble_injected_reads(); assert_eq!(injected.len(), 2); } +/// A timeout entry that comes before a passive result times the barrier +/// out, whatever arrives after it. #[test] -fn dependent_barrier_timeout_detected() { - let txn = make_dependent_txn(7); - let lock_owner = TxnId::new(txn.epoch, txn.position); - let barrier = PendingDependentBarrier { - txn, - lock_owner, - waiting_for: { - let mut s = BTreeSet::new(); - s.insert(7u32); - s - }, - received: BTreeMap::new(), - // Timeout already in the past. - timeout_at: Instant::now() - Duration::from_millis(1), - }; - - assert!( - barrier.is_timed_out(), - "barrier should report timeout when timeout_at is in the past" - ); +fn a_logged_timeout_before_a_result_times_the_barrier_out() { + let passive: BTreeSet = [10].into_iter().collect(); + let mut log = BarrierLog::default(); + log.note(BarrierEvent::Timeout); + log.note(read(10, "coll_10", b"a")); + assert_eq!(log.outcome(&passive), BarrierOutcome::TimedOut); } -/// Happy-path test: passive delivers, active assembles injected_reads correctly. +/// Values that match the coordinator's read pass. A moved value drifts. #[test] -fn read_result_event_assembles_correctly() { - let event = ReadResultEvent { - epoch: 5, - position: 2, - passive_vshard: 17, - tenant_id: TenantId::new(3), - values: vec![( - PassiveReadKeyId { - collection: QualifiedCollection::new(DatabaseId::DEFAULT, "coll"), - surrogate: 100, - }, - Value::Float(std::f64::consts::PI), - )], +fn read_values_are_checked_against_the_coordinators_read() { + let spec = DependentReadSpec { + passive_reads: BTreeMap::new(), + expected: BTreeMap::from([(row("coll", 1), Some(b"v1".to_vec()))]), }; - - assert_eq!(event.epoch, 5); - assert_eq!(event.passive_vshard, 17); - assert_eq!(event.values.len(), 1); - assert_eq!(event.values[0].0.collection.as_str(), "coll"); + let same = BTreeMap::from([(row("coll", 1), Value::Bytes(b"v1".to_vec()))]); + let moved = BTreeMap::from([(row("coll", 1), Value::Bytes(b"v2".to_vec()))]); + assert_eq!(expected_reads_drift(&spec, &same), None); + assert!(expected_reads_drift(&spec, &moved).is_some()); } diff --git a/nodedb/tests/inproc/cases/calvin_executor_panic_recovery.rs b/nodedb/tests/inproc/cases/calvin_executor_panic_recovery.rs index 4329149c8..095cc11e6 100644 --- a/nodedb/tests/inproc/cases/calvin_executor_panic_recovery.rs +++ b/nodedb/tests/inproc/cases/calvin_executor_panic_recovery.rs @@ -1,10 +1,10 @@ // SPDX-License-Identifier: BUSL-1.1 -//! Executor panic-recovery test for a committed Calvin transaction's flush. +//! Executor panic-recovery test for a committed Calvin slice's stamped install. //! -//! Compiled only with `--features failpoints`. Tests the flush's -//! panic-recovery path: when a panic fires while the flush installs the -//! transaction's redo record (`replay::between_standalone_and_redo`, after +//! Compiled only with `--features failpoints`. Tests the install's +//! panic-recovery path: when a panic fires while the stamped install writes +//! the transaction's redo record (`replay::between_standalone_and_redo`, after //! the KV writes landed), the install must: //! //! - Catch the panic and roll back every write the install made. @@ -13,12 +13,12 @@ //! //! ## Failure model alignment //! -//! Per the Calvin failure model: a panic while the flush installs causes the -//! shard to return `Status::Error`. The lock-manager invariant ("locks NOT -//! released on a failed flush") is enforced by the scheduler layer, which -//! halts. The executor's contract is: +//! Per the Calvin failure model: a panic while the stamped install writes +//! causes the shard to return `Status::Error`. The apply loop reports the +//! refusal, and the scheduler halts with the txn's locks held. The executor's +//! contract is: //! -//! 1. Rolled-back writes are not visible after the failed flush. +//! 1. Rolled-back writes are not visible after the failed install. //! 2. The error response is a retryable refusal naming the panic. //! 3. Subsequent operations on the same `CoreLoop` succeed (no state //! corruption from the unwind). @@ -43,7 +43,9 @@ use nodedb::bridge::envelope::{ErrorCode, Status}; #[cfg(feature = "failpoints")] use nodedb::fail_point::{FailAction, FailGuard}; #[cfg(feature = "failpoints")] -use nodedb_physical::physical_plan::{KvOp, MetaOp, PhysicalPlan}; +use nodedb_physical::physical_plan::{ + CalvinInstall, CalvinResolved, KvOp, MetaOp, PhysicalPlan, RedoOrigin, +}; #[cfg(feature = "failpoints")] use nodedb_types::TenantId as NodedbTenantId; #[cfg(feature = "failpoints")] @@ -60,7 +62,6 @@ fn calvin_static(epoch: u64, plans: Vec) -> PhysicalPlan { tenant_id: NodedbTenantId::new(1), plans, epoch_system_ms: 1_700_000_000_000, - is_group_leader: true, versioned_reads: vec![], body_plans: vec![], }) @@ -71,11 +72,11 @@ fn calvin_static(epoch: u64, plans: Vec) -> PhysicalPlan { const INSTALL_FAIL_POINT: &str = "replay::between_standalone_and_redo"; /// Stage a static Calvin transaction (validate + stage), resolve it into its -/// redo record, and flush the record at `lsn`, returning the flush response -/// (where any install-time panic surfaces). The stage and resolve steps must -/// always return `Status::Ok`. +/// redo record, and install the stamped record at `epoch * 10`, returning the +/// install response (where any install-time panic surfaces). The stage and +/// resolve steps must always return `Status::Ok`. #[cfg(feature = "failpoints")] -fn stage_then_flush( +fn stage_then_install( core: &mut nodedb::data::executor::core_loop::CoreLoop, tx: &mut nodedb_bridge::buffer::Producer, rx: &mut nodedb_bridge::buffer::Consumer, @@ -101,12 +102,20 @@ fn stage_then_flush( "resolve must succeed; got {:?}", resolved.error_code ); - let mut request = make_request(PhysicalPlan::Meta(MetaOp::CalvinFlush { - epoch, - position: 0, - redo: resolved.payload.to_vec(), + let answer: CalvinResolved = + zerompk::from_msgpack(resolved.payload.as_bytes()).expect("decode resolved answer"); + let mut request = make_request(PhysicalPlan::Meta(MetaOp::ApplyTransactionRedo { + redo: answer.redo, collections: Vec::new(), sum_targets: Vec::new(), + origin: RedoOrigin::Commit, + calvin: Some(CalvinInstall { + epoch, + position: 0, + epoch_system_ms: 1_700_000_000_000, + reply: answer.reply, + user_write: true, + }), })); request.wal_lsn = Some(nodedb::types::Lsn::new(epoch * 10)); tx.try_push(BridgeRequest::unfloored(request)).unwrap(); @@ -157,7 +166,8 @@ fn kv_get_in(coll: &str, key: &[u8]) -> PhysicalPlan { // ── Test 1: install panic caught, typed response returned ───────────────────── -/// Panic injected while the flush installs the transaction's redo record. +/// Panic injected while the stamped install writes the transaction's redo +/// record. /// /// The record carries two KV puts, which land before the fail point fires. /// The install must catch the unwind, roll both writes back, and answer with @@ -169,8 +179,8 @@ fn calvin_static_panic_returns_internal_error() { let _guard = FailGuard::install(INSTALL_FAIL_POINT, FailAction::Panic); - // Stage and resolve write nothing; the panic fires in the flush's install. - let resp = stage_then_flush( + // Stage and resolve write nothing; the panic fires in the install. + let resp = stage_then_install( &mut core, &mut tx, &mut rx, @@ -185,7 +195,7 @@ fn calvin_static_panic_returns_internal_error() { // ── Test 2: rolled-back writes not visible after Calvin panic ───────────────── -/// After a Calvin executor panic, writes from the failed flush must not be +/// After a Calvin executor panic, writes from the failed install must not be /// visible. The CoreLoop remains functional for subsequent requests. #[cfg(feature = "failpoints")] #[test] @@ -203,7 +213,7 @@ fn calvin_static_panic_rollback_not_visible() { // Inject a panic on the second sub-apply. let _guard = FailGuard::install(INSTALL_FAIL_POINT, FailAction::Panic); - let resp = stage_then_flush( + let resp = stage_then_install( &mut core, &mut tx, &mut rx, @@ -216,7 +226,7 @@ fn calvin_static_panic_rollback_not_visible() { assert_eq!( resp.status, Status::Error, - "a panicking flush must return Error; got {:?}", + "a panicking install must return Error; got {:?}", resp.status ); @@ -260,16 +270,16 @@ fn calvin_static_panic_rollback_not_visible() { /// After clearing the fail point, Calvin transactions must commit /// successfully. This confirms there is no state corruption from the earlier -/// panicking flush. +/// panicking install. #[cfg(feature = "failpoints")] #[test] fn calvin_static_normal_operation_resumes_after_panic() { let (mut core, mut tx, mut rx, _dir) = make_core(); - // Trigger a panicking flush. + // Trigger a panicking install. { let _guard = FailGuard::install(INSTALL_FAIL_POINT, FailAction::Panic); - let resp = stage_then_flush( + let resp = stage_then_install( &mut core, &mut tx, &mut rx, @@ -282,14 +292,14 @@ fn calvin_static_normal_operation_resumes_after_panic() { assert_eq!( resp.status, Status::Error, - "a panicking flush must return Error; got {:?}", + "a panicking install must return Error; got {:?}", resp.status ); // Guard drops here, clearing the fail point. } // A normal Calvin transaction after the fail point is cleared. - let success_resp = stage_then_flush( + let success_resp = stage_then_install( &mut core, &mut tx, &mut rx, @@ -299,7 +309,7 @@ fn calvin_static_normal_operation_resumes_after_panic() { assert_eq!( success_resp.status, Status::Ok, - "a Calvin flush after the fail point is cleared must succeed; got {:?}", + "a Calvin install after the fail point is cleared must succeed; got {:?}", success_resp.error_code ); @@ -324,13 +334,13 @@ fn calvin_static_normal_operation_resumes_after_panic() { // ── Test 4: WAL replay correctness — fresh CoreLoop sees only committed data ── -/// After a panicking Calvin flush, create a fresh `CoreLoop` at the same data -/// directory. The fresh core must see only data that was committed before the -/// panicking flush — the rolled-back writes must not appear after replay. +/// After a panicking Calvin install, create a fresh `CoreLoop` at the same +/// data directory. The fresh core must see only data that was committed before +/// the panicking install — the rolled-back writes must not appear after replay. /// /// The install rolled its writes back before the response returned, so a /// fresh core opened over the same directory holds none of them. The redo -/// record itself lives in the WAL the scheduler appends, which this +/// record itself lives in the WAL the data group's apply appends, which this /// core-level test does not write. #[cfg(feature = "failpoints")] #[test] @@ -360,36 +370,17 @@ fn calvin_static_replay_sees_only_committed_data() { (core, req_tx, resp_rx) }; - // Commit a reference write before the panicking flush. + // Commit a reference write before the panicking install. { - use nodedb::bridge::dispatch::BridgeRequest; - use nodedb::bridge::envelope::{Priority, Request}; + use nodedb::bridge::envelope::Request; use nodedb_physical::physical_plan::PhysicalPlan; - use std::time::{Duration, Instant}; let make_req = |plan: PhysicalPlan| Request { request_id: nodedb::types::RequestId::new(42), - tenant_id: nodedb::types::TenantId::new(1), - vshard_id: nodedb::types::VShardId::new(0), - database_id: nodedb::types::DatabaseId::DEFAULT, - plan, - deadline: Instant::now() + Duration::from_secs(5), - priority: Priority::Normal, - trace_id: nodedb_types::TraceId::ZERO, - consistency: nodedb::types::ReadConsistency::Strong, - idempotency_key: None, - event_source: nodedb::event::EventSource::User, - user_roles: Vec::new(), - user_id: None, - statement_digest: None, - txn_id: None, - wal_lsn: None, - resolved_now_ms: None, - commit_hlc: None, - admission: nodedb::bridge::envelope::Admission::Admitted, + ..make_request(plan) }; - // Commit a value before the panicking flush. + // Commit a value before the panicking install. tx.try_push(BridgeRequest::unfloored(make_req(kv_put_in( "replay_coll", b"pre_commit", @@ -406,9 +397,9 @@ fn calvin_static_replay_sees_only_committed_data() { ); // Panicking install — writes must not persist. Stage and resolve - // write nothing; the panic fires while the flush installs. + // write nothing; the panic fires while the install writes. let _guard = FailGuard::install(INSTALL_FAIL_POINT, FailAction::Panic); - let panic_resp = stage_then_flush( + let panic_resp = stage_then_install( &mut core, &mut tx, &mut rx, @@ -443,33 +434,15 @@ fn calvin_static_replay_sees_only_committed_data() { (core, req_tx, resp_rx) }; - use nodedb::bridge::dispatch::BridgeRequest; - use nodedb::bridge::envelope::{Priority, Request}; + use nodedb::bridge::envelope::Request; use nodedb_physical::physical_plan::PhysicalPlan; - use std::time::{Duration, Instant}; let make_req2 = |plan: PhysicalPlan| Request { request_id: nodedb::types::RequestId::new(43), - tenant_id: nodedb::types::TenantId::new(1), - vshard_id: nodedb::types::VShardId::new(0), - database_id: nodedb::types::DatabaseId::DEFAULT, - plan, - deadline: Instant::now() + Duration::from_secs(5), - priority: Priority::Normal, - trace_id: nodedb_types::TraceId::ZERO, - consistency: nodedb::types::ReadConsistency::Strong, - idempotency_key: None, - event_source: nodedb::event::EventSource::User, - user_roles: Vec::new(), - user_id: None, - statement_digest: None, - txn_id: None, - wal_lsn: None, - resolved_now_ms: None, - commit_hlc: None, admission: nodedb::bridge::envelope::Admission::Exempt( nodedb::bridge::envelope::ExemptReason::Read, ), + ..make_request(plan) }; // Note: this test does NOT assert that the pre-panic committed value is @@ -478,7 +451,7 @@ fn calvin_static_replay_sees_only_committed_data() { // on a single-CoreLoop reopen. WAL-driven KV state recovery is a property // of the cluster apply path (replicated entries → applier → engine), not // of CoreLoop::open. The meaningful invariant exercised below is that the - // rolled-back writes from the panicking flush do NOT appear, which holds + // rolled-back writes from the panicking install do NOT appear, which holds // trivially under empty-replay state and confirms the panic-rollback path // never let the bad writes reach durable storage. diff --git a/nodedb/tests/inproc/cases/calvin_executor_static_multishard.rs b/nodedb/tests/inproc/cases/calvin_executor_static_multishard.rs index 8116b1e73..f73ab597b 100644 --- a/nodedb/tests/inproc/cases/calvin_executor_static_multishard.rs +++ b/nodedb/tests/inproc/cases/calvin_executor_static_multishard.rs @@ -21,7 +21,6 @@ fn calvin_execute_static_round_trip_msgpack() { tenant_id: TenantId::new(1), plans: vec![], epoch_system_ms: 0, - is_group_leader: true, versioned_reads: vec![], body_plans: vec![], }; @@ -59,7 +58,6 @@ fn calvin_execute_static_and_active_are_distinct_variants() { tenant_id: TenantId::new(1), plans: vec![], epoch_system_ms: 0, - is_group_leader: true, versioned_reads: vec![], body_plans: vec![], }; diff --git a/nodedb/tests/inproc/cases/calvin_scheduler_restart.rs b/nodedb/tests/inproc/cases/calvin_scheduler_restart.rs index e745f51c4..af28640ed 100644 --- a/nodedb/tests/inproc/cases/calvin_scheduler_restart.rs +++ b/nodedb/tests/inproc/cases/calvin_scheduler_restart.rs @@ -3,10 +3,11 @@ //! Calvin scheduler restart idempotency tests. //! //! Covers the WAL-recovery layer: after N epochs, a freshly-opened -//! `WalManager` reads back the correct applied `(epoch, position)` markers, -//! including out-of-order writes, vshard isolation, the greenfield sentinel, -//! and — critically — a MULTI-POSITION epoch where only some positions -//! committed before the crash (a torn transaction must be re-applied, not lost). +//! `WalManager` reads back the correct applied `(epoch, position)` pairs from +//! the stamped `TransactionRedo` records, including out-of-order writes, +//! vshard isolation, the greenfield sentinel, and — critically — a +//! MULTI-POSITION epoch where only some positions committed before the crash +//! (a torn transaction must be re-applied, not lost). //! //! End-to-end scheduler rebuild via `MultiRaft::read_committed_entries` //! is covered by `nodedb-cluster/tests/calvin_3node_shard_failover.rs:: @@ -14,9 +15,14 @@ use tempfile::TempDir; -use nodedb::control::cluster::calvin::scheduler::{NOT_YET_APPLIED_EPOCH, read_applied_recovery}; -use nodedb::types::VShardId; +use nodedb::control::cluster::calvin::scheduler::{ + AppliedRecovery, NOT_YET_APPLIED_EPOCH, recover_all_applied, +}; +use nodedb::control::security::catalog::SystemCatalog; +use nodedb::event::EventSource; +use nodedb::types::{DatabaseId, TenantId, VShardId}; use nodedb::wal::manager::{NO_APPLY_KEY, WalManager}; +use nodedb::wal::{CalvinStamp, RedoRecord}; // ── Helper ──────────────────────────────────────────────────────────────────── @@ -24,11 +30,53 @@ fn open_wal(dir: &TempDir) -> WalManager { WalManager::open_for_testing(dir.path()).expect("open wal") } +/// Append the stamped redo record of the Calvin slice at `(epoch, position)` +/// on `vshard_id`: the WAL record a committed slice's install writes. +fn append_stamped(wal: &WalManager, vshard_id: u32, epoch: u64, position: u32) { + let record = RedoRecord { + version: 1, + ops: Vec::new(), + calvin_stamp: Some(CalvinStamp { + epoch, + position, + vshard_id, + }), + cross_shard_applied: None, + row_sources: Vec::new(), + publishes: Vec::new(), + row_changes: Vec::new(), + }; + wal.appender(NO_APPLY_KEY) + .with_event_source(EventSource::User) + .append_transaction_redo( + TenantId::new(0), + VShardId::new(vshard_id), + DatabaseId::DEFAULT, + &record, + ) + .expect("append stamped redo"); +} + +/// The recovered state of `vshard_id` from a catalog with nothing saved. A +/// vShard with no stamped redo record recovers as the greenfield state: the +/// `NOT_YET_APPLIED_EPOCH` sentinel and an empty tail. +fn recover_vshard(wal: &WalManager, vshard_id: u32) -> AppliedRecovery { + let catalog = SystemCatalog::open_in_memory().expect("in-memory catalog"); + recover_all_applied(wal, &catalog, &|_| None) + .expect("recovery scan should succeed") + .remove(&vshard_id) + .unwrap_or(AppliedRecovery { + fully_applied_epoch: NOT_YET_APPLIED_EPOCH, + applied_tail: Default::default(), + max_applied_epoch: NOT_YET_APPLIED_EPOCH, + }) +} + // ── Tests ───────────────────────────────────────────────────────────────────── -/// Simulating 5 epochs (each with a single position 0): append CalvinApplied +/// Simulating 5 epochs (each with a single position 0): append stamped redo /// records, sync, then verify that a freshly-opened WalManager reads back all -/// five markers and a max-applied epoch of 5. +/// five positions and a max-applied epoch of 5. #[test] fn scheduler_restart_reads_applied_markers_after_five_epochs() { let dir = TempDir::new().unwrap(); @@ -37,16 +85,14 @@ fn scheduler_restart_reads_applied_markers_after_five_epochs() { { let wal = open_wal(&dir); for epoch in 1u64..=5 { - wal.appender(NO_APPLY_KEY) - .append_calvin_applied(VShardId::new(vshard_id), epoch, 0) - .unwrap(); + append_stamped(&wal, vshard_id, epoch, 0); } wal.sync().unwrap(); } { let wal = open_wal(&dir); - let rec = read_applied_recovery(&wal, vshard_id).expect("recovery scan should succeed"); + let rec = recover_vshard(&wal, vshard_id); assert_eq!(rec.max_applied_epoch, 5, "max applied epoch should be 5"); for epoch in 1u64..=5 { assert!( @@ -66,24 +112,15 @@ fn scheduler_restart_reports_max_and_all_positions_regardless_of_order() { { let wal = open_wal(&dir); - wal.appender(NO_APPLY_KEY) - .append_calvin_applied(VShardId::new(vshard_id), 3, 0) - .unwrap(); - wal.appender(NO_APPLY_KEY) - .append_calvin_applied(VShardId::new(vshard_id), 1, 0) - .unwrap(); - wal.appender(NO_APPLY_KEY) - .append_calvin_applied(VShardId::new(vshard_id), 5, 0) - .unwrap(); - wal.appender(NO_APPLY_KEY) - .append_calvin_applied(VShardId::new(vshard_id), 2, 0) - .unwrap(); + for epoch in [3u64, 1, 5, 2] { + append_stamped(&wal, vshard_id, epoch, 0); + } wal.sync().unwrap(); } { let wal = open_wal(&dir); - let rec = read_applied_recovery(&wal, vshard_id).unwrap(); + let rec = recover_vshard(&wal, vshard_id); assert_eq!( rec.max_applied_epoch, 5, "max should be 5, not last-written 2" @@ -112,19 +149,15 @@ fn scheduler_restart_multi_position_epoch_does_not_lose_uncommitted_position() { { let wal = open_wal(&dir); // Prior epoch fully applied. - wal.appender(NO_APPLY_KEY) - .append_calvin_applied(VShardId::new(vshard_id), 8, 0) - .unwrap(); + append_stamped(&wal, vshard_id, 8, 0); // Torn epoch: position 0 committed, position 1 did NOT (crash between). - wal.appender(NO_APPLY_KEY) - .append_calvin_applied(VShardId::new(vshard_id), torn_epoch, 0) - .unwrap(); + append_stamped(&wal, vshard_id, torn_epoch, 0); wal.sync().unwrap(); } { let wal = open_wal(&dir); - let rec = read_applied_recovery(&wal, vshard_id).unwrap(); + let rec = recover_vshard(&wal, vshard_id); assert!( rec.applied_tail.contains(&(torn_epoch, 0)), "committed position 0 must stay applied (not re-applied on restart)" @@ -134,8 +167,7 @@ fn scheduler_restart_multi_position_epoch_does_not_lose_uncommitted_position() { "uncommitted position 1 must be reported NOT applied so it is \ re-delivered and applied on restart — else a lost/torn transaction" ); - // The fully-applied watermark stays below the torn epoch: the per-epoch - // collapse that caused the bug is gone. + // The fully-applied watermark stays below the torn epoch. assert!( rec.fully_applied_epoch == NOT_YET_APPLIED_EPOCH || rec.fully_applied_epoch < torn_epoch, @@ -144,7 +176,7 @@ fn scheduler_restart_multi_position_epoch_does_not_lose_uncommitted_position() { } } -/// Greenfield: a WAL with no CalvinApplied records returns the +/// Greenfield: a WAL with no stamped redo record returns the /// `NOT_YET_APPLIED_EPOCH` sentinel and an empty tail. Epoch 0 is a valid real /// epoch, so a distinct sentinel distinguishes "never applied" from "applied /// epoch 0". @@ -152,35 +184,29 @@ fn scheduler_restart_multi_position_epoch_does_not_lose_uncommitted_position() { fn scheduler_restart_greenfield_returns_sentinel() { let dir = TempDir::new().unwrap(); let wal = open_wal(&dir); - let rec = read_applied_recovery(&wal, 1).unwrap(); + let rec = recover_vshard(&wal, 1); assert_eq!(rec.fully_applied_epoch, NOT_YET_APPLIED_EPOCH); assert_eq!(rec.max_applied_epoch, NOT_YET_APPLIED_EPOCH); assert!(rec.applied_tail.is_empty()); } -/// Multiple vshards: recovery for one vshard must not see another's markers. +/// Multiple vshards: recovery for one vshard must not see another's records. #[test] fn scheduler_restart_vshard_isolation() { let dir = TempDir::new().unwrap(); { let wal = open_wal(&dir); - wal.appender(NO_APPLY_KEY) - .append_calvin_applied(VShardId::new(1), 10, 0) - .unwrap(); - wal.appender(NO_APPLY_KEY) - .append_calvin_applied(VShardId::new(2), 99, 0) - .unwrap(); - wal.appender(NO_APPLY_KEY) - .append_calvin_applied(VShardId::new(1), 20, 0) - .unwrap(); + append_stamped(&wal, 1, 10, 0); + append_stamped(&wal, 2, 99, 0); + append_stamped(&wal, 1, 20, 0); wal.sync().unwrap(); } { let wal = open_wal(&dir); - let r1 = read_applied_recovery(&wal, 1).unwrap(); - let r2 = read_applied_recovery(&wal, 2).unwrap(); + let r1 = recover_vshard(&wal, 1); + let r2 = recover_vshard(&wal, 2); assert_eq!(r1.max_applied_epoch, 20, "vshard 1 max is 20"); assert!(r1.applied_tail.contains(&(10, 0)) && r1.applied_tail.contains(&(20, 0))); assert!(!r1.applied_tail.contains(&(99, 0)), "must not see vshard 2"); diff --git a/nodedb/tests/inproc/cases/calvin_two_phase_apply.rs b/nodedb/tests/inproc/cases/calvin_two_phase_apply.rs deleted file mode 100644 index c65106ffd..000000000 --- a/nodedb/tests/inproc/cases/calvin_two_phase_apply.rs +++ /dev/null @@ -1,901 +0,0 @@ -// SPDX-License-Identifier: BUSL-1.1 - -//! Staged Calvin apply on the Data Plane: `MetaOp::CalvinExecuteStatic` -//! VALIDATES + STAGES a transaction's write plans into the commit-pending -//! buffer WITHOUT mutating base, returning the local commit vote on -//! `read_set_valid`. `MetaOp::CalvinResolve` then resolves the staged plans -//! into the transaction's redo record, and `MetaOp::CalvinFlush` installs it -//! at its LSN (making the write visible), or `MetaOp::CalvinDrop` discards -//! the staged state (leaving base unchanged). -//! -//! These drive a `CoreLoop` directly through the SPSC ring so the atomicity -//! seam is observed without any scheduler timing: nothing a stage writes is -//! visible until the flush, and a drop never makes it visible. - -use std::sync::Arc; -use std::time::{Duration, Instant}; - -use nodedb::bridge::dispatch::{BridgeRequest, BridgeResponse}; -use nodedb::bridge::envelope::{Priority, Request, Response, Status}; -use nodedb::data::executor::core_loop::CoreLoop; -use nodedb::types::*; -use nodedb_bridge::buffer::{Consumer, Producer, RingBuffer}; -use nodedb_physical::physical_plan::{DocumentOp, KvOp, MetaOp, PhysicalPlan}; -use nodedb_types::QualifiedCollection; -use nodedb_types::Surrogate; -use nodedb_types::calvin::{EngineTag, ReadKeyIdent, VersionedReadEntry}; - -fn make_core() -> ( - CoreLoop, - Producer, - Consumer, - tempfile::TempDir, -) { - let dir = tempfile::tempdir().unwrap(); - let (req_tx, req_rx) = RingBuffer::channel::(64); - let (resp_tx, resp_rx) = RingBuffer::channel::(64); - let core = CoreLoop::open( - 0, - req_rx, - resp_tx, - dir.path(), - Arc::new(nodedb_types::OrdinalClock::new()), - nodedb::data::executor::core_loop::test_governor(), - ) - .unwrap(); - (core, req_tx, resp_rx, dir) -} - -/// Build a request for `plan` on `vshard`, carrying an optional committed WAL -/// LSN (present on the seed write so its version is recorded). -fn make_request(plan: PhysicalPlan, vshard: u32, wal_lsn: Option) -> Request { - Request { - request_id: RequestId::new(1), - tenant_id: TenantId::new(1), - vshard_id: VShardId::new(vshard), - database_id: DatabaseId::DEFAULT, - plan, - deadline: Instant::now() + Duration::from_secs(5), - priority: Priority::Normal, - trace_id: nodedb_types::TraceId::ZERO, - consistency: ReadConsistency::Strong, - idempotency_key: None, - event_source: nodedb::event::EventSource::RaftFollower, - user_roles: Vec::new(), - user_id: None, - statement_digest: None, - txn_id: None, - wal_lsn, - resolved_now_ms: None, - commit_hlc: None, - admission: nodedb::bridge::envelope::Admission::Admitted, - } -} - -fn send( - core: &mut CoreLoop, - tx: &mut Producer, - rx: &mut Consumer, - plan: PhysicalPlan, - vshard: u32, - wal_lsn: Option, -) -> Response { - tx.try_push(BridgeRequest::unfloored(make_request( - plan, vshard, wal_lsn, - ))) - .unwrap(); - core.tick(); - rx.try_pop().unwrap().inner -} - -/// Resolve the transaction staged at `(epoch, 0)` on `vshard` and build the -/// flush that installs its redo record. -fn resolved_flush( - core: &mut CoreLoop, - tx: &mut Producer, - rx: &mut Consumer, - epoch: u64, - vshard: u32, -) -> PhysicalPlan { - let resolved = send( - core, - tx, - rx, - PhysicalPlan::Meta(MetaOp::CalvinResolve { epoch, position: 0 }), - vshard, - None, - ); - assert_eq!( - resolved.status, - Status::Ok, - "resolve must succeed: {resolved:?}" - ); - PhysicalPlan::Meta(MetaOp::CalvinFlush { - epoch, - position: 0, - redo: resolved.payload.to_vec(), - collections: Vec::new(), - sum_targets: Vec::new(), - }) -} - -/// A write committed through the Calvin path to seed a write version. -struct CalvinSeed<'a> { - epoch: u64, - vshard: u32, - /// The collection `plans` write; the install records its floor at `lsn`. - collection: &'a str, - plans: Vec, - lsn: u64, -} - -/// Commit `seed.plans` as the Calvin transaction at `(seed.epoch, 0)` on -/// `seed.vshard` and install its redo record at `seed.lsn`: the path a -/// committed multi-shard write takes. The install records each written key -/// and the collection floor at that LSN. -fn commit_calvin( - core: &mut CoreLoop, - tx: &mut Producer, - rx: &mut Consumer, - seed: CalvinSeed<'_>, -) -> Response { - let staged = send( - core, - tx, - rx, - stage_static(seed.epoch, 0, seed.plans, Vec::new()), - seed.vshard, - None, - ); - assert_eq!( - staged.status, - Status::Ok, - "seed stage must succeed: {staged:?}" - ); - let resolved = send( - core, - tx, - rx, - PhysicalPlan::Meta(MetaOp::CalvinResolve { - epoch: seed.epoch, - position: 0, - }), - seed.vshard, - None, - ); - assert_eq!( - resolved.status, - Status::Ok, - "seed resolve must succeed: {resolved:?}" - ); - send( - core, - tx, - rx, - PhysicalPlan::Meta(MetaOp::CalvinFlush { - epoch: seed.epoch, - position: 0, - redo: resolved.payload.to_vec(), - collections: vec![seed.collection.to_string()], - sum_targets: Vec::new(), - }), - seed.vshard, - Some(Lsn::new(seed.lsn)), - ) -} - -fn kv_put(coll: &str, key: &[u8], value: &[u8]) -> PhysicalPlan { - PhysicalPlan::Kv(KvOp::Put { - collection: QualifiedCollection::new(DatabaseId::DEFAULT, coll), - key: key.to_vec(), - value: value.to_vec(), - ttl_ms: 0, - surrogate: nodedb_test_support::kv_rows::kv_row_surrogate(key), - returning: None, - rls_filters: Vec::new(), - provenance: None, - }) -} - -fn kv_get(coll: &str, key: &[u8]) -> PhysicalPlan { - PhysicalPlan::Kv(KvOp::Get { - collection: QualifiedCollection::new(DatabaseId::DEFAULT, coll), - key: key.to_vec(), - rls_filters: Vec::new(), - surrogate_ceiling: None, - }) -} - -/// A minimal msgpack-encoded document body, `{"a": "1"}`. -fn doc_value() -> Vec { - let mut obj = std::collections::HashMap::new(); - obj.insert("a".to_string(), nodedb_types::Value::String("1".into())); - zerompk::to_msgpack_vec(&nodedb_types::Value::Object(obj)).unwrap() -} - -/// A minimal document `PointInsert` plan, used to record a write-version at a -/// specific surrogate (the document engine's per-key identity). -fn doc_insert(coll: &str, document_id: &str, surrogate: u32) -> PhysicalPlan { - PhysicalPlan::Document(DocumentOp::PointInsert { - collection: QualifiedCollection::new(DatabaseId::DEFAULT, coll), - document_id: document_id.to_string(), - value: doc_value(), - if_absent: false, - surrogate: Surrogate::new(surrogate), - returning: None, - rls_filters: Vec::new(), - resolved_sum_targets: Vec::new(), - deferred_sum_targets: Vec::new(), - }) -} - -fn stage_static( - epoch: u64, - position: u32, - plans: Vec, - versioned_reads: Vec, -) -> PhysicalPlan { - PhysicalPlan::Meta(MetaOp::CalvinExecuteStatic { - epoch, - position, - tenant_id: TenantId::new(1), - plans, - epoch_system_ms: 0, - is_group_leader: true, - versioned_reads, - body_plans: Vec::new(), - }) -} - -/// A valid (empty read-set) staged write is NOT visible until the flush, and -/// the flush makes it visible — the stage/flush atomicity seam. -#[test] -fn flush_makes_staged_calvin_write_visible() { - let (mut core, mut tx, mut rx, _dir) = make_core(); - - // Stage a write with an empty read-set → vote is valid (commit). - let staged = send( - &mut core, - &mut tx, - &mut rx, - stage_static(6, 0, vec![kv_put("flushcoll", b"fk", b"fv")], Vec::new()), - 0, - None, - ); - assert_eq!(staged.status, Status::Ok, "stage must succeed"); - assert_eq!( - staged.read_set_valid, - Some(true), - "empty read-set is vacuously current → commit vote" - ); - - // Not yet applied: the staged write is invisible before the flush. - let before = send( - &mut core, - &mut tx, - &mut rx, - kv_get("flushcoll", b"fk"), - 0, - None, - ); - assert!( - before.payload.is_empty() || before.status == Status::Error, - "staged write must NOT be visible before flush; got {before:?}" - ); - - // The flush installs the resolved redo record to base. - let flush_plan = resolved_flush(&mut core, &mut tx, &mut rx, 6, 0); - let flush = send( - &mut core, - &mut tx, - &mut rx, - flush_plan, - 0, - Some(Lsn::new(60)), - ); - assert_eq!(flush.status, Status::Ok, "flush must succeed: {flush:?}"); - - // Now visible. - let after = send( - &mut core, - &mut tx, - &mut rx, - kv_get("flushcoll", b"fk"), - 0, - None, - ); - assert_eq!( - after.status, - Status::Ok, - "read after flush must succeed: {after:?}" - ); - assert!( - !after.payload.is_empty(), - "flushed write must be visible after flush" - ); -} - -/// An invalid vote (a stale versioned read against a newer local write) STAGES -/// but never applies; the drop discards it and base stays unchanged. -#[test] -fn drop_discards_invalid_staged_calvin_write() { - let (mut core, mut tx, mut rx, _dir) = make_core(); - - // Seed a committed write to `dropcoll` at LSN 100 so its collection write - // version floor is 100. The seed carries a WAL LSN so the version records. - let seed = commit_calvin( - &mut core, - &mut tx, - &mut rx, - CalvinSeed { - epoch: 1, - vshard: 0, - collection: "dropcoll", - plans: vec![kv_put("dropcoll", b"seed", b"v")], - lsn: 100, - }, - ); - assert_eq!(seed.status, Status::Ok, "seed write must commit: {seed:?}"); - - // The read entry's collection must home to the staged request's vShard for - // the read-set check to consider it. - let read_vshard = nodedb_types::CollectionKey::from_bare(DatabaseId::DEFAULT, "dropcoll") - .vshard() - .as_u32(); - - // A read of `dropcoll` observed at LSN 50 — stale against the seed's write - // at LSN 100 → the read-set is no longer current → abort vote. - let stale_read = VersionedReadEntry { - engine: EngineTag::Kv, - collection: "dropcoll".to_string(), - key: ReadKeyIdent::Predicate, - read_lsn: Lsn::new(50), - home_vshard: None, - served_by: 0, - }; - - let staged = send( - &mut core, - &mut tx, - &mut rx, - stage_static( - 7, - 0, - vec![kv_put("targetcoll", b"tk", b"tv")], - vec![stale_read], - ), - read_vshard, - None, - ); - assert_eq!( - staged.status, - Status::Ok, - "stage must succeed even on abort vote" - ); - assert_eq!( - staged.read_set_valid, - Some(false), - "stale read-set must produce an abort vote" - ); - - // Staged, not applied: the target write is invisible. - let before = send( - &mut core, - &mut tx, - &mut rx, - kv_get("targetcoll", b"tk"), - 0, - None, - ); - assert!( - before.payload.is_empty() || before.status == Status::Error, - "aborted staged write must NOT be visible; got {before:?}" - ); - - // Drop discards the staged plans and fires nothing. The drop must target - // the SAME vShard the stage keyed under (as production dispatches it), so it - // actually pops this participant's staged slice. - let dropped = send( - &mut core, - &mut tx, - &mut rx, - PhysicalPlan::Meta(MetaOp::CalvinDrop { - epoch: 7, - position: 0, - }), - read_vshard, - None, - ); - assert_eq!(dropped.status, Status::Ok, "drop must succeed: {dropped:?}"); - - // Still invisible after the drop — base was never mutated. - let after = send( - &mut core, - &mut tx, - &mut rx, - kv_get("targetcoll", b"tk"), - 0, - None, - ); - assert!( - after.payload.is_empty() || after.status == Status::Error, - "dropped write must never be visible; got {after:?}" - ); -} - -/// A `Point` read (not just a `Predicate` read) at or after the recorded write -/// LSN is current → commit vote, and the flush applies the staged write. -#[test] -fn point_read_at_write_lsn_commits_and_flush_applies() { - let (mut core, mut tx, mut rx, _dir) = make_core(); - - // Seed a committed write to key `pk` in `pointcoll` at LSN 10. - let seed = commit_calvin( - &mut core, - &mut tx, - &mut rx, - CalvinSeed { - epoch: 1, - vshard: 0, - collection: "pointcoll", - plans: vec![kv_put("pointcoll", b"pk", b"v1")], - lsn: 10, - }, - ); - assert_eq!(seed.status, Status::Ok, "seed write must commit: {seed:?}"); - - let point_vshard = nodedb_types::CollectionKey::from_bare(DatabaseId::DEFAULT, "pointcoll") - .vshard() - .as_u32(); - - // A Point read of the exact same key observed at LSN 10 (== the write) is - // still current: no write happened AFTER the read. - let current_read = VersionedReadEntry { - engine: EngineTag::Kv, - collection: "pointcoll".to_string(), - key: ReadKeyIdent::Point(KeyRepr::KvKey(Box::from(b"pk".as_slice()))), - read_lsn: Lsn::new(10), - home_vshard: None, - served_by: 0, - }; - - let staged = send( - &mut core, - &mut tx, - &mut rx, - stage_static( - 8, - 0, - vec![kv_put("committarget", b"ck", b"cv")], - vec![current_read], - ), - point_vshard, - None, - ); - assert_eq!(staged.status, Status::Ok, "stage must succeed: {staged:?}"); - assert_eq!( - staged.read_set_valid, - Some(true), - "a read at or after the last write LSN must be current -> commit vote" - ); - - let flush_plan = resolved_flush(&mut core, &mut tx, &mut rx, 8, point_vshard); - let flush = send( - &mut core, - &mut tx, - &mut rx, - flush_plan, - point_vshard, - Some(Lsn::new(80)), - ); - assert_eq!(flush.status, Status::Ok, "flush must succeed: {flush:?}"); - - let after = send( - &mut core, - &mut tx, - &mut rx, - kv_get("committarget", b"ck"), - 0, - None, - ); - assert_eq!(after.status, Status::Ok, "read after flush must succeed"); - assert!( - !after.payload.is_empty(), - "flushed write must be visible after flush" - ); -} - -/// A `Point` read of a key STALE against a later write to that same key -/// (read_lsn=5 vs. a committed write at LSN=10) aborts the stage vote, and the -/// drop discards the staged write with no base mutation — mirrors -/// `drop_discards_invalid_staged_calvin_write` but for a Point key (not a -/// collection-scoped Predicate). -#[test] -fn stale_point_read_of_kv_key_aborts_stage_and_drop_discards() { - let (mut core, mut tx, mut rx, _dir) = make_core(); - - // Seed a committed write to key `pk` in `stalecoll` at LSN 10. - let seed = commit_calvin( - &mut core, - &mut tx, - &mut rx, - CalvinSeed { - epoch: 1, - vshard: 0, - collection: "stalecoll", - plans: vec![kv_put("stalecoll", b"pk", b"v1")], - lsn: 10, - }, - ); - assert_eq!(seed.status, Status::Ok, "seed write must commit: {seed:?}"); - - let stale_vshard = nodedb_types::CollectionKey::from_bare(DatabaseId::DEFAULT, "stalecoll") - .vshard() - .as_u32(); - - // A Point read of the exact same key observed at LSN 5 — stale against the - // write at LSN 10 → the read-set is no longer current → abort vote. - let stale_read = VersionedReadEntry { - engine: EngineTag::Kv, - collection: "stalecoll".to_string(), - key: ReadKeyIdent::Point(KeyRepr::KvKey(Box::from(b"pk".as_slice()))), - read_lsn: Lsn::new(5), - home_vshard: None, - served_by: 0, - }; - - let staged = send( - &mut core, - &mut tx, - &mut rx, - stage_static( - 9, - 0, - vec![kv_put("aborttarget", b"ak", b"av")], - vec![stale_read], - ), - stale_vshard, - None, - ); - assert_eq!( - staged.status, - Status::Ok, - "stage must succeed even on abort vote" - ); - assert_eq!( - staged.read_set_valid, - Some(false), - "a stale Point read (write after read_lsn) must produce an abort vote" - ); - - let before = send( - &mut core, - &mut tx, - &mut rx, - kv_get("aborttarget", b"ak"), - 0, - None, - ); - assert!( - before.payload.is_empty() || before.status == Status::Error, - "aborted staged write must NOT be visible; got {before:?}" - ); - - let dropped = send( - &mut core, - &mut tx, - &mut rx, - PhysicalPlan::Meta(MetaOp::CalvinDrop { - epoch: 9, - position: 0, - }), - stale_vshard, - None, - ); - assert_eq!(dropped.status, Status::Ok, "drop must succeed: {dropped:?}"); - - let after = send( - &mut core, - &mut tx, - &mut rx, - kv_get("aborttarget", b"ak"), - 0, - None, - ); - assert!( - after.payload.is_empty() || after.status == Status::Error, - "dropped write must never be visible; got {after:?}" - ); -} - -/// Phantom-insert proof for the KV engine: a Point read observed a key as -/// ABSENT at LSN 5; a concurrent INSERT of that EXACT key commits at LSN 8. -/// The KV read-key identity is the raw key bytes — the same identity the -/// insert writes under — so the per-key write-version check catches the -/// phantom: validating the stale absent-read against the now-present key -/// aborts the stage. -#[test] -fn absent_kv_key_phantom_insert_causes_abort() { - let (mut core, mut tx, mut rx, _dir) = make_core(); - - let phantom_vshard = nodedb_types::CollectionKey::from_bare(DatabaseId::DEFAULT, "phantomkv") - .vshard() - .as_u32(); - - // The key was absent when read at (the then-current watermark) LSN 5 — no - // write is seeded yet. - let absent_read = VersionedReadEntry { - engine: EngineTag::Kv, - collection: "phantomkv".to_string(), - key: ReadKeyIdent::Point(KeyRepr::KvKey(Box::from(b"newkey".as_slice()))), - read_lsn: Lsn::new(5), - home_vshard: None, - served_by: 0, - }; - - // Concurrently, the exact same key is inserted and commits at LSN 8. - let insert = commit_calvin( - &mut core, - &mut tx, - &mut rx, - CalvinSeed { - epoch: 1, - vshard: phantom_vshard, - collection: "phantomkv", - plans: vec![PhysicalPlan::Kv(KvOp::Insert { - collection: QualifiedCollection::new(DatabaseId::DEFAULT, "phantomkv"), - key: b"newkey".to_vec(), - value: b"v".to_vec(), - ttl_ms: 0, - surrogate: nodedb_test_support::kv_rows::kv_row_surrogate(b"newkey".as_ref()), - returning: None, - rls_filters: Vec::new(), - })], - lsn: 8, - }, - ); - assert_eq!( - insert.status, - Status::Ok, - "phantom insert must commit: {insert:?}" - ); - - // Validating the (now stale) absent-read against the current write index - // must abort: the insert at LSN 8 is AFTER the read at LSN 5. - let staged = send( - &mut core, - &mut tx, - &mut rx, - stage_static( - 10, - 0, - vec![kv_put("phantomtarget", b"tk", b"tv")], - vec![absent_read], - ), - phantom_vshard, - None, - ); - assert_eq!(staged.status, Status::Ok, "stage must succeed: {staged:?}"); - assert_eq!( - staged.read_set_valid, - Some(false), - "a phantom insert into a key observed absent must abort the stage vote" - ); -} - -/// Absent-DOCUMENT phantom safety: a document `PointGet` on an absent document -/// cannot record `Point(KeyRepr::Surrogate(s))`, because the placeholder -/// surrogate `s` a miss carries is allocated from a monotonic counter unrelated -/// to `document_id` — it never coincides with the freshly-minted surrogate a -/// subsequent INSERT of that `document_id` receives, so a per-key OCC check -/// would never catch the phantom. The capture layer therefore degrades an -/// absent document read to `Predicate` (collection floor). A concurrent INSERT -/// into that collection advances `coll_write_lsn` past the read_lsn, so -/// `WriteVersionIndex::read_is_valid` (predicate branch) judges the stale read -/// invalid and the stage VOTES ABORT — collection-granular phantom safety. -#[test] -fn absent_document_phantom_insert_is_caught() { - let (mut core, mut tx, mut rx, _dir) = make_core(); - - let doc_vshard = nodedb_types::CollectionKey::from_bare(DatabaseId::DEFAULT, "phantomdocs") - .vshard() - .as_u32(); - - // The document was absent when read: capture degraded the miss to a - // collection-scoped predicate on "phantomdocs" at read_lsn 5. - let absent_doc_read = VersionedReadEntry { - engine: EngineTag::Document, - collection: "phantomdocs".to_string(), - key: ReadKeyIdent::Predicate, - read_lsn: Lsn::new(5), - home_vshard: None, - served_by: 0, - }; - - // Concurrently, a document with the same document_id the read targeted is - // actually inserted -- allocated a freshly-minted surrogate (42), committing - // at LSN 8. Its collection floor advance (phantomdocs -> 8) is what the - // predicate read validates against. - const NEWLY_ALLOCATED_SURROGATE: u32 = 42; - let insert = commit_calvin( - &mut core, - &mut tx, - &mut rx, - CalvinSeed { - epoch: 1, - vshard: doc_vshard, - collection: "phantomdocs", - plans: vec![doc_insert( - "phantomdocs", - "the-doc-id", - NEWLY_ALLOCATED_SURROGATE, - )], - lsn: 8, - }, - ); - assert_eq!( - insert.status, - Status::Ok, - "phantom insert must commit: {insert:?}" - ); - - // Validate the stale absent-read: the predicate check sees the phantomdocs - // collection floor (LSN 8) is AFTER the read (LSN 5), so the read is no - // longer current and the stage must abort. - let staged = send( - &mut core, - &mut tx, - &mut rx, - stage_static( - 11, - 0, - vec![kv_put("docphantomtarget", b"tk", b"tv")], - vec![absent_doc_read], - ), - doc_vshard, - None, - ); - assert_eq!(staged.status, Status::Ok, "stage must succeed: {staged:?}"); - assert_eq!( - staged.read_set_valid, - Some(false), - "a phantom insert into a collection observed absent must abort the stage \ - vote via the collection floor" - ); -} - -/// No-over-abort companion to `absent_document_phantom_insert_is_caught`: the -/// collection-floor degrade must NOT abort every absent-read transaction. An -/// absent document read (predicate on "phantomdocs" at LSN 5) stays valid when -/// no insert lands in THAT collection — a concurrent insert into an UNRELATED -/// collection advances only its own floor, leaving phantomdocs at zero, so the -/// stale read is still current and the stage VOTES COMMIT. -#[test] -fn absent_document_read_without_matching_insert_still_commits() { - let (mut core, mut tx, mut rx, _dir) = make_core(); - - let doc_vshard = nodedb_types::CollectionKey::from_bare(DatabaseId::DEFAULT, "phantomdocs") - .vshard() - .as_u32(); - - let absent_doc_read = VersionedReadEntry { - engine: EngineTag::Document, - collection: "phantomdocs".to_string(), - key: ReadKeyIdent::Predicate, - read_lsn: Lsn::new(5), - home_vshard: None, - served_by: 0, - }; - - // A concurrent insert into a DIFFERENT collection commits at LSN 8. It - // advances only "othercoll"'s floor; phantomdocs is untouched. - const UNRELATED_SURROGATE: u32 = 42; - let insert = commit_calvin( - &mut core, - &mut tx, - &mut rx, - CalvinSeed { - epoch: 1, - vshard: doc_vshard, - collection: "othercoll", - plans: vec![doc_insert("othercoll", "other-id", UNRELATED_SURROGATE)], - lsn: 8, - }, - ); - assert_eq!( - insert.status, - Status::Ok, - "unrelated insert must commit: {insert:?}" - ); - - // The predicate read on phantomdocs is still current: its collection floor - // never advanced past read_lsn 5, so the stage must commit. - let staged = send( - &mut core, - &mut tx, - &mut rx, - stage_static( - 11, - 0, - vec![kv_put("docphantomtarget", b"tk", b"tv")], - vec![absent_doc_read], - ), - doc_vshard, - None, - ); - assert_eq!(staged.status, Status::Ok, "stage must succeed: {staged:?}"); - assert_eq!( - staged.read_set_valid, - Some(true), - "an absent-document read must NOT over-abort when no insert lands in its \ - collection" - ); -} - -/// Push one prebuilt request through the ring and return its response. -fn send_request( - core: &mut CoreLoop, - tx: &mut Producer, - rx: &mut Consumer, - request: Request, -) -> Response { - tx.try_push(BridgeRequest::unfloored(request)).unwrap(); - core.tick(); - rx.try_pop().unwrap().inner -} - -/// Calvin sub-operations are already ordered: every replica must run them. -/// A stage and a flush whose envelope deadline has already passed still -/// execute, and the write becomes visible. They never answer -/// `DeadlineExceeded`. -#[test] -fn already_ordered_stage_and_flush_run_past_their_deadline() { - let (mut core, mut tx, mut rx, _dir) = make_core(); - let already_ordered = |plan: PhysicalPlan| Request { - deadline: Instant::now() - Duration::from_secs(1), - admission: nodedb::bridge::envelope::Admission::Exempt( - nodedb::bridge::envelope::ExemptReason::AlreadyOrdered, - ), - ..make_request(plan, 0, None) - }; - - let staged = send_request( - &mut core, - &mut tx, - &mut rx, - already_ordered(stage_static( - 9, - 0, - vec![kv_put("latecoll", b"lk", b"lv")], - Vec::new(), - )), - ); - assert_eq!(staged.status, Status::Ok, "late stage must run: {staged:?}"); - assert_eq!(staged.read_set_valid, Some(true)); - - let flush_plan = resolved_flush(&mut core, &mut tx, &mut rx, 9, 0); - let flush = send_request( - &mut core, - &mut tx, - &mut rx, - Request { - wal_lsn: Some(Lsn::new(90)), - ..already_ordered(flush_plan) - }, - ); - assert_eq!(flush.status, Status::Ok, "late flush must run: {flush:?}"); - - let after = send( - &mut core, - &mut tx, - &mut rx, - kv_get("latecoll", b"lk"), - 0, - None, - ); - assert_eq!(after.status, Status::Ok, "read after flush: {after:?}"); - assert!( - !after.payload.is_empty(), - "the late flush must make the staged write visible" - ); -} diff --git a/nodedb/tests/inproc/cases/calvin_two_phase_apply/install.rs b/nodedb/tests/inproc/cases/calvin_two_phase_apply/install.rs new file mode 100644 index 000000000..7e92a4772 --- /dev/null +++ b/nodedb/tests/inproc/cases/calvin_two_phase_apply/install.rs @@ -0,0 +1,147 @@ +// SPDX-License-Identifier: BUSL-1.1 + +//! A staged write becomes visible only at its stamped install. + +use std::time::{Duration, Instant}; + +use nodedb::bridge::envelope::{Request, StageVote, Status}; +use nodedb::types::*; +use nodedb_physical::physical_plan::PhysicalPlan; + +use super::support::*; + +/// A valid (empty read-set) staged write is NOT visible until the stamped +/// install, and the install makes it visible: the stage/install atomicity seam. +#[test] +fn the_stamped_install_makes_a_staged_calvin_write_visible() { + let (mut core, mut tx, mut rx, _dir) = make_core(); + + // Stage a write with an empty read-set → vote is valid (commit). + let staged = send( + &mut core, + &mut tx, + &mut rx, + stage_static(6, 0, vec![kv_put("flushcoll", b"fk", b"fv")], Vec::new()), + 0, + None, + ); + assert_eq!(staged.status, Status::Ok, "stage must succeed"); + assert_eq!( + staged.stage_vote, + Some(StageVote::Commit), + "empty read-set is vacuously current → commit vote" + ); + + // Not yet applied: the staged write is invisible before the install. + let before = send( + &mut core, + &mut tx, + &mut rx, + kv_get("flushcoll", b"fk"), + 0, + None, + ); + assert!( + before.payload.is_empty() || before.status == Status::Error, + "staged write must NOT be visible before the install; got {before:?}" + ); + + // The stamped install writes the resolved redo record to base. + let install_plan = resolved_install(&mut core, &mut tx, &mut rx, 6, 0); + let installed = send( + &mut core, + &mut tx, + &mut rx, + install_plan, + 0, + Some(Lsn::new(60)), + ); + assert_eq!( + installed.status, + Status::Ok, + "install must succeed: {installed:?}" + ); + + // Now visible. + let after = send( + &mut core, + &mut tx, + &mut rx, + kv_get("flushcoll", b"fk"), + 0, + None, + ); + assert_eq!( + after.status, + Status::Ok, + "read after the install must succeed: {after:?}" + ); + assert!( + !after.payload.is_empty(), + "the installed write must be visible after the install" + ); +} + +/// Calvin sub-operations are already ordered: every replica must run them. +/// A stage and a stamped install whose envelope deadline has already passed +/// still execute, and the write becomes visible. They never answer +/// `DeadlineExceeded`. +#[test] +fn already_ordered_stage_and_install_run_past_their_deadline() { + let (mut core, mut tx, mut rx, _dir) = make_core(); + let already_ordered = |plan: PhysicalPlan| Request { + deadline: Instant::now() - Duration::from_secs(1), + admission: nodedb::bridge::envelope::Admission::Exempt( + nodedb::bridge::envelope::ExemptReason::AlreadyOrdered, + ), + ..make_request(plan, 0, None) + }; + + let staged = send_request( + &mut core, + &mut tx, + &mut rx, + already_ordered(stage_static( + 9, + 0, + vec![kv_put("latecoll", b"lk", b"lv")], + Vec::new(), + )), + ); + assert_eq!(staged.status, Status::Ok, "late stage must run: {staged:?}"); + assert_eq!(staged.stage_vote, Some(StageVote::Commit)); + + let install_plan = resolved_install(&mut core, &mut tx, &mut rx, 9, 0); + let installed = send_request( + &mut core, + &mut tx, + &mut rx, + Request { + wal_lsn: Some(Lsn::new(90)), + ..already_ordered(install_plan) + }, + ); + assert_eq!( + installed.status, + Status::Ok, + "late install must run: {installed:?}" + ); + + let after = send( + &mut core, + &mut tx, + &mut rx, + kv_get("latecoll", b"lk"), + 0, + None, + ); + assert_eq!( + after.status, + Status::Ok, + "read after the install: {after:?}" + ); + assert!( + !after.payload.is_empty(), + "the late install must make the staged write visible" + ); +} diff --git a/nodedb/tests/inproc/cases/calvin_two_phase_apply/mod.rs b/nodedb/tests/inproc/cases/calvin_two_phase_apply/mod.rs new file mode 100644 index 000000000..aa222a58d --- /dev/null +++ b/nodedb/tests/inproc/cases/calvin_two_phase_apply/mod.rs @@ -0,0 +1,19 @@ +// SPDX-License-Identifier: BUSL-1.1 + +//! Staged Calvin apply on the Data Plane: `MetaOp::CalvinExecuteStatic` +//! VALIDATES + STAGES a transaction's write plans into the commit-pending +//! buffer WITHOUT mutating base, returning the local commit vote on +//! `stage_vote`. `MetaOp::CalvinResolve` then resolves the staged plans +//! into the transaction's redo record. The data group's apply of the slice's +//! stamped redo entry (`MetaOp::ApplyTransactionRedo` with a `CalvinInstall`) +//! installs it at its LSN (making the write visible), or `MetaOp::CalvinDrop` +//! discards the staged state (leaving base unchanged). +//! +//! These drive a `CoreLoop` directly through the SPSC ring so the atomicity +//! seam is observed without any scheduler timing: nothing a stage writes is +//! visible until the install, and a drop never makes it visible. + +mod install; +mod phantom; +mod read_set; +mod support; diff --git a/nodedb/tests/inproc/cases/calvin_two_phase_apply/phantom.rs b/nodedb/tests/inproc/cases/calvin_two_phase_apply/phantom.rs new file mode 100644 index 000000000..190a659e9 --- /dev/null +++ b/nodedb/tests/inproc/cases/calvin_two_phase_apply/phantom.rs @@ -0,0 +1,233 @@ +// SPDX-License-Identifier: BUSL-1.1 + +//! A phantom insert into a key or collection a slice read as absent +//! aborts the stage vote. An unrelated insert does not. + +use nodedb::bridge::envelope::{StageVote, Status}; +use nodedb::types::*; +use nodedb_physical::physical_plan::{KvOp, PhysicalPlan}; +use nodedb_types::QualifiedCollection; +use nodedb_types::calvin::{EngineTag, ReadKeyIdent, VersionedReadEntry}; + +use super::support::*; + +/// Phantom-insert proof for the KV engine: a Point read observed a key as +/// ABSENT at LSN 5; a concurrent INSERT of that EXACT key commits at LSN 8. +/// The KV read-key identity is the raw key bytes — the same identity the +/// insert writes under — so the per-key write-version check catches the +/// phantom: validating the stale absent-read against the now-present key +/// aborts the stage. +#[test] +fn absent_kv_key_phantom_insert_causes_abort() { + let (mut core, mut tx, mut rx, _dir) = make_core(); + + let phantom_vshard = nodedb_types::CollectionKey::from_bare(DatabaseId::DEFAULT, "phantomkv") + .vshard() + .as_u32(); + + // The key was absent when read at (the then-current watermark) LSN 5 — no + // write is seeded yet. + let absent_read = VersionedReadEntry { + engine: EngineTag::Kv, + collection: "phantomkv".to_string(), + key: ReadKeyIdent::Point(KeyRepr::KvKey(Box::from(b"newkey".as_slice()))), + read_version: local_version(5), + home_vshard: None, + }; + + // Concurrently, the exact same key is inserted and commits at LSN 8. + let insert = commit_calvin( + &mut core, + &mut tx, + &mut rx, + CalvinSeed { + epoch: 1, + vshard: phantom_vshard, + collection: "phantomkv", + plans: vec![PhysicalPlan::Kv(KvOp::Insert { + collection: QualifiedCollection::new(DatabaseId::DEFAULT, "phantomkv"), + key: b"newkey".to_vec(), + value: b"v".to_vec(), + ttl_ms: 0, + surrogate: nodedb_test_support::kv_rows::kv_row_surrogate(b"newkey".as_ref()), + returning: None, + rls_filters: Vec::new(), + })], + lsn: 8, + }, + ); + assert_eq!( + insert.status, + Status::Ok, + "phantom insert must commit: {insert:?}" + ); + + // Validating the (now stale) absent-read against the current write index + // must abort: the insert at LSN 8 is AFTER the read at LSN 5. + let staged = send( + &mut core, + &mut tx, + &mut rx, + stage_static( + 10, + 0, + vec![kv_put("phantomtarget", b"tk", b"tv")], + vec![absent_read], + ), + phantom_vshard, + None, + ); + assert_eq!(staged.status, Status::Ok, "stage must succeed: {staged:?}"); + assert_eq!( + staged.stage_vote, + Some(StageVote::SerializationConflict), + "a phantom insert into a key observed absent must abort the stage vote" + ); +} + +/// Absent-DOCUMENT phantom safety: a document `PointGet` on an absent document +/// cannot record `Point(KeyRepr::Surrogate(s))`, because the placeholder +/// surrogate `s` a miss carries is allocated from a monotonic counter unrelated +/// to `document_id` — it never coincides with the freshly-minted surrogate a +/// subsequent INSERT of that `document_id` receives, so a per-key OCC check +/// would never catch the phantom. The capture layer therefore degrades an +/// absent document read to `Predicate` (collection floor). A concurrent INSERT +/// into that collection advances the collection's version past the read's, so +/// `WriteVersionIndex::read_is_valid` (predicate branch) judges the stale read +/// invalid and the stage VOTES ABORT — collection-granular phantom safety. +#[test] +fn absent_document_phantom_insert_is_caught() { + let (mut core, mut tx, mut rx, _dir) = make_core(); + + let doc_vshard = nodedb_types::CollectionKey::from_bare(DatabaseId::DEFAULT, "phantomdocs") + .vshard() + .as_u32(); + + // The document was absent when read: capture degraded the miss to a + // collection-scoped predicate on "phantomdocs" at the version of LSN 5. + let absent_doc_read = VersionedReadEntry { + engine: EngineTag::Document, + collection: "phantomdocs".to_string(), + key: ReadKeyIdent::Predicate, + read_version: local_version(5), + home_vshard: None, + }; + + // Concurrently, a document with the same document_id the read targeted is + // actually inserted -- allocated a freshly-minted surrogate (42), committing + // at LSN 8. Its collection floor advance (phantomdocs -> 8) is what the + // predicate read validates against. + const NEWLY_ALLOCATED_SURROGATE: u32 = 42; + let insert = commit_calvin( + &mut core, + &mut tx, + &mut rx, + CalvinSeed { + epoch: 1, + vshard: doc_vshard, + collection: "phantomdocs", + plans: vec![doc_insert( + "phantomdocs", + "the-doc-id", + NEWLY_ALLOCATED_SURROGATE, + )], + lsn: 8, + }, + ); + assert_eq!( + insert.status, + Status::Ok, + "phantom insert must commit: {insert:?}" + ); + + // Validate the stale absent-read: the predicate check sees the phantomdocs + // collection floor (LSN 8) is AFTER the read (LSN 5), so the read is no + // longer current and the stage must abort. + let staged = send( + &mut core, + &mut tx, + &mut rx, + stage_static( + 11, + 0, + vec![kv_put("docphantomtarget", b"tk", b"tv")], + vec![absent_doc_read], + ), + doc_vshard, + None, + ); + assert_eq!(staged.status, Status::Ok, "stage must succeed: {staged:?}"); + assert_eq!( + staged.stage_vote, + Some(StageVote::SerializationConflict), + "a phantom insert into a collection observed absent must abort the stage \ + vote via the collection floor" + ); +} + +/// No-over-abort companion to `absent_document_phantom_insert_is_caught`: the +/// collection-floor degrade must NOT abort every absent-read transaction. An +/// absent document read (predicate on "phantomdocs" at LSN 5) stays valid when +/// no insert lands in THAT collection — a concurrent insert into an UNRELATED +/// collection advances only its own floor, leaving phantomdocs at zero, so the +/// stale read is still current and the stage VOTES COMMIT. +#[test] +fn absent_document_read_without_matching_insert_still_commits() { + let (mut core, mut tx, mut rx, _dir) = make_core(); + + let doc_vshard = nodedb_types::CollectionKey::from_bare(DatabaseId::DEFAULT, "phantomdocs") + .vshard() + .as_u32(); + + let absent_doc_read = VersionedReadEntry { + engine: EngineTag::Document, + collection: "phantomdocs".to_string(), + key: ReadKeyIdent::Predicate, + read_version: local_version(5), + home_vshard: None, + }; + + // A concurrent insert into a DIFFERENT collection commits at LSN 8. It + // advances only "othercoll"'s floor; phantomdocs is untouched. + const UNRELATED_SURROGATE: u32 = 42; + let insert = commit_calvin( + &mut core, + &mut tx, + &mut rx, + CalvinSeed { + epoch: 1, + vshard: doc_vshard, + collection: "othercoll", + plans: vec![doc_insert("othercoll", "other-id", UNRELATED_SURROGATE)], + lsn: 8, + }, + ); + assert_eq!( + insert.status, + Status::Ok, + "unrelated insert must commit: {insert:?}" + ); + + // The predicate read on phantomdocs is still current: its collection floor + // never advanced past the read's version, so the stage must commit. + let staged = send( + &mut core, + &mut tx, + &mut rx, + stage_static( + 11, + 0, + vec![kv_put("docphantomtarget", b"tk", b"tv")], + vec![absent_doc_read], + ), + doc_vshard, + None, + ); + assert_eq!(staged.status, Status::Ok, "stage must succeed: {staged:?}"); + assert_eq!( + staged.stage_vote, + Some(StageVote::Commit), + "an absent-document read must NOT over-abort when no insert lands in its \ + collection" + ); +} diff --git a/nodedb/tests/inproc/cases/calvin_two_phase_apply/read_set.rs b/nodedb/tests/inproc/cases/calvin_two_phase_apply/read_set.rs new file mode 100644 index 000000000..bd656cc3f --- /dev/null +++ b/nodedb/tests/inproc/cases/calvin_two_phase_apply/read_set.rs @@ -0,0 +1,305 @@ +// SPDX-License-Identifier: BUSL-1.1 + +//! The stage vote judges the slice's read-set against the local write +//! versions: a current read commits, a stale one aborts and drops. + +use nodedb::bridge::envelope::{StageVote, Status}; +use nodedb::types::*; +use nodedb_physical::physical_plan::{MetaOp, PhysicalPlan}; +use nodedb_types::calvin::{EngineTag, ReadKeyIdent, VersionedReadEntry}; + +use super::support::*; + +/// An invalid vote (a stale versioned read against a newer local write) STAGES +/// but never applies; the drop discards it and base stays unchanged. +#[test] +fn drop_discards_invalid_staged_calvin_write() { + let (mut core, mut tx, mut rx, _dir) = make_core(); + + // Seed a committed write to `dropcoll` at LSN 100 on its home vShard, so + // its collection version there is the version of LSN 100. + let seed = commit_calvin( + &mut core, + &mut tx, + &mut rx, + CalvinSeed { + epoch: 1, + vshard: home_vshard("dropcoll"), + collection: "dropcoll", + plans: vec![kv_put("dropcoll", b"seed", b"v")], + lsn: 100, + }, + ); + assert_eq!(seed.status, Status::Ok, "seed write must commit: {seed:?}"); + + // The read entry's collection must home to the staged request's vShard for + // the read-set check to consider it. + let read_vshard = home_vshard("dropcoll"); + + // A read of `dropcoll` observed at the version of LSN 50 — stale against + // the seed's write at LSN 100 → the read-set is no longer current → abort + // vote. + let stale_read = VersionedReadEntry { + engine: EngineTag::Kv, + collection: "dropcoll".to_string(), + key: ReadKeyIdent::Predicate, + read_version: local_version(50), + home_vshard: None, + }; + + let staged = send( + &mut core, + &mut tx, + &mut rx, + stage_static( + 7, + 0, + vec![kv_put("targetcoll", b"tk", b"tv")], + vec![stale_read], + ), + read_vshard, + None, + ); + assert_eq!( + staged.status, + Status::Ok, + "stage must succeed even on abort vote" + ); + assert_eq!( + staged.stage_vote, + Some(StageVote::SerializationConflict), + "stale read-set must produce an abort vote" + ); + + // Staged, not applied: the target write is invisible. + let before = send( + &mut core, + &mut tx, + &mut rx, + kv_get("targetcoll", b"tk"), + 0, + None, + ); + assert!( + before.payload.is_empty() || before.status == Status::Error, + "aborted staged write must NOT be visible; got {before:?}" + ); + + // Drop discards the staged plans and fires nothing. The drop must target + // the SAME vShard the stage keyed under (as production dispatches it), so it + // actually pops this participant's staged slice. + let dropped = send( + &mut core, + &mut tx, + &mut rx, + PhysicalPlan::Meta(MetaOp::CalvinDrop { + epoch: 7, + position: 0, + }), + read_vshard, + None, + ); + assert_eq!(dropped.status, Status::Ok, "drop must succeed: {dropped:?}"); + + // Still invisible after the drop — base was never mutated. + let after = send( + &mut core, + &mut tx, + &mut rx, + kv_get("targetcoll", b"tk"), + 0, + None, + ); + assert!( + after.payload.is_empty() || after.status == Status::Error, + "dropped write must never be visible; got {after:?}" + ); +} + +/// A `Point` read (not just a `Predicate` read) at or after the recorded write +/// version is current → commit vote, and the stamped install applies the +/// staged write. +#[test] +fn point_read_at_write_version_commits_and_the_install_applies() { + let (mut core, mut tx, mut rx, _dir) = make_core(); + + // Seed a committed write to key `pk` in `pointcoll` at LSN 10. + let seed = commit_calvin( + &mut core, + &mut tx, + &mut rx, + CalvinSeed { + epoch: 1, + vshard: home_vshard("pointcoll"), + collection: "pointcoll", + plans: vec![kv_put("pointcoll", b"pk", b"v1")], + lsn: 10, + }, + ); + assert_eq!(seed.status, Status::Ok, "seed write must commit: {seed:?}"); + + let point_vshard = home_vshard("pointcoll"); + + // A Point read of the exact same key observed at the write's version is + // still current: no write happened AFTER the read. + let current_read = VersionedReadEntry { + engine: EngineTag::Kv, + collection: "pointcoll".to_string(), + key: ReadKeyIdent::Point(KeyRepr::KvKey(Box::from(b"pk".as_slice()))), + read_version: local_version(10), + home_vshard: None, + }; + + let staged = send( + &mut core, + &mut tx, + &mut rx, + stage_static( + 8, + 0, + vec![kv_put("committarget", b"ck", b"cv")], + vec![current_read], + ), + point_vshard, + None, + ); + assert_eq!(staged.status, Status::Ok, "stage must succeed: {staged:?}"); + assert_eq!( + staged.stage_vote, + Some(StageVote::Commit), + "a read at or after the last write's version must be current -> commit vote" + ); + + let install_plan = resolved_install(&mut core, &mut tx, &mut rx, 8, point_vshard); + let installed = send( + &mut core, + &mut tx, + &mut rx, + install_plan, + point_vshard, + Some(Lsn::new(80)), + ); + assert_eq!( + installed.status, + Status::Ok, + "install must succeed: {installed:?}" + ); + + let after = send( + &mut core, + &mut tx, + &mut rx, + kv_get("committarget", b"ck"), + 0, + None, + ); + assert_eq!( + after.status, + Status::Ok, + "read after the install must succeed" + ); + assert!( + !after.payload.is_empty(), + "the installed write must be visible after the install" + ); +} + +/// A `Point` read of a key STALE against a later write to that same key +/// (read at LSN 5 vs. a committed write at LSN 10) aborts the stage vote, and the +/// drop discards the staged write with no base mutation — mirrors +/// `drop_discards_invalid_staged_calvin_write` but for a Point key (not a +/// collection-scoped Predicate). +#[test] +fn stale_point_read_of_kv_key_aborts_stage_and_drop_discards() { + let (mut core, mut tx, mut rx, _dir) = make_core(); + + // Seed a committed write to key `pk` in `stalecoll` at LSN 10. + let seed = commit_calvin( + &mut core, + &mut tx, + &mut rx, + CalvinSeed { + epoch: 1, + vshard: home_vshard("stalecoll"), + collection: "stalecoll", + plans: vec![kv_put("stalecoll", b"pk", b"v1")], + lsn: 10, + }, + ); + assert_eq!(seed.status, Status::Ok, "seed write must commit: {seed:?}"); + + let stale_vshard = home_vshard("stalecoll"); + + // A Point read of the exact same key observed at LSN 5 — stale against the + // write at LSN 10 → the read-set is no longer current → abort vote. + let stale_read = VersionedReadEntry { + engine: EngineTag::Kv, + collection: "stalecoll".to_string(), + key: ReadKeyIdent::Point(KeyRepr::KvKey(Box::from(b"pk".as_slice()))), + read_version: local_version(5), + home_vshard: None, + }; + + let staged = send( + &mut core, + &mut tx, + &mut rx, + stage_static( + 9, + 0, + vec![kv_put("aborttarget", b"ak", b"av")], + vec![stale_read], + ), + stale_vshard, + None, + ); + assert_eq!( + staged.status, + Status::Ok, + "stage must succeed even on abort vote" + ); + assert_eq!( + staged.stage_vote, + Some(StageVote::SerializationConflict), + "a stale Point read (write after the read) must produce an abort vote" + ); + + let before = send( + &mut core, + &mut tx, + &mut rx, + kv_get("aborttarget", b"ak"), + 0, + None, + ); + assert!( + before.payload.is_empty() || before.status == Status::Error, + "aborted staged write must NOT be visible; got {before:?}" + ); + + let dropped = send( + &mut core, + &mut tx, + &mut rx, + PhysicalPlan::Meta(MetaOp::CalvinDrop { + epoch: 9, + position: 0, + }), + stale_vshard, + None, + ); + assert_eq!(dropped.status, Status::Ok, "drop must succeed: {dropped:?}"); + + let after = send( + &mut core, + &mut tx, + &mut rx, + kv_get("aborttarget", b"ak"), + 0, + None, + ); + assert!( + after.payload.is_empty() || after.status == Status::Error, + "dropped write must never be visible; got {after:?}" + ); +} diff --git a/nodedb/tests/inproc/cases/calvin_two_phase_apply/support.rs b/nodedb/tests/inproc/cases/calvin_two_phase_apply/support.rs new file mode 100644 index 000000000..a2d54c933 --- /dev/null +++ b/nodedb/tests/inproc/cases/calvin_two_phase_apply/support.rs @@ -0,0 +1,278 @@ +// SPDX-License-Identifier: BUSL-1.1 + +//! Helpers every staged Calvin apply case drives a `CoreLoop` with. + +use std::sync::Arc; +use std::time::{Duration, Instant}; + +use nodedb::bridge::dispatch::{BridgeRequest, BridgeResponse}; +use nodedb::bridge::envelope::{Priority, Request, Response, Status}; +use nodedb::data::executor::core_loop::CoreLoop; +use nodedb::types::*; +use nodedb_bridge::buffer::{Consumer, Producer, RingBuffer}; +use nodedb_physical::physical_plan::{ + CalvinInstall, CalvinResolved, DocumentOp, KvOp, MetaOp, PhysicalPlan, RedoOrigin, +}; +use nodedb_types::QualifiedCollection; +use nodedb_types::Surrogate; +use nodedb_types::calvin::VersionedReadEntry; + +/// The version a single-node write at WAL LSN `lsn` records: no data-group +/// entry applied it, so its version is its LSN on top of an empty vShard. +pub(super) fn local_version(lsn: u64) -> nodedb_types::WriteVersion { + nodedb_types::WriteVersion::local_after(nodedb_types::WriteVersion::ZERO, lsn) +} + +/// The vShard a Calvin participant writes `collection` on: the collection's +/// home vShard. +pub(super) fn home_vshard(collection: &str) -> u32 { + nodedb_types::CollectionKey::from_bare(DatabaseId::DEFAULT, collection) + .vshard() + .as_u32() +} + +pub(super) fn make_core() -> ( + CoreLoop, + Producer, + Consumer, + tempfile::TempDir, +) { + let dir = tempfile::tempdir().unwrap(); + let (req_tx, req_rx) = RingBuffer::channel::(64); + let (resp_tx, resp_rx) = RingBuffer::channel::(64); + let core = CoreLoop::open( + 0, + req_rx, + resp_tx, + dir.path(), + Arc::new(nodedb_types::OrdinalClock::new()), + nodedb::data::executor::core_loop::test_governor(), + ) + .unwrap(); + (core, req_tx, resp_rx, dir) +} + +/// Build a request for `plan` on `vshard`, carrying an optional committed WAL +/// LSN (present on the seed write so its version is recorded). +pub(super) fn make_request(plan: PhysicalPlan, vshard: u32, wal_lsn: Option) -> Request { + Request { + request_id: RequestId::new(1), + tenant_id: TenantId::new(1), + vshard_id: VShardId::new(vshard), + database_id: DatabaseId::DEFAULT, + plan, + deadline: Instant::now() + Duration::from_secs(5), + priority: Priority::Normal, + trace_id: nodedb_types::TraceId::ZERO, + consistency: ReadConsistency::Strong, + idempotency_key: None, + event_source: nodedb::event::EventSource::RaftFollower, + user_roles: Vec::new(), + user_id: None, + statement_digest: None, + txn_id: None, + wal_lsn, + resolved_now_ms: None, + commit_hlc: None, + entry_version: None, + admission: nodedb::bridge::envelope::Admission::Admitted, + } +} + +pub(super) fn send( + core: &mut CoreLoop, + tx: &mut Producer, + rx: &mut Consumer, + plan: PhysicalPlan, + vshard: u32, + wal_lsn: Option, +) -> Response { + tx.try_push(BridgeRequest::unfloored(make_request( + plan, vshard, wal_lsn, + ))) + .unwrap(); + core.tick(); + rx.try_pop().unwrap().inner +} + +/// Resolve the transaction staged at `(epoch, 0)` on `vshard` and build the +/// stamped install of its redo record. +pub(super) fn resolved_install( + core: &mut CoreLoop, + tx: &mut Producer, + rx: &mut Consumer, + epoch: u64, + vshard: u32, +) -> PhysicalPlan { + let resolved = send( + core, + tx, + rx, + PhysicalPlan::Meta(MetaOp::CalvinResolve { epoch, position: 0 }), + vshard, + None, + ); + assert_eq!( + resolved.status, + Status::Ok, + "resolve must succeed: {resolved:?}" + ); + stamped_install(epoch, &resolved, Vec::new()) +} + +/// The stamped install of the slice at `(epoch, 0)` whose `CalvinResolve` +/// answered `resolved`. +pub(super) fn stamped_install( + epoch: u64, + resolved: &Response, + collections: Vec, +) -> PhysicalPlan { + let answer: CalvinResolved = + zerompk::from_msgpack(resolved.payload.as_bytes()).expect("decode resolved answer"); + PhysicalPlan::Meta(MetaOp::ApplyTransactionRedo { + redo: answer.redo, + collections, + sum_targets: Vec::new(), + origin: RedoOrigin::Commit, + calvin: Some(CalvinInstall { + epoch, + position: 0, + epoch_system_ms: 0, + reply: answer.reply, + user_write: true, + }), + }) +} + +/// A write committed through the Calvin path to seed a write version. +pub(super) struct CalvinSeed<'a> { + pub(super) epoch: u64, + pub(super) vshard: u32, + /// The collection `plans` write; the install records its floor at `lsn`. + pub(super) collection: &'a str, + pub(super) plans: Vec, + pub(super) lsn: u64, +} + +/// Commit `seed.plans` as the Calvin transaction at `(seed.epoch, 0)` on +/// `seed.vshard` and install its redo record at `seed.lsn`: the path a +/// committed multi-shard write takes. The install records each written key +/// and the collection floor at that LSN. +pub(super) fn commit_calvin( + core: &mut CoreLoop, + tx: &mut Producer, + rx: &mut Consumer, + seed: CalvinSeed<'_>, +) -> Response { + let staged = send( + core, + tx, + rx, + stage_static(seed.epoch, 0, seed.plans, Vec::new()), + seed.vshard, + None, + ); + assert_eq!( + staged.status, + Status::Ok, + "seed stage must succeed: {staged:?}" + ); + let resolved = send( + core, + tx, + rx, + PhysicalPlan::Meta(MetaOp::CalvinResolve { + epoch: seed.epoch, + position: 0, + }), + seed.vshard, + None, + ); + assert_eq!( + resolved.status, + Status::Ok, + "seed resolve must succeed: {resolved:?}" + ); + send( + core, + tx, + rx, + stamped_install(seed.epoch, &resolved, vec![seed.collection.to_string()]), + seed.vshard, + Some(Lsn::new(seed.lsn)), + ) +} + +pub(super) fn kv_put(coll: &str, key: &[u8], value: &[u8]) -> PhysicalPlan { + PhysicalPlan::Kv(KvOp::Put { + collection: QualifiedCollection::new(DatabaseId::DEFAULT, coll), + key: key.to_vec(), + value: value.to_vec(), + ttl_ms: 0, + surrogate: nodedb_test_support::kv_rows::kv_row_surrogate(key), + returning: None, + rls_filters: Vec::new(), + provenance: None, + }) +} + +pub(super) fn kv_get(coll: &str, key: &[u8]) -> PhysicalPlan { + PhysicalPlan::Kv(KvOp::Get { + collection: QualifiedCollection::new(DatabaseId::DEFAULT, coll), + key: key.to_vec(), + rls_filters: Vec::new(), + surrogate_ceiling: None, + }) +} + +/// A minimal msgpack-encoded document body, `{"a": "1"}`. +pub(super) fn doc_value() -> Vec { + let mut obj = std::collections::HashMap::new(); + obj.insert("a".to_string(), nodedb_types::Value::String("1".into())); + zerompk::to_msgpack_vec(&nodedb_types::Value::Object(obj)).unwrap() +} + +/// A minimal document `PointInsert` plan, used to record a write-version at a +/// specific surrogate (the document engine's per-key identity). +pub(super) fn doc_insert(coll: &str, document_id: &str, surrogate: u32) -> PhysicalPlan { + PhysicalPlan::Document(DocumentOp::PointInsert { + collection: QualifiedCollection::new(DatabaseId::DEFAULT, coll), + document_id: document_id.to_string(), + value: doc_value(), + if_absent: false, + surrogate: Surrogate::new(surrogate), + returning: None, + rls_filters: Vec::new(), + resolved_sum_targets: Vec::new(), + deferred_sum_targets: Vec::new(), + }) +} + +pub(super) fn stage_static( + epoch: u64, + position: u32, + plans: Vec, + versioned_reads: Vec, +) -> PhysicalPlan { + PhysicalPlan::Meta(MetaOp::CalvinExecuteStatic { + epoch, + position, + tenant_id: TenantId::new(1), + plans, + epoch_system_ms: 0, + versioned_reads, + body_plans: Vec::new(), + }) +} + +/// Push one prebuilt request through the ring and return its response. +pub(super) fn send_request( + core: &mut CoreLoop, + tx: &mut Producer, + rx: &mut Consumer, + request: Request, +) -> Response { + tx.try_push(BridgeRequest::unfloored(request)).unwrap(); + core.tick(); + rx.try_pop().unwrap().inner +} diff --git a/nodedb/tests/inproc/cases/cross_engine_bitmap_currency.rs b/nodedb/tests/inproc/cases/cross_engine_bitmap_currency.rs index 016b6b2a5..8c034ab21 100644 --- a/nodedb/tests/inproc/cases/cross_engine_bitmap_currency.rs +++ b/nodedb/tests/inproc/cases/cross_engine_bitmap_currency.rs @@ -76,6 +76,7 @@ fn make_req(plan: PhysicalPlan) -> Request { wal_lsn: None, resolved_now_ms: None, commit_hlc: None, + entry_version: None, admission: nodedb::bridge::envelope::Admission::Admitted, } } diff --git a/nodedb/tests/inproc/cases/cross_engine_three_way_fts_vector_doc.rs b/nodedb/tests/inproc/cases/cross_engine_three_way_fts_vector_doc.rs index 1c9391db5..f1718a139 100644 --- a/nodedb/tests/inproc/cases/cross_engine_three_way_fts_vector_doc.rs +++ b/nodedb/tests/inproc/cases/cross_engine_three_way_fts_vector_doc.rs @@ -79,6 +79,7 @@ fn make_req(plan: PhysicalPlan) -> Request { wal_lsn: None, resolved_now_ms: None, commit_hlc: None, + entry_version: None, admission: nodedb::bridge::envelope::Admission::Admitted, } } diff --git a/nodedb/tests/inproc/cases/event_restart_watermark_catchup.rs b/nodedb/tests/inproc/cases/event_restart_watermark_catchup.rs index 83b179d54..3ebebcdb1 100644 --- a/nodedb/tests/inproc/cases/event_restart_watermark_catchup.rs +++ b/nodedb/tests/inproc/cases/event_restart_watermark_catchup.rs @@ -187,8 +187,9 @@ async fn restart_fires_held_events_the_lane_had_not_fired_once() { let gates = tempfile::tempdir().unwrap(); let release = gates.path().join("before_firing"); let parked = gates.path().join("before_firing.parked"); - let gate = FailGuard::install( - &format!("trigger::before_firing::node{}", server.shared.node_id), + let gate = FailGuard::for_node( + server.shared.node_id, + "trigger::before_firing", FailAction::WaitForFile(release), ); wait_until("the node's firing parks", || parked.exists()).await; diff --git a/nodedb/tests/inproc/cases/executor_tests/helpers.rs b/nodedb/tests/inproc/cases/executor_tests/helpers.rs index a725cedda..47cf1d129 100644 --- a/nodedb/tests/inproc/cases/executor_tests/helpers.rs +++ b/nodedb/tests/inproc/cases/executor_tests/helpers.rs @@ -94,6 +94,7 @@ pub fn make_request(plan: PhysicalPlan) -> Request { wal_lsn: None, resolved_now_ms: None, commit_hlc: None, + entry_version: None, admission: nodedb::bridge::envelope::Admission::Admitted, } } diff --git a/nodedb/tests/inproc/cases/mod.rs b/nodedb/tests/inproc/cases/mod.rs index 6bd9f9821..1dbd62460 100644 --- a/nodedb/tests/inproc/cases/mod.rs +++ b/nodedb/tests/inproc/cases/mod.rs @@ -214,4 +214,5 @@ mod wal_catchup; mod warm_storage_object_store; mod wire_server_version; mod write_admission_fence; +mod write_admission_leader_gate; mod ws_live_lifecycle; diff --git a/nodedb/tests/inproc/cases/request_tracker_backpressure.rs b/nodedb/tests/inproc/cases/request_tracker_backpressure.rs index 706e8dca0..c481facf4 100644 --- a/nodedb/tests/inproc/cases/request_tracker_backpressure.rs +++ b/nodedb/tests/inproc/cases/request_tracker_backpressure.rs @@ -24,9 +24,9 @@ fn partial(id: u64, data: &[u8]) -> Response { partial: true, payload: Payload::from_vec(data.to_vec()), watermark_lsn: Lsn::ZERO, - read_version_lsn: Lsn::ZERO, + read_versions: nodedb::types::ReadVersions::new(), error_code: None, - read_set_valid: None, + stage_vote: None, write_set: Vec::new(), } } diff --git a/nodedb/tests/inproc/cases/startup_gate_pgwire.rs b/nodedb/tests/inproc/cases/startup_gate_pgwire.rs index b4425008c..4c68d6494 100644 --- a/nodedb/tests/inproc/cases/startup_gate_pgwire.rs +++ b/nodedb/tests/inproc/cases/startup_gate_pgwire.rs @@ -84,9 +84,9 @@ fn spawn_fake_data_plane(mut data_side: CoreChannelDataSide) { partial: false, payload, watermark_lsn: Lsn::ZERO, - read_version_lsn: Lsn::ZERO, + read_versions: nodedb::types::ReadVersions::new(), error_code: None, - read_set_valid: None, + stage_vote: None, write_set: Vec::new(), }, }; diff --git a/nodedb/tests/inproc/cases/surrogate_round_trip.rs b/nodedb/tests/inproc/cases/surrogate_round_trip.rs index 841399318..fe0b72849 100644 --- a/nodedb/tests/inproc/cases/surrogate_round_trip.rs +++ b/nodedb/tests/inproc/cases/surrogate_round_trip.rs @@ -90,6 +90,7 @@ fn make_req(plan: PhysicalPlan) -> Request { wal_lsn: None, resolved_now_ms: None, commit_hlc: None, + entry_version: None, admission: nodedb::bridge::envelope::Admission::Admitted, } } diff --git a/nodedb/tests/inproc/cases/wal_catchup/startup_replay.rs b/nodedb/tests/inproc/cases/wal_catchup/startup_replay.rs index acf1277c7..56ce142ca 100644 --- a/nodedb/tests/inproc/cases/wal_catchup/startup_replay.rs +++ b/nodedb/tests/inproc/cases/wal_catchup/startup_replay.rs @@ -114,6 +114,7 @@ fn startup_replay_recovers_all_wal_data() { wal_lsn: None, resolved_now_ms: None, commit_hlc: None, + entry_version: None, admission: nodedb::bridge::envelope::Admission::Exempt( nodedb::bridge::envelope::ExemptReason::Read, ), diff --git a/nodedb/tests/inproc/cases/write_admission_fence.rs b/nodedb/tests/inproc/cases/write_admission_fence.rs index 5a4687eca..1d380e298 100644 --- a/nodedb/tests/inproc/cases/write_admission_fence.rs +++ b/nodedb/tests/inproc/cases/write_admission_fence.rs @@ -18,12 +18,12 @@ //! test suite); these tests assert the gate's fence DECISION, which is the piece //! this change introduces. -use std::collections::BTreeSet; +use std::collections::BTreeMap; use std::sync::{Arc, Mutex}; use nodedb::bridge::dispatch::Dispatcher; use nodedb::control::cluster::calvin::scheduler::lock_manager::{ - AcquireOutcome, LockKey, LockManager, TxnId, + AcquireOutcome, LockKey, LockManager, LockMode, TxnId, }; use nodedb::control::server::shared::write_admission::{ WriteAdmission, WriteTarget, admit, cp_routed_to_calvin, @@ -36,7 +36,7 @@ use nodedb_types::QualifiedCollection; /// Build a single-node `SharedState` over a throwaway WAL. The returned /// `TempDir` must be kept alive for the WAL's lifetime. -fn build_shared() -> (Arc, tempfile::TempDir) { +pub(super) fn build_shared() -> (Arc, tempfile::TempDir) { let dir = tempfile::tempdir().expect("tempdir"); let wal = Arc::new(WalManager::open_for_testing(&dir.path().join("fence.wal")).expect("open wal")); @@ -47,7 +47,7 @@ fn build_shared() -> (Arc, tempfile::TempDir) { /// Register an empty lock table for `collection`'s vShard and return both the /// shared `Arc` and the vShard the gate will resolve the plan to. -fn register_lock_manager( +pub(super) fn register_lock_manager( shared: &SharedState, collection: &str, ) -> (Arc>, VShardId) { @@ -65,7 +65,7 @@ fn register_lock_manager( /// Register a promotion channel for `vshard` and return the receiver. The gate /// clones the sender into any fast-path guard it builds for this vShard, so the /// guard's drop delivers promoted scheduler waiters here. -fn register_promotion_channel( +pub(super) fn register_promotion_channel( shared: &SharedState, vshard: VShardId, ) -> tokio::sync::mpsc::UnboundedReceiver> { @@ -80,7 +80,7 @@ fn register_promotion_channel( } /// A single-key KV point write to `collection` / `key`. -fn kv_put(collection: &str, key: &[u8]) -> PhysicalPlan { +pub(super) fn kv_put(collection: &str, key: &[u8]) -> PhysicalPlan { PhysicalPlan::Kv(KvOp::Put { collection: QualifiedCollection::new(DatabaseId::DEFAULT, collection), key: key.to_vec(), @@ -120,7 +120,7 @@ async fn fence_write_blocks_behind_held_commit_lock() { // A pending commit (a normal Calvin-band txn) holds the fence on key K. let commit_txn = TxnId::new(5, 0); - let held: BTreeSet = [kv_lock_key(coll, b"K")].into(); + let held: BTreeMap = [(kv_lock_key(coll, b"K"), LockMode::Exclusive)].into(); assert_eq!( lm.lock().expect("lm").acquire(commit_txn, held), AcquireOutcome::Ready, @@ -131,7 +131,7 @@ async fn fence_write_blocks_behind_held_commit_lock() { // holder via the scheduler's FIFO queue. let plan = kv_put(coll, b"K"); let before = cp_routed_to_calvin(); - match admit(&shared, &target(vshard, &plan)) { + match admit(&shared, &target(vshard, &plan)).expect("admit") { WriteAdmission::RouteToCalvin => {} _ => panic!("a point write behind a held commit lock must route to Calvin"), } @@ -142,7 +142,7 @@ async fn fence_write_blocks_behind_held_commit_lock() { // The commit releases; the same write is now admitted to the fast path. let _ = lm.lock().expect("lm").release(commit_txn); - match admit(&shared, &target(vshard, &plan)) { + match admit(&shared, &target(vshard, &plan)).expect("admit") { WriteAdmission::FastPath { guard: Some(_) } => {} _ => panic!("after release, the point write must fast-path with a real lock guard"), } @@ -159,14 +159,14 @@ async fn two_concurrent_same_key_point_writes_serialize() { let plan = kv_put(coll, b"K"); // First write takes the fast path and holds the lock via its RAII guard. - let guard1 = match admit(&shared, &target(vshard, &plan)) { + let guard1 = match admit(&shared, &target(vshard, &plan)).expect("admit") { WriteAdmission::FastPath { guard: Some(g) } => g, _ => panic!("first same-key write must fast-path with a real lock guard"), }; // Second write to the same key observes the held lock and is routed. let before = cp_routed_to_calvin(); - match admit(&shared, &target(vshard, &plan)) { + match admit(&shared, &target(vshard, &plan)).expect("admit") { WriteAdmission::RouteToCalvin => {} _ => panic!("second same-key write must route to Calvin behind the first"), } @@ -177,7 +177,7 @@ async fn two_concurrent_same_key_point_writes_serialize() { // First write completes (guard drops, releasing the lock); the key is free. drop(guard1); - match admit(&shared, &target(vshard, &plan)) { + match admit(&shared, &target(vshard, &plan)).expect("admit") { WriteAdmission::FastPath { guard: Some(_) } => {} _ => panic!("after the first write releases, the key must fast-path again"), } @@ -195,7 +195,7 @@ async fn single_node_point_write_uses_global_keyed_order_lock() { let vshard = nodedb_types::CollectionKey::from_bare(DatabaseId::DEFAULT, coll).vshard(); let plan = kv_put(coll, b"K"); - let lock = match admit(&shared, &target(vshard, &plan)) { + let lock = match admit(&shared, &target(vshard, &plan)).expect("admit") { WriteAdmission::FastPathBlocking { key, keyed_lock } => { assert_eq!( key, @@ -222,7 +222,7 @@ async fn single_node_same_key_serializes_fifo() { let vshard = nodedb_types::CollectionKey::from_bare(DatabaseId::DEFAULT, coll).vshard(); let plan = kv_put(coll, b"K"); - let (key, lock) = match admit(&shared, &target(vshard, &plan)) { + let (key, lock) = match admit(&shared, &target(vshard, &plan)).expect("admit") { WriteAdmission::FastPathBlocking { key, keyed_lock } => (key, keyed_lock), _ => panic!("single-node point write must return FastPathBlocking"), }; @@ -281,7 +281,7 @@ async fn fast_path_drop_delivers_promoted_scheduler_txn() { // A fast-path point write on K takes the fence via its RAII guard (an // autocommit-band holder). K was uncontended at acquire time. let plan = kv_put(coll, b"K"); - let guard = match admit(&shared, &target(vshard, &plan)) { + let guard = match admit(&shared, &target(vshard, &plan)).expect("admit") { WriteAdmission::FastPath { guard: Some(g) } => g, _ => panic!("uncontended point write must fast-path with a real lock guard"), }; @@ -290,7 +290,7 @@ async fn fast_path_drop_delivers_promoted_scheduler_txn() { // K and queues behind the fast-path holder (Blocked) — the exact contention // the old "fast-path keys are never contended" assumption ignored. let scheduler_txn = TxnId::new(7, 0); - let want: BTreeSet = [kv_lock_key(coll, b"K")].into(); + let want: BTreeMap = [(kv_lock_key(coll, b"K"), LockMode::Exclusive)].into(); assert_eq!( lm.lock().expect("lm").acquire(scheduler_txn, want.clone()), AcquireOutcome::Blocked, @@ -340,3 +340,93 @@ async fn single_node_distinct_keys_do_not_block() { drop(g_b); drop(g_a); } + +/// The collection lock key a truncate holds `Exclusive` and a row writer +/// holds `Intent`. +fn collection_lock_key(collection: &str) -> LockKey { + LockKey::Collection { + collection: Arc::from(collection), + } +} + +/// A Calvin truncate and an autocommit point write of one collection +/// serialize at the gate, whichever comes first. A truncate holds the +/// collection `Exclusive`, and a point write holds it `Intent`. +#[tokio::test] +async fn calvin_truncate_and_autocommit_point_write_serialize() { + let (shared, _dir) = build_shared(); + let coll = "truncate_coll"; + let (lm, vshard) = register_lock_manager(&shared, coll); + let mut promotion_rx = register_promotion_channel(&shared, vshard); + let truncate_keys: BTreeMap = + [(collection_lock_key(coll), LockMode::Exclusive)].into(); + let plan = kv_put(coll, b"K"); + + // The truncate is sequenced first: the point write routes behind it. + let truncate = TxnId::new(9, 0); + assert_eq!( + lm.lock() + .expect("lm") + .acquire(truncate, truncate_keys.clone()), + AcquireOutcome::Ready + ); + assert!( + matches!( + admit(&shared, &target(vshard, &plan)).expect("admit"), + WriteAdmission::RouteToCalvin + ), + "a point write of a collection a truncate holds must route behind it" + ); + let _ = lm.lock().expect("lm").release(truncate); + + // The point write is admitted first: the truncate waits for its guard. + let guard = match admit(&shared, &target(vshard, &plan)).expect("admit") { + WriteAdmission::FastPath { guard: Some(guard) } => guard, + _ => panic!("an uncontended point write must fast-path with a real lock guard"), + }; + let later_truncate = TxnId::new(9, 1); + assert_eq!( + lm.lock() + .expect("lm") + .acquire(later_truncate, truncate_keys.clone()), + AcquireOutcome::Blocked, + "a truncate must wait for the point write's collection intent" + ); + drop(guard); + let promoted = promotion_rx + .try_recv() + .expect("the guard's drop must deliver the promoted truncate"); + assert!(promoted.contains(&later_truncate)); + assert!( + lm.lock() + .expect("lm") + .is_ready(later_truncate, &truncate_keys) + ); +} + +/// An autocommit truncate locks its collection `Exclusive`: it routes +/// behind a Calvin row writer of the collection. +#[tokio::test] +async fn autocommit_truncate_routes_behind_a_calvin_row_writer() { + let (shared, _dir) = build_shared(); + let coll = "truncate_after_row"; + let (lm, vshard) = register_lock_manager(&shared, coll); + let row_writer = TxnId::new(4, 0); + let row_keys: BTreeMap = [ + (kv_lock_key(coll, b"K"), LockMode::Exclusive), + (collection_lock_key(coll), LockMode::Intent), + ] + .into(); + assert_eq!( + lm.lock().expect("lm").acquire(row_writer, row_keys), + AcquireOutcome::Ready + ); + let truncate = PhysicalPlan::Kv(KvOp::Truncate { + collection: QualifiedCollection::new(DatabaseId::DEFAULT, coll), + restart_identity: false, + }); + assert!(matches!( + admit(&shared, &target(vshard, &truncate)).expect("admit"), + WriteAdmission::RouteToCalvin + )); +} diff --git a/nodedb/tests/inproc/cases/write_admission_leader_gate.rs b/nodedb/tests/inproc/cases/write_admission_leader_gate.rs new file mode 100644 index 000000000..d20016061 --- /dev/null +++ b/nodedb/tests/inproc/cases/write_admission_leader_gate.rs @@ -0,0 +1,231 @@ +// SPDX-License-Identifier: BUSL-1.1 + +//! The data-group leader's write gate. +//! +//! A replicated write takes its lock keys on its leader before its propose. +//! These tests drive `LeaderWriteGate` with encoded entries against the lock +//! table a Calvin scheduler shares: +//! +//! - An uncontended entry holds its keys until the leader's apply loop starts +//! the entry. A Calvin truncate of the collection waits until then. +//! - A contended entry a Calvin transaction can sequence is refused with +//! `RouteToSequencer`. +//! - A contended entry no Calvin transaction sequences waits for its keys, +//! until the proposer's deadline at the latest. + +use std::collections::BTreeMap; +use std::sync::Arc; +use std::time::Duration; + +use nodedb::control::cluster::calvin::scheduler::lock_manager::{ + AcquireOutcome, LockKey, LockMode, TxnId, +}; +use nodedb::control::server::shared::write_admission::LeaderWriteGate; +use nodedb::control::wal_replication::{ReplicableWrite, to_replicated_entry}; +use nodedb::types::{DatabaseId, TenantId, VShardId}; +use nodedb_cluster::{CalvinError, ClusterError, DataProposeGate}; +use nodedb_physical::physical_plan::{KvOp, KvResolvedMutation, PhysicalPlan}; +use nodedb_types::{QualifiedCollection, Surrogate}; + +use super::write_admission_fence::{build_shared, kv_put, register_lock_manager}; + +/// The encoded replicated entry of `plan` on `vshard`. +fn entry_bytes(vshard: VShardId, plan: &PhysicalPlan) -> Vec { + let replicable = ReplicableWrite::decide_for_replication(plan).expect("replicable"); + to_replicated_entry(TenantId::new(1), DatabaseId::DEFAULT, vshard, &replicable) + .expect("encode") + .expect("the plan has a replicated form") + .encode() + .expect("entry bytes") +} + +/// A proposer deadline no test reaches. +fn far_deadline() -> tokio::time::Instant { + tokio::time::Instant::now() + Duration::from_secs(60) +} + +fn truncate_keys(collection: &str) -> BTreeMap { + [( + LockKey::Collection { + collection: Arc::from(collection), + }, + LockMode::Exclusive, + )] + .into() +} + +/// An admitted entry holds its keys until the leader starts its apply. A +/// Calvin truncate sequenced meanwhile runs after it. +#[tokio::test] +async fn an_admitted_entry_holds_its_keys_until_its_apply_starts() { + let (shared, _dir) = build_shared(); + let coll = "leader_gate_hold"; + let (lm, vshard) = register_lock_manager(&shared, coll); + let gate = LeaderWriteGate::new(&shared); + + let hold = gate + .admit( + vshard.as_u32(), + &entry_bytes(vshard, &kv_put(coll, b"K")), + far_deadline(), + ) + .await + .expect("admit") + .expect("an uncontended row write holds its keys"); + let (group_id, log_index) = (3, 10); + hold.landed(group_id, log_index); + + let truncate = TxnId::new(9, 0); + let keys = truncate_keys(coll); + assert_eq!( + lm.lock().expect("lm").acquire(truncate, keys.clone()), + AcquireOutcome::Blocked, + "a Calvin truncate waits for the admitted write" + ); + shared + .calvin + .admission_holds + .release_through(group_id, log_index - 1); + assert!( + !lm.lock().expect("lm").is_ready(truncate, &keys), + "an earlier entry's start releases nothing of this one" + ); + shared + .calvin + .admission_holds + .release_through(group_id, log_index); + assert!( + lm.lock().expect("lm").is_ready(truncate, &keys), + "the entry's start releases its keys to the truncate" + ); +} + +/// A contended entry a Calvin transaction can sequence is refused, so its +/// proposer submits it to the sequencer. +#[tokio::test] +async fn a_contended_sequenced_entry_routes_to_the_sequencer() { + let (shared, _dir) = build_shared(); + let coll = "leader_gate_route"; + let (lm, vshard) = register_lock_manager(&shared, coll); + assert_eq!( + lm.lock() + .expect("lm") + .acquire(TxnId::new(9, 0), truncate_keys(coll)), + AcquireOutcome::Ready + ); + let gate = LeaderWriteGate::new(&shared); + let refused = gate + .admit( + vshard.as_u32(), + &entry_bytes(vshard, &kv_put(coll, b"K")), + far_deadline(), + ) + .await; + assert!(matches!( + refused, + Err(ClusterError::Calvin(CalvinError::RouteToSequencer)) + )); +} + +/// A contended resolved write, which no Calvin transaction sequences, waits +/// for its keys and takes them once the holder releases. +#[tokio::test] +async fn a_contended_resolved_entry_waits_for_its_keys() { + let (shared, _dir) = build_shared(); + let coll = "leader_gate_wait"; + let (lm, vshard) = register_lock_manager(&shared, coll); + let truncate = TxnId::new(9, 0); + assert_eq!( + lm.lock() + .expect("lm") + .acquire(truncate, truncate_keys(coll)), + AcquireOutcome::Ready + ); + let resolved = PhysicalPlan::Kv(KvOp::ResolvedWrite { + mutations: vec![KvResolvedMutation::Put { + collection: QualifiedCollection::new(DatabaseId::DEFAULT, coll), + key: b"K".to_vec(), + value: b"v".to_vec(), + ttl_ms: 0, + expire_at_ms: 0, + surrogate: Surrogate::new(5), + precondition: None, + }], + response_payload: Vec::new(), + rls_write_check: nodedb_types::RlsWriteCheck::decided_earlier_in_request(), + }); + let bytes = entry_bytes(vshard, &resolved); + + let waiter = { + let gate = LeaderWriteGate::new(&shared); + let vshard = vshard.as_u32(); + tokio::spawn(async move { gate.admit(vshard, &bytes, far_deadline()).await }) + }; + tokio::task::yield_now().await; + assert!( + !waiter.is_finished(), + "the entry waits while the truncate holds" + ); + let _ = lm.lock().expect("lm").release(truncate); + let hold = waiter + .await + .expect("join") + .expect("the keys free up") + .expect("the resolved write holds its row"); + drop(hold); + assert_eq!( + lm.lock() + .expect("lm") + .acquire(TxnId::new(10, 0), truncate_keys(coll)), + AcquireOutcome::Ready, + "dropping the hold frees every key it held" + ); +} + +/// A contended entry waits no longer than the proposer's deadline. A +/// forwarded entry carries a short budget, and the gate refuses it +/// unproposed once the budget runs out. +#[tokio::test] +async fn a_contended_entry_waits_no_longer_than_its_budget() { + let (shared, _dir) = build_shared(); + let coll = "leader_gate_budget"; + let (lm, vshard) = register_lock_manager(&shared, coll); + assert_eq!( + lm.lock() + .expect("lm") + .acquire(TxnId::new(9, 0), truncate_keys(coll)), + AcquireOutcome::Ready + ); + let resolved = PhysicalPlan::Kv(KvOp::ResolvedWrite { + mutations: vec![KvResolvedMutation::Put { + collection: QualifiedCollection::new(DatabaseId::DEFAULT, coll), + key: b"K".to_vec(), + value: b"v".to_vec(), + ttl_ms: 0, + expire_at_ms: 0, + surrogate: Surrogate::new(5), + precondition: None, + }], + response_payload: Vec::new(), + rls_write_check: nodedb_types::RlsWriteCheck::decided_earlier_in_request(), + }); + let budget_ms = 50; + let deadline = nodedb_cluster::rpc_codec::forwarded_deadline(budget_ms).expect("a live budget"); + let gate = LeaderWriteGate::new(&shared); + // The node default deadline is seconds long. The gate must end at the + // budget, so this bound fails a gate that waits for the default. + let refused = tokio::time::timeout( + Duration::from_secs(5), + gate.admit(vshard.as_u32(), &entry_bytes(vshard, &resolved), deadline), + ) + .await + .expect("the gate ends at the forwarded budget, not the node default"); + assert!(matches!( + refused, + Err(ClusterError::Calvin(CalvinError::AdmissionTimedOut)) + )); + assert!( + tokio::time::Instant::now() >= deadline, + "the gate waits until the budget runs out" + ); +} diff --git a/nodedb/tests/wire/cases/materialized_sum_cross_shard_expanders.rs b/nodedb/tests/wire/cases/materialized_sum_cross_shard_expanders.rs new file mode 100644 index 000000000..0757cee4b --- /dev/null +++ b/nodedb/tests/wire/cases/materialized_sum_cross_shard_expanders.rs @@ -0,0 +1,262 @@ +// SPDX-License-Identifier: BUSL-1.1 + +//! `INSERT ... SELECT`, `UPDATE ... FROM` and `MERGE` into the source of a +//! materialized sum whose target lives on another vShard. +//! +//! Each statement expands into point writes on the source. A balance on a +//! cross-shard target cannot fold on the source's core: that core owns no +//! target row. Each expansion ships the balance on its own task, homed on the +//! target's vShard, and COMMIT applies it with the source rows. +//! +//! The server runs two Data Plane cores, and the pair below lands on +//! different cores. A balance folded on the source's core then finds no +//! target row. + +use nodedb_types::id::DatabaseId; +use tokio_postgres::SimpleQueryMessage; + +use crate::harness::TestServer; + +/// The sum's target collection. +const ACCOUNTS: &str = "ptv_acct"; +/// The sum's source collection. +const POSTINGS: &str = "ptv_post"; +/// Rows the statements copy or join from. Its home does not matter. +const FEED: &str = "ptv_feed"; +/// Data Plane cores the server runs. +const CORES: usize = 2; + +fn vshard(name: &str) -> u32 { + nodedb_types::CollectionKey::from_bare(DatabaseId::DEFAULT, name) + .vshard() + .as_u32() +} + +/// The premise every test rests on: the target and the source hash to +/// different vShards, and those vShards live on different cores. +#[test] +fn the_sum_pair_spans_two_vshards_and_two_cores() { + assert_ne!( + vshard(ACCOUNTS), + vshard(POSTINGS), + "the pair must plan a separate balance task" + ); + assert_ne!( + vshard(ACCOUNTS) as usize % CORES, + vshard(POSTINGS) as usize % CORES, + "a balance folded on the source's core must find no target row" + ); +} + +/// Every `CommandComplete` count in `sql`'s simple-query response, in wire +/// order. +async fn command_counts(server: &TestServer, sql: &str) -> Vec { + let messages = server + .client + .simple_query(sql) + .await + .unwrap_or_else(|e| panic!("run {sql}: {e:?}")); + messages + .into_iter() + .filter_map(|m| match m { + SimpleQueryMessage::CommandComplete(n) => Some(n), + _ => None, + }) + .collect() +} + +async fn exec(server: &TestServer, sql: &str) { + server + .exec(sql) + .await + .unwrap_or_else(|e| panic!("run {sql}: {e}")); +} + +/// Start a server and create the sum pair and the feed. Accounts `acc` and +/// `acc2` start empty. Posting `e0` puts 10 on `acc`, posting `e1` puts 5 on +/// `acc`. +pub(super) async fn sum_fixture() -> TestServer { + let server = TestServer::start_multicores(CORES).await; + exec( + &server, + &format!( + "CREATE COLLECTION {ACCOUNTS} (id TEXT PRIMARY KEY, owner TEXT) \ + WITH (engine='document_strict')" + ), + ) + .await; + exec( + &server, + &format!( + "CREATE COLLECTION {POSTINGS} (id TEXT PRIMARY KEY, account_id TEXT, amount TEXT) \ + WITH (engine='document_strict')" + ), + ) + .await; + exec( + &server, + &format!( + "ALTER COLLECTION {ACCOUNTS} ADD COLUMN balance TEXT \ + MATERIALIZED_SUM SOURCE {POSTINGS} \ + ON {POSTINGS}.account_id = {ACCOUNTS}.id VALUE {POSTINGS}.amount" + ), + ) + .await; + exec( + &server, + &format!( + "CREATE COLLECTION {FEED} (id TEXT PRIMARY KEY, account_id TEXT, amount TEXT) \ + WITH (engine='document_strict')" + ), + ) + .await; + for account in ["acc", "acc2"] { + exec( + &server, + &format!( + "INSERT INTO {ACCOUNTS} (id, owner, balance) VALUES ('{account}', 'alice', '0')" + ), + ) + .await; + } + for (id, amount) in [("e0", "10"), ("e1", "5")] { + exec( + &server, + &format!( + "INSERT INTO {POSTINGS} (id, account_id, amount) VALUES ('{id}', 'acc', '{amount}')" + ), + ) + .await; + } + assert_eq!( + balance(&server, "acc").await, + "15", + "the seed postings land" + ); + server +} + +/// Insert one feed row. +pub(super) async fn feed(server: &TestServer, id: &str, account: &str, amount: &str) { + exec( + server, + &format!( + "INSERT INTO {FEED} (id, account_id, amount) VALUES ('{id}', '{account}', '{amount}')" + ), + ) + .await; +} + +/// The stored balance of `account`. +pub(super) async fn balance(server: &TestServer, account: &str) -> String { + let rows = server + .query_text(&format!( + "SELECT balance FROM {ACCOUNTS} WHERE id = '{account}'" + )) + .await + .unwrap_or_else(|e| panic!("read the balance of {account}: {e}")); + match rows.as_slice() { + [value] => value.clone(), + other => panic!("one balance row for {account}, got {other:?}"), + } +} + +pub(super) const INSERT_SELECT: &str = "INSERT INTO ptv_post (id, account_id, amount) \ + SELECT id, account_id, amount FROM ptv_feed"; + +/// The amount of `e0` changes in place. `e1` moves from `acc` to `acc2`. +const UPDATE_FROM: &str = "UPDATE ptv_post SET amount = f.amount, account_id = f.account_id \ + FROM ptv_feed f WHERE ptv_post.id = f.id"; + +/// Seed the feed `UPDATE_FROM` joins. +async fn update_feed(server: &TestServer) { + feed(server, "e0", "acc", "25").await; + feed(server, "e1", "acc2", "5").await; +} + +/// `BEGIN; INSERT ... SELECT; COMMIT` credits each copied row's amount to its +/// cross-shard account, and tags only the copied rows. +#[tokio::test(flavor = "multi_thread", worker_threads = 4)] +async fn a_staged_insert_select_moves_a_cross_shard_sum() { + let server = sum_fixture().await; + feed(&server, "f1", "acc", "4").await; + feed(&server, "f2", "acc2", "6").await; + + assert_eq!( + command_counts(&server, &format!("BEGIN; {INSERT_SELECT}; COMMIT")).await, + vec![0, 2, 0], + "BEGIN, INSERT 0 2, COMMIT: a balance task adds no count" + ); + + assert_eq!(balance(&server, "acc").await, "19", "10 + 5 + 4"); + assert_eq!(balance(&server, "acc2").await, "6"); +} + +/// `BEGIN; UPDATE ... FROM; COMMIT` moves a cross-shard balance by the +/// amount's difference, and moves a whole posting between accounts when the +/// join column changes. +#[tokio::test(flavor = "multi_thread", worker_threads = 4)] +async fn a_staged_update_from_moves_a_cross_shard_sum() { + let server = sum_fixture().await; + update_feed(&server).await; + + assert_eq!( + command_counts(&server, &format!("BEGIN; {UPDATE_FROM}; COMMIT")).await, + vec![0, 2, 0], + "BEGIN, UPDATE 2, COMMIT: a balance task adds no count" + ); + + assert_eq!(balance(&server, "acc").await, "25", "e0 at 25, e1 left"); + assert_eq!(balance(&server, "acc2").await, "5", "e1 joined"); +} + +/// `BEGIN; MERGE; COMMIT` moves a cross-shard balance for its UPDATE arm and +/// its INSERT arm alike. +#[tokio::test(flavor = "multi_thread", worker_threads = 4)] +async fn a_staged_merge_moves_a_cross_shard_sum() { + let server = sum_fixture().await; + feed(&server, "e0", "acc", "30").await; + feed(&server, "f1", "acc2", "7").await; + + exec( + &server, + &format!( + "BEGIN; \ + MERGE INTO {POSTINGS} p USING {FEED} f ON p.id = f.id \ + WHEN MATCHED THEN UPDATE SET amount = f.amount \ + WHEN NOT MATCHED THEN INSERT (id, account_id, amount) \ + VALUES (f.id, f.account_id, f.amount); \ + COMMIT" + ), + ) + .await; + + assert_eq!(balance(&server, "acc").await, "35", "e0 at 30, e1 at 5"); + assert_eq!(balance(&server, "acc2").await, "7", "f1 inserted"); +} + +/// Outside a transaction block the same statements run in an implicit +/// transaction, and move the cross-shard balance the same way. +#[tokio::test(flavor = "multi_thread", worker_threads = 4)] +async fn autocommit_insert_select_and_update_from_move_a_cross_shard_sum() { + let server = sum_fixture().await; + feed(&server, "f1", "acc2", "6").await; + + assert_eq!( + command_counts(&server, INSERT_SELECT).await, + vec![1], + "INSERT 0 1: a balance task adds no count" + ); + assert_eq!(balance(&server, "acc2").await, "6", "f1 copied"); + + exec(&server, &format!("DELETE FROM {FEED} WHERE id = 'f1'")).await; + update_feed(&server).await; + assert_eq!( + command_counts(&server, UPDATE_FROM).await, + vec![2], + "UPDATE 2: a balance task adds no count" + ); + + assert_eq!(balance(&server, "acc").await, "25", "e0 at 25, e1 left"); + assert_eq!(balance(&server, "acc2").await, "11", "f1 and e1"); +} diff --git a/nodedb/tests/wire/cases/materialized_sum_cross_shard_http.rs b/nodedb/tests/wire/cases/materialized_sum_cross_shard_http.rs new file mode 100644 index 000000000..4a48c9746 --- /dev/null +++ b/nodedb/tests/wire/cases/materialized_sum_cross_shard_http.rs @@ -0,0 +1,177 @@ +// SPDX-License-Identifier: BUSL-1.1 + +//! A plain `INSERT` and an `INSERT ... SELECT` into the source of a +//! materialized sum whose target lives on another vShard, over HTTP +//! `/v1/query`, NDJSON `/v1/query/stream` and WebSocket RPC `/v1/ws`. +//! +//! The plain `INSERT` plans the source write and an `ApplyBalanceDelta` task +//! homed on the target's vShard. The two commit together through Calvin. +//! The `INSERT ... SELECT` runs in an implicit transaction that expands it +//! into point writes and ships each balance to its target. Each route +//! answers one `{"affected": n}` row: the balance task adds no count, as on +//! pgwire and native. +//! +//! The fixture is the one the pgwire cases use: two Data Plane cores, and a +//! target and source that land on different cores. + +use futures::{SinkExt, StreamExt}; +use tokio_tungstenite::tungstenite::Message; + +use super::materialized_sum_cross_shard_expanders::{INSERT_SELECT, balance, feed, sum_fixture}; +use crate::harness::TestServer; + +/// A plain write into the source. It credits 7 to `acc2`. +const PLAIN_INSERT: &str = + "INSERT INTO ptv_post (id, account_id, amount) VALUES ('h1', 'acc2', '7')"; + +/// The answer a write that touched `n` rows gives on these routes. +fn affected(n: u64) -> serde_json::Value { + serde_json::json!({ "affected": n }) +} + +/// Seed the feed `INSERT_SELECT` copies: 4 for `acc`, 6 for `acc2`. +async fn seed_feed(server: &TestServer) { + feed(server, "f1", "acc", "4").await; + feed(server, "f2", "acc2", "6").await; +} + +/// The balances after `PLAIN_INSERT` and `INSERT_SELECT` both ran. +async fn assert_balances(server: &TestServer) { + assert_eq!(balance(server, "acc").await, "19", "10 + 5 + 4"); + assert_eq!(balance(server, "acc2").await, "13", "7 + 6"); +} + +/// POST `sql` to `/v1/query` and return its `rows`. +async fn http_rows(http_port: u16, sql: &str) -> Vec { + let response = reqwest::Client::new() + .post(format!("http://127.0.0.1:{http_port}/v1/query")) + .json(&serde_json::json!({ "sql": sql })) + .send() + .await + .unwrap_or_else(|e| panic!("POST /v1/query {sql}: {e}")); + let status = response.status(); + let body: serde_json::Value = response + .json() + .await + .unwrap_or_else(|e| panic!("decode the /v1/query body for {sql}: {e}")); + assert!(status.is_success(), "{sql} answered {status}: {body}"); + body["rows"] + .as_array() + .unwrap_or_else(|| panic!("a `rows` array for {sql}: {body}")) + .clone() +} + +/// POST `sql` to `/v1/query/stream` and return its lines, parsed. +async fn ndjson_lines(http_port: u16, sql: &str) -> Vec { + let response = reqwest::Client::new() + .post(format!("http://127.0.0.1:{http_port}/v1/query/stream")) + .json(&serde_json::json!({ "sql": sql })) + .send() + .await + .unwrap_or_else(|e| panic!("POST /v1/query/stream {sql}: {e}")); + let status = response.status(); + let body = response + .text() + .await + .unwrap_or_else(|e| panic!("read the /v1/query/stream body for {sql}: {e}")); + assert!(status.is_success(), "{sql} answered {status}: {body}"); + body.lines() + .filter(|line| !line.trim().is_empty()) + .map(|line| { + serde_json::from_str(line) + .unwrap_or_else(|e| panic!("an NDJSON line for {sql}: {e}: {line}")) + }) + .collect() +} + +type WsStream = + tokio_tungstenite::WebSocketStream>; + +/// Send `sql` as RPC `id` and return its `result`. +async fn ws_result(ws: &mut WsStream, id: u64, sql: &str) -> serde_json::Value { + let request = serde_json::json!({ "id": id, "method": "query", "params": { "sql": sql } }); + ws.send(Message::Text(request.to_string().into())) + .await + .unwrap_or_else(|e| panic!("send {sql}: {e}")); + loop { + let message = tokio::time::timeout(std::time::Duration::from_secs(30), ws.next()) + .await + .unwrap_or_else(|_| panic!("no answer to {sql}")) + .unwrap_or_else(|| panic!("the socket closed before answering {sql}")) + .unwrap_or_else(|e| panic!("read the answer to {sql}: {e}")); + let Message::Text(text) = message else { + continue; + }; + let frame: serde_json::Value = serde_json::from_str(&text) + .unwrap_or_else(|e| panic!("a JSON frame for {sql}: {e}: {text}")); + if frame["id"] != id { + continue; + } + assert!(frame.get("error").is_none(), "{sql} failed: {frame}"); + return frame["result"].clone(); + } +} + +#[tokio::test(flavor = "multi_thread", worker_threads = 4)] +async fn http_query_moves_a_cross_shard_sum_atomically() { + let server = sum_fixture().await; + seed_feed(&server).await; + + assert_eq!( + http_rows(server.http_port, PLAIN_INSERT).await, + vec![affected(1)], + "one row: the balance task adds no count" + ); + assert_eq!(balance(&server, "acc2").await, "7", "h1 credited"); + + assert_eq!( + http_rows(server.http_port, INSERT_SELECT).await, + vec![affected(2)], + "two copied rows: a balance task adds no count" + ); + assert_balances(&server).await; +} + +#[tokio::test(flavor = "multi_thread", worker_threads = 4)] +async fn ndjson_query_moves_a_cross_shard_sum_atomically() { + let server = sum_fixture().await; + seed_feed(&server).await; + + assert_eq!( + ndjson_lines(server.http_port, PLAIN_INSERT).await, + vec![affected(1)], + "one line: the balance task adds no count" + ); + assert_eq!(balance(&server, "acc2").await, "7", "h1 credited"); + + assert_eq!( + ndjson_lines(server.http_port, INSERT_SELECT).await, + vec![affected(2)], + "two copied rows: a balance task adds no count" + ); + assert_balances(&server).await; +} + +#[tokio::test(flavor = "multi_thread", worker_threads = 4)] +async fn ws_rpc_query_moves_a_cross_shard_sum_atomically() { + let server = sum_fixture().await; + seed_feed(&server).await; + let (mut ws, _) = + tokio_tungstenite::connect_async(format!("ws://127.0.0.1:{}/v1/ws", server.http_port)) + .await + .expect("connect the WebSocket RPC endpoint"); + + assert_eq!( + ws_result(&mut ws, 1, PLAIN_INSERT).await, + affected(1), + "one row: the balance task adds no count" + ); + assert_eq!(balance(&server, "acc2").await, "7", "h1 credited"); + + assert_eq!( + ws_result(&mut ws, 2, INSERT_SELECT).await, + affected(2), + "two copied rows: a balance task adds no count" + ); + assert_balances(&server).await; +} diff --git a/nodedb/tests/wire/cases/materialized_sum_txn_command_tag.rs b/nodedb/tests/wire/cases/materialized_sum_txn_command_tag.rs new file mode 100644 index 000000000..7a9deb404 --- /dev/null +++ b/nodedb/tests/wire/cases/materialized_sum_txn_command_tag.rs @@ -0,0 +1,127 @@ +// SPDX-License-Identifier: BUSL-1.1 + +//! A write to a materialized-sum source in a transaction block answers with +//! the command tag of its own rows only. +//! +//! A sum-source write whose target lives on another vShard plans a second +//! task: the balance move on the target row. That task is a derived write. +//! Its verb and its count never reach the statement's tag. An `INSERT` +//! answers `INSERT 0 1`, never `INSERT 0 2`. A `DELETE` answers `DELETE 1`, +//! never an error for mixing `DELETE` with the balance move's `UPDATE`. + +use nodedb_types::id::DatabaseId; +use tokio_postgres::SimpleQueryMessage; + +use crate::harness::TestServer; + +/// A target and source that share one vShard: the balance rides the source +/// write's own transaction. +const CO_RESIDENT: (&str, &str) = ("uo_accounts", "uo_entries"); + +/// A target and source on two vShards: the balance travels on its own task. +const CROSS_SHARD: (&str, &str) = ("ptv_acct", "ptv_post"); + +fn vshard(name: &str) -> u32 { + nodedb_types::CollectionKey::from_bare(DatabaseId::DEFAULT, name) + .vshard() + .as_u32() +} + +/// The premise both paths rest on: the names hash as the constants say. +#[test] +fn the_pairs_hash_to_the_paths_they_name() { + assert_eq!(vshard(CO_RESIDENT.0), vshard(CO_RESIDENT.1)); + assert_ne!( + vshard(CROSS_SHARD.0), + vshard(CROSS_SHARD.1), + "the cross-shard pair must plan a separate balance task" + ); +} + +/// Every `CommandComplete` count in `sql`'s simple-query response, in wire +/// order. +async fn command_counts(server: &TestServer, sql: &str) -> Vec { + let messages = server + .client + .simple_query(sql) + .await + .unwrap_or_else(|e| panic!("run {sql}: {e:?}")); + messages + .into_iter() + .filter_map(|m| match m { + SimpleQueryMessage::CommandComplete(n) => Some(n), + _ => None, + }) + .collect() +} + +/// Create `target` with a `balance` summed from `source.amount`, and seed +/// one account and one entry. +async fn sum_pair(server: &TestServer, target: &str, source: &str) { + server + .exec(&format!( + "CREATE COLLECTION {target} (id TEXT PRIMARY KEY, owner TEXT) \ + WITH (engine='document_strict')" + )) + .await + .unwrap(); + server + .exec(&format!( + "CREATE COLLECTION {source} (id TEXT PRIMARY KEY, account_id TEXT, amount TEXT) \ + WITH (engine='document_strict')" + )) + .await + .unwrap(); + server + .exec(&format!( + "ALTER COLLECTION {target} ADD COLUMN balance TEXT \ + MATERIALIZED_SUM SOURCE {source} \ + ON {source}.account_id = {target}.id VALUE {source}.amount" + )) + .await + .unwrap(); + server + .exec(&format!( + "INSERT INTO {target} (id, owner, balance) VALUES ('acc', 'alice', '0')" + )) + .await + .unwrap(); + server + .exec(&format!( + "INSERT INTO {source} (id, account_id, amount) VALUES ('e0', 'acc', '10')" + )) + .await + .unwrap(); +} + +/// `BEGIN; INSERT; DELETE; COMMIT` on a sum source tags each write with its +/// own row count, whether the target shares the source's vShard or not. +#[tokio::test(flavor = "multi_thread", worker_threads = 4)] +async fn a_sum_source_write_in_a_block_tags_only_its_own_rows() { + let server = TestServer::start().await; + for (target, source) in [CO_RESIDENT, CROSS_SHARD] { + sum_pair(&server, target, source).await; + + let sql = format!( + "BEGIN; \ + INSERT INTO {source} (id, account_id, amount) VALUES ('e1', 'acc', '4'); \ + DELETE FROM {source} WHERE id = 'e0'; \ + COMMIT" + ); + assert_eq!( + command_counts(&server, &sql).await, + vec![0, 1, 1, 0], + "BEGIN, INSERT 0 1, DELETE 1, COMMIT for {source}" + ); + + let balance = server + .query_text(&format!("SELECT balance FROM {target} WHERE id = 'acc'")) + .await + .unwrap(); + assert_eq!( + balance, + vec!["4".to_string()], + "{target} holds the inserted entry and not the deleted one" + ); + } +} diff --git a/nodedb/tests/wire/cases/materialized_sum_txn_preimage.rs b/nodedb/tests/wire/cases/materialized_sum_txn_preimage.rs new file mode 100644 index 000000000..c1295497c --- /dev/null +++ b/nodedb/tests/wire/cases/materialized_sum_txn_preimage.rs @@ -0,0 +1,208 @@ +// SPDX-License-Identifier: BUSL-1.1 + +//! A write to a materialized-sum source in a transaction block settles its +//! balance from the row as the transaction sees it. +//! +//! An UPDATE or DELETE of a source row owes the target the difference between +//! the row's pre-image and its post-image. When an earlier statement of the +//! same transaction inserted or rewrote that row, the pre-image is the staged +//! row, not the committed one. A pre-image read from committed base alone +//! misses a staged insert and returns the base value of a staged rewrite. The +//! balance is then wrong by the staged row's contribution. +//! +//! Each test runs on both pairs. On the cross-shard pair the balance travels +//! on its own task, settled from the pre-image at statement time. On the +//! co-resident pair the source write folds it at commit, from the target rows +//! the statement resolved off the same pre-image. + +use nodedb_types::id::DatabaseId; + +use crate::harness::TestServer; + +/// A target and source on one vShard: the balance rides the source write. +const CO_RESIDENT: (&str, &str) = ("uo_accounts", "uo_entries"); +/// A target and source on two vShards: the balance travels on its own task. +const CROSS_SHARD: (&str, &str) = ("ptv_acct", "ptv_post"); +/// Data Plane cores the server runs. +const CORES: usize = 2; + +fn vshard(name: &str) -> u32 { + nodedb_types::CollectionKey::from_bare(DatabaseId::DEFAULT, name) + .vshard() + .as_u32() +} + +/// The premise both paths rest on: the names hash as the constants say, and +/// the cross-shard pair lands on two cores. +#[test] +fn the_pairs_hash_to_the_paths_they_name() { + assert_eq!(vshard(CO_RESIDENT.0), vshard(CO_RESIDENT.1)); + assert_ne!( + vshard(CROSS_SHARD.0), + vshard(CROSS_SHARD.1), + "the cross-shard pair must plan a separate balance task" + ); + assert_ne!( + vshard(CROSS_SHARD.0) as usize % CORES, + vshard(CROSS_SHARD.1) as usize % CORES, + "a balance folded on the source's core must find no target row" + ); +} + +async fn exec(server: &TestServer, sql: &str) { + server + .exec(sql) + .await + .unwrap_or_else(|e| panic!("run {sql}: {e}")); +} + +/// Start a server and create both sum pairs. Each pair has accounts `acc-0` +/// and `acc-1`, and posting `e0` puts 10 on `acc-0`. +async fn fixture() -> TestServer { + let server = TestServer::start_multicores(CORES).await; + for (target, source) in [CO_RESIDENT, CROSS_SHARD] { + exec( + &server, + &format!( + "CREATE COLLECTION {target} (id TEXT PRIMARY KEY, owner TEXT) \ + WITH (engine='document_strict')" + ), + ) + .await; + exec( + &server, + &format!( + "CREATE COLLECTION {source} (id TEXT PRIMARY KEY, account_id TEXT, amount TEXT) \ + WITH (engine='document_strict')" + ), + ) + .await; + exec( + &server, + &format!( + "ALTER COLLECTION {target} ADD COLUMN balance TEXT \ + MATERIALIZED_SUM SOURCE {source} \ + ON {source}.account_id = {target}.id VALUE {source}.amount" + ), + ) + .await; + for account in ["acc-0", "acc-1"] { + exec( + &server, + &format!( + "INSERT INTO {target} (id, owner, balance) VALUES ('{account}', 'alice', '0')" + ), + ) + .await; + } + exec( + &server, + &format!("INSERT INTO {source} (id, account_id, amount) VALUES ('e0', 'acc-0', '10')"), + ) + .await; + assert_eq!( + balance(&server, target, "acc-0").await, + "10", + "the seed posting lands on {target}" + ); + } + server +} + +/// The stored balance of `account` in `target`. +async fn balance(server: &TestServer, target: &str, account: &str) -> String { + let rows = server + .query_text(&format!( + "SELECT balance FROM {target} WHERE id = '{account}'" + )) + .await + .unwrap_or_else(|e| panic!("read the balance of {target}.{account}: {e}")); + match rows.as_slice() { + [value] => value.clone(), + other => panic!("one balance row for {target}.{account}, got {other:?}"), + } +} + +/// Run `statements` in one transaction block on every pair, then check each +/// pair's balances of `acc-0` and `acc-1`. +async fn run_block(statements: &[&str], expected: (&str, &str)) { + let server = fixture().await; + for (target, source) in [CO_RESIDENT, CROSS_SHARD] { + let body: Vec = statements + .iter() + .map(|statement| statement.replace("{source}", source)) + .collect(); + let sql = format!("BEGIN; {}; COMMIT", body.join("; ")); + exec(&server, &sql).await; + assert_eq!( + balance(&server, target, "acc-0").await, + expected.0, + "acc-0 of {target} after {sql}" + ); + assert_eq!( + balance(&server, target, "acc-1").await, + expected.1, + "acc-1 of {target} after {sql}" + ); + } +} + +/// A row inserted and deleted in one block leaves the balance unchanged. The +/// delete's pre-image is the staged row, so it debits the amount the insert +/// credited. +#[tokio::test(flavor = "multi_thread", worker_threads = 4)] +async fn a_staged_insert_then_delete_leaves_the_balance_unchanged() { + run_block( + &[ + "INSERT INTO {source} (id, account_id, amount) VALUES ('e1', 'acc-0', '5')", + "DELETE FROM {source} WHERE id = 'e1'", + ], + ("10", "0"), + ) + .await; +} + +/// A row inserted and then updated in one block credits the updated amount. +/// The update's pre-image is the staged row, so it moves the difference. +#[tokio::test(flavor = "multi_thread", worker_threads = 4)] +async fn a_staged_insert_then_update_credits_the_updated_amount() { + run_block( + &[ + "INSERT INTO {source} (id, account_id, amount) VALUES ('e1', 'acc-0', '5')", + "UPDATE {source} SET amount = '7' WHERE id = 'e1'", + ], + ("17", "0"), + ) + .await; +} + +/// A row moved to another account and then updated in one block: the old +/// account loses the original amount and the new account holds the updated +/// one. The second update's pre-image names the account the first moved the +/// row to. +#[tokio::test(flavor = "multi_thread", worker_threads = 4)] +async fn a_staged_move_then_update_settles_on_the_new_account() { + run_block( + &[ + "UPDATE {source} SET account_id = 'acc-1' WHERE id = 'e0'", + "UPDATE {source} SET amount = '9' WHERE id = 'e0'", + ], + ("0", "9"), + ) + .await; +} + +/// A row inserted and then moved to another account in one block credits +/// only the account it ends on. The move's pre-image is the staged row, so +/// the statement resolves and debits the account the insert credited. +#[tokio::test(flavor = "multi_thread", worker_threads = 4)] +async fn a_staged_insert_then_move_credits_only_the_final_account() { + run_block( + &[ + "INSERT INTO {source} (id, account_id, amount) VALUES ('e1', 'acc-0', '5')", + "UPDATE {source} SET account_id = 'acc-1' WHERE id = 'e1'", + ], + ("10", "5"), + ) + .await; +} diff --git a/nodedb/tests/wire/cases/mod.rs b/nodedb/tests/wire/cases/mod.rs index 5696b97e4..96e339a60 100644 --- a/nodedb/tests/wire/cases/mod.rs +++ b/nodedb/tests/wire/cases/mod.rs @@ -127,6 +127,10 @@ mod kv_sql_select; mod kv_transfer_decimal; mod kv_write_row_level_security; mod kv_write_row_level_security_atomics; +mod materialized_sum_cross_shard_expanders; +mod materialized_sum_cross_shard_http; +mod materialized_sum_txn_command_tag; +mod materialized_sum_txn_preimage; mod merge_insert_atomicity; mod merge_insert_cross_engine_visibility; mod merge_insert_renamed_source_column; @@ -169,6 +173,7 @@ mod query_function_engine_gate; mod quota_bitemporal_composition; mod quota_sieve_routing_composition; mod redaction_policy_database_scope; +mod redo_chunked_session_commit; mod reindex_concurrent; mod restart_refused_write_not_resurrected; mod rls_policy_database_scope; diff --git a/nodedb/tests/wire/cases/redo_chunked_session_commit.rs b/nodedb/tests/wire/cases/redo_chunked_session_commit.rs new file mode 100644 index 000000000..2ffc1b04a --- /dev/null +++ b/nodedb/tests/wire/cases/redo_chunked_session_commit.rs @@ -0,0 +1,80 @@ +// SPDX-License-Identifier: BUSL-1.1 + +//! A session COMMIT whose redo passes `tuning.calvin.max_redo_entry_bytes` +//! travels through its data-group log as chunk entries and a final entry. +//! The commit installs every row once, live and after a WAL-only restart. + +use crate::harness::TestServer; + +/// The redo entry limit the server boots with: the smallest the config takes. +const ENTRY_LIMIT: usize = 64 * 1024; + +/// Rows one statement inserts. +const ROWS_PER_STATEMENT: usize = 100; + +/// Statements one transaction runs. With 400-byte values the redo holds +/// several times `ENTRY_LIMIT`. +const STATEMENTS: usize = 6; + +async fn count(srv: &TestServer, coll: &str) -> usize { + let rows = srv + .query_text(&format!("SELECT COUNT(*) FROM {coll}")) + .await + .unwrap(); + rows[0].parse().expect("COUNT(*) is a number") +} + +/// Insert `STATEMENTS * ROWS_PER_STATEMENT` rows with ids from `prefix` in +/// one transaction. +async fn commit_large_transaction(srv: &TestServer, coll: &str, prefix: &str) { + let filler = "x".repeat(400); + srv.exec("BEGIN").await.unwrap(); + for statement in 0..STATEMENTS { + let values: Vec = (0..ROWS_PER_STATEMENT) + .map(|row| { + let id = statement * ROWS_PER_STATEMENT + row; + format!("('{prefix}{id}', '{filler}')") + }) + .collect(); + srv.exec(&format!( + "INSERT INTO {coll} (id, value) VALUES {}", + values.join(", ") + )) + .await + .unwrap(); + } + srv.exec("COMMIT").await.unwrap(); +} + +#[tokio::test(flavor = "multi_thread")] +async fn a_chunked_session_commit_installs_once() { + let srv = TestServer::start_with_max_redo_entry_bytes(ENTRY_LIMIT).await; + srv.exec( + "CREATE COLLECTION chunked_commit (id STRING PRIMARY KEY, value STRING) \ + WITH (engine='document_strict')", + ) + .await + .unwrap(); + + commit_large_transaction(&srv, "chunked_commit", "a").await; + let expected = STATEMENTS * ROWS_PER_STATEMENT; + assert_eq!( + count(&srv, "chunked_commit").await, + expected, + "a chunked commit must install every row exactly once" + ); + + // The finished stream frees its bytes and its floor: a second large + // commit chunks and installs too. + commit_large_transaction(&srv, "chunked_commit", "b").await; + assert_eq!(count(&srv, "chunked_commit").await, 2 * expected); + + let (srv, dir) = srv.take_dir(); + srv.graceful_shutdown().await; + let (srv, _dir) = TestServer::open_on_path_with_max_redo_entry_bytes(dir, ENTRY_LIMIT).await; + assert_eq!( + count(&srv, "chunked_commit").await, + 2 * expected, + "WAL replay of a chunked commit must restore every row exactly once" + ); +} diff --git a/nodedb/tests/wire/cases/sql_backup_calvin_cut.rs b/nodedb/tests/wire/cases/sql_backup_calvin_cut.rs index 2eac7356b..2d64c00de 100644 --- a/nodedb/tests/wire/cases/sql_backup_calvin_cut.rs +++ b/nodedb/tests/wire/cases/sql_backup_calvin_cut.rs @@ -3,11 +3,12 @@ //! A backup's consistent cut covers Calvin transactions. //! //! A transaction that writes two collections on two vShards commits through -//! the Calvin scheduler. Its install records the transaction's commit HLC on -//! the tenant's write mark before the COMMIT is acknowledged, so a Calvin -//! commit after a backup refuses a restore of it. A Calvin transaction held -//! at its flush while a backup starts was sequenced before the backup's cut, -//! so the backup waits for its install and holds its rows. +//! the Calvin scheduler. The data-group apply of its stamped redo records the +//! transaction's commit HLC on the tenant's write mark before the COMMIT is +//! acknowledged, so a Calvin commit after a backup refuses a restore of it. A +//! Calvin transaction whose redo proposal is held while a backup starts was +//! sequenced before the backup's cut, so the backup waits for its install and +//! holds its rows. use super::backup_support::{drain_backup, names_on_two_vshards, push_restore}; use crate::harness::TestServer; @@ -65,30 +66,32 @@ async fn a_calvin_commit_after_the_backup_refuses_the_restore() { "expected the staleness refusal, got: {error}" ); assert!( - error.contains("calvin flush"), - "the refusal must name the Calvin commit as the newer write, got: {error}" + error.contains("replicated apply"), + "the refusal must name the Calvin commit's data-group apply as the newer write, \ + got: {error}" ); } #[cfg(feature = "failpoints")] #[tokio::test(flavor = "multi_thread", worker_threads = 4)] -async fn a_calvin_commit_held_at_its_flush_is_in_the_backup_or_refuses_its_restore() { +async fn a_calvin_commit_held_at_its_redo_is_in_the_backup_or_refuses_its_restore() { use std::time::Duration; const PARKED_FOR: Duration = Duration::from_millis(1500); let (first, second) = names_on_two_vshards("calvin_cut"); let gate_dir = tempfile::tempdir().expect("gate tempdir"); - let release = gate_dir.path().join("release-flush"); + let release = gate_dir.path().join("release-redo"); let server = TestServer::start_with_failpoints(&format!( - "calvin::before_flush::{first}=wait_file({})", + "calvin::before_redo_propose::{first}=wait_file({})", release.display() )) .await; create_kv(&server, &first).await; create_kv(&server, &second).await; - // Hold the Calvin transaction at its flush on `first`'s vShard. + // Hold the redo proposal of the Calvin transaction's slice on `first`'s + // vShard. let (committer, committer_handle) = server .connect_as("nodedb", "nodedb") .await @@ -106,7 +109,7 @@ async fn a_calvin_commit_held_at_its_flush_is_in_the_backup_or_refuses_its_resto tokio::time::sleep(PARKED_FOR).await; assert!( !commit.is_finished(), - "the COMMIT was not held at its flush" + "the COMMIT was not held at its redo proposal" ); // Back up while the transaction is sequenced but not installed. @@ -121,7 +124,7 @@ async fn a_calvin_commit_held_at_its_flush_is_in_the_backup_or_refuses_its_resto "the backup snapshotted before a Calvin transaction sequenced ahead of its cut installed" ); - std::fs::write(&release, b"release").expect("release the flush"); + std::fs::write(&release, b"release").expect("release the redo proposal"); commit .await .expect("commit task") diff --git a/nodedb/tests/wire/cases/sql_backup_restore_durable_marks.rs b/nodedb/tests/wire/cases/sql_backup_restore_durable_marks.rs index 1f5f8847d..27171307a 100644 --- a/nodedb/tests/wire/cases/sql_backup_restore_durable_marks.rs +++ b/nodedb/tests/wire/cases/sql_backup_restore_durable_marks.rs @@ -103,7 +103,7 @@ async fn a_calvin_commit_after_the_backup_refuses_the_restore_after_a_restart() "expected the staleness refusal, got: {error}" ); assert!( - error.contains("calvin flush"), + error.contains("replicated apply"), "the refusal must name the Calvin commit as the newer write, got: {error}" ); } diff --git a/nodedb/tests/wire/harness/config_toml.rs b/nodedb/tests/wire/harness/config_toml.rs index 64425e0ca..86f860b40 100644 --- a/nodedb/tests/wire/harness/config_toml.rs +++ b/nodedb/tests/wire/harness/config_toml.rs @@ -49,6 +49,9 @@ pub(super) struct TuningOverrides { /// Overrides `[tuning.vector] seal_threshold` so a test can observe HNSW /// segment builds on a few hundred vectors. pub(super) vector_seal_threshold: Option, + /// Overrides `[tuning.calvin] max_redo_entry_bytes` so a test can drive + /// chunked redo with a commit of a few hundred kilobytes. + pub(super) max_redo_entry_bytes: Option, } impl TuningOverrides { @@ -89,6 +92,14 @@ impl TuningOverrides { } } + /// Boot with a lowered redo entry limit. + pub(super) fn max_redo_entry_bytes(bytes: usize) -> Self { + Self { + max_redo_entry_bytes: Some(bytes), + ..Self::default() + } + } + /// Boot with a lowered vector seal threshold. pub(super) fn vector_seal_threshold(vectors: usize) -> Self { Self { @@ -141,6 +152,11 @@ pub(super) fn write_config(dir: &Path, auth_mode: AuthMode, tuning: TuningOverri if let Some(vectors) = tuning.vector_seal_threshold { toml.push_str(&format!("\n[tuning.vector]\nseal_threshold = {vectors}\n")); } + if let Some(bytes) = tuning.max_redo_entry_bytes { + toml.push_str(&format!( + "\n[tuning.calvin]\nmax_redo_entry_bytes = {bytes}\n" + )); + } toml.push_str(&format!( "\n[backup_encryption]\nkey_path = {}\n", toml_quote(&write_backup_kek(dir)) diff --git a/nodedb/tests/wire/harness/lifecycle.rs b/nodedb/tests/wire/harness/lifecycle.rs index ae661f291..799837cf4 100644 --- a/nodedb/tests/wire/harness/lifecycle.rs +++ b/nodedb/tests/wire/harness/lifecycle.rs @@ -162,6 +162,36 @@ impl TestServer { (server, dir) } + /// Spawn a single-core NodeDB server with a lowered redo entry limit, so + /// a commit of a few hundred kilobytes travels as a chunked redo. + pub async fn start_with_max_redo_entry_bytes(bytes: usize) -> Self { + let dir = tempfile::tempdir().expect("tempdir"); + let spawned = process::spawn( + dir.path(), + AuthMode::Trust, + TuningOverrides::max_redo_entry_bytes(bytes), + 1, + ); + Self::connect_and_build(spawned, dir, AuthMode::Trust).await + } + + /// Reopen `dir` with the lowered redo entry limit a server started by + /// [`Self::start_with_max_redo_entry_bytes`] used. + pub async fn open_on_path_with_max_redo_entry_bytes( + dir: TestDataDir, + bytes: usize, + ) -> (Self, TestDataDir) { + let spawned = process::spawn( + dir.path(), + AuthMode::Trust, + TuningOverrides::max_redo_entry_bytes(bytes), + 1, + ); + let placeholder = tempfile::tempdir().expect("placeholder tempdir"); + let server = Self::connect_and_build(spawned, placeholder, AuthMode::Trust).await; + (server, dir) + } + /// Open a server backed by an existing data directory, reopened in place /// so a previous server's data is visible after boot. `dir` is not /// consumed — ownership stays with the caller.